RDMA/irdma: Fix RQ completion opcode

The opcode written by HW, in the RQ CQE, is the
RoCEv2/iWARP protocol opcode from the received
packet and not the SW opcode as currently assumed.
Fix this by returning the raw operation type and
queue type in the CQE to irdma_process_cqe and add
2 helpers set_ib_wc_op_sq set_ib_wc_op_rq to map
IRDMA HW op types to IB op types.

Note that for iWARP, only Write with Immediate is
supported so the opcode can only be IB_WC_RECV_RDMA_WITH_IMM
when there is immediate data present.

Fixes: b48c24c2d710 ("RDMA/irdma: Implement device supported verb APIs")
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
Link: https://lore.kernel.org/r/20221115011701.1379-3-shiraz.saleem@intel.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c
index 1a57ed9d77ff3260b563c35b3f8b7a27bb5bc1ac..16183e894da77b1742291591e15321c56405b06e 100644 (file)
--- a/drivers/infiniband/hw/irdma/uk.c
+++ b/drivers/infiniband/hw/irdma/uk.c
@@ -1049,11 +1049,10 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
        __le64 *cqe;
        struct irdma_qp_uk *qp;
        struct irdma_ring *pring = NULL;
-       u32 wqe_idx, q_type;
+       u32 wqe_idx;
        int ret_code;
        bool move_cq_head = true;
        u8 polarity;
-       u8 op_type;
        bool ext_valid;
        __le64 *ext_cqe;
 
@@ -1121,7 +1120,7 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
                info->ud_vlan_valid = false;
        }
 
-       q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
+       info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
        info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
        info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3);
        info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
@@ -1160,8 +1159,9 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
        }
        wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
        info->qp_handle = (irdma_qp_handle)(unsigned long)qp;
+       info->op_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
 
-       if (q_type == IRDMA_CQE_QTYPE_RQ) {
+       if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
                u32 array_idx;
 
                array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
@@ -1181,10 +1181,6 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
 
                info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
 
-               if (info->imm_valid)
-                       info->op_type = IRDMA_OP_TYPE_REC_IMM;
-               else
-                       info->op_type = IRDMA_OP_TYPE_REC;
                if (qword3 & IRDMACQ_STAG) {
                        info->stag_invalid_set = true;
                        info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);
@@ -1242,17 +1238,18 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
                                sw_wqe = qp->sq_base[tail].elem;
                                get_64bit_val(sw_wqe, 24,
                                              &wqe_qword);
-                               op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
-                               info->op_type = op_type;
+                               info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE,
+                                                             wqe_qword);
                                IRDMA_RING_SET_TAIL(qp->sq_ring,
                                                    tail + qp->sq_wrtrk_array[tail].quanta);
-                               if (op_type != IRDMAQP_OP_NOP) {
+                               if (info->op_type != IRDMAQP_OP_NOP) {
                                        info->wr_id = qp->sq_wrtrk_array[tail].wrid;
                                        info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
                                        break;
                                }
                        } while (1);
-                       if (op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR)
+                       if (info->op_type == IRDMA_OP_TYPE_BIND_MW &&
+                           info->minor_err == FLUSH_PROT_ERR)
                                info->minor_err = FLUSH_MW_BIND_ERR;
                        qp->sq_flush_seen = true;
                        if (!IRDMA_RING_MORE_WORK(qp->sq_ring))

diff --git a/drivers/infiniband/hw/irdma/user.h b/drivers/infiniband/hw/irdma/user.h
index 424d4aa8cdcd50afd30e9fc0efcd319293ee55af..d0cdf609f5e06aa71c24a5471b63b8f8aab2ba36 100644 (file)
--- a/drivers/infiniband/hw/irdma/user.h
+++ b/drivers/infiniband/hw/irdma/user.h
@@ -245,6 +245,7 @@ struct irdma_cq_poll_info {
        u16 ud_vlan;
        u8 ud_smac[6];
        u8 op_type;
+       u8 q_type;
        bool stag_invalid_set:1; /* or L_R_Key set */
        bool push_dropped:1;
        bool error:1;

diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c
index 8dfc9e154d733aba6dbfc47b82276dfdfe710c75..445e69e864097ef85adbf351387eccd08150ffcb 100644 (file)
--- a/drivers/infiniband/hw/irdma/utils.c
+++ b/drivers/infiniband/hw/irdma/utils.c
@@ -2591,6 +2591,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp)
                        sw_wqe = qp->sq_base[wqe_idx].elem;
                        get_64bit_val(sw_wqe, 24, &wqe_qword);
                        cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
+                       cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
                        /* remove the SQ WR by moving SQ tail*/
                        IRDMA_RING_SET_TAIL(*sq_ring,
                                sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
@@ -2629,6 +2630,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp)
 
                        cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
                        cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
+                       cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
                        /* remove the RQ WR by moving RQ tail */
                        IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
                        ibdev_dbg(iwqp->iwrcq->ibcq.device,

diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index e252f431e2aca4b370e76617031a930f5a7b61be..01d0dc4b564902e9c77ddb1401d52b879d8fffbc 100644 (file)
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -3334,7 +3334,6 @@ static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode
 static void irdma_process_cqe(struct ib_wc *entry,
                              struct irdma_cq_poll_info *cq_poll_info)
 {
-       struct irdma_qp *iwqp;
        struct irdma_sc_qp *qp;
 
        entry->wc_flags = 0;
@@ -3342,7 +3341,6 @@ static void irdma_process_cqe(struct ib_wc *entry,
        entry->wr_id = cq_poll_info->wr_id;
 
        qp = cq_poll_info->qp_handle;
-       iwqp = qp->qp_uk.back_qp;
        entry->qp = qp->qp_uk.back_qp;
 
        if (cq_poll_info->error) {
@@ -3375,42 +3373,17 @@ static void irdma_process_cqe(struct ib_wc *entry,
                }
        }
 
-       switch (cq_poll_info->op_type) {
-       case IRDMA_OP_TYPE_RDMA_WRITE:
-       case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
-               entry->opcode = IB_WC_RDMA_WRITE;
-               break;
-       case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
-       case IRDMA_OP_TYPE_RDMA_READ:
-               entry->opcode = IB_WC_RDMA_READ;
-               break;
-       case IRDMA_OP_TYPE_SEND_INV:
-       case IRDMA_OP_TYPE_SEND_SOL:
-       case IRDMA_OP_TYPE_SEND_SOL_INV:
-       case IRDMA_OP_TYPE_SEND:
-               entry->opcode = IB_WC_SEND;
-               break;
-       case IRDMA_OP_TYPE_FAST_REG_NSMR:
-               entry->opcode = IB_WC_REG_MR;
-               break;
-       case IRDMA_OP_TYPE_INV_STAG:
-               entry->opcode = IB_WC_LOCAL_INV;
-               break;
-       case IRDMA_OP_TYPE_REC_IMM:
-       case IRDMA_OP_TYPE_REC:
-               entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ?
-                       IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
+       if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) {
+               set_ib_wc_op_sq(cq_poll_info, entry);
+       } else {
+               set_ib_wc_op_rq(cq_poll_info, entry,
+                               qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM ?
+                               true : false);
                if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD &&
                    cq_poll_info->stag_invalid_set) {
                        entry->ex.invalidate_rkey = cq_poll_info->inv_stag;
                        entry->wc_flags |= IB_WC_WITH_INVALIDATE;
                }
-               break;
-       default:
-               ibdev_err(&iwqp->iwdev->ibdev,
-                         "Invalid opcode = %d in CQE\n", cq_poll_info->op_type);
-               entry->status = IB_WC_GENERAL_ERR;
-               return;
        }
 
        if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) {

diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h
index 4309b7159f42c8c634cb1e4293c3d5e5f64985c7..a536e9fa85ebf1b84a5865401e787dc3278b9e8e 100644 (file)
--- a/drivers/infiniband/hw/irdma/verbs.h
+++ b/drivers/infiniband/hw/irdma/verbs.h
@@ -232,6 +232,59 @@ static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev)
        return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
 }
 
+static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
+                                  struct ib_wc *entry)
+{
+       switch (cq_poll_info->op_type) {
+       case IRDMA_OP_TYPE_RDMA_WRITE:
+       case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
+               entry->opcode = IB_WC_RDMA_WRITE;
+               break;
+       case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
+       case IRDMA_OP_TYPE_RDMA_READ:
+               entry->opcode = IB_WC_RDMA_READ;
+               break;
+       case IRDMA_OP_TYPE_SEND_SOL:
+       case IRDMA_OP_TYPE_SEND_SOL_INV:
+       case IRDMA_OP_TYPE_SEND_INV:
+       case IRDMA_OP_TYPE_SEND:
+               entry->opcode = IB_WC_SEND;
+               break;
+       case IRDMA_OP_TYPE_FAST_REG_NSMR:
+               entry->opcode = IB_WC_REG_MR;
+               break;
+       case IRDMA_OP_TYPE_INV_STAG:
+               entry->opcode = IB_WC_LOCAL_INV;
+               break;
+       default:
+               entry->status = IB_WC_GENERAL_ERR;
+       }
+}
+
+static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
+                                  struct ib_wc *entry, bool send_imm_support)
+{
+       /**
+        * iWARP does not support sendImm, so the presence of Imm data
+        * must be WriteImm.
+        */
+       if (!send_imm_support) {
+               entry->opcode = cq_poll_info->imm_valid ?
+                                       IB_WC_RECV_RDMA_WITH_IMM :
+                                       IB_WC_RECV;
+               return;
+       }
+
+       switch (cq_poll_info->op_type) {
+       case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
+       case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+               entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+               break;
+       default:
+               entry->opcode = IB_WC_RECV;
+       }
+}
+
 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4);
 int irdma_ib_register_device(struct irdma_device *iwdev);
 void irdma_ib_unregister_device(struct irdma_device *iwdev);