RDMA/irdma: Add wait for suspend on SQD

Currently, there is no wait for the QP suspend to complete on a modify
to SQD state. Add a wait, after the modify to SQD state, for the Suspend
Complete AE. While we are at it, update the suspend timeout value in
irdma_prep_tc_change to use IRDMA_EVENT_TIMEOUT_MS too.

Fixes: b48c24c2d710 ("RDMA/irdma: Implement device supported verb APIs")
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
Link: https://lore.kernel.org/r/20231114170246.238-3-shiraz.saleem@intel.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
index 8fa7e4a18e737ae55243f0e160af0b2cbad2277c..74f6bc7b7ad1269ce8803e229a67e06d00e4799c 100644 (file)
--- a/drivers/infiniband/hw/irdma/hw.c
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -321,7 +321,11 @@ static void irdma_process_aeq(struct irdma_pci_f *rf)
                        break;
                case IRDMA_AE_QP_SUSPEND_COMPLETE:
                        if (iwqp->iwdev->vsi.tc_change_pending) {
-                               atomic_dec(&iwqp->sc_qp.vsi->qp_suspend_reqs);
+                               if (!atomic_dec_return(&qp->vsi->qp_suspend_reqs))
+                                       wake_up(&iwqp->iwdev->suspend_wq);
+                       }
+                       if (iwqp->suspend_pending) {
+                               iwqp->suspend_pending = false;
                                wake_up(&iwqp->iwdev->suspend_wq);
                        }
                        break;

diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c
index 9ac48b4dab413d4f4eef6a51ebe7b3fd13e97a6c..3f13200ff71bc03a3f5e7817af12390eded1bc3b 100644 (file)
--- a/drivers/infiniband/hw/irdma/main.c
+++ b/drivers/infiniband/hw/irdma/main.c
@@ -48,7 +48,7 @@ static void irdma_prep_tc_change(struct irdma_device *iwdev)
        /* Wait for all qp's to suspend */
        wait_event_timeout(iwdev->suspend_wq,
                           !atomic_read(&iwdev->vsi.qp_suspend_reqs),
-                          IRDMA_EVENT_TIMEOUT);
+                          msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS));
        irdma_ws_reset(&iwdev->vsi);
 }
 

diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h
index d66d87bb8bc4d8935de8a2f7a498555b506e3ce0..b65bc2ea542f56b2e72bae1db491dd5969c620e2 100644 (file)
--- a/drivers/infiniband/hw/irdma/main.h
+++ b/drivers/infiniband/hw/irdma/main.h
@@ -78,7 +78,7 @@ extern struct auxiliary_driver i40iw_auxiliary_drv;
 
 #define MAX_DPC_ITERATIONS     128
 
-#define IRDMA_EVENT_TIMEOUT            50000
+#define IRDMA_EVENT_TIMEOUT_MS         5000
 #define IRDMA_VCHNL_EVENT_TIMEOUT      100000
 #define IRDMA_RST_TIMEOUT_HZ           4
 

diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index 36e69e6ca9f8aff6759b720ead19340160800722..5fa88e6cca4e907c83ffbe8f076266812a9baf0f 100644 (file)
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -1157,6 +1157,21 @@ exit:
        return prio;
 }
 
+static int irdma_wait_for_suspend(struct irdma_qp *iwqp)
+{
+       if (!wait_event_timeout(iwqp->iwdev->suspend_wq,
+                               !iwqp->suspend_pending,
+                               msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) {
+               iwqp->suspend_pending = false;
+               ibdev_warn(&iwqp->iwdev->ibdev,
+                          "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n",
+                          iwqp->ibqp.qp_num, iwqp->last_aeq);
+               return -EBUSY;
+       }
+
+       return 0;
+}
+
 /**
  * irdma_modify_qp_roce - modify qp request
  * @ibqp: qp's pointer for modify
@@ -1420,6 +1435,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 
                        info.next_iwarp_state = IRDMA_QP_STATE_SQD;
                        issue_modify_qp = 1;
+                       iwqp->suspend_pending = true;
                        break;
                case IB_QPS_SQE:
                case IB_QPS_ERR:
@@ -1460,6 +1476,11 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
                        ctx_info->rem_endpoint_idx = udp_info->arp_idx;
                        if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))
                                return -EINVAL;
+                       if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) {
+                               ret = irdma_wait_for_suspend(iwqp);
+                               if (ret)
+                                       return ret;
+                       }
                        spin_lock_irqsave(&iwqp->lock, flags);
                        if (iwqp->iwarp_state == info.curr_iwarp_state) {
                                iwqp->iwarp_state = info.next_iwarp_state;

diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h
index c42ac22de00e9372f7db62fdc5b8b691fac99c45..cfa140b36395ae9f49a9b928baa4d5a1a0aaf336 100644 (file)
--- a/drivers/infiniband/hw/irdma/verbs.h
+++ b/drivers/infiniband/hw/irdma/verbs.h
@@ -198,6 +198,7 @@ struct irdma_qp {
        u8 flush_issued : 1;
        u8 sig_all : 1;
        u8 pau_mode : 1;
+       u8 suspend_pending : 1;
        u8 rsvd : 1;
        u8 iwarp_state;
        u16 term_sq_flush_code;