RDMA/rtrs-clt: Use minimal max_send_sge when create qp

We use device limit max_send_sge, which is suboptimal for memory usage.
We don't need that much for User Con, 1 is enough. And for IO con,
sess->max_segments + 1 is enough

Link: https://lore.kernel.org/r/20210614090337.29557-3-jinpu.wang@ionos.com
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
Signed-off-by: Guoqing Jiang <guoqing.jiang@cloud.ionos.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index f1fd7ae9ac5378fe767d37b3d448c17a5cd86fac..cd53edddfe1f7714a67a885e4b5ade203cfe85bb 100644 (file)
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -1417,7 +1417,6 @@ static void query_fast_reg_mode(struct rtrs_clt_sess *sess)
        sess->max_pages_per_mr =
                min3(sess->max_pages_per_mr, (u32)max_pages_per_mr,
                     ib_dev->attrs.max_fast_reg_page_list_len);
-       sess->max_send_sge = ib_dev->attrs.max_send_sge;
 }
 
 static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess,
@@ -1573,7 +1572,7 @@ static void destroy_con(struct rtrs_clt_con *con)
 static int create_con_cq_qp(struct rtrs_clt_con *con)
 {
        struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
-       u32 max_send_wr, max_recv_wr, cq_size;
+       u32 max_send_wr, max_recv_wr, cq_size, max_send_sge;
        int err, cq_vector;
        struct rtrs_msg_rkey_rsp *rsp;
 
@@ -1587,6 +1586,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con)
                 */
                max_send_wr = SERVICE_CON_QUEUE_DEPTH * 2 + 2;
                max_recv_wr = SERVICE_CON_QUEUE_DEPTH * 2 + 2;
+               max_send_sge = 1;
                /* We must be the first here */
                if (WARN_ON(sess->s.dev))
                        return -EINVAL;
@@ -1625,25 +1625,27 @@ static int create_con_cq_qp(struct rtrs_clt_con *con)
                max_recv_wr =
                        min_t(int, sess->s.dev->ib_dev->attrs.max_qp_wr,
                              sess->queue_depth * 3 + 1);
+               max_send_sge = sess->clt->max_segments + 1;
        }
+       cq_size = max_send_wr + max_recv_wr;
        /* alloc iu to recv new rkey reply when server reports flags set */
        if (sess->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) {
-               con->rsp_ius = rtrs_iu_alloc(max_recv_wr, sizeof(*rsp),
+               con->rsp_ius = rtrs_iu_alloc(cq_size, sizeof(*rsp),
                                              GFP_KERNEL, sess->s.dev->ib_dev,
                                              DMA_FROM_DEVICE,
                                              rtrs_clt_rdma_done);
                if (!con->rsp_ius)
                        return -ENOMEM;
-               con->queue_size = max_recv_wr;
+               con->queue_size = cq_size;
        }
        cq_size = max_send_wr + max_recv_wr;
        cq_vector = con->cpu % sess->s.dev->ib_dev->num_comp_vectors;
        if (con->c.cid >= sess->s.irq_con_num)
-               err = rtrs_cq_qp_create(&sess->s, &con->c, sess->max_send_sge,
+               err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge,
                                        cq_vector, cq_size, max_send_wr,
                                        max_recv_wr, IB_POLL_DIRECT);
        else
-               err = rtrs_cq_qp_create(&sess->s, &con->c, sess->max_send_sge,
+               err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge,
                                        cq_vector, cq_size, max_send_wr,
                                        max_recv_wr, IB_POLL_SOFTIRQ);
        /*

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
index 919c9f96f25b70708891377a3383e2552364ecd5..822a820540d4633a574731485686a398f33b65c8 100644 (file)
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
@@ -141,7 +141,6 @@ struct rtrs_clt_sess {
        u32                     chunk_size;
        size_t                  queue_depth;
        u32                     max_pages_per_mr;
-       int                     max_send_sge;
        u32                     flags;
        struct kobject          kobj;
        u8                      for_new_clt;