IB/core: add max_send_sge and max_recv_sge attributes

This patch replaces the ib_device_attr.max_sge with max_send_sge and
max_recv_sge. It allows ulps to take advantage of devices that have very
different send and recv sge depths.  For example cxgb4 has a max_recv_sge
of 4, yet a max_send_sge of 16.  Splitting out these attributes allows
much more efficient use of the SQ for cxgb4 with ulps that use the RDMA_RW
API. Consider a large RDMA WRITE that has 16 scattergather entries.
With max_sge of 4, the ulp would send 4 WRITE WRs, but with max_sge of
16, it can be done with 1 WRITE WR.

Acked-by: Sagi Grimberg <sagi@grimberg.me>
Acked-by: Christoph Hellwig <hch@lst.de>
Acked-by: Selvin Xavier <selvin.xavier@broadcom.com>
Acked-by: Shiraz Saleem <shiraz.saleem@intel.com>
Acked-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 5733d0fb06737b3c1a0ea73d683e593b6b9b432e..908ee8ab32972fa8bc3ef3fcb70521ad10643405 100644 (file)
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -189,7 +189,7 @@ static void copy_query_dev_fields(struct ib_uverbs_file *file,
        resp->max_qp            = attr->max_qp;
        resp->max_qp_wr         = attr->max_qp_wr;
        resp->device_cap_flags  = lower_32_bits(attr->device_cap_flags);
-       resp->max_sge           = attr->max_sge;
+       resp->max_sge           = min(attr->max_send_sge, attr->max_recv_sge);
        resp->max_sge_rd        = attr->max_sge_rd;
        resp->max_cq            = attr->max_cq;
        resp->max_cqe           = attr->max_cqe;

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 136eaa78ad4ad5174d108c6c42ef4e8cf5eac329..6c0c6d3426e077c473d482693f06a2d2fb9ab8de 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -166,7 +166,8 @@ int bnxt_re_query_device(struct ib_device *ibdev,
                                    | IB_DEVICE_MEM_WINDOW
                                    | IB_DEVICE_MEM_WINDOW_TYPE_2B
                                    | IB_DEVICE_MEM_MGT_EXTENSIONS;
-       ib_attr->max_sge = dev_attr->max_qp_sges;
+       ib_attr->max_send_sge = dev_attr->max_qp_sges;
+       ib_attr->max_recv_sge = dev_attr->max_qp_sges;
        ib_attr->max_sge_rd = dev_attr->max_qp_sges;
        ib_attr->max_cq = dev_attr->max_cq;
        ib_attr->max_cqe = dev_attr->max_cq_wqes;

diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index be097c6723c0f2b56d2af959ac671bc2fe1821dc..68bc2f9a532f82ee087291880f8cfa0445eac59d 100644 (file)
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -1103,7 +1103,8 @@ static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *pro
        props->max_mr_size = dev->attr.max_mr_size;
        props->max_qp = dev->attr.max_qps;
        props->max_qp_wr = dev->attr.max_wrs;
-       props->max_sge = dev->attr.max_sge_per_wr;
+       props->max_send_sge = dev->attr.max_sge_per_wr;
+       props->max_recv_sge = dev->attr.max_sge_per_wr;
        props->max_sge_rd = 1;
        props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
        props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;

diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 1feade8bb4b3b2b56e38af281362d7aa4a4796aa..61b8bdb9423d9a7575092f5546506bc2e255f41c 100644 (file)
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -343,7 +343,8 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro
        props->max_mr_size = T4_MAX_MR_SIZE;
        props->max_qp = dev->rdev.lldi.vr->qp.size / 2;
        props->max_qp_wr = dev->rdev.hw_queue.t4_max_qp_depth;
-       props->max_sge = T4_MAX_RECV_SGE;
+       props->max_send_sge = min(T4_MAX_SEND_SGE, T4_MAX_WRITE_SGE);
+       props->max_recv_sge = T4_MAX_RECV_SGE;
        props->max_sge_rd = 1;
        props->max_res_rd_atom = dev->rdev.lldi.max_ird_adapter;
        props->max_qp_rd_atom = min(dev->rdev.lldi.max_ordird_qp,

diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 08991874c0e2c4d12918dfdfed059cb1c6eacf78..b7c75b63f887604d350136243a0012bae76b4fea 100644 (file)
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -1410,7 +1410,8 @@ static void hfi1_fill_device_attr(struct hfi1_devdata *dd)
        rdi->dparms.props.max_fast_reg_page_list_len = UINT_MAX;
        rdi->dparms.props.max_qp = hfi1_max_qps;
        rdi->dparms.props.max_qp_wr = hfi1_max_qp_wrs;
-       rdi->dparms.props.max_sge = hfi1_max_sges;
+       rdi->dparms.props.max_send_sge = hfi1_max_sges;
+       rdi->dparms.props.max_recv_sge = hfi1_max_sges;
        rdi->dparms.props.max_sge_rd = hfi1_max_sges;
        rdi->dparms.props.max_cq = hfi1_max_cqs;
        rdi->dparms.props.max_ah = hfi1_max_ahs;

diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 24a2ea0018d9a599c2aabd67d92afef326d3cf62..850032de86763ff82c21d97a78e2f3d340285c5e 100644 (file)
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -206,7 +206,8 @@ static int hns_roce_query_device(struct ib_device *ib_dev,
        props->max_qp_wr = hr_dev->caps.max_wqes;
        props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT |
                                  IB_DEVICE_RC_RNR_NAK_GEN;
-       props->max_sge = max(hr_dev->caps.max_sq_sg, hr_dev->caps.max_rq_sg);
+       props->max_send_sge = hr_dev->caps.max_sq_sg;
+       props->max_recv_sge = hr_dev->caps.max_rq_sg;
        props->max_sge_rd = 1;
        props->max_cq = hr_dev->caps.num_cqs;
        props->max_cqe = hr_dev->caps.max_cqes;

diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 68679ad4c6daf7b83b98d2d1113df522260e0d52..8884ff71a63468aceb78ee8774752f2a5cf3283c 100644 (file)
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -71,7 +71,8 @@ static int i40iw_query_device(struct ib_device *ibdev,
        props->max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
        props->max_qp = iwdev->max_qp - iwdev->used_qps;
        props->max_qp_wr = I40IW_MAX_QP_WRS;
-       props->max_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
+       props->max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
+       props->max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
        props->max_cq = iwdev->max_cq - iwdev->used_cqs;
        props->max_cqe = iwdev->max_cqe;
        props->max_mr = iwdev->max_mr - iwdev->used_mrs;

diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 908b8e5c5acb20446d32dbf228850ee17f791dd6..87de1a467d6053597fabcaf418f1a699fee03371 100644 (file)
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -517,8 +517,8 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
        props->page_size_cap       = dev->dev->caps.page_size_cap;
        props->max_qp              = dev->dev->quotas.qp;
        props->max_qp_wr           = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE;
-       props->max_sge             = min(dev->dev->caps.max_sq_sg,
-                                        dev->dev->caps.max_rq_sg);
+       props->max_send_sge        = dev->dev->caps.max_sq_sg;
+       props->max_recv_sge        = dev->dev->caps.max_rq_sg;
        props->max_sge_rd          = MLX4_MAX_SGE_RD;
        props->max_cq              = dev->dev->quotas.cq;
        props->max_cqe             = dev->dev->caps.max_cqes;

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index e6d88f32391b63b9763e7569aac106d3f5919e7a..e46cda7404793dab09602ecdda7ec9936a8c3310 100644 (file)
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -888,7 +888,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
        max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
                     sizeof(struct mlx5_wqe_raddr_seg)) /
                sizeof(struct mlx5_wqe_data_seg);
-       props->max_sge = min(max_rq_sg, max_sq_sg);
+       props->max_send_sge = max_sq_sg;
+       props->max_recv_sge = max_rq_sg;
        props->max_sge_rd          = MLX5_MAX_SGE_RD;
        props->max_cq              = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
        props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;

diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 541f237965c750dee9c81616d0213e7e32ad93fa..20febafc1fdd6ccf8257d188b50fab1015e72bdd 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -96,8 +96,9 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr
        props->page_size_cap       = mdev->limits.page_size_cap;
        props->max_qp              = mdev->limits.num_qps - mdev->limits.reserved_qps;
        props->max_qp_wr           = mdev->limits.max_wqes;
-       props->max_sge             = mdev->limits.max_sg;
-       props->max_sge_rd          = props->max_sge;
+       props->max_send_sge        = mdev->limits.max_sg;
+       props->max_recv_sge        = mdev->limits.max_sg;
+       props->max_sge_rd          = mdev->limits.max_sg;
        props->max_cq              = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
        props->max_cqe             = mdev->limits.max_cqes;
        props->max_mr              = mdev->limits.num_mpts - mdev->limits.reserved_mrws;

diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 32f26556c808016f2d6f68f975a502724b714084..82b8f9630ee8dc37b9583837060652475e3de38f 100644 (file)
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -436,7 +436,8 @@ static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *prop
        props->max_mr_size = 0x80000000;
        props->max_qp = nesibdev->max_qp;
        props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2;
-       props->max_sge = nesdev->nesadapter->max_sge;
+       props->max_send_sge = nesdev->nesadapter->max_sge;
+       props->max_recv_sge = nesdev->nesadapter->max_sge;
        props->max_cq = nesibdev->max_cq;
        props->max_cqe = nesdev->nesadapter->max_cqe;
        props->max_mr = nesibdev->max_mr;

diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index 82e20fc32890eca3fcc25729f6887fee9399fbad..1f057fdb3a8c35a393ab1872a61883edd77b5756 100644 (file)
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -89,7 +89,8 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
                                        IB_DEVICE_SYS_IMAGE_GUID |
                                        IB_DEVICE_LOCAL_DMA_LKEY |
                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
-       attr->max_sge = min(dev->attr.max_send_sge, dev->attr.max_recv_sge);
+       attr->max_send_sge = dev->attr.max_send_sge;
+       attr->max_recv_sge = dev->attr.max_recv_sge;
        attr->max_sge_rd = dev->attr.max_rdma_sge;
        attr->max_cq = dev->attr.max_cq;
        attr->max_cqe = dev->attr.max_cqe;

diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 10d8f4134ec03621fdc17c706a454b093c6888e4..0c41d54f586b4fc5dcc5596c8eeff1ff1cfce530 100644 (file)
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -112,7 +112,8 @@ int qedr_query_device(struct ib_device *ibdev,
            IB_DEVICE_RC_RNR_NAK_GEN |
            IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 
-       attr->max_sge = qattr->max_sge;
+       attr->max_send_sge = qattr->max_sge;
+       attr->max_recv_sge = qattr->max_sge;
        attr->max_sge_rd = qattr->max_sge;
        attr->max_cq = qattr->max_cq;
        attr->max_cqe = qattr->max_cqe;

diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index 14b4057a2b8f95306c322d6b3c5a98a77ac6f6a7..41babbc0db583c96e5def53fafe9596ee3ae01ee 100644 (file)
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -1489,7 +1489,8 @@ static void qib_fill_device_attr(struct qib_devdata *dd)
        rdi->dparms.props.max_mr_size = ~0ULL;
        rdi->dparms.props.max_qp = ib_qib_max_qps;
        rdi->dparms.props.max_qp_wr = ib_qib_max_qp_wrs;
-       rdi->dparms.props.max_sge = ib_qib_max_sges;
+       rdi->dparms.props.max_send_sge = ib_qib_max_sges;
+       rdi->dparms.props.max_recv_sge = ib_qib_max_sges;
        rdi->dparms.props.max_sge_rd = ib_qib_max_sges;
        rdi->dparms.props.max_cq = ib_qib_max_cqs;
        rdi->dparms.props.max_cqe = ib_qib_max_cqes;

diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index a51463cd2f374729cf57a448a26d4be06fc0c342..816cc285daf60be7950d2160997e259850882967 100644 (file)
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -82,7 +82,8 @@ int pvrdma_query_device(struct ib_device *ibdev,
        props->max_qp = dev->dsr->caps.max_qp;
        props->max_qp_wr = dev->dsr->caps.max_qp_wr;
        props->device_cap_flags = dev->dsr->caps.device_cap_flags;
-       props->max_sge = dev->dsr->caps.max_sge;
+       props->max_send_sge = dev->dsr->caps.max_sge;
+       props->max_recv_sge = dev->dsr->caps.max_sge;
        props->max_sge_rd = PVRDMA_GET_CAP(dev, dev->dsr->caps.max_sge,
                                           dev->dsr->caps.max_sge_rd);
        props->max_srq = dev->dsr->caps.max_srq;

diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 815f94c17c4850b3baef64fa7e2462da94e0a256..d29e3c9433996cb8d5bbdafb360f81c0e5a574bc 100644 (file)
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -780,14 +780,15 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
        if (!rdi)
                return ERR_PTR(-EINVAL);
 
-       if (init_attr->cap.max_send_sge > rdi->dparms.props.max_sge ||
+       if (init_attr->cap.max_send_sge > rdi->dparms.props.max_send_sge ||
            init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr ||
            init_attr->create_flags)
                return ERR_PTR(-EINVAL);
 
        /* Check receive queue parameters if no SRQ is specified. */
        if (!init_attr->srq) {
-               if (init_attr->cap.max_recv_sge > rdi->dparms.props.max_sge ||
+               if (init_attr->cap.max_recv_sge >
+                   rdi->dparms.props.max_recv_sge ||
                    init_attr->cap.max_recv_wr > rdi->dparms.props.max_qp_wr)
                        return ERR_PTR(-EINVAL);
 

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 7121e1b1eb898eba0e161a809ebc4f919a14546a..10999fa6928181f4f99e5e2cb60b6bea8b358eed 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -91,7 +91,8 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
        rxe->attr.max_qp                        = RXE_MAX_QP;
        rxe->attr.max_qp_wr                     = RXE_MAX_QP_WR;
        rxe->attr.device_cap_flags              = RXE_DEVICE_CAP_FLAGS;
-       rxe->attr.max_sge                       = RXE_MAX_SGE;
+       rxe->attr.max_send_sge                  = RXE_MAX_SGE;
+       rxe->attr.max_recv_sge                  = RXE_MAX_SGE;
        rxe->attr.max_sge_rd                    = RXE_MAX_SGE_RD;
        rxe->attr.max_cq                        = RXE_MAX_CQ;
        rxe->attr.max_cqe                       = (1 << RXE_MAX_LOG_CQE) - 1;

diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index 9f83fc982f3195c94187a65a064c52ed546cb7b9..c58452daffc7449875f362269b7cd51bb9f8803e 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -49,9 +49,9 @@ static int rxe_qp_chk_cap(struct rxe_dev *rxe, struct ib_qp_cap *cap,
                goto err1;
        }
 
-       if (cap->max_send_sge > rxe->attr.max_sge) {
+       if (cap->max_send_sge > rxe->attr.max_send_sge) {
                pr_warn("invalid send sge = %d > %d\n",
-                       cap->max_send_sge, rxe->attr.max_sge);
+                       cap->max_send_sge, rxe->attr.max_send_sge);
                goto err1;
        }
 
@@ -62,9 +62,9 @@ static int rxe_qp_chk_cap(struct rxe_dev *rxe, struct ib_qp_cap *cap,
                        goto err1;
                }
 
-               if (cap->max_recv_sge > rxe->attr.max_sge) {
+               if (cap->max_recv_sge > rxe->attr.max_recv_sge) {
                        pr_warn("invalid recv sge = %d > %d\n",
-                               cap->max_recv_sge, rxe->attr.max_sge);
+                               cap->max_recv_sge, rxe->attr.max_recv_sge);
                        goto err1;
                }
        }

diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index 6535d9beb24d29675ec8c6c513909a2ee0e6332d..23cb1adc636f741815653ea0748059ac18721a57 100644 (file)
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -1068,8 +1068,8 @@ static struct ib_qp *ipoib_cm_create_tx_qp(struct net_device *dev, struct ipoib_
        struct ib_qp *tx_qp;
 
        if (dev->features & NETIF_F_SG)
-               attr.cap.max_send_sge =
-                       min_t(u32, priv->ca->attrs.max_sge, MAX_SKB_FRAGS + 1);
+               attr.cap.max_send_sge = min_t(u32, priv->ca->attrs.max_send_sge,
+                                             MAX_SKB_FRAGS + 1);
 
        tx_qp = ib_create_qp(priv->pd, &attr);
        tx->max_send_sge = attr.cap.max_send_sge;

diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index 984a88096f3961eef1353d4db5cae58799a0626e..ba4669f24014d490b9a5b43c446dd90b7798fe38 100644 (file)
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -147,7 +147,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
                .cap = {
                        .max_send_wr  = ipoib_sendq_size,
                        .max_recv_wr  = ipoib_recvq_size,
-                       .max_send_sge = min_t(u32, priv->ca->attrs.max_sge,
+                       .max_send_sge = min_t(u32, priv->ca->attrs.max_send_sge,
                                              MAX_SKB_FRAGS + 1),
                        .max_recv_sge = IPOIB_UD_RX_SG
                },

diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index cccbcf0eb035a31124066c21ae4de7e525055aaf..7e056f3c82a0c890941d0359c8ad2585cdcdbdcb 100644 (file)
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -136,7 +136,7 @@ isert_create_qp(struct isert_conn *isert_conn,
        attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
        attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
        attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
-       attr.cap.max_send_sge = device->ib_device->attrs.max_sge;
+       attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
        attr.cap.max_recv_sge = 1;
        attr.sq_sig_type = IB_SIGNAL_REQ_WR;
        attr.qp_type = IB_QPT_RC;
@@ -299,7 +299,8 @@ isert_create_device_ib_res(struct isert_device *device)
        struct ib_device *ib_dev = device->ib_device;
        int ret;
 
-       isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge);
+       isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n",
+                 ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
        isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
 
        ret = isert_alloc_comps(device);

diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 698f7779e231c37af8d52535a7bb4c7dafff5e27..1b0b285a0ae021794915d8950f7ab6b62e7cb74b 100644 (file)
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -1753,13 +1753,15 @@ retry:
         */
        qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
        qp_init->cap.max_rdma_ctxs = sq_size / 2;
-       qp_init->cap.max_send_sge = min(attrs->max_sge, SRPT_MAX_SG_PER_WQE);
+       qp_init->cap.max_send_sge = min(attrs->max_send_sge,
+                                       SRPT_MAX_SG_PER_WQE);
        qp_init->port_num = ch->sport->port;
        if (sdev->use_srq) {
                qp_init->srq = sdev->srq;
        } else {
                qp_init->cap.max_recv_wr = ch->rq_size;
-               qp_init->cap.max_recv_sge = qp_init->cap.max_send_sge;
+               qp_init->cap.max_recv_sge = min(attrs->max_recv_sge,
+                                               SRPT_MAX_SG_PER_WQE);
        }
 
        if (ch->using_rdma_cm) {

diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index 52e0c5d579a7aad180472faa0d53cd106e35c615..0d7f3d603f1d13ae95f1cdc5adb90e9ff6c24309 100644 (file)
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -874,7 +874,7 @@ static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue)
        qp_attr.cap.max_send_wr = queue->send_queue_size + 1;
        qp_attr.cap.max_rdma_ctxs = queue->send_queue_size;
        qp_attr.cap.max_send_sge = max(ndev->device->attrs.max_sge_rd,
-                                       ndev->device->attrs.max_sge);
+                                       ndev->device->attrs.max_send_sge);
 
        if (ndev->srq) {
                qp_attr.srq = ndev->srq;

diff --git a/fs/cifs/smbdirect.c b/fs/cifs/smbdirect.c
index e459c97151b34e684dc3f3cbbc36772fee5aaee5..c5a1cddd8856a93738ce376d5413756594bfc3e5 100644 (file)
--- a/fs/cifs/smbdirect.c
+++ b/fs/cifs/smbdirect.c
@@ -1661,9 +1661,16 @@ static struct smbd_connection *_smbd_get_connection(
        info->max_receive_size = smbd_max_receive_size;
        info->keep_alive_interval = smbd_keep_alive_interval;
 
-       if (info->id->device->attrs.max_sge < SMBDIRECT_MAX_SGE) {
-               log_rdma_event(ERR, "warning: device max_sge = %d too small\n",
-                       info->id->device->attrs.max_sge);
+       if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SGE) {
+               log_rdma_event(ERR,
+                       "warning: device max_send_sge = %d too small\n",
+                       info->id->device->attrs.max_send_sge);
+               log_rdma_event(ERR, "Queue Pair creation may fail\n");
+       }
+       if (info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_SGE) {
+               log_rdma_event(ERR,
+                       "warning: device max_recv_sge = %d too small\n",
+                       info->id->device->attrs.max_recv_sge);
                log_rdma_event(ERR, "Queue Pair creation may fail\n");
        }
 

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 0232c0f9f7171433ddc07b51871d442c6b19701e..dc5d262739e595552764310a537ddf0e5affc71a 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -345,7 +345,8 @@ struct ib_device_attr {
        int                     max_qp;
        int                     max_qp_wr;
        u64                     device_cap_flags;
-       int                     max_sge;
+       int                     max_send_sge;
+       int                     max_recv_sge;
        int                     max_sge_rd;
        int                     max_cq;
        int                     max_cqe;

diff --git a/net/rds/ib.c b/net/rds/ib.c
index b6ad38e48f62692fa9dc6cc9f7c7081c706394a7..683b55d4e2b07c8d6ba8c35792bf6458c9cdf96c 100644 (file)
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -143,7 +143,7 @@ static void rds_ib_add_one(struct ib_device *device)
        INIT_WORK(&rds_ibdev->free_work, rds_ib_dev_free);
 
        rds_ibdev->max_wrs = device->attrs.max_qp_wr;
-       rds_ibdev->max_sge = min(device->attrs.max_sge, RDS_IB_MAX_SGE);
+       rds_ibdev->max_sge = min(device->attrs.max_send_sge, RDS_IB_MAX_SGE);
 
        has_fr = (device->attrs.device_cap_flags &
                  IB_DEVICE_MEM_MGT_EXTENSIONS);

diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index e9535a66bab0044de5aee1fa3a3df0f837430701..547b2cdf14274f4267ad6ea8ddc605ff0da4a7b6 100644 (file)
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -476,7 +476,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 
        /* Qualify the transport resource defaults with the
         * capabilities of this particular device */
-       newxprt->sc_max_send_sges = dev->attrs.max_sge;
+       newxprt->sc_max_send_sges = dev->attrs.max_send_sge;
        /* transport hdr, head iovec, one page list entry, tail iovec */
        if (newxprt->sc_max_send_sges < 4) {
                pr_err("svcrdma: too few Send SGEs available (%d)\n",

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 16161a36dc739896b6712f14ccdbf5cc6106ec8a..112a15abc4a4af24a025caebfe4e8a0884cb1ae9 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -508,7 +508,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
        unsigned int max_sge;
        int rc;
 
-       max_sge = min_t(unsigned int, ia->ri_device->attrs.max_sge,
+       max_sge = min_t(unsigned int, ia->ri_device->attrs.max_send_sge,
                        RPCRDMA_MAX_SEND_SGES);
        if (max_sge < RPCRDMA_MIN_SEND_SGES) {
                pr_warn("rpcrdma: HCA provides only %d send SGEs\n", max_sge);