RDMA/rxe: Enable MW object pool

Currently the rxe driver has a rxe_mw struct object but nothing about
memory windows is enabled. This patch turns on memory windows and some
minor cleanup.

Set device attribute in rxe.c so max_mw = MAX_MW.  Change parameters in
rxe_param.h so that MAX_MW is the same as MAX_MR.  Reduce the number of
MRs and MWs to 4K from 256K.  Add device capability bits for 2a and 2b
memory windows.  Removed RXE_MR_TYPE_MW from the rxe_mr_type enum.

Link: https://lore.kernel.org/r/20210608042552.33275-4-rpearsonhpe@gmail.com
Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 95f0de0c8b49ca95300ba37c7617ca974813cfd4..8e0f9c489cab2940dcf3ea03f3b0569ccf96616a 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -54,6 +54,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
        rxe->attr.max_cq                        = RXE_MAX_CQ;
        rxe->attr.max_cqe                       = (1 << RXE_MAX_LOG_CQE) - 1;
        rxe->attr.max_mr                        = RXE_MAX_MR;
+       rxe->attr.max_mw                        = RXE_MAX_MW;
        rxe->attr.max_pd                        = RXE_MAX_PD;
        rxe->attr.max_qp_rd_atom                = RXE_MAX_QP_RD_ATOM;
        rxe->attr.max_res_rd_atom               = RXE_MAX_RES_RD_ATOM;

diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
index 25ab50d9b7c283f9e928ed47eaffab7bbf7ab41e..742e6ec93686cc96a4ff5d00def4db38751c453f 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_param.h
+++ b/drivers/infiniband/sw/rxe/rxe_param.h
@@ -37,7 +37,6 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
 enum rxe_device_param {
        RXE_MAX_MR_SIZE                 = -1ull,
        RXE_PAGE_SIZE_CAP               = 0xfffff000,
-       RXE_MAX_QP                      = 0x10000,
        RXE_MAX_QP_WR                   = 0x4000,
        RXE_DEVICE_CAP_FLAGS            = IB_DEVICE_BAD_PKEY_CNTR
                                        | IB_DEVICE_BAD_QKEY_CNTR
@@ -49,7 +48,10 @@ enum rxe_device_param {
                                        | IB_DEVICE_RC_RNR_NAK_GEN
                                        | IB_DEVICE_SRQ_RESIZE
                                        | IB_DEVICE_MEM_MGT_EXTENSIONS
-                                       | IB_DEVICE_ALLOW_USER_UNREG,
+                                       | IB_DEVICE_ALLOW_USER_UNREG
+                                       | IB_DEVICE_MEM_WINDOW
+                                       | IB_DEVICE_MEM_WINDOW_TYPE_2A
+                                       | IB_DEVICE_MEM_WINDOW_TYPE_2B,
        RXE_MAX_SGE                     = 32,
        RXE_MAX_WQE_SIZE                = sizeof(struct rxe_send_wqe) +
                                          sizeof(struct ib_sge) * RXE_MAX_SGE,
@@ -58,7 +60,6 @@ enum rxe_device_param {
        RXE_MAX_SGE_RD                  = 32,
        RXE_MAX_CQ                      = 16384,
        RXE_MAX_LOG_CQE                 = 15,
-       RXE_MAX_MR                      = 256 * 1024,
        RXE_MAX_PD                      = 0x7ffc,
        RXE_MAX_QP_RD_ATOM              = 128,
        RXE_MAX_RES_RD_ATOM             = 0x3f000,
@@ -67,7 +68,6 @@ enum rxe_device_param {
        RXE_MAX_MCAST_QP_ATTACH         = 56,
        RXE_MAX_TOT_MCAST_QP_ATTACH     = 0x70000,
        RXE_MAX_AH                      = 100,
-       RXE_MAX_SRQ                     = 960,
        RXE_MAX_SRQ_WR                  = 0x4000,
        RXE_MIN_SRQ_WR                  = 1,
        RXE_MAX_SRQ_SGE                 = 27,
@@ -80,16 +80,21 @@ enum rxe_device_param {
 
        RXE_NUM_PORT                    = 1,
 
+       RXE_MAX_QP                      = 0x10000,
        RXE_MIN_QP_INDEX                = 16,
        RXE_MAX_QP_INDEX                = 0x00020000,
 
+       RXE_MAX_SRQ                     = 0x00001000,
        RXE_MIN_SRQ_INDEX               = 0x00020001,
        RXE_MAX_SRQ_INDEX               = 0x00040000,
 
+       RXE_MAX_MR                      = 0x00001000,
+       RXE_MAX_MW                      = 0x00001000,
        RXE_MIN_MR_INDEX                = 0x00000001,
-       RXE_MAX_MR_INDEX                = 0x00040000,
-       RXE_MIN_MW_INDEX                = 0x00040001,
-       RXE_MAX_MW_INDEX                = 0x00060000,
+       RXE_MAX_MR_INDEX                = 0x00010000,
+       RXE_MIN_MW_INDEX                = 0x00010001,
+       RXE_MAX_MW_INDEX                = 0x00020000,
+
        RXE_MAX_PKT_PER_ACK             = 64,
 
        RXE_MAX_UNACKED_PSNS            = 128,

diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index e0ba5723ff20fb0576159f03d5cb15bda8577204..207bbd5014e9cd03b36c6fa88dde6d73771a46bf 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -276,7 +276,6 @@ enum rxe_mr_type {
        RXE_MR_TYPE_NONE,
        RXE_MR_TYPE_DMA,
        RXE_MR_TYPE_MR,
-       RXE_MR_TYPE_MW,
 };
 
 #define RXE_BUF_PER_MAP                (PAGE_SIZE / sizeof(struct rxe_phys_buf))