RDMA/efa: Add support for dmabuf memory regions

Implement a dmabuf importer for the EFA driver. As ODP is not supported,
the pinned dmabuf are used to prevent the move_notify callback from being
called.

Link: https://lore.kernel.org/r/20211012120903.96933-4-galpress@amazon.com
Signed-off-by: Gal Pressman <galpress@amazon.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h
index e0c9ba4c70f4f113325073335e475f5dae5611da..7352a1f5d81112ef3a2d866d061273f0ce5a15b2 100644 (file)
--- a/drivers/infiniband/hw/efa/efa.h
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -154,6 +154,10 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
                         u64 virt_addr, int access_flags,
                         struct ib_udata *udata);
+struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
+                                    u64 length, u64 virt_addr,
+                                    int fd, int access_flags,
+                                    struct ib_udata *udata);
 int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
 int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num,
                           struct ib_port_immutable *immutable);

diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index d2a445f6f8e5df46462d0a036a8fc51087682ebc..94b94cca487092fdf026ce749e4db58ef28b6444 100644 (file)
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -380,6 +380,7 @@ static const struct ib_device_ops efa_dev_ops = {
        .query_port = efa_query_port,
        .query_qp = efa_query_qp,
        .reg_user_mr = efa_reg_mr,
+       .reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf,
 
        INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),
        INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq),

diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 8335a6f44869d805fcd67ed5fc42b2fea4011a57..ecfe70eb5efbebf3cef26f3844ed0dcf8c4b4ebd 100644 (file)
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -3,6 +3,8 @@
  * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
+#include <linux/dma-buf.h>
+#include <linux/dma-resv.h>
 #include <linux/vmalloc.h>
 #include <linux/log2.h>
 
@@ -1544,26 +1546,18 @@ static int efa_create_pbl(struct efa_dev *dev,
        return 0;
 }
 
-struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
-                        u64 virt_addr, int access_flags,
-                        struct ib_udata *udata)
+static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags,
+                                  struct ib_udata *udata)
 {
        struct efa_dev *dev = to_edev(ibpd->device);
-       struct efa_com_reg_mr_params params = {};
-       struct efa_com_reg_mr_result result = {};
-       struct pbl_context pbl;
        int supp_access_flags;
-       unsigned int pg_sz;
        struct efa_mr *mr;
-       int inline_size;
-       int err;
 
        if (udata && udata->inlen &&
            !ib_is_udata_cleared(udata, 0, sizeof(udata->inlen))) {
                ibdev_dbg(&dev->ibdev,
                          "Incompatible ABI params, udata not cleared\n");
-               err = -EINVAL;
-               goto err_out;
+               return ERR_PTR(-EINVAL);
        }
 
        supp_access_flags =
@@ -1575,23 +1569,26 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
                ibdev_dbg(&dev->ibdev,
                          "Unsupported access flags[%#x], supported[%#x]\n",
                          access_flags, supp_access_flags);
-               err = -EOPNOTSUPP;
-               goto err_out;
+               return ERR_PTR(-EOPNOTSUPP);
        }
 
        mr = kzalloc(sizeof(*mr), GFP_KERNEL);
-       if (!mr) {
-               err = -ENOMEM;
-               goto err_out;
-       }
+       if (!mr)
+               return ERR_PTR(-ENOMEM);
 
-       mr->umem = ib_umem_get(ibpd->device, start, length, access_flags);
-       if (IS_ERR(mr->umem)) {
-               err = PTR_ERR(mr->umem);
-               ibdev_dbg(&dev->ibdev,
-                         "Failed to pin and map user space memory[%d]\n", err);
-               goto err_free;
-       }
+       return mr;
+}
+
+static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,
+                          u64 length, u64 virt_addr, int access_flags)
+{
+       struct efa_dev *dev = to_edev(ibpd->device);
+       struct efa_com_reg_mr_params params = {};
+       struct efa_com_reg_mr_result result = {};
+       struct pbl_context pbl;
+       unsigned int pg_sz;
+       int inline_size;
+       int err;
 
        params.pd = to_epd(ibpd)->pdn;
        params.iova = virt_addr;
@@ -1602,10 +1599,9 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
                                       dev->dev_attr.page_size_cap,
                                       virt_addr);
        if (!pg_sz) {
-               err = -EOPNOTSUPP;
                ibdev_dbg(&dev->ibdev, "Failed to find a suitable page size in page_size_cap %#llx\n",
                          dev->dev_attr.page_size_cap);
-               goto err_unmap;
+               return -EOPNOTSUPP;
        }
 
        params.page_shift = order_base_2(pg_sz);
@@ -1619,21 +1615,21 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
        if (params.page_num <= inline_size) {
                err = efa_create_inline_pbl(dev, mr, &params);
                if (err)
-                       goto err_unmap;
+                       return err;
 
                err = efa_com_register_mr(&dev->edev, &params, &result);
                if (err)
-                       goto err_unmap;
+                       return err;
        } else {
                err = efa_create_pbl(dev, &pbl, mr, &params);
                if (err)
-                       goto err_unmap;
+                       return err;
 
                err = efa_com_register_mr(&dev->edev, &params, &result);
                pbl_destroy(dev, &pbl);
 
                if (err)
-                       goto err_unmap;
+                       return err;
        }
 
        mr->ibmr.lkey = result.l_key;
@@ -1641,9 +1637,78 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
        mr->ibmr.length = length;
        ibdev_dbg(&dev->ibdev, "Registered mr[%d]\n", mr->ibmr.lkey);
 
+       return 0;
+}
+
+struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
+                                    u64 length, u64 virt_addr,
+                                    int fd, int access_flags,
+                                    struct ib_udata *udata)
+{
+       struct efa_dev *dev = to_edev(ibpd->device);
+       struct ib_umem_dmabuf *umem_dmabuf;
+       struct efa_mr *mr;
+       int err;
+
+       mr = efa_alloc_mr(ibpd, access_flags, udata);
+       if (IS_ERR(mr)) {
+               err = PTR_ERR(mr);
+               goto err_out;
+       }
+
+       umem_dmabuf = ib_umem_dmabuf_get_pinned(ibpd->device, start, length, fd,
+                                               access_flags);
+       if (IS_ERR(umem_dmabuf)) {
+               err = PTR_ERR(umem_dmabuf);
+               ibdev_dbg(&dev->ibdev, "Failed to get dmabuf umem[%d]\n", err);
+               goto err_free;
+       }
+
+       mr->umem = &umem_dmabuf->umem;
+       err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags);
+       if (err)
+               goto err_release;
+
+       return &mr->ibmr;
+
+err_release:
+       ib_umem_release(mr->umem);
+err_free:
+       kfree(mr);
+err_out:
+       atomic64_inc(&dev->stats.reg_mr_err);
+       return ERR_PTR(err);
+}
+
+struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
+                        u64 virt_addr, int access_flags,
+                        struct ib_udata *udata)
+{
+       struct efa_dev *dev = to_edev(ibpd->device);
+       struct efa_mr *mr;
+       int err;
+
+       mr = efa_alloc_mr(ibpd, access_flags, udata);
+       if (IS_ERR(mr)) {
+               err = PTR_ERR(mr);
+               goto err_out;
+       }
+
+       mr->umem = ib_umem_get(ibpd->device, start, length, access_flags);
+       if (IS_ERR(mr->umem)) {
+               err = PTR_ERR(mr->umem);
+               ibdev_dbg(&dev->ibdev,
+                         "Failed to pin and map user space memory[%d]\n", err);
+               goto err_free;
+       }
+
+       err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags);
+       if (err)
+               goto err_release;
+
        return &mr->ibmr;
 
-err_unmap:
+err_release:
        ib_umem_release(mr->umem);
 err_free:
        kfree(mr);