IB/mlx5: Introduce MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD

Introduce MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD and its initial implementation.

This object is from type class FD and will be used to read DEVX async
commands completion.

The core layer should allow the driver to set object from type FD in a
safe mode, this option was added with a matching comment in place.

Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c
index 6c4747e61d2b5ba9196d874450f2ee94fcc3fcbc..a260d2f8e0b77119d6b135a593ea7897b0366916 100644 (file)
--- a/drivers/infiniband/core/rdma_core.c
+++ b/drivers/infiniband/core/rdma_core.c
@@ -801,6 +801,7 @@ void uverbs_close_fd(struct file *f)
        /* Pairs with filp->private_data in alloc_begin_fd_uobject */
        uverbs_uobject_put(uobj);
 }
+EXPORT_SYMBOL(uverbs_close_fd);
 
 /*
  * Drop the ucontext off the ufile and completely disconnect it from the

diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c
index 9ae08e4b78a301f36714a65f86b3afd5a0d216ea..7a987acf0c0bbdf0b48460371ca164b6cb34a194 100644 (file)
--- a/drivers/infiniband/core/uverbs_uapi.c
+++ b/drivers/infiniband/core/uverbs_uapi.c
@@ -188,13 +188,18 @@ static int uapi_merge_obj_tree(struct uverbs_api *uapi,
                obj_elm->type_attrs = obj->type_attrs;
                obj_elm->type_class = obj->type_attrs->type_class;
                /*
-                * Today drivers are only permitted to use idr_class
-                * types. They cannot use FD types because we currently have
-                * no way to revoke the fops pointer after device
-                * disassociation.
+                * Today drivers are only permitted to use idr_class and
+                * fd_class types. We can revoke the IDR types during
+                * disassociation, and the FD types require the driver to use
+                * struct file_operations.owner to prevent the driver module
+                * code from unloading while the file is open. This provides
+                * enough safety that uverbs_close_fd() will continue to work.
+                * Drivers using FD are responsible to handle disassociation of
+                * the device on their own.
                 */
                if (WARN_ON(is_driver &&
-                           obj->type_attrs->type_class != &uverbs_idr_class))
+                           obj->type_attrs->type_class != &uverbs_idr_class &&
+                           obj->type_attrs->type_class != &uverbs_fd_class))
                        return -EINVAL;
        }
 

diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 5a588f3cfb1bb622d18d4d42e51fc358ee9ca846..9933bcf83a6b1dbf69e033a60f1828be3714372e 100644 (file)
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -1168,6 +1168,38 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
                              cmd_out, cmd_out_len);
 }
 
+struct devx_async_event_queue {
+       spinlock_t              lock;
+       wait_queue_head_t       poll_wait;
+       struct list_head        event_list;
+};
+
+struct devx_async_cmd_event_file {
+       struct ib_uobject               uobj;
+       struct devx_async_event_queue   ev_queue;
+};
+
+static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
+{
+       spin_lock_init(&ev_queue->lock);
+       INIT_LIST_HEAD(&ev_queue->event_list);
+       init_waitqueue_head(&ev_queue->poll_wait);
+}
+
+static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
+       struct uverbs_attr_bundle *attrs)
+{
+       struct devx_async_cmd_event_file *ev_file;
+
+       struct ib_uobject *uobj = uverbs_attr_get_uobject(
+               attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
+
+       ev_file = container_of(uobj, struct devx_async_cmd_event_file,
+                              uobj);
+       devx_init_event_queue(&ev_file->ev_queue);
+       return 0;
+}
+
 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
                         struct uverbs_attr_bundle *attrs,
                         struct devx_umem *obj)
@@ -1313,6 +1345,38 @@ static int devx_umem_cleanup(struct ib_uobject *uobject,
        return 0;
 }
 
+static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
+                                        size_t count, loff_t *pos)
+{
+       return -EINVAL;
+}
+
+static int devx_async_cmd_event_close(struct inode *inode, struct file *filp)
+{
+       uverbs_close_fd(filp);
+       return 0;
+}
+
+static __poll_t devx_async_cmd_event_poll(struct file *filp,
+                                             struct poll_table_struct *wait)
+{
+       return 0;
+}
+
+const struct file_operations devx_async_cmd_event_fops = {
+       .owner   = THIS_MODULE,
+       .read    = devx_async_cmd_event_read,
+       .poll    = devx_async_cmd_event_poll,
+       .release = devx_async_cmd_event_close,
+       .llseek  = no_llseek,
+};
+
+static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj,
+                                                  enum rdma_remove_reason why)
+{
+       return 0;
+};
+
 DECLARE_UVERBS_NAMED_METHOD(
        MLX5_IB_METHOD_DEVX_UMEM_REG,
        UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
@@ -1440,6 +1504,22 @@ DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
                            &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
                            &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
 
+
+DECLARE_UVERBS_NAMED_METHOD(
+       MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
+       UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
+                       MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
+                       UVERBS_ACCESS_NEW,
+                       UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(
+       MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
+       UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
+                            devx_hot_unplug_async_cmd_event_file,
+                            &devx_async_cmd_event_fops, "[devx_async_cmd]",
+                            O_RDONLY),
+       &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
+
 static bool devx_is_supported(struct ib_device *device)
 {
        struct mlx5_ib_dev *dev = to_mdev(device);
@@ -1457,5 +1537,8 @@ const struct uapi_definition mlx5_ib_devx_defs[] = {
        UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
                MLX5_IB_OBJECT_DEVX_UMEM,
                UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
+       UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
+               MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
+               UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
        {},
 };

diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h
index acb1bfa3cc99a5b2d82f70e547acce8ea93d5928..175d761695e1b45e087ee805f42792a50a7a70d2 100644 (file)
--- a/include/rdma/uverbs_types.h
+++ b/include/rdma/uverbs_types.h
@@ -157,6 +157,7 @@ struct uverbs_obj_fd_type {
 
 extern const struct uverbs_obj_type_class uverbs_idr_class;
 extern const struct uverbs_obj_type_class uverbs_fd_class;
+void uverbs_close_fd(struct file *f);
 
 #define UVERBS_BUILD_BUG_ON(cond) (sizeof(char[1 - 2 * !!(cond)]) -    \
                                   sizeof(char))

diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
index b8d121d457f1511ded3e163741ac82996ee659ce..6ceae29d77cdfc4576013f7fb663dc614ad7ad02 100644 (file)
--- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -113,11 +113,20 @@ enum mlx5_ib_devx_umem_methods {
        MLX5_IB_METHOD_DEVX_UMEM_DEREG,
 };
 
+enum mlx5_ib_devx_async_cmd_fd_alloc_attrs {
+       MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+};
+
+enum mlx5_ib_devx_async_cmd_fd_methods {
+       MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
+};
+
 enum mlx5_ib_objects {
        MLX5_IB_OBJECT_DEVX = (1U << UVERBS_ID_NS_SHIFT),
        MLX5_IB_OBJECT_DEVX_OBJ,
        MLX5_IB_OBJECT_DEVX_UMEM,
        MLX5_IB_OBJECT_FLOW_MATCHER,
+       MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
 };
 
 enum mlx5_ib_flow_matcher_create_attrs {