iommufd: Add IOMMU_HWPT_INVALIDATE

In nested translation, the stage-1 page table is user-managed but cached
by the IOMMU hardware, so an update on present page table entries in the
stage-1 page table should be followed with a cache invalidation.

Add an IOMMU_HWPT_INVALIDATE ioctl to support such a cache invalidation.
It takes hwpt_id to specify the iommu_domain, and a multi-entry array to
support multiple invalidation data in one ioctl.

enum iommu_hwpt_invalidate_data_type is defined to tag the data type of
the entries in the multi-entry array.

Link: https://lore.kernel.org/r/20240111041015.47920-3-yi.l.liu@intel.com
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Co-developed-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c
index cbb5df0a6c32f835b50535a84bde3f44bfb4d6db..4e8711f19f721565964fbbbee82bc87278d33858 100644 (file)
--- a/drivers/iommu/iommufd/hw_pagetable.c
+++ b/drivers/iommu/iommufd/hw_pagetable.c
@@ -371,3 +371,44 @@ int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd)
        iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
        return rc;
 }
+
+int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd)
+{
+       struct iommu_hwpt_invalidate *cmd = ucmd->cmd;
+       struct iommu_user_data_array data_array = {
+               .type = cmd->data_type,
+               .uptr = u64_to_user_ptr(cmd->data_uptr),
+               .entry_len = cmd->entry_len,
+               .entry_num = cmd->entry_num,
+       };
+       struct iommufd_hw_pagetable *hwpt;
+       u32 done_num = 0;
+       int rc;
+
+       if (cmd->__reserved) {
+               rc = -EOPNOTSUPP;
+               goto out;
+       }
+
+       if (cmd->entry_num && (!cmd->data_uptr || !cmd->entry_len)) {
+               rc = -EINVAL;
+               goto out;
+       }
+
+       hwpt = iommufd_get_hwpt_nested(ucmd, cmd->hwpt_id);
+       if (IS_ERR(hwpt)) {
+               rc = PTR_ERR(hwpt);
+               goto out;
+       }
+
+       rc = hwpt->domain->ops->cache_invalidate_user(hwpt->domain,
+                                                     &data_array);
+       done_num = data_array.entry_num;
+
+       iommufd_put_object(ucmd->ictx, &hwpt->obj);
+out:
+       cmd->entry_num = done_num;
+       if (iommufd_ucmd_respond(ucmd, sizeof(*cmd)))
+               return -EFAULT;
+       return rc;
+}

diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index abae041e256f7ed1a0a6fcc68a48087098effc6b..991f864d1f9bc175b9acc9b9c445d32ed837f878 100644 (file)
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -328,6 +328,15 @@ iommufd_get_hwpt_paging(struct iommufd_ucmd *ucmd, u32 id)
                                               IOMMUFD_OBJ_HWPT_PAGING),
                            struct iommufd_hwpt_paging, common.obj);
 }
+
+static inline struct iommufd_hw_pagetable *
+iommufd_get_hwpt_nested(struct iommufd_ucmd *ucmd, u32 id)
+{
+       return container_of(iommufd_get_object(ucmd->ictx, id,
+                                              IOMMUFD_OBJ_HWPT_NESTED),
+                           struct iommufd_hw_pagetable, obj);
+}
+
 int iommufd_hwpt_set_dirty_tracking(struct iommufd_ucmd *ucmd);
 int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd);
 
@@ -345,6 +354,7 @@ void iommufd_hwpt_paging_abort(struct iommufd_object *obj);
 void iommufd_hwpt_nested_destroy(struct iommufd_object *obj);
 void iommufd_hwpt_nested_abort(struct iommufd_object *obj);
 int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd);
+int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd);
 
 static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx,
                                            struct iommufd_hw_pagetable *hwpt)

diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c
index c9091e46d208abeea14aea1c649a016c39a077ba..39b32932c61ee4e924e24ab18fc05ec4149829c9 100644 (file)
--- a/drivers/iommu/iommufd/main.c
+++ b/drivers/iommu/iommufd/main.c
@@ -322,6 +322,7 @@ union ucmd_buffer {
        struct iommu_hw_info info;
        struct iommu_hwpt_alloc hwpt;
        struct iommu_hwpt_get_dirty_bitmap get_dirty_bitmap;
+       struct iommu_hwpt_invalidate cache;
        struct iommu_hwpt_set_dirty_tracking set_dirty_tracking;
        struct iommu_ioas_alloc alloc;
        struct iommu_ioas_allow_iovas allow_iovas;
@@ -360,6 +361,8 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
                 __reserved),
        IOCTL_OP(IOMMU_HWPT_GET_DIRTY_BITMAP, iommufd_hwpt_get_dirty_bitmap,
                 struct iommu_hwpt_get_dirty_bitmap, data),
+       IOCTL_OP(IOMMU_HWPT_INVALIDATE, iommufd_hwpt_invalidate,
+                struct iommu_hwpt_invalidate, __reserved),
        IOCTL_OP(IOMMU_HWPT_SET_DIRTY_TRACKING, iommufd_hwpt_set_dirty_tracking,
                 struct iommu_hwpt_set_dirty_tracking, __reserved),
        IOCTL_OP(IOMMU_IOAS_ALLOC, iommufd_ioas_alloc_ioctl,

diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h
index 0b2bc6252e2ca2840b556ee6dd858ae123f22c9b..824560c50ec6df0928ef4b40094577ba73ef6e88 100644 (file)
--- a/include/uapi/linux/iommufd.h
+++ b/include/uapi/linux/iommufd.h
@@ -49,6 +49,7 @@ enum {
        IOMMUFD_CMD_GET_HW_INFO,
        IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING,
        IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP,
+       IOMMUFD_CMD_HWPT_INVALIDATE,
 };
 
 /**
@@ -613,4 +614,46 @@ struct iommu_hwpt_get_dirty_bitmap {
 #define IOMMU_HWPT_GET_DIRTY_BITMAP _IO(IOMMUFD_TYPE, \
                                        IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP)
 
+/**
+ * enum iommu_hwpt_invalidate_data_type - IOMMU HWPT Cache Invalidation
+ *                                        Data Type
+ * @IOMMU_HWPT_INVALIDATE_DATA_VTD_S1: Invalidation data for VTD_S1
+ */
+enum iommu_hwpt_invalidate_data_type {
+       IOMMU_HWPT_INVALIDATE_DATA_VTD_S1,
+};
+
+/**
+ * struct iommu_hwpt_invalidate - ioctl(IOMMU_HWPT_INVALIDATE)
+ * @size: sizeof(struct iommu_hwpt_invalidate)
+ * @hwpt_id: ID of a nested HWPT for cache invalidation
+ * @data_uptr: User pointer to an array of driver-specific cache invalidation
+ *             data.
+ * @data_type: One of enum iommu_hwpt_invalidate_data_type, defining the data
+ *             type of all the entries in the invalidation request array. It
+ *             should be a type supported by the hwpt pointed by @hwpt_id.
+ * @entry_len: Length (in bytes) of a request entry in the request array
+ * @entry_num: Input the number of cache invalidation requests in the array.
+ *             Output the number of requests successfully handled by kernel.
+ * @__reserved: Must be 0.
+ *
+ * Invalidate the iommu cache for user-managed page table. Modifications on a
+ * user-managed page table should be followed by this operation to sync cache.
+ * Each ioctl can support one or more cache invalidation requests in the array
+ * that has a total size of @entry_len * @entry_num.
+ *
+ * An empty invalidation request array by setting @entry_num==0 is allowed, and
+ * @entry_len and @data_uptr would be ignored in this case. This can be used to
+ * check if the given @data_type is supported or not by kernel.
+ */
+struct iommu_hwpt_invalidate {
+       __u32 size;
+       __u32 hwpt_id;
+       __aligned_u64 data_uptr;
+       __u32 data_type;
+       __u32 entry_len;
+       __u32 entry_num;
+       __u32 __reserved;
+};
+#define IOMMU_HWPT_INVALIDATE _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_INVALIDATE)
 #endif