drm/xe: VM LRU bulk move

Use the TTM LRU bulk move for BOs tied to a VM. Update the bulk moves
LRU position on every exec.

v2: Bulk move for compute VMs, use WARN rather than BUG

Reviewed-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
Signed-off-by: Rodrigo Vivi <rodrigo.vivi@intel.com>

diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 8bac1717ca78f63d4b4145abfdfb488d6cd88f41..43801994f069da04ff630fffe6b5beaf5010e11f 100644 (file)
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -992,6 +992,22 @@ static void xe_gem_object_free(struct drm_gem_object *obj)
        ttm_bo_put(container_of(obj, struct ttm_buffer_object, base));
 }
 
+static void xe_gem_object_close(struct drm_gem_object *obj,
+                               struct drm_file *file_priv)
+{
+       struct xe_bo *bo = gem_to_xe_bo(obj);
+
+       if (bo->vm && !xe_vm_in_fault_mode(bo->vm)) {
+               struct ww_acquire_ctx ww;
+
+               XE_WARN_ON(!xe_bo_is_user(bo));
+
+               xe_bo_lock(bo, &ww, 0, false);
+               ttm_bo_set_bulk_move(&bo->ttm, NULL);
+               xe_bo_unlock(bo, &ww);
+       }
+}
+
 static bool should_migrate_to_system(struct xe_bo *bo)
 {
        struct xe_device *xe = xe_bo_device(bo);
@@ -1047,6 +1063,7 @@ static const struct vm_operations_struct xe_gem_vm_ops = {
 
 static const struct drm_gem_object_funcs xe_gem_object_funcs = {
        .free = xe_gem_object_free,
+       .close = xe_gem_object_close,
        .mmap = drm_gem_ttm_mmap,
        .export = xe_gem_prime_export,
        .vm_ops = &xe_gem_vm_ops,
@@ -1088,8 +1105,8 @@ void xe_bo_free(struct xe_bo *bo)
 
 struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
                                    struct xe_tile *tile, struct dma_resv *resv,
-                                   size_t size, enum ttm_bo_type type,
-                                   u32 flags)
+                                   struct ttm_lru_bulk_move *bulk, size_t size,
+                                   enum ttm_bo_type type, u32 flags)
 {
        struct ttm_operation_ctx ctx = {
                .interruptible = true,
@@ -1156,7 +1173,10 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
                return ERR_PTR(err);
 
        bo->created = true;
-       ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
+       if (bulk)
+               ttm_bo_set_bulk_move(&bo->ttm, bulk);
+       else
+               ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
 
        return bo;
 }
@@ -1226,7 +1246,10 @@ xe_bo_create_locked_range(struct xe_device *xe,
                }
        }
 
-       bo = __xe_bo_create_locked(xe, bo, tile, vm ? &vm->resv : NULL, size,
+       bo = __xe_bo_create_locked(xe, bo, tile, vm ? &vm->resv : NULL,
+                                  vm && !xe_vm_in_fault_mode(vm) &&
+                                  flags & XE_BO_CREATE_USER_BIT ?
+                                  &vm->lru_bulk_move : NULL, size,
                                   type, flags);
        if (IS_ERR(bo))
                return bo;

diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 3a148cc6e811e5f1394da42117c5d3d46f3c6ee3..08ca1d06bf7728a9fb5de08b80033f8ef0acb6e8 100644 (file)
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -81,8 +81,8 @@ void xe_bo_free(struct xe_bo *bo);
 
 struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
                                    struct xe_tile *tile, struct dma_resv *resv,
-                                   size_t size, enum ttm_bo_type type,
-                                   u32 flags);
+                                   struct ttm_lru_bulk_move *bulk, size_t size,
+                                   enum ttm_bo_type type, u32 flags);
 struct xe_bo *
 xe_bo_create_locked_range(struct xe_device *xe,
                          struct xe_tile *tile, struct xe_vm *vm,

diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index 9b252cc782b7831de34e92a705f33929dab00be6..975dee1f770ffd22f9323f4f86ab7f1a0a25a4b2 100644 (file)
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -199,7 +199,7 @@ xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
        int ret;
 
        dma_resv_lock(resv, NULL);
-       bo = __xe_bo_create_locked(xe, storage, NULL, resv, dma_buf->size,
+       bo = __xe_bo_create_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
                                   ttm_bo_type_sg, XE_BO_CREATE_SYSTEM_BIT);
        if (IS_ERR(bo)) {
                ret = PTR_ERR(bo);

diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 700f65b66d408bea53c3a7fe4fa9e64fa984d435..c52edff9a358402604a260b81d5d5d789edf15a6 100644 (file)
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -399,6 +399,12 @@ retry:
        xe_sched_job_push(job);
        xe_vm_reactivate_rebind(vm);
 
+       if (!err && !xe_vm_no_dma_fences(vm)) {
+               spin_lock(&xe->ttm.lru_lock);
+               ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
+               spin_unlock(&xe->ttm.lru_lock);
+       }
+
 err_repin:
        if (!xe_vm_no_dma_fences(vm))
                up_read(&vm->userptr.notifier_lock);

diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index c98801ee3f55819f722af9ef071a1a9bf064c68c..06ebc1cfc4f715fed4faa97491039d16de34bb7e 100644 (file)
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -632,6 +632,10 @@ retry:
 
 #undef retry_required
 
+       spin_lock(&vm->xe->ttm.lru_lock);
+       ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
+       spin_unlock(&vm->xe->ttm.lru_lock);
+
        /* Point of no return. */
        arm_preempt_fences(vm, &preempt_fences);
        resume_and_reinstall_preempt_fences(vm);

diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index 9b39c5f64afad43909a23d00b60faa719b0d60c7..c148dd49a6ca295d6cc0d7d3271f1f6ed3f8ca29 100644 (file)
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -166,6 +166,9 @@ struct xe_vm {
        /** Protects @rebind_list and the page-table structures */
        struct dma_resv resv;
 
+       /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */
+       struct ttm_lru_bulk_move lru_bulk_move;
+
        u64 size;
        struct rb_root vmas;