drm/xe: Add XE_BO_GGTT_INVALIDATE flag

Add XE_BO_GGTT_INVALIDATE flag which indicates the GGTT should be
invalidated when a BO is added / removed from the GGTT. This is
typically set when a BO is used by the GuC as the GuC has GGTT TLBs.

Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
[mlankhorst: Small fix to only inherit GGTT_INVALIDATE from src bo]
[mlankhorst: Remove _BIT from name]
Signed-off-by: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240306052002.311196-4-matthew.brost@intel.com

diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c
index 311ffad6e2c063eebc7a3c725a8f7192b89452fc..2a50a7eaaa3102f183805b3afaf5a8b71d1a9180 100644 (file)
--- a/drivers/gpu/drm/xe/display/xe_fb_pin.c
+++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c
@@ -323,7 +323,7 @@ static void __xe_unpin_fb_vma(struct i915_vma *vma)
                xe_bo_unpin_map_no_vm(vma->dpt);
        else if (!drm_mm_node_allocated(&vma->bo->ggtt_node) ||
                 vma->bo->ggtt_node.start != vma->node.start)
-               xe_ggtt_remove_node(ggtt, &vma->node);
+               xe_ggtt_remove_node(ggtt, &vma->node, false);
 
        ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);
        ttm_bo_unpin(&vma->bo->ttm);

diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 8383c56c99602ec8d38da179814781598322d457..9298546909b5231a5423c7c9d29a370095025523 100644 (file)
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -1580,13 +1580,15 @@ struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_til
 int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src)
 {
        struct xe_bo *bo;
+       u32 dst_flags = XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT;
+
+       dst_flags |= (*src)->flags & XE_BO_GGTT_INVALIDATE;
 
        xe_assert(xe, IS_DGFX(xe));
        xe_assert(xe, !(*src)->vmap.is_iomem);
 
-       bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr, (*src)->size,
-                                           XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-                                           XE_BO_CREATE_GGTT_BIT);
+       bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr,
+                                           (*src)->size, dst_flags);
        if (IS_ERR(bo))
                return PTR_ERR(bo);
 

diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 87ae8ee7c3162583256f7df156f28d902e9eae5c..52e441f77e96a674ca5b8c57313297121d295b6b 100644 (file)
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -46,6 +46,7 @@
 #define XE_BO_NEEDS_CPU_ACCESS         BIT(13)
 #define XE_BO_NEEDS_UC                 BIT(14)
 #define XE_BO_NEEDS_64K                        BIT(15)
+#define XE_BO_GGTT_INVALIDATE          BIT(16)
 /* this one is trigger internally only */
 #define XE_BO_INTERNAL_TEST            BIT(30)
 #define XE_BO_INTERNAL_64K             BIT(31)

diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index cc60ea08fc8495bda4f14734f36b53bf8e8f7811..9947ded57cf11208beca3f4575f2dc2183e3e5fb 100644 (file)
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -390,7 +390,8 @@ void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
                xe_ggtt_set_pte(ggtt, start + offset, pte);
        }
 
-       xe_ggtt_invalidate(ggtt);
+       if (bo->flags & XE_BO_GGTT_INVALIDATE)
+               xe_ggtt_invalidate(ggtt);
 }
 
 static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
@@ -435,7 +436,8 @@ int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
        return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
 }
 
-void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node)
+void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
+                        bool invalidate)
 {
        xe_device_mem_access_get(tile_to_xe(ggtt->tile));
        mutex_lock(&ggtt->lock);
@@ -444,7 +446,8 @@ void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node)
        drm_mm_remove_node(node);
        node->size = 0;
 
-       xe_ggtt_invalidate(ggtt);
+       if (invalidate)
+               xe_ggtt_invalidate(ggtt);
 
        mutex_unlock(&ggtt->lock);
        xe_device_mem_access_put(tile_to_xe(ggtt->tile));
@@ -458,7 +461,8 @@ void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
        /* This BO is not currently in the GGTT */
        xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
 
-       xe_ggtt_remove_node(ggtt, &bo->ggtt_node);
+       xe_ggtt_remove_node(ggtt, &bo->ggtt_node,
+                           bo->flags & XE_BO_GGTT_INVALIDATE);
 }
 
 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)

diff --git a/drivers/gpu/drm/xe/xe_ggtt.h b/drivers/gpu/drm/xe/xe_ggtt.h
index dc9631d20720bcccf85512f7481c765226007934..8306ef74abc6e7aecdbd453bf4e8d5e47856af68 100644 (file)
--- a/drivers/gpu/drm/xe/xe_ggtt.h
+++ b/drivers/gpu/drm/xe/xe_ggtt.h
@@ -23,7 +23,8 @@ int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
 int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt,
                                       struct drm_mm_node *node,
                                       u32 size, u32 align, u32 mm_flags);
-void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node);
+void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
+                        bool invalidate);
 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo);
 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo);
 int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,

diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index 6ad4c1a90a787bfb3fab63d266b86d2cfc9d1b7e..df2bffb7e22047bbfdbe085a9e7436d9c86acd08 100644 (file)
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -274,7 +274,8 @@ int xe_guc_ads_init(struct xe_guc_ads *ads)
 
        bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE,
                                          XE_BO_CREATE_SYSTEM_BIT |
-                                         XE_BO_CREATE_GGTT_BIT);
+                                         XE_BO_CREATE_GGTT_BIT |
+                                         XE_BO_GGTT_INVALIDATE);
        if (IS_ERR(bo))
                return PTR_ERR(bo);
 

diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c
index 355edd4d758af7cf1e4c15daf00ed86bbd4de898..d9fa81900ff5302011700efa6ca950ebe0740ed9 100644 (file)
--- a/drivers/gpu/drm/xe/xe_guc_ct.c
+++ b/drivers/gpu/drm/xe/xe_guc_ct.c
@@ -156,7 +156,8 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
 
        bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
                                          XE_BO_CREATE_SYSTEM_BIT |
-                                         XE_BO_CREATE_GGTT_BIT);
+                                         XE_BO_CREATE_GGTT_BIT |
+                                         XE_BO_GGTT_INVALIDATE);
        if (IS_ERR(bo))
                return PTR_ERR(bo);
 

diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.c b/drivers/gpu/drm/xe/xe_guc_hwconfig.c
index ea49f3885c108d261317b3b0a52ed0f5a08f53b2..1173453c403948e805a36fb041de659651aa5bd5 100644 (file)
--- a/drivers/gpu/drm/xe/xe_guc_hwconfig.c
+++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.c
@@ -79,7 +79,8 @@ int xe_guc_hwconfig_init(struct xe_guc *guc)
 
        bo = xe_managed_bo_create_pin_map(xe, tile, PAGE_ALIGN(size),
                                          XE_BO_CREATE_SYSTEM_BIT |
-                                         XE_BO_CREATE_GGTT_BIT);
+                                         XE_BO_CREATE_GGTT_BIT |
+                                         XE_BO_GGTT_INVALIDATE);
        if (IS_ERR(bo))
                return PTR_ERR(bo);
        guc->hwconfig.bo = bo;

diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c
index 45135c3520e5499d8761932215faa21b090ac5d7..9302a7faaf0bf7cc022b5f3012aa4a3cbd677f9d 100644 (file)
--- a/drivers/gpu/drm/xe/xe_guc_log.c
+++ b/drivers/gpu/drm/xe/xe_guc_log.c
@@ -85,7 +85,8 @@ int xe_guc_log_init(struct xe_guc_log *log)
 
        bo = xe_managed_bo_create_pin_map(xe, tile, guc_log_size(),
                                          XE_BO_CREATE_SYSTEM_BIT |
-                                         XE_BO_CREATE_GGTT_BIT);
+                                         XE_BO_CREATE_GGTT_BIT |
+                                         XE_BO_GGTT_INVALIDATE);
        if (IS_ERR(bo))
                return PTR_ERR(bo);
 

diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c
index f4b031b8d9deb12a9682b93d6ef5241e16699c74..eb174547c8f2ddbe26aa3a2a0bad26af1e583bc8 100644 (file)
--- a/drivers/gpu/drm/xe/xe_guc_pc.c
+++ b/drivers/gpu/drm/xe/xe_guc_pc.c
@@ -955,7 +955,8 @@ int xe_guc_pc_init(struct xe_guc_pc *pc)
 
        bo = xe_managed_bo_create_pin_map(xe, tile, size,
                                          XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-                                         XE_BO_CREATE_GGTT_BIT);
+                                         XE_BO_CREATE_GGTT_BIT |
+                                         XE_BO_GGTT_INVALIDATE);
        if (IS_ERR(bo))
                return PTR_ERR(bo);
 

diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c
index 2c5615130a3875cbf3bfc02be85f9a46fb45c26f..b94924a4f31902c66dda56c8395689e7d9e83e47 100644 (file)
--- a/drivers/gpu/drm/xe/xe_hw_engine.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine.c
@@ -519,7 +519,8 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe,
 
        hwe->hwsp = xe_managed_bo_create_pin_map(xe, tile, SZ_4K,
                                                 XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-                                                XE_BO_CREATE_GGTT_BIT);
+                                                XE_BO_CREATE_GGTT_BIT |
+                                                XE_BO_GGTT_INVALIDATE);
        if (IS_ERR(hwe->hwsp)) {
                err = PTR_ERR(hwe->hwsp);
                goto err_name;

diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c
index 3c4d317032077003c61806f120c7c1a96f6048f0..95e0f7b1ec3f6ea3664ac10425d07109f87f1506 100644 (file)
--- a/drivers/gpu/drm/xe/xe_lrc.c
+++ b/drivers/gpu/drm/xe/xe_lrc.c
@@ -746,7 +746,8 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
                                      ring_size + xe_lrc_size(xe, hwe->class),
                                      ttm_bo_type_kernel,
                                      XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-                                     XE_BO_CREATE_GGTT_BIT);
+                                     XE_BO_CREATE_GGTT_BIT |
+                                     XE_BO_GGTT_INVALIDATE);
        if (IS_ERR(lrc->bo))
                return PTR_ERR(lrc->bo);
 

diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c
index 76e95535d7f60a24bc5d7cf1f7184d9d38be7fe8..0eb28681bec7d0396651009681b2494329d1dffe 100644 (file)
--- a/drivers/gpu/drm/xe/xe_memirq.c
+++ b/drivers/gpu/drm/xe/xe_memirq.c
@@ -129,6 +129,7 @@ static int memirq_alloc_pages(struct xe_memirq *memirq)
                                  ttm_bo_type_kernel,
                                  XE_BO_CREATE_SYSTEM_BIT |
                                  XE_BO_CREATE_GGTT_BIT |
+                                 XE_BO_GGTT_INVALIDATE |
                                  XE_BO_NEEDS_UC |
                                  XE_BO_NEEDS_CPU_ACCESS);
        if (IS_ERR(bo)) {

diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c
index 2c4632259edd0528dadbd96267468dc65add1eb5..164202ac64544e8f17e7c6eb74c4baee9787bc89 100644 (file)
--- a/drivers/gpu/drm/xe/xe_sa.c
+++ b/drivers/gpu/drm/xe/xe_sa.c
@@ -49,7 +49,8 @@ struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32
 
        bo = xe_bo_create_pin_map(xe, tile, NULL, size, ttm_bo_type_kernel,
                                  XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-                                 XE_BO_CREATE_GGTT_BIT);
+                                 XE_BO_CREATE_GGTT_BIT |
+                                 XE_BO_GGTT_INVALIDATE);
        if (IS_ERR(bo)) {
                drm_err(&xe->drm, "failed to allocate bo for sa manager: %ld\n",
                        PTR_ERR(bo));

diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c
index 99c5490132d5656e0c6e3d9f4882d4c6279f9b82..400651485b8598ef087cebe8ef3dc0663ed11676 100644 (file)
--- a/drivers/gpu/drm/xe/xe_uc_fw.c
+++ b/drivers/gpu/drm/xe/xe_uc_fw.c
@@ -763,7 +763,8 @@ int xe_uc_fw_init(struct xe_uc_fw *uc_fw)
                return 0;
 
        err = uc_fw_copy(uc_fw, fw->data, fw->size,
-                        XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_GGTT_BIT);
+                        XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_GGTT_BIT |
+                        XE_BO_GGTT_INVALIDATE);
 
        uc_fw_release(fw);