drm/amdkfd: Use KIQ to unmap HIQ

Currently, we unmap HIQ by directly writing to HQD
registers. This doesn't work for GFX9.4.3. Instead,
use KIQ to unmap HIQ, similar to how we use KIQ to
map HIQ. Using KIQ to unmap HIQ works for all GFX
series post GFXv9.

Signed-off-by: Mukul Joshi <mukul.joshi@amd.com>
Reviewed-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index b4fcad0e62f7ec23b22a1f27b3a8b352a3e0b1aa..0040c63e2356ddf236b703796504240b53292919 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -830,3 +830,39 @@ u64 amdgpu_amdkfd_xcp_memory_size(struct amdgpu_device *adev, int xcp_id)
                return adev->gmc.real_vram_size;
        }
 }
+
+int amdgpu_amdkfd_unmap_hiq(struct amdgpu_device *adev, u32 doorbell_off,
+                           u32 inst)
+{
+       struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst];
+       struct amdgpu_ring *kiq_ring = &kiq->ring;
+       struct amdgpu_ring_funcs ring_funcs;
+       struct amdgpu_ring ring;
+       int r = 0;
+
+       if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
+               return -EINVAL;
+
+       memset(&ring, 0x0, sizeof(struct amdgpu_ring));
+       memset(&ring_funcs, 0x0, sizeof(struct amdgpu_ring_funcs));
+
+       ring_funcs.type = AMDGPU_RING_TYPE_COMPUTE;
+       ring.doorbell_index = doorbell_off;
+       ring.funcs = &ring_funcs;
+
+       spin_lock(&kiq->ring_lock);
+
+       if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
+               spin_unlock(&kiq->ring_lock);
+               return -ENOMEM;
+       }
+
+       kiq->pmf->kiq_unmap_queues(kiq_ring, &ring, RESET_QUEUES, 0, 0);
+
+       if (kiq_ring->sched.ready && !adev->job_hang)
+               r = amdgpu_ring_test_helper(kiq_ring);
+
+       spin_unlock(&kiq->ring_lock);
+
+       return r;
+}

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 2d0406bff84ecbeca3651d7e74fa3d588cfd161c..b34418e3e006ed08ed3c5c385435940daf99ca32 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -252,6 +252,8 @@ int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct amdgpu_device *dst,
 int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min);
 int amdgpu_amdkfd_send_close_event_drain_irq(struct amdgpu_device *adev,
                                        uint32_t *payload);
+int amdgpu_amdkfd_unmap_hiq(struct amdgpu_device *adev, u32 doorbell_off,
+                               u32 inst);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware when

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index 94c0fc2e57b7f8e6afafe438e1d75ee4f05eae75..83699392c8089cd12d8bafa274c72e6e0d8bcca3 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -318,6 +318,26 @@ static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
                        1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
 }
 
+static int destroy_hiq_mqd(struct mqd_manager *mm, void *mqd,
+                       enum kfd_preempt_type type, unsigned int timeout,
+                       uint32_t pipe_id, uint32_t queue_id)
+{
+       int err;
+       struct v10_compute_mqd *m;
+       u32 doorbell_off;
+
+       m = get_mqd(mqd);
+
+       doorbell_off = m->cp_hqd_pq_doorbell_control >>
+                       CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+
+       err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, 0);
+       if (err)
+               pr_debug("Destroy HIQ MQD failed: %d\n", err);
+
+       return err;
+}
+
 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
                struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
                struct queue_properties *q)
@@ -460,7 +480,7 @@ struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
                mqd->free_mqd = free_mqd_hiq_sdma;
                mqd->load_mqd = kfd_hiq_load_mqd_kiq;
                mqd->update_mqd = update_mqd;
-               mqd->destroy_mqd = kfd_destroy_mqd_cp;
+               mqd->destroy_mqd = destroy_hiq_mqd;
                mqd->is_occupied = kfd_is_occupied_cp;
                mqd->mqd_size = sizeof(struct v10_compute_mqd);
                mqd->mqd_stride = kfd_mqd_stride;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index 31fec5e70d13523a36781a8f78b0d32df5b2b537..2319467d2d9575cb365055a4ee4fab83f3cfda3a 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -335,6 +335,26 @@ static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
                        1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
 }
 
+static int destroy_hiq_mqd(struct mqd_manager *mm, void *mqd,
+                       enum kfd_preempt_type type, unsigned int timeout,
+                       uint32_t pipe_id, uint32_t queue_id)
+{
+       int err;
+       struct v11_compute_mqd *m;
+       u32 doorbell_off;
+
+       m = get_mqd(mqd);
+
+       doorbell_off = m->cp_hqd_pq_doorbell_control >>
+                       CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+
+       err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, 0);
+       if (err)
+               pr_debug("Destroy HIQ MQD failed: %d\n", err);
+
+       return err;
+}
+
 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
                struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
                struct queue_properties *q)
@@ -449,7 +469,7 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
                mqd->free_mqd = free_mqd_hiq_sdma;
                mqd->load_mqd = kfd_hiq_load_mqd_kiq;
                mqd->update_mqd = update_mqd;
-               mqd->destroy_mqd = kfd_destroy_mqd_cp;
+               mqd->destroy_mqd = destroy_hiq_mqd;
                mqd->is_occupied = kfd_is_occupied_cp;
                mqd->mqd_size = sizeof(struct v11_compute_mqd);
 #if defined(CONFIG_DEBUG_FS)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 601bb9f68048ccfc13f7cc5dcab641e7ca727e73..e23d32f356077225f06b76008b8b75cc6ab22a87 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -405,6 +405,25 @@ static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
                        1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
 }
 
+static int destroy_hiq_mqd(struct mqd_manager *mm, void *mqd,
+                       enum kfd_preempt_type type, unsigned int timeout,
+                       uint32_t pipe_id, uint32_t queue_id)
+{
+       int err;
+       struct v9_mqd *m;
+       u32 doorbell_off;
+
+       m = get_mqd(mqd);
+
+       doorbell_off = m->cp_hqd_pq_doorbell_control >>
+                       CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+       err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, 0);
+       if (err)
+               pr_debug("Destroy HIQ MQD failed: %d\n", err);
+
+       return err;
+}
+
 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
                struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
                struct queue_properties *q)
@@ -548,16 +567,19 @@ static int destroy_hiq_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 {
        uint32_t xcc_mask = mm->dev->xcc_mask;
        int xcc_id, err, inst = 0;
-       void *xcc_mqd;
        uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);
+       struct v9_mqd *m;
+       u32 doorbell_off;
 
        for_each_inst(xcc_id, xcc_mask) {
-               xcc_mqd = mqd + hiq_mqd_size * inst;
-               err = mm->dev->kfd2kgd->hqd_destroy(mm->dev->adev, xcc_mqd,
-                                                   type, timeout, pipe_id,
-                                                   queue_id, xcc_id);
+               m = get_mqd(mqd + hiq_mqd_size * inst);
+
+               doorbell_off = m->cp_hqd_pq_doorbell_control >>
+                               CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+
+               err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, xcc_id);
                if (err) {
-                       pr_debug("Destroy MQD failed for xcc: %d\n", inst);
+                       pr_debug("Destroy HIQ MQD failed for xcc: %d\n", inst);
                        break;
                }
                ++inst;
@@ -846,7 +868,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
                } else {
                        mqd->init_mqd = init_mqd_hiq;
                        mqd->load_mqd = kfd_hiq_load_mqd_kiq;
-                       mqd->destroy_mqd = kfd_destroy_mqd_cp;
+                       mqd->destroy_mqd = destroy_hiq_mqd;
                }
                break;
        case KFD_MQD_TYPE_DIQ: