drm/amdkfd: Use partial hmm page walk during buffer validation in SVM

SVM uses hmm page walk to valid buffer before map to gpu vm. After have partial
migration/mapping do validation on same vm range as migration/map do instead of
whole svm range that can be very large. This change is expected to improve svm
code performance.

Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
Reviewed-by: Philip Yang <philip.yang@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index b854cbf06dcee9144b77d7eff71ea1a9843b148b..d630100b9e91b8588dc2e9611e279fba909e54c5 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -260,19 +260,6 @@ static void svm_migrate_put_sys_page(unsigned long addr)
        put_page(page);
 }
 
-static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)
-{
-       unsigned long cpages = 0;
-       unsigned long i;
-
-       for (i = 0; i < migrate->npages; i++) {
-               if (migrate->src[i] & MIGRATE_PFN_VALID &&
-                   migrate->src[i] & MIGRATE_PFN_MIGRATE)
-                       cpages++;
-       }
-       return cpages;
-}
-
 static unsigned long svm_migrate_unsuccessful_pages(struct migrate_vma *migrate)
 {
        unsigned long upages = 0;
@@ -402,6 +389,7 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
        struct dma_fence *mfence = NULL;
        struct migrate_vma migrate = { 0 };
        unsigned long cpages = 0;
+       unsigned long mpages = 0;
        dma_addr_t *scratch;
        void *buf;
        int r = -ENOMEM;
@@ -450,12 +438,13 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
        r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);
        migrate_vma_pages(&migrate);
 
-       pr_debug("successful/cpages/npages 0x%lx/0x%lx/0x%lx\n",
-               svm_migrate_successful_pages(&migrate), cpages, migrate.npages);
-
        svm_migrate_copy_done(adev, mfence);
        migrate_vma_finalize(&migrate);
 
+       mpages = cpages - svm_migrate_unsuccessful_pages(&migrate);
+       pr_debug("successful/cpages/npages 0x%lx/0x%lx/0x%lx\n",
+                        mpages, cpages, migrate.npages);
+
        kfd_smi_event_migration_end(node, p->lead_thread->pid,
                                    start >> PAGE_SHIFT, end >> PAGE_SHIFT,
                                    0, node->id, trigger);
@@ -465,12 +454,12 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
 out_free:
        kvfree(buf);
 out:
-       if (!r && cpages) {
+       if (!r && mpages) {
                pdd = svm_range_get_pdd_by_node(prange, node);
                if (pdd)
-                       WRITE_ONCE(pdd->page_in, pdd->page_in + cpages);
+                       WRITE_ONCE(pdd->page_in, pdd->page_in + mpages);
 
-               return cpages;
+               return mpages;
        }
        return r;
 }
@@ -498,7 +487,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
        struct vm_area_struct *vma;
        uint64_t ttm_res_offset;
        struct kfd_node *node;
-       unsigned long cpages = 0;
+       unsigned long mpages = 0;
        long r = 0;
 
        if (start_mgr < prange->start || last_mgr > prange->last) {
@@ -540,15 +529,15 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
                        pr_debug("failed %ld to migrate\n", r);
                        break;
                } else {
-                       cpages += r;
+                       mpages += r;
                }
                ttm_res_offset += next - addr;
                addr = next;
        }
 
-       if (cpages) {
+       if (mpages) {
                prange->actual_loc = best_loc;
-               prange->vram_pages = prange->vram_pages + cpages;
+               prange->vram_pages += mpages;
        } else if (!prange->actual_loc) {
                /* if no page migrated and all pages from prange are at
                 * sys ram drop svm_bo got from svm_range_vram_node_new

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index fb0011f8027714bbc075041245b6f1c65a19a38f..aacd85a4fce1e2c4557af9ff2a53bd36827ba6a9 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -158,13 +158,12 @@ svm_is_valid_dma_mapping_addr(struct device *dev, dma_addr_t dma_addr)
 static int
 svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,
                      unsigned long offset, unsigned long npages,
-                     unsigned long *hmm_pfns, uint32_t gpuidx, uint64_t *vram_pages)
+                     unsigned long *hmm_pfns, uint32_t gpuidx)
 {
        enum dma_data_direction dir = DMA_BIDIRECTIONAL;
        dma_addr_t *addr = prange->dma_addr[gpuidx];
        struct device *dev = adev->dev;
        struct page *page;
-       uint64_t vram_pages_dev;
        int i, r;
 
        if (!addr) {
@@ -174,7 +173,6 @@ svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,
                prange->dma_addr[gpuidx] = addr;
        }
 
-       vram_pages_dev = 0;
        addr += offset;
        for (i = 0; i < npages; i++) {
                if (svm_is_valid_dma_mapping_addr(dev, addr[i]))
@@ -184,7 +182,6 @@ svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,
                if (is_zone_device_page(page)) {
                        struct amdgpu_device *bo_adev = prange->svm_bo->node->adev;
 
-                       vram_pages_dev++;
                        addr[i] = (hmm_pfns[i] << PAGE_SHIFT) +
                                   bo_adev->vm_manager.vram_base_offset -
                                   bo_adev->kfd.pgmap.range.start;
@@ -201,14 +198,14 @@ svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping 0x%llx for page addr 0x%lx\n",
                                     addr[i] >> PAGE_SHIFT, page_to_pfn(page));
        }
-       *vram_pages = vram_pages_dev;
+
        return 0;
 }
 
 static int
 svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap,
                  unsigned long offset, unsigned long npages,
-                 unsigned long *hmm_pfns, uint64_t *vram_pages)
+                 unsigned long *hmm_pfns)
 {
        struct kfd_process *p;
        uint32_t gpuidx;
@@ -227,7 +224,7 @@ svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap,
                }
 
                r = svm_range_dma_map_dev(pdd->dev->adev, prange, offset, npages,
-                                         hmm_pfns, gpuidx, vram_pages);
+                                         hmm_pfns, gpuidx);
                if (r)
                        break;
        }
@@ -885,14 +882,29 @@ static void svm_range_debug_dump(struct svm_range_list *svms)
 
 static void *
 svm_range_copy_array(void *psrc, size_t size, uint64_t num_elements,
-                    uint64_t offset)
+                    uint64_t offset, uint64_t *vram_pages)
 {
+       unsigned char *src = (unsigned char *)psrc + offset;
        unsigned char *dst;
+       uint64_t i;
 
        dst = kvmalloc_array(num_elements, size, GFP_KERNEL);
        if (!dst)
                return NULL;
-       memcpy(dst, (unsigned char *)psrc + offset, num_elements * size);
+
+       if (!vram_pages) {
+               memcpy(dst, src, num_elements * size);
+               return (void *)dst;
+       }
+
+       *vram_pages = 0;
+       for (i = 0; i < num_elements; i++) {
+               dma_addr_t *temp;
+               temp = (dma_addr_t *)dst + i;
+               *temp = *((dma_addr_t *)src + i);
+               if (*temp&SVM_RANGE_VRAM_DOMAIN)
+                       (*vram_pages)++;
+       }
 
        return (void *)dst;
 }
@@ -906,7 +918,7 @@ svm_range_copy_dma_addrs(struct svm_range *dst, struct svm_range *src)
                if (!src->dma_addr[i])
                        continue;
                dst->dma_addr[i] = svm_range_copy_array(src->dma_addr[i],
-                                       sizeof(*src->dma_addr[i]), src->npages, 0);
+                                       sizeof(*src->dma_addr[i]), src->npages, 0, NULL);
                if (!dst->dma_addr[i])
                        return -ENOMEM;
        }
@@ -917,7 +929,7 @@ svm_range_copy_dma_addrs(struct svm_range *dst, struct svm_range *src)
 static int
 svm_range_split_array(void *ppnew, void *ppold, size_t size,
                      uint64_t old_start, uint64_t old_n,
-                     uint64_t new_start, uint64_t new_n)
+                     uint64_t new_start, uint64_t new_n, uint64_t *new_vram_pages)
 {
        unsigned char *new, *old, *pold;
        uint64_t d;
@@ -929,11 +941,12 @@ svm_range_split_array(void *ppnew, void *ppold, size_t size,
                return 0;
 
        d = (new_start - old_start) * size;
-       new = svm_range_copy_array(pold, size, new_n, d);
+       /* get dma addr array for new range and calculte its vram page number */
+       new = svm_range_copy_array(pold, size, new_n, d, new_vram_pages);
        if (!new)
                return -ENOMEM;
        d = (new_start == old_start) ? new_n * size : 0;
-       old = svm_range_copy_array(pold, size, old_n, d);
+       old = svm_range_copy_array(pold, size, old_n, d, NULL);
        if (!old) {
                kvfree(new);
                return -ENOMEM;
@@ -955,10 +968,13 @@ svm_range_split_pages(struct svm_range *new, struct svm_range *old,
        for (i = 0; i < MAX_GPU_INSTANCE; i++) {
                r = svm_range_split_array(&new->dma_addr[i], &old->dma_addr[i],
                                          sizeof(*old->dma_addr[i]), old->start,
-                                         npages, new->start, new->npages);
+                                         npages, new->start, new->npages,
+                                         old->actual_loc ? &new->vram_pages : NULL);
                if (r)
                        return r;
        }
+       if (old->actual_loc)
+               old->vram_pages -= new->vram_pages;
 
        return 0;
 }
@@ -982,11 +998,6 @@ svm_range_split_nodes(struct svm_range *new, struct svm_range *old,
        new->svm_bo = svm_range_bo_ref(old->svm_bo);
        new->ttm_res = old->ttm_res;
 
-       /* set new's vram_pages as old range's now, the acurate vram_pages
-        * will be updated during mapping
-        */
-       new->vram_pages = min(old->vram_pages, new->npages);
-
        spin_lock(&new->svm_bo->list_lock);
        list_add(&new->svm_bo_list, &new->svm_bo->range_list);
        spin_unlock(&new->svm_bo->list_lock);
@@ -1109,7 +1120,7 @@ static int
 svm_range_split_tail(struct svm_range *prange, uint64_t new_last,
                     struct list_head *insert_list, struct list_head *remap_list)
 {
-       struct svm_range *tail;
+       struct svm_range *tail = NULL;
        int r = svm_range_split(prange, prange->start, new_last, &tail);
 
        if (!r) {
@@ -1124,7 +1135,7 @@ static int
 svm_range_split_head(struct svm_range *prange, uint64_t new_start,
                     struct list_head *insert_list, struct list_head *remap_list)
 {
-       struct svm_range *head;
+       struct svm_range *head = NULL;
        int r = svm_range_split(prange, new_start, prange->last, &head);
 
        if (!r) {
@@ -1573,7 +1584,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
        struct svm_validate_context *ctx;
        unsigned long start, end, addr;
        struct kfd_process *p;
-       uint64_t vram_pages;
        void *owner;
        int32_t idx;
        int r = 0;
@@ -1648,15 +1658,13 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
                }
        }
 
-       vram_pages = 0;
-       start = prange->start << PAGE_SHIFT;
-       end = (prange->last + 1) << PAGE_SHIFT;
+       start = map_start << PAGE_SHIFT;
+       end = (map_last + 1) << PAGE_SHIFT;
        for (addr = start; !r && addr < end; ) {
                struct hmm_range *hmm_range;
                unsigned long map_start_vma;
                unsigned long map_last_vma;
                struct vm_area_struct *vma;
-               uint64_t vram_pages_vma;
                unsigned long next = 0;
                unsigned long offset;
                unsigned long npages;
@@ -1683,13 +1691,11 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
                }
 
                if (!r) {
-                       offset = (addr - start) >> PAGE_SHIFT;
+                       offset = (addr >> PAGE_SHIFT) - prange->start;
                        r = svm_range_dma_map(prange, ctx->bitmap, offset, npages,
-                                             hmm_range->hmm_pfns, &vram_pages_vma);
+                                             hmm_range->hmm_pfns);
                        if (r)
                                pr_debug("failed %d to dma map range\n", r);
-                       else
-                               vram_pages += vram_pages_vma;
                }
 
                svm_range_lock(prange);
@@ -1722,19 +1728,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
                addr = next;
        }
 
-       if (addr == end) {
-               prange->vram_pages = vram_pages;
-
-               /* if prange does not include any vram page and it
-                * has not released svm_bo drop its svm_bo reference
-                * and set its actaul_loc to sys ram
-                */
-               if (!vram_pages && prange->ttm_res) {
-                       prange->actual_loc = 0;
-                       svm_range_vram_node_free(prange);
-               }
-       }
-
        svm_range_unreserve_bos(ctx);
        if (!r)
                prange->validate_timestamp = ktime_get_boottime();