[PATCH v2 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver

[PATCH v2 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver

[Xiaogang.Chen]

From: Xiaogang Chen <xiaogang.chen@amd.com>

Use following work of device-private THP support [1] in kfd driver to add
2MB/THP support in migration. The implementation enables efficient migration
of large folios between system memory and device-private memory.

[1]https://lore.kernel.org/linux-mm/20251001065707.920170-1-balbirs@nvidia.com/

v2:
- Reset is_large each 4K page iteration
- Unmap the THP DMA mapping once as DMA mapping, at its real size
- Leave migrate->dst[i] zero when the source is not migrating
- Count THP migrations as HPAGE_PMD_NR pages in prange->vram_pages
- Implement folio_split and hold one svm_bo ref per subpage

Xiaogang Chen (4):
  drm/amdkfd: Add awareness of THP of device and system RAM in kfd svm
    driver
  drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP
    size
  drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd
    driver
  drm/amdkfd: Apply AMDGPU_PTE_FRAG to pte of gart page table for THP
    mapping

 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 348 ++++++++++++++++++-----
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c     |  28 +-
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h     |  13 +
 3 files changed, 315 insertions(+), 74 deletions(-)

-- 
2.34.1

[PATCH v2 1/4] drm/amdkfd: Add awareness of THP of device and system RAM in kfd svm driver

[Xiaogang.Chen]

From: Xiaogang Chen <xiaogang.chen@amd.com>

Extend kfd/svm function to allocate HPAGE_PMD_SIZE based device memory by buddy
allocator, each drm_buddy_block is HPAGE_PMD_SIZE aligned and to allocate THP
system ram by vma_alloc_folio.

Introduce SVM_RANGE_DMA_THP flag that indicates dma map of THP. THP dma
addresss will use this flag.

Add dev_pagemap_ops->folio_split callback that is called by folio_split when
core MM splits device memory folio.

These are preparations for following support for (THP) migration of zone
device-private memory, no function change.

Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 50 +++++++++++++++++++-----
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c     | 15 +++++--
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h     |  1 +
 3 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 253365a8257e..813f3c1d29dc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -217,14 +217,19 @@ svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr)
 }
 
 static void
-svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
+svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn,
+			   int order)
 {
 	struct page *page;
+	struct folio *folio;
 
 	page = pfn_to_page(pfn);
+	folio = page_folio(page);
+
+	zone_device_folio_init(folio, folio->pgmap, order);
+
+	folio_set_zone_device_data(folio, prange->svm_bo);
 	svm_range_bo_ref(prange->svm_bo);
-	page->zone_device_data = prange->svm_bo;
-	zone_device_page_init(page, page_pgmap(page), 0);
 }
 
 static void
@@ -247,11 +252,17 @@ svm_migrate_addr(struct amdgpu_device *adev, struct page *page)
 }
 
 static struct page *
-svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr)
+svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr,
+			  unsigned long order)
 {
 	struct page *page;
 
-	page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+	if (order)
+		page = folio_page(vma_alloc_folio(GFP_HIGHUSER,
+						  order, vma, addr), 0);
+	else
+		page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+
 	if (page)
 		lock_page(page);
 
@@ -265,8 +276,12 @@ static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)
 
 	for (i = 0; i < migrate->npages; i++) {
 		if (migrate->dst[i] & MIGRATE_PFN_VALID &&
-		    migrate->src[i] & MIGRATE_PFN_MIGRATE)
-			mpages++;
+			migrate->src[i] & MIGRATE_PFN_MIGRATE) {
+				if (migrate->dst[i] & MIGRATE_PFN_COMPOUND)
+					mpages += HPAGE_PMD_NR;
+				else
+					mpages++;
+			}
 	}
 	return mpages;
 }
@@ -300,7 +315,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 		if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
 			dst[i] = cursor.start + (j << PAGE_SHIFT);
 			migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-			svm_migrate_get_vram_page(prange, migrate->dst[i]);
+			svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
 			migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 			mpages++;
 		}
@@ -568,6 +583,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
 	return r < 0 ? r : 0;
 }
 
+/* folio can be compound folio or single page */
 static void svm_migrate_folio_free(struct folio *folio)
 {
 	struct page *page = &folio->page;
@@ -630,7 +646,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 			j = 0;
 		}
 
-		dpage = svm_migrate_get_sys_page(migrate->vma, addr);
+		dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
 		if (!dpage) {
 			pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
 				 prange->svms, prange->start, prange->last);
@@ -1033,9 +1049,25 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
 	return r ? VM_FAULT_SIGBUS : 0;
 }
 
+static void svm_migrate_folio_split(struct folio *head, struct folio *tail)
+{
+	struct svm_range_bo *svm_bo;
+
+	if (tail == NULL)
+		return;
+
+	tail->pgmap = head->pgmap;
+	tail->mapping = head->mapping;
+
+	svm_bo = folio_zone_device_data(head);
+	folio_set_zone_device_data(tail, svm_bo);
+	svm_range_bo_ref(svm_bo);
+}
+
 static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
 	.folio_free		= svm_migrate_folio_free,
 	.migrate_to_ram		= svm_migrate_to_ram,
+	.folio_split		= svm_migrate_folio_split,
 };
 
 /* Each VRAM page uses sizeof(struct page) on system memory */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index fa4054d51f60..6b783d12bce4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -245,7 +245,16 @@ void svm_range_dma_unmap_dev(struct device *dev, dma_addr_t *dma_addr,
 		if (!svm_is_valid_dma_mapping_addr(dev, dma_addr[i]))
 			continue;
 		pr_debug_ratelimited("unmap 0x%llx\n", dma_addr[i] >> PAGE_SHIFT);
-		dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
+
+		/* dma unmap of THP */
+		if (dma_addr[i] & SVM_RANGE_DMA_THP) {
+
+			dma_addr[i] &= ~SVM_RANGE_DMA_THP;
+			dma_unmap_page(dev, dma_addr[i], PAGE_SIZE*HPAGE_PMD_NR,
+                                   DMA_BIDIRECTIONAL);
+		} else
+			dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
+
 		dma_addr[i] = 0;
 	}
 }
@@ -578,10 +587,10 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
 	}
 
 	memset(&bp, 0, sizeof(bp));
-	bp.size = prange->npages * PAGE_SIZE;
+	bp.size = ALIGN(prange->npages * PAGE_SIZE, HPAGE_PMD_SIZE);
 	bp.bo_ptr_size = sizeof(struct svm_range_bo);
 	bp.destroy = svm_range_bo_destroy;
-	bp.byte_align = PAGE_SIZE;
+	bp.byte_align = HPAGE_PMD_SIZE;
 	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
 	bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
 	bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index c7d7adae4476..f2b3a05cd8cf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -35,6 +35,7 @@
 #include "kfd_priv.h"
 
 #define SVM_RANGE_VRAM_DOMAIN (1UL << 0)
+#define SVM_RANGE_DMA_THP (1UL << 1)
 #define SVM_ADEV_PGMAP_OWNER(adev)\
 			((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
 
-- 
2.34.1

[PATCH v2 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size

[Xiaogang.Chen]

From: Xiaogang Chen <xiaogang.chen@amd.com>

When use HPAGE_PMD_SIZE based device private pages during migration core HMM
treats device private memory in HPAGE_PMD_SIZE compound folio if possible.
Current kfd driver uses prange->granularity that can be changed by user. Need
have migration size in CPU and GPU page fault handler in HPAGE_PMD_SIZE based.

For AMD GPU that exposes private device memory choose HPAGE_PMD_SIZE as
minimums migration size in CPU and GPU page fault handler. For x86 it is
same as default prange->granularity.

Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c |  6 ++++--
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c     | 13 +++++++++++--
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h     | 12 ++++++++++++
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 813f3c1d29dc..bbf0fefd5722 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -1026,8 +1026,10 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
 	if (!prange->actual_loc)
 		goto out_unlock_prange;
 
-	/* Align migration range start and size to granularity size */
-	size = 1UL << prange->granularity;
+	/* Align migration range start and size to max of
+	 * THP with HPAGE_PMD_ORDER and granularity size
+	 */
+	size = 1UL << max(prange->granularity, HPAGE_PMD_ORDER);
 	start = max(ALIGN_DOWN(addr, size), prange->start);
 	last = min(ALIGN(addr + 1, size) - 1, prange->last);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 6b783d12bce4..482cd4e7eee5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -3071,6 +3071,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 	struct kfd_node *node;
 	int32_t best_loc;
 	int32_t gpuid, gpuidx = MAX_GPU_INSTANCE;
+	bool is_private_device = false;
 	bool write_locked = false;
 	struct vm_area_struct *vma;
 	bool migration = false;
@@ -3087,6 +3088,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 		return 0;
 	}
 	svms = &p->svms;
+	is_private_device = svm_is_private_zone(adev);
 
 	pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
 
@@ -3224,8 +3226,15 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 	kfd_smi_event_page_fault_start(node, p->lead_thread, addr,
 				       write_fault, timestamp);
 
-	/* Align migration range start and size to granularity size */
-	size = 1UL << prange->granularity;
+	if (is_private_device)
+		/* Align migration range start and size to max of
+		 * THP and granularity size
+		 */
+		size = 1UL << max(prange->granularity, HPAGE_PMD_ORDER);
+	else
+		/* Align migration range start and size to granularity size */
+		size = 1UL << prange->granularity;
+
 	start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);
 	last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);
 	if (prange->actual_loc != 0 || best_loc != 0) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index f2b3a05cd8cf..e78ee94ba33c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -215,6 +215,13 @@ void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
 void svm_range_set_max_pages(struct amdgpu_device *adev);
 int svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled);
 
+/* check adev has device private zone memory */
+static inline bool svm_is_private_zone(struct amdgpu_device *adev)
+{
+	struct amdgpu_kfd_dev *kfddev = &adev->kfd;
+	return (kfddev->pgmap.type == MEMORY_DEVICE_PRIVATE);
+}
+
 #else
 
 struct kfd_process;
@@ -277,6 +284,11 @@ static inline void svm_range_set_max_pages(struct amdgpu_device *adev)
 {
 }
 
+static inline bool svm_is_private_zone(struct amdgpu_device *adev)
+{
+	return false;
+}
+
 #define KFD_IS_SVM_API_SUPPORTED(dev) false
 
 #endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
-- 
2.34.1

[PATCH v2 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver

[Xiaogang.Chen]

From: Xiaogang Chen <xiaogang.chen@amd.com>

Update kfd svm driver to migrate device-private THP introduced from HMM core
migration function. Select this function by flag MIGRATE_VMA_SELECT_COMPOUND
when call migrate_vma_setup. kfd migration procedure is updated according to
collected page type that can be either compound folio(physical continuous) or
normal size page.

Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 271 ++++++++++++++++++-----
 1 file changed, 219 insertions(+), 52 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index bbf0fefd5722..af39e547c1fa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -298,7 +298,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 	u64 mpages = 0;
 	dma_addr_t *src;
 	u64 *dst;
-	u64 i, j;
+	u64 i, j, k, l, m;
 	int r = 0;
 
 	pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,
@@ -309,59 +309,158 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 
 	amdgpu_res_first(prange->ttm_res, ttm_res_offset,
 			 npages << PAGE_SHIFT, &cursor);
-	for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {
+	for (i = j = m = 0; (i < npages) && (mpages < migrate->cpages);) {
 		struct page *spage;
+		unsigned long cur_dst_pfn;
+		bool is_large = false;
 
-		if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
-			dst[i] = cursor.start + (j << PAGE_SHIFT);
-			migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-			svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
-			migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+		cur_dst_pfn = svm_migrate_addr_to_pfn(adev,
+						      cursor.start + (m << PAGE_SHIFT));
+
+		/* when migrate->src[i] has MIGRATE_PFN_COMPOUND set the src page
+		 * is compound THP; its vm address is HPAGE_PMD_SIZE aligned and
+		 * its MIGRATE_PFN_MIGRATE is set
+		 */
+		if ((m + HPAGE_PMD_NR) <= (cursor.size >> PAGE_SHIFT) &&
+		    (i + HPAGE_PMD_NR) <= npages &&
+		    (migrate->src[i] & MIGRATE_PFN_COMPOUND) &&
+		    IS_ALIGNED(cur_dst_pfn, HPAGE_PMD_NR)) {
+
+			is_large = true;
+			k = HPAGE_PMD_NR;
+		} else
+			 k = 1;
+
+		/* for THP src[0] alwas MIGRATE_PFN_MIGRATE
+		 * just the first migrate->dst need be setup, others are zero
+		 */
+		if (is_large) {
+
+			dst[i] = cursor.start + (m << PAGE_SHIFT);
+			svm_migrate_get_vram_page(prange, cur_dst_pfn,
+						  HPAGE_PMD_ORDER);
+
+			migrate->dst[i] = migrate_pfn(cur_dst_pfn);
+			migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
+
+			for (l=1; l < k; l++)
+				migrate->dst[i+l] = 0;
+
+			mpages++;
+
+		} else if ((migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
+			dst[i] = cursor.start + (m << PAGE_SHIFT);
+			svm_migrate_get_vram_page(prange, cur_dst_pfn, 0);
+			migrate->dst[i] = migrate_pfn(cur_dst_pfn);
 			mpages++;
 		}
-		spage = migrate_pfn_to_page(migrate->src[i]);
-		if (spage && !is_zone_device_page(spage)) {
-			src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
-					      DMA_BIDIRECTIONAL);
-			r = dma_mapping_error(dev, src[i]);
-			if (r) {
-				src[i] = 0;
-				dev_err(dev, "%s: fail %d dma_map_page\n",
-					__func__, r);
-				goto out_free_vram_pages;
+
+		if (is_large) {
+			if (j) {
+				/* migrate previous accumulated pages */
+				r = svm_migrate_copy_memory_gart(
+						adev, src + i - j,
+						dst + i - j, j,
+						FROM_RAM_TO_VRAM,
+						mfence);
+
+				if (r)
+					goto out_free_vram_pages;
+
+				j = 0;
+			}
+
+			/* for THP check if the first src page is valid
+			 * if not valid skip following HPAGE_PMD_NR - 1 pages
+			 */
+			spage = migrate_pfn_to_page(migrate->src[i]);
+			if (spage && !is_zone_device_page(spage)) {
+				/* dma_map continuous HPAGE_PMD_NR sys ram pages */
+				src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE*HPAGE_PMD_NR,
+						      DMA_BIDIRECTIONAL);
+
+				r = dma_mapping_error(dev, src[i]);
+				if (r) {
+					dev_err(dev, "%s: fail %d dma_map_page\n",
+							__func__, r);
+					goto out_free_vram_pages;
+				}
+
+				/* get dma address for following HPAGE_PMD_NR-1 pages
+				 * since src pages are continuous their dma addresses
+				 * are continuous too.
+				 */
+				for (l=1; l < k; l++)
+					src[i + l] = src[i] + l*PAGE_SIZE;
+
+				/* migrate the HPAGE_PMD_NR pages above */
+				r = svm_migrate_copy_memory_gart(
+						adev, src + i,
+						dst + i, HPAGE_PMD_NR,
+						FROM_RAM_TO_VRAM,
+						mfence);
+
+				/* mark head page dma mapping as THP, tail pages dma addr
+				 * are set to 0 for following dma_unmap
+				 */
+				src[i] |= SVM_RANGE_DMA_THP;
+				for (l = 1; l < k; l++)
+					src[i + l] = 0;
+
+				if (r)
+					goto out_free_vram_pages;
+
+				j = 0;
 			}
 		} else {
-			if (j) {
+			/* single normal page case */
+			spage = migrate_pfn_to_page(migrate->src[i]);
+			if (spage && !is_zone_device_page(spage)) {
+				src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
+						      DMA_BIDIRECTIONAL);
+
+				r = dma_mapping_error(dev, src[i]);
+
+				if (r) {
+					dev_err(dev, "%s: fail %d dma_map_page\n",
+							__func__, r);
+					goto out_free_vram_pages;
+				}
+				j += 1;
+
+			} else if (j) {
 				r = svm_migrate_copy_memory_gart(
 						adev, src + i - j,
 						dst + i - j, j,
 						FROM_RAM_TO_VRAM,
 						mfence);
+
 				if (r)
 					goto out_free_vram_pages;
-				amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
+
 				j = 0;
-			} else {
-				amdgpu_res_next(&cursor, PAGE_SIZE);
 			}
-			continue;
 		}
 
-		pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
-				     src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+		pr_debug_ratelimited("dma mapping %lld pages, src to 0x%llx, pfn 0x%lx\n",
+				     k, src[i] >> PAGE_SHIFT, migrate->src[i] >> MIGRATE_PFN_SHIFT);
+		i += k;
+		m += k;
+
+		if (m >= (cursor.size >> PAGE_SHIFT)) {
+			if (j > 0) {
+				r = svm_migrate_copy_memory_gart(adev, src + i - j,
+								 dst + i - j, j,
+								 FROM_RAM_TO_VRAM,
+								 mfence);
+				if (r)
+					goto out_free_vram_pages;
+			}
+
+			amdgpu_res_next(&cursor, m*PAGE_SIZE);
 
-		/* accumulated j + 1 pages reach end of current drm_buddy_block */
-		if (j + 1 >= (cursor.size >> PAGE_SHIFT)) {
-			r = svm_migrate_copy_memory_gart(adev, src + i - j,
-							 dst + i - j, j + 1,
-							 FROM_RAM_TO_VRAM,
-							 mfence);
-			if (r)
-				goto out_free_vram_pages;
-			amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
 			j = 0;
-		} else {
-			j++;
+			m = 0;
 		}
 	}
 
@@ -410,17 +509,24 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
 	struct kfd_process_device *pdd;
 	struct dma_fence *mfence = NULL;
 	struct migrate_vma migrate = { 0 };
+	bool is_private_device = false;
 	unsigned long cpages = 0;
 	unsigned long mpages = 0;
 	dma_addr_t *scratch;
 	void *buf;
 	int r = -ENOMEM;
 
+	is_private_device = svm_is_private_zone(adev);
+
 	memset(&migrate, 0, sizeof(migrate));
 	migrate.vma = vma;
 	migrate.start = start;
 	migrate.end = end;
 	migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
+
+	if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
+		migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
+
 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
 
 	buf = kvcalloc(npages,
@@ -609,6 +715,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 	u64 addr;
 	int r = 0;
 
+	u64 l, k;
+	bool is_large = false;
+
 	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
 		 prange->last);
 
@@ -616,8 +725,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 
 	src = (u64 *)(scratch + npages);
 	dst = scratch;
-
-	for (i = 0, j = 0; i < npages; i++, addr += PAGE_SIZE) {
+	for (i = 0, j = 0; i < npages;) {
 		struct page *spage;
 
 		spage = migrate_pfn_to_page(migrate->src[i]);
@@ -633,6 +741,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 					goto out_oom;
 				j = 0;
 			}
+
+			addr += PAGE_SIZE;
+			i++;
 			continue;
 		}
 		src[i] = svm_migrate_addr(adev, spage);
@@ -646,7 +757,22 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 			j = 0;
 		}
 
-		dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
+		if(IS_ALIGNED(page_to_pfn(spage), HPAGE_PMD_NR) &&
+		   (addr + HPAGE_PMD_SIZE) <= migrate->end &&
+		   IS_ALIGNED (addr, HPAGE_PMD_SIZE) &&
+		   migrate->src[i] & MIGRATE_PFN_COMPOUND) {
+
+			is_large = true;
+			k = HPAGE_PMD_NR;
+
+			dpage = svm_migrate_get_sys_page(migrate->vma, addr,
+							 HPAGE_PMD_ORDER);
+		} else {
+			k = 1;
+			is_large = false;
+			dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
+		}
+
 		if (!dpage) {
 			pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
 				 prange->svms, prange->start, prange->last);
@@ -654,21 +780,59 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 			goto out_oom;
 		}
 
-		dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+		dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE*k, DMA_BIDIRECTIONAL);
 		r = dma_mapping_error(dev, dst[i]);
 		if (r) {
 			dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r);
-			dst[i] = 0;
 			goto out_oom;
 		}
 
-		pr_debug_ratelimited("dma mapping dst to 0x%llx, pfn 0x%lx\n",
-				     dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
-
 		migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
+		if (is_large)
+			migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
 
-		dpage = NULL;
-		j++;
+		if (is_large) {
+			/* migrate previous accumulated pages */
+			if(j) {
+				r = svm_migrate_copy_memory_gart(adev, dst + i - j,
+								 src + i - j, j, FROM_VRAM_TO_RAM, mfence);
+				if (r)
+					goto out_oom;
+				j = 0;
+			}
+
+			for (l = 1; l < k; l++) {
+
+				src[i + l] = src[i] + l*PAGE_SIZE;
+				dst[i + l] = dst[i] + l*PAGE_SIZE;
+				migrate->dst[i + l] = 0;
+			}
+
+			/* migrate the HPAGE_PMD_NR pages above */
+			/* svm_migrate_copy_memory_gart will add a paramter to indicate
+			 * the migration is for 2MB THP
+			 */
+			r = svm_migrate_copy_memory_gart(
+						adev, dst + i,
+						src + i, HPAGE_PMD_NR,
+						FROM_VRAM_TO_RAM,
+						mfence);
+
+			/* mark head page dma mapping as THP, tail pages dma addr
+			 * are set to 0 for following dma_unmap
+			 */
+			dst[i] |= SVM_RANGE_DMA_THP;
+			for (l = 1; l < k; l++)
+				dst[i + l] = 0;
+
+			if (r)
+				goto out_oom;
+
+		} else
+			j++;
+
+		addr += PAGE_SIZE*k;
+		i += k;
 	}
 
 	if (j > 0)
@@ -687,12 +851,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 		/* release previous allocated sys pages and unmap dma address */
 		while (i--) {
 
-			if (dst[i]) {
-				dma_unmap_page(dev, dst[i], PAGE_SIZE,
-					       DMA_BIDIRECTIONAL);
-				dst[i] = 0;
-			}
-
+			/* follwing svm_range_dma_unmap_dev will do dma unmap anyway
+			 * not need do dma unmap here
+			 */
 			dpage = migrate_pfn_to_page(migrate->dst[i]);
 			if (!dpage)
 				continue;
@@ -733,6 +894,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
 	unsigned long cpages = 0;
 	unsigned long mpages = 0;
 	struct amdgpu_device *adev = node->adev;
+	bool is_private_device = false;
 	struct kfd_process_device *pdd;
 	struct dma_fence *mfence = NULL;
 	struct migrate_vma migrate = { 0 };
@@ -740,6 +902,8 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
 	void *buf;
 	int r = -ENOMEM;
 
+	is_private_device = svm_is_private_zone(adev);
+
 	memset(&migrate, 0, sizeof(migrate));
 	migrate.vma = vma;
 	migrate.start = start;
@@ -750,6 +914,9 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
 	else
 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
 
+	if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
+		migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
+
 	buf = kvcalloc(npages,
 		       2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),
 		       GFP_KERNEL);
@@ -1132,7 +1299,7 @@ int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
 
 	amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
 
-	pr_info("HMM registered %ldMB device memory\n", size >> 20);
+	pr_info("---XCHEN 3.2 HMM registered %ldMB device memory\n", size >> 20);
 
 	return 0;
 }
-- 
2.34.1

[PATCH v2 4/4] drm/amdkfd: Apply AMDGPU_PTE_FRAG to pte of gart page table for THP mapping

[Xiaogang.Chen]

From: Xiaogang Chen <xiaogang.chen@amd.com>

When both sys ram and vram are physical continuous HPAGE_PMD_NR pages during
migration set AMDGPU_PTE_FRAG(HPAGE_PMD_ORDER) at pte in gart page table to
let hardware know the migrating pages are HPAGE_PMD_NR size THP. That will
reduce hardware TLB pressure or increase TLB hit ration.

Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 45 ++++++++++++++----------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index af39e547c1fa..8a2fbe3a7613 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -48,7 +48,7 @@ static int
 svm_migrate_gart_map(struct amdgpu_ring *ring,
 		     struct amdgpu_ttm_buffer_entity *entity,
 		     u64 npages,
-		     dma_addr_t *addr, u64 *gart_addr, u64 flags)
+		     dma_addr_t *addr, u64 *gart_addr, u64 flags, bool is_thp)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_job *job;
@@ -90,6 +90,9 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
 		pte_flags |= AMDGPU_PTE_WRITEABLE;
 	pte_flags |= adev->gart.gart_pte_flags;
 
+	if (is_thp)
+		pte_flags |= AMDGPU_PTE_FRAG(HPAGE_PMD_ORDER);
+
 	cpu_addr = &job->ibs[0].ptr[num_dw];
 
 	amdgpu_gart_map(adev, 0, npages, addr, pte_flags, cpu_addr);
@@ -108,6 +111,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
  * @npages: number of pages to copy
  * @direction: enum MIGRATION_COPY_DIR
  * @mfence: output, sdma fence to signal after sdma is done
+ * @is_thp: both sys and vram are physical continuous HPAGE_PMD_NR pages
  *
  * ram address uses GART table continuous entries mapping to ram pages,
  * vram address uses direct mapping of vram pages, which must have npages
@@ -126,7 +130,7 @@ static int
 svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 			     u64 *vram, u64 npages,
 			     enum MIGRATION_COPY_DIR direction,
-			     struct dma_fence **mfence)
+			     struct dma_fence **mfence, bool is_thp)
 {
 	const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT);
 	struct amdgpu_ring *ring;
@@ -136,6 +140,12 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 	u64 size;
 	int r = 0;
 
+	if (is_thp && npages != HPAGE_PMD_NR) {
+		dev_warn(adev->dev, "THP migration should have %d pages\n",
+			 HPAGE_PMD_NR);
+		is_thp = false;
+	}
+
 	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
 	entity = &adev->mman.move_entities[0];
 
@@ -146,11 +156,12 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 
 		if (direction == FROM_VRAM_TO_RAM) {
 			gart_s = svm_migrate_direct_mapping_addr(adev, *vram);
-			r = svm_migrate_gart_map(ring, entity, size, sys, &gart_d, 0);
+			r = svm_migrate_gart_map(ring, entity, size, sys, &gart_d, 0,
+						 is_thp);
 
 		} else if (direction == FROM_RAM_TO_VRAM) {
 			r = svm_migrate_gart_map(ring, entity, size, sys, &gart_s,
-						 KFD_IOCTL_SVM_FLAG_GPU_RO);
+						 KFD_IOCTL_SVM_FLAG_GPU_RO, is_thp);
 			gart_d = svm_migrate_direct_mapping_addr(adev, *vram);
 		}
 		if (r) {
@@ -362,7 +373,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 						adev, src + i - j,
 						dst + i - j, j,
 						FROM_RAM_TO_VRAM,
-						mfence);
+						mfence, false);
 
 				if (r)
 					goto out_free_vram_pages;
@@ -398,7 +409,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 						adev, src + i,
 						dst + i, HPAGE_PMD_NR,
 						FROM_RAM_TO_VRAM,
-						mfence);
+						mfence, true);
 
 				/* mark head page dma mapping as THP, tail pages dma addr
 				 * are set to 0 for following dma_unmap
@@ -433,7 +444,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 						adev, src + i - j,
 						dst + i - j, j,
 						FROM_RAM_TO_VRAM,
-						mfence);
+						mfence, false);
 
 				if (r)
 					goto out_free_vram_pages;
@@ -452,7 +463,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 				r = svm_migrate_copy_memory_gart(adev, src + i - j,
 								 dst + i - j, j,
 								 FROM_RAM_TO_VRAM,
-								 mfence);
+								 mfence, false);
 				if (r)
 					goto out_free_vram_pages;
 			}
@@ -466,7 +477,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 
 	if (j > 0)
 		r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
-					 FROM_RAM_TO_VRAM, mfence);
+					 FROM_RAM_TO_VRAM, mfence, false);
 
 out_free_vram_pages:
 	if (r) {
@@ -736,7 +747,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 				r = svm_migrate_copy_memory_gart(adev, dst + i - j,
 								 src + i - j, j,
 								 FROM_VRAM_TO_RAM,
-								 mfence);
+								 mfence, false);
 				if (r)
 					goto out_oom;
 				j = 0;
@@ -751,7 +762,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 			r = svm_migrate_copy_memory_gart(adev, dst + i - j,
 							 src + i - j, j,
 							 FROM_VRAM_TO_RAM,
-							 mfence);
+							 mfence, false);
 			if (r)
 				goto out_oom;
 			j = 0;
@@ -795,7 +806,8 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 			/* migrate previous accumulated pages */
 			if(j) {
 				r = svm_migrate_copy_memory_gart(adev, dst + i - j,
-								 src + i - j, j, FROM_VRAM_TO_RAM, mfence);
+								 src + i - j, j, FROM_VRAM_TO_RAM,
+								 mfence, false);
 				if (r)
 					goto out_oom;
 				j = 0;
@@ -809,14 +821,11 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 			}
 
 			/* migrate the HPAGE_PMD_NR pages above */
-			/* svm_migrate_copy_memory_gart will add a paramter to indicate
-			 * the migration is for 2MB THP
-			 */
 			r = svm_migrate_copy_memory_gart(
 						adev, dst + i,
 						src + i, HPAGE_PMD_NR,
 						FROM_VRAM_TO_RAM,
-						mfence);
+						mfence, true);
 
 			/* mark head page dma mapping as THP, tail pages dma addr
 			 * are set to 0 for following dma_unmap
@@ -837,7 +846,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 
 	if (j > 0)
 		r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
-						 FROM_VRAM_TO_RAM, mfence);
+						 FROM_VRAM_TO_RAM, mfence, false);
 out_oom:
 	if (r) {
 		pr_debug("failed %d copy to ram\n", r);
@@ -1299,7 +1308,7 @@ int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
 
 	amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
 
-	pr_info("---XCHEN 3.2 HMM registered %ldMB device memory\n", size >> 20);
+	pr_info("HMM registered %ldMB device memory\n", size >> 20);
 
 	return 0;
 }
-- 
2.34.1