* [PATCH 1/4] drm/amdkfd: Add awareness of THP of device and system RAM in kfd svm driver
2026-08-31 14:24 [PATCH 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
@ 2026-08-31 14:24 ` Xiaogang.Chen
2026-08-31 14:24 ` [PATCH 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size Xiaogang.Chen
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Xiaogang.Chen @ 2026-08-31 14:24 UTC (permalink / raw)
To: amd-gfx; +Cc: Xiaogang Chen
From: Xiaogang Chen <xiaogang.chen@amd.com>
Extend kfd/svm function to allocate HPAGE_PMD_SIZE based device memory by buddy
allocator, each drm_buddy_block is HPAGE_PMD_SIZE aligned and to allocate THP
system ram by vma_alloc_folio.
It is preparation for following support for (THP) migration in zone
device-private memory.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 22 ++++++++++++++++------
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 4 ++--
2 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 253365a8257e..49a231e60dfa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -217,14 +217,18 @@ svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr)
}
static void
-svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
+svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn,
+ int order)
{
struct page *page;
+ struct folio *folio;
page = pfn_to_page(pfn);
svm_range_bo_ref(prange->svm_bo);
page->zone_device_data = prange->svm_bo;
- zone_device_page_init(page, page_pgmap(page), 0);
+
+ folio = page_folio(page);
+ zone_device_folio_init(folio, folio->pgmap, order);
}
static void
@@ -247,11 +251,17 @@ svm_migrate_addr(struct amdgpu_device *adev, struct page *page)
}
static struct page *
-svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr)
+svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr,
+ unsigned long order)
{
struct page *page;
- page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+ if (order)
+ page = folio_page(vma_alloc_folio(GFP_HIGHUSER,
+ order, vma, addr), 0);
+ else
+ page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+
if (page)
lock_page(page);
@@ -300,7 +310,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
dst[i] = cursor.start + (j << PAGE_SHIFT);
migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
- svm_migrate_get_vram_page(prange, migrate->dst[i]);
+ svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
migrate->dst[i] = migrate_pfn(migrate->dst[i]);
mpages++;
}
@@ -630,7 +640,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
j = 0;
}
- dpage = svm_migrate_get_sys_page(migrate->vma, addr);
+ dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
if (!dpage) {
pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
prange->svms, prange->start, prange->last);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index fa4054d51f60..280c38932217 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -578,10 +578,10 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
}
memset(&bp, 0, sizeof(bp));
- bp.size = prange->npages * PAGE_SIZE;
+ bp.size = ALIGN(prange->npages * PAGE_SIZE, HPAGE_PMD_SIZE);
bp.bo_ptr_size = sizeof(struct svm_range_bo);
bp.destroy = svm_range_bo_destroy;
- bp.byte_align = PAGE_SIZE;
+ bp.byte_align = HPAGE_PMD_SIZE;
bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size
2026-08-31 14:24 [PATCH 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
2026-08-31 14:24 ` [PATCH 1/4] drm/amdkfd: Add awareness of THP of device and system RAM " Xiaogang.Chen
@ 2026-08-31 14:24 ` Xiaogang.Chen
2026-08-31 14:24 ` [PATCH 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver Xiaogang.Chen
2026-08-31 14:24 ` [PATCH 4/4] drm/amdkfd: Apply AMDGPU_PTE_FRAG to pte of gart page table for THP mapping Xiaogang.Chen
3 siblings, 0 replies; 5+ messages in thread
From: Xiaogang.Chen @ 2026-08-31 14:24 UTC (permalink / raw)
To: amd-gfx; +Cc: Xiaogang Chen
From: Xiaogang Chen <xiaogang.chen@amd.com>
When use HPAGE_PMD_SIZE based device private pages during migration core HMM
treats device private memory in HPAGE_PMD_SIZE compound folio if possible.
Current kfd driver uses prange->granularity that can be changed by user. Need
have migration size in CPU and GPU page fault handler in HPAGE_PMD_SIZE based.
For AMD GPU that exposes private device memory choose HPAGE_PMD_SIZE as
minimums migration size in CPU and GPU page fault handler. For x86 it is
same as default prange->granularity.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 6 ++++--
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 13 +++++++++++--
drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 12 ++++++++++++
3 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 49a231e60dfa..f1399dd90d2f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -1020,8 +1020,10 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
if (!prange->actual_loc)
goto out_unlock_prange;
- /* Align migration range start and size to granularity size */
- size = 1UL << prange->granularity;
+ /* Align migration range start and size to max of
+ * THP with HPAGE_PMD_ORDER and granularity size
+ */
+ size = 1UL << max(prange->granularity, HPAGE_PMD_ORDER);
start = max(ALIGN_DOWN(addr, size), prange->start);
last = min(ALIGN(addr + 1, size) - 1, prange->last);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 280c38932217..c66f83abedd0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -3062,6 +3062,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
struct kfd_node *node;
int32_t best_loc;
int32_t gpuid, gpuidx = MAX_GPU_INSTANCE;
+ bool is_private_device = false;
bool write_locked = false;
struct vm_area_struct *vma;
bool migration = false;
@@ -3078,6 +3079,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
return 0;
}
svms = &p->svms;
+ is_private_device = svm_is_private_zone(adev);
pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
@@ -3215,8 +3217,15 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
kfd_smi_event_page_fault_start(node, p->lead_thread, addr,
write_fault, timestamp);
- /* Align migration range start and size to granularity size */
- size = 1UL << prange->granularity;
+ if (is_private_device)
+ /* Align migration range start and size to max of
+ * THP and granularity size
+ */
+ size = 1UL << max(prange->granularity, HPAGE_PMD_ORDER);
+ else
+ /* Align migration range start and size to granularity size */
+ size = 1UL << prange->granularity;
+
start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);
last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);
if (prange->actual_loc != 0 || best_loc != 0) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index c7d7adae4476..0c9bb9cfd0b5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -214,6 +214,13 @@ void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
void svm_range_set_max_pages(struct amdgpu_device *adev);
int svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled);
+/* check adev has device private zone memory */
+static inline bool svm_is_private_zone(struct amdgpu_device *adev)
+{
+ struct amdgpu_kfd_dev *kfddev = &adev->kfd;
+ return (kfddev->pgmap.type == MEMORY_DEVICE_PRIVATE);
+}
+
#else
struct kfd_process;
@@ -276,6 +283,11 @@ static inline void svm_range_set_max_pages(struct amdgpu_device *adev)
{
}
+static inline bool svm_is_private_zone(struct amdgpu_device *adev)
+{
+ return false;
+}
+
#define KFD_IS_SVM_API_SUPPORTED(dev) false
#endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver
2026-08-31 14:24 [PATCH 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
2026-08-31 14:24 ` [PATCH 1/4] drm/amdkfd: Add awareness of THP of device and system RAM " Xiaogang.Chen
2026-08-31 14:24 ` [PATCH 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size Xiaogang.Chen
@ 2026-08-31 14:24 ` Xiaogang.Chen
2026-08-31 14:24 ` [PATCH 4/4] drm/amdkfd: Apply AMDGPU_PTE_FRAG to pte of gart page table for THP mapping Xiaogang.Chen
3 siblings, 0 replies; 5+ messages in thread
From: Xiaogang.Chen @ 2026-08-31 14:24 UTC (permalink / raw)
To: amd-gfx; +Cc: Xiaogang Chen
From: Xiaogang Chen <xiaogang.chen@amd.com>
Update kfd svm driver to migrate device-private THP introduced from HMM core
migration function. Select this function by flag MIGRATE_VMA_SELECT_COMPOUND
when call migrate_vma_setup. kfd migration procedure is updated according to
collected page type that can be either compound folio(physical continuous) or
normal size page.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 237 +++++++++++++++++++----
1 file changed, 194 insertions(+), 43 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index f1399dd90d2f..30eac5fbcde5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -293,7 +293,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
u64 mpages = 0;
dma_addr_t *src;
u64 *dst;
- u64 i, j;
+ u64 i, j, k, l, m;
int r = 0;
pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,
@@ -304,59 +304,147 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
amdgpu_res_first(prange->ttm_res, ttm_res_offset,
npages << PAGE_SHIFT, &cursor);
- for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {
+ for (i = j = m = 0; (i < npages) && (mpages < migrate->cpages);) {
struct page *spage;
+ bool is_large = false;
- if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
- dst[i] = cursor.start + (j << PAGE_SHIFT);
- migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+ dst[i] = cursor.start + (m << PAGE_SHIFT);
+ migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+
+ /* when migrate->src[i] has MIGRATE_PFN_COMPOUND set the src page
+ * is compound THP; its vm address is HPAGE_PMD_SIZE aligned and
+ * its MIGRATE_PFN_MIGRATE is set
+ */
+ if ((m + HPAGE_PMD_NR) <= (cursor.size >> PAGE_SHIFT) &&
+ (i + HPAGE_PMD_NR) <= npages &&
+ (migrate->src[i] & MIGRATE_PFN_COMPOUND) &&
+ IS_ALIGNED(migrate->dst[i], HPAGE_PMD_NR)) {
+
+ is_large = true;
+ k = HPAGE_PMD_NR;
+ } else
+ k = 1;
+
+ /* for THP src[0] alwas MIGRATE_PFN_MIGRATE
+ * just the first migrate->dst need be setup, others are zero
+ */
+ if (is_large) {
+ svm_migrate_get_vram_page(prange, migrate->dst[i],
+ HPAGE_PMD_ORDER);
+
+ migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+ migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
+
+ for (l=1; l < k; l++)
+ migrate->dst[i+l] = 0;
+
+ mpages++;
+
+ } else if ((migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
migrate->dst[i] = migrate_pfn(migrate->dst[i]);
mpages++;
}
- spage = migrate_pfn_to_page(migrate->src[i]);
- if (spage && !is_zone_device_page(spage)) {
- src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
- DMA_BIDIRECTIONAL);
- r = dma_mapping_error(dev, src[i]);
- if (r) {
- src[i] = 0;
- dev_err(dev, "%s: fail %d dma_map_page\n",
- __func__, r);
- goto out_free_vram_pages;
+
+ if (is_large) {
+ if (j) {
+ /* migrate previous accumulated pages */
+ r = svm_migrate_copy_memory_gart(
+ adev, src + i - j,
+ dst + i - j, j,
+ FROM_RAM_TO_VRAM,
+ mfence);
+
+ if (r)
+ goto out_free_vram_pages;
+
+ j = 0;
+ }
+
+ /* for THP check if the first src page is valid
+ * if not valid skip following HPAGE_PMD_NR - 1 pages
+ */
+ spage = migrate_pfn_to_page(migrate->src[i]);
+ if (spage && !is_zone_device_page(spage)) {
+ /* dma_map continuous HPAGE_PMD_NR sys ram pages */
+ src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE*HPAGE_PMD_NR,
+ DMA_BIDIRECTIONAL);
+
+ r = dma_mapping_error(dev, src[i]);
+ if (r) {
+ dev_err(dev, "%s: fail %d dma_map_page\n",
+ __func__, r);
+ goto out_free_vram_pages;
+ }
+
+ /* get dma address for following HPAGE_PMD_NR-1 pages
+ * since src pages are continuous their dma addresses
+ * are continuous too.
+ */
+ for (l=1; l < k; l++)
+ src[i + l] = src[i] + l*PAGE_SIZE;
+
+ /* migrate the HPAGE_PMD_NR pages above */
+ r = svm_migrate_copy_memory_gart(
+ adev, src + i,
+ dst + i, HPAGE_PMD_NR,
+ FROM_RAM_TO_VRAM,
+ mfence);
+
+ if (r)
+ goto out_free_vram_pages;
+
+ j = 0;
}
} else {
- if (j) {
+ /* single normal page case */
+ spage = migrate_pfn_to_page(migrate->src[i]);
+ if (spage && !is_zone_device_page(spage)) {
+ src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+
+ r = dma_mapping_error(dev, src[i]);
+
+ if (r) {
+ dev_err(dev, "%s: fail %d dma_map_page\n",
+ __func__, r);
+ goto out_free_vram_pages;
+ }
+ j += 1;
+
+ } else if (j) {
r = svm_migrate_copy_memory_gart(
adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
mfence);
+
if (r)
goto out_free_vram_pages;
- amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
+
j = 0;
- } else {
- amdgpu_res_next(&cursor, PAGE_SIZE);
}
- continue;
}
- pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
- src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+ pr_debug_ratelimited("dma mapping %lld pages, src to 0x%llx, pfn 0x%lx\n",
+ k, src[i] >> PAGE_SHIFT, migrate->src[i] >> MIGRATE_PFN_SHIFT);
+ i += k;
+ m += k;
+
+ if (m >= (cursor.size >> PAGE_SHIFT)) {
+ if (j > 0) {
+ r = svm_migrate_copy_memory_gart(adev, src + i - j,
+ dst + i - j, j,
+ FROM_RAM_TO_VRAM,
+ mfence);
+ if (r)
+ goto out_free_vram_pages;
+ }
+
+ amdgpu_res_next(&cursor, m*PAGE_SIZE);
- /* accumulated j + 1 pages reach end of current drm_buddy_block */
- if (j + 1 >= (cursor.size >> PAGE_SHIFT)) {
- r = svm_migrate_copy_memory_gart(adev, src + i - j,
- dst + i - j, j + 1,
- FROM_RAM_TO_VRAM,
- mfence);
- if (r)
- goto out_free_vram_pages;
- amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
j = 0;
- } else {
- j++;
+ m = 0;
}
}
@@ -405,17 +493,24 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
struct kfd_process_device *pdd;
struct dma_fence *mfence = NULL;
struct migrate_vma migrate = { 0 };
+ bool is_private_device = false;
unsigned long cpages = 0;
unsigned long mpages = 0;
dma_addr_t *scratch;
void *buf;
int r = -ENOMEM;
+ is_private_device = svm_is_private_zone(adev);
+
memset(&migrate, 0, sizeof(migrate));
migrate.vma = vma;
migrate.start = start;
migrate.end = end;
migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
+
+ if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
+ migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
+
migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
buf = kvcalloc(npages,
@@ -603,6 +698,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
u64 addr;
int r = 0;
+ u64 l, k;
+ bool is_large = false;
+
pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
prange->last);
@@ -610,8 +708,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
src = (u64 *)(scratch + npages);
dst = scratch;
-
- for (i = 0, j = 0; i < npages; i++, addr += PAGE_SIZE) {
+ for (i = 0, j = 0; i < npages;) {
struct page *spage;
spage = migrate_pfn_to_page(migrate->src[i]);
@@ -627,6 +724,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
goto out_oom;
j = 0;
}
+
+ addr += PAGE_SIZE;
+ i++;
continue;
}
src[i] = svm_migrate_addr(adev, spage);
@@ -640,7 +740,21 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
j = 0;
}
- dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
+ if(IS_ALIGNED(page_to_pfn(spage), HPAGE_PMD_NR) &&
+ (addr + HPAGE_PMD_SIZE) <= migrate->end &&
+ IS_ALIGNED (addr, HPAGE_PMD_SIZE) &&
+ migrate->src[i] & MIGRATE_PFN_COMPOUND) {
+
+ is_large = true;
+ k = HPAGE_PMD_NR;
+
+ dpage = svm_migrate_get_sys_page(migrate->vma, addr,
+ HPAGE_PMD_ORDER);
+ } else {
+ k = 1;
+ dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
+ }
+
if (!dpage) {
pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
prange->svms, prange->start, prange->last);
@@ -648,21 +762,52 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
goto out_oom;
}
- dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+ dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE*k, DMA_BIDIRECTIONAL);
r = dma_mapping_error(dev, dst[i]);
if (r) {
dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r);
- dst[i] = 0;
goto out_oom;
}
- pr_debug_ratelimited("dma mapping dst to 0x%llx, pfn 0x%lx\n",
- dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
-
migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
+ if (is_large)
+ migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
+
+ if (is_large) {
+ /* migrate previous accumulated pages */
+ if(j) {
+ r = svm_migrate_copy_memory_gart(adev, dst + i - j,
+ src + i - j, j, FROM_VRAM_TO_RAM, mfence);
+ if (r)
+ goto out_oom;
+ j = 0;
+ }
+
+ for (l = 1; l < k; l++) {
+
+ src[i + l] = src[i] + l*PAGE_SIZE;
+ dst[i + l] = dst[i] + l*PAGE_SIZE;
+ migrate->dst[i + l] = 0;
+ }
+
+ /* migrate the HPAGE_PMD_NR pages above */
+ /* svm_migrate_copy_memory_gart will add a paramter to indicate
+ * the migration is for 2MB THP
+ */
+ r = svm_migrate_copy_memory_gart(
+ adev, dst + i,
+ src + i, HPAGE_PMD_NR,
+ FROM_VRAM_TO_RAM,
+ mfence);
+
+ if (r)
+ goto out_oom;
+
+ } else
+ j++;
- dpage = NULL;
- j++;
+ addr += PAGE_SIZE*k;
+ i += k;
}
if (j > 0)
@@ -727,6 +872,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
unsigned long cpages = 0;
unsigned long mpages = 0;
struct amdgpu_device *adev = node->adev;
+ bool is_private_device = false;
struct kfd_process_device *pdd;
struct dma_fence *mfence = NULL;
struct migrate_vma migrate = { 0 };
@@ -734,6 +880,8 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
void *buf;
int r = -ENOMEM;
+ is_private_device = svm_is_private_zone(adev);
+
memset(&migrate, 0, sizeof(migrate));
migrate.vma = vma;
migrate.start = start;
@@ -744,6 +892,9 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
else
migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
+ if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
+ migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
+
buf = kvcalloc(npages,
2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),
GFP_KERNEL);
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 4/4] drm/amdkfd: Apply AMDGPU_PTE_FRAG to pte of gart page table for THP mapping
2026-08-31 14:24 [PATCH 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
` (2 preceding siblings ...)
2026-08-31 14:24 ` [PATCH 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver Xiaogang.Chen
@ 2026-08-31 14:24 ` Xiaogang.Chen
3 siblings, 0 replies; 5+ messages in thread
From: Xiaogang.Chen @ 2026-08-31 14:24 UTC (permalink / raw)
To: amd-gfx; +Cc: Xiaogang Chen
From: Xiaogang Chen <xiaogang.chen@amd.com>
When both sys ram and vram are physical continuous HPAGE_PMD_NR pages during
migration set AMDGPU_PTE_FRAG(HPAGE_PMD_ORDER) at pte in gart page table to
let hardware know the migrating pages are HPAGE_PMD_NR size THP. That will
reduce hardware TLB pressure or increase TLB hit ration.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 43 ++++++++++++++----------
1 file changed, 26 insertions(+), 17 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 30eac5fbcde5..6863be041ba5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -48,7 +48,7 @@ static int
svm_migrate_gart_map(struct amdgpu_ring *ring,
struct amdgpu_ttm_buffer_entity *entity,
u64 npages,
- dma_addr_t *addr, u64 *gart_addr, u64 flags)
+ dma_addr_t *addr, u64 *gart_addr, u64 flags, bool is_thp)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_job *job;
@@ -90,6 +90,9 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
pte_flags |= AMDGPU_PTE_WRITEABLE;
pte_flags |= adev->gart.gart_pte_flags;
+ if (is_thp)
+ pte_flags |= AMDGPU_PTE_FRAG(HPAGE_PMD_ORDER);
+
cpu_addr = &job->ibs[0].ptr[num_dw];
amdgpu_gart_map(adev, 0, npages, addr, pte_flags, cpu_addr);
@@ -108,6 +111,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
* @npages: number of pages to copy
* @direction: enum MIGRATION_COPY_DIR
* @mfence: output, sdma fence to signal after sdma is done
+ * @is_thp: both sys and vram are physical continuous HPAGE_PMD_NR pages
*
* ram address uses GART table continuous entries mapping to ram pages,
* vram address uses direct mapping of vram pages, which must have npages
@@ -126,7 +130,7 @@ static int
svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
u64 *vram, u64 npages,
enum MIGRATION_COPY_DIR direction,
- struct dma_fence **mfence)
+ struct dma_fence **mfence, bool is_thp)
{
const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT);
struct amdgpu_ring *ring;
@@ -136,6 +140,12 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
u64 size;
int r = 0;
+ if (is_thp && npages != HPAGE_PMD_NR) {
+ dev_warn(adev->dev, "THP migration should have %d pages\n",
+ HPAGE_PMD_NR);
+ is_thp = false;
+ }
+
ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
entity = &adev->mman.move_entities[0];
@@ -146,11 +156,12 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
if (direction == FROM_VRAM_TO_RAM) {
gart_s = svm_migrate_direct_mapping_addr(adev, *vram);
- r = svm_migrate_gart_map(ring, entity, size, sys, &gart_d, 0);
+ r = svm_migrate_gart_map(ring, entity, size, sys, &gart_d, 0,
+ is_thp);
} else if (direction == FROM_RAM_TO_VRAM) {
r = svm_migrate_gart_map(ring, entity, size, sys, &gart_s,
- KFD_IOCTL_SVM_FLAG_GPU_RO);
+ KFD_IOCTL_SVM_FLAG_GPU_RO, is_thp);
gart_d = svm_migrate_direct_mapping_addr(adev, *vram);
}
if (r) {
@@ -353,7 +364,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
- mfence);
+ mfence, false);
if (r)
goto out_free_vram_pages;
@@ -389,7 +400,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
adev, src + i,
dst + i, HPAGE_PMD_NR,
FROM_RAM_TO_VRAM,
- mfence);
+ mfence, true);
if (r)
goto out_free_vram_pages;
@@ -417,7 +428,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
- mfence);
+ mfence, false);
if (r)
goto out_free_vram_pages;
@@ -436,7 +447,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
r = svm_migrate_copy_memory_gart(adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
- mfence);
+ mfence, false);
if (r)
goto out_free_vram_pages;
}
@@ -450,7 +461,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
if (j > 0)
r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
- FROM_RAM_TO_VRAM, mfence);
+ FROM_RAM_TO_VRAM, mfence, false);
out_free_vram_pages:
if (r) {
@@ -719,7 +730,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
r = svm_migrate_copy_memory_gart(adev, dst + i - j,
src + i - j, j,
FROM_VRAM_TO_RAM,
- mfence);
+ mfence, false);
if (r)
goto out_oom;
j = 0;
@@ -734,7 +745,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
r = svm_migrate_copy_memory_gart(adev, dst + i - j,
src + i - j, j,
FROM_VRAM_TO_RAM,
- mfence);
+ mfence, false);
if (r)
goto out_oom;
j = 0;
@@ -777,7 +788,8 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
/* migrate previous accumulated pages */
if(j) {
r = svm_migrate_copy_memory_gart(adev, dst + i - j,
- src + i - j, j, FROM_VRAM_TO_RAM, mfence);
+ src + i - j, j, FROM_VRAM_TO_RAM,
+ mfence, false);
if (r)
goto out_oom;
j = 0;
@@ -791,14 +803,11 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
}
/* migrate the HPAGE_PMD_NR pages above */
- /* svm_migrate_copy_memory_gart will add a paramter to indicate
- * the migration is for 2MB THP
- */
r = svm_migrate_copy_memory_gart(
adev, dst + i,
src + i, HPAGE_PMD_NR,
FROM_VRAM_TO_RAM,
- mfence);
+ mfence, true);
if (r)
goto out_oom;
@@ -812,7 +821,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
if (j > 0)
r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
- FROM_VRAM_TO_RAM, mfence);
+ FROM_VRAM_TO_RAM, mfence, false);
out_oom:
if (r) {
pr_debug("failed %d copy to ram\n", r);
--
2.34.1
^ permalink raw reply related [flat|nested] 5+ messages in thread