From: Xiaogang.Chen <xiaogang.chen@amd.com>
To: <amd-gfx@lists.freedesktop.org>
Cc: Xiaogang Chen <xiaogang.chen@amd.com>
Subject: [PATCH v3 3/3] drm/amdkfd: Enable HMM device-private THP migration in KFD SVM driver
Date: Mon, 19 Oct 2026 08:38:45 -0500 [thread overview]
Message-ID: <20261019133845.166916-4-xiaogang.chen@amd.com> (raw)
In-Reply-To: <20261019133845.166916-1-xiaogang.chen@amd.com>
From: Xiaogang Chen <xiaogang.chen@amd.com>
HMM migrates a PMD-mapped THP as one compound folio when
migrate_vma_setup() is passed MIGRATE_VMA_SELECT_COMPOUND.
Without that flag the collect path splits the source THP to 4K pages.
If split_folio() fails because of extra refs or pins, those pages are
skipped.
Set SELECT_COMPOUND for device-private ranges whose migrate window is
at least HPAGE_PMD_NR pages. The RAM<->VRAM copy path follows
MIGRATE_PFN_COMPOUND in src[] (HPAGE_PMD_ORDER device folio, THP DMA)
and keeps the existing 4K path otherwise.
Coherent zone-device memory is unchanged.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 253 ++++++++++++++++++-----
1 file changed, 203 insertions(+), 50 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 2febe3bb690e..5c98028bfe73 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -324,7 +324,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
u64 mpages = 0;
dma_addr_t *src;
u64 *dst;
- u64 i, j;
+ u64 i, j, k, l, m;
int r = 0;
pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,
@@ -335,59 +335,155 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
amdgpu_res_first(prange->ttm_res, ttm_res_offset,
npages << PAGE_SHIFT, &cursor);
- for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {
+ for (i = j = m = 0; (i < npages) && (mpages < migrate->cpages);) {
struct page *spage;
+ unsigned long cur_dst_pfn;
- if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
- dst[i] = cursor.start + (j << PAGE_SHIFT);
- migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
- svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
- migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+ cur_dst_pfn = svm_migrate_addr_to_pfn(adev,
+ cursor.start + (m << PAGE_SHIFT));
+
+ /* when migrate->src[i] has MIGRATE_PFN_COMPOUND set the src page
+ * is compound THP; its vm address is HPAGE_PMD_SIZE aligned and
+ * its MIGRATE_PFN_MIGRATE is set
+ */
+ if ((m + HPAGE_PMD_NR) <= (cursor.size >> PAGE_SHIFT) &&
+ (i + HPAGE_PMD_NR) <= npages &&
+ (migrate->src[i] & MIGRATE_PFN_COMPOUND) &&
+ IS_ALIGNED(cur_dst_pfn, HPAGE_PMD_NR)) {
+ k = HPAGE_PMD_NR;
+ } else
+ k = 1;
+
+ /* for THP src[0] alwas MIGRATE_PFN_MIGRATE
+ * just the first migrate->dst need be setup, others are zero
+ */
+ if (k > 1) {
+
+ dst[i] = cursor.start + (m << PAGE_SHIFT);
+ svm_migrate_get_vram_page(prange, cur_dst_pfn,
+ HPAGE_PMD_ORDER);
+
+ migrate->dst[i] = migrate_pfn(cur_dst_pfn);
+ migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
+
+ for (l = 1; l < k; l++)
+ migrate->dst[i+l] = 0;
+
+ mpages++;
+
+ } else if ((migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
+ dst[i] = cursor.start + (m << PAGE_SHIFT);
+ svm_migrate_get_vram_page(prange, cur_dst_pfn, 0);
+ migrate->dst[i] = migrate_pfn(cur_dst_pfn);
mpages++;
}
- spage = migrate_pfn_to_page(migrate->src[i]);
- if (spage && !is_zone_device_page(spage)) {
- src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
- DMA_BIDIRECTIONAL);
- r = dma_mapping_error(dev, src[i]);
- if (r) {
- src[i] = 0;
- dev_err(dev, "%s: fail %d dma_map_page\n",
- __func__, r);
- goto out_free_vram_pages;
+
+ if (k > 1) {
+ if (j) {
+ /* migrate previous accumulated pages */
+ r = svm_migrate_copy_memory_gart(
+ adev, src + i - j,
+ dst + i - j, j,
+ FROM_RAM_TO_VRAM,
+ mfence);
+
+ if (r)
+ goto out_free_vram_pages;
+
+ j = 0;
+ }
+
+ /* for THP check if the first src page is valid
+ * if not valid skip following HPAGE_PMD_NR - 1 pages
+ */
+ spage = migrate_pfn_to_page(migrate->src[i]);
+ if (spage && !is_zone_device_page(spage)) {
+ /* dma_map continuous HPAGE_PMD_NR sys ram pages */
+ src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE*HPAGE_PMD_NR,
+ DMA_BIDIRECTIONAL);
+
+ r = dma_mapping_error(dev, src[i]);
+ if (r) {
+ dev_err(dev, "%s: fail %d dma_map_page\n",
+ __func__, r);
+ goto out_free_vram_pages;
+ }
+
+ /* get dma address for following HPAGE_PMD_NR-1 pages
+ * since src pages are continuous their dma addresses
+ * are continuous too.
+ */
+ for (l = 1; l < k; l++)
+ src[i + l] = src[i] + l*PAGE_SIZE;
+
+ /* migrate the HPAGE_PMD_NR pages above */
+ r = svm_migrate_copy_memory_gart(
+ adev, src + i,
+ dst + i, HPAGE_PMD_NR,
+ FROM_RAM_TO_VRAM,
+ mfence);
+
+ /* mark head page dma mapping as THP, tail pages dma addr
+ * are set to 0 for following dma_unmap
+ */
+ src[i] |= SVM_RANGE_DMA_THP;
+ for (l = 1; l < k; l++)
+ src[i + l] = 0;
+
+ if (r)
+ goto out_free_vram_pages;
+
+ j = 0;
}
} else {
- if (j) {
+ /* single normal page case */
+ spage = migrate_pfn_to_page(migrate->src[i]);
+ if (spage && !is_zone_device_page(spage)) {
+ src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+
+ r = dma_mapping_error(dev, src[i]);
+
+ if (r) {
+ dev_err(dev, "%s: fail %d dma_map_page\n",
+ __func__, r);
+ goto out_free_vram_pages;
+ }
+ j += 1;
+
+ } else if (j) {
r = svm_migrate_copy_memory_gart(
adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
mfence);
+
if (r)
goto out_free_vram_pages;
- amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
+
j = 0;
- } else {
- amdgpu_res_next(&cursor, PAGE_SIZE);
}
- continue;
}
- pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
- src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+ pr_debug_ratelimited("dma mapping %lld pages, src to 0x%llx, pfn 0x%lx\n",
+ k, src[i] >> PAGE_SHIFT, migrate->src[i] >> MIGRATE_PFN_SHIFT);
+ i += k;
+ m += k;
+
+ if (m >= (cursor.size >> PAGE_SHIFT)) {
+ if (j > 0) {
+ r = svm_migrate_copy_memory_gart(adev, src + i - j,
+ dst + i - j, j,
+ FROM_RAM_TO_VRAM,
+ mfence);
+ if (r)
+ goto out_free_vram_pages;
+ }
+
+ amdgpu_res_next(&cursor, m*PAGE_SIZE);
- /* accumulated j + 1 pages reach end of current drm_buddy_block */
- if (j + 1 >= (cursor.size >> PAGE_SHIFT)) {
- r = svm_migrate_copy_memory_gart(adev, src + i - j,
- dst + i - j, j + 1,
- FROM_RAM_TO_VRAM,
- mfence);
- if (r)
- goto out_free_vram_pages;
- amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
j = 0;
- } else {
- j++;
+ m = 0;
}
}
@@ -476,6 +572,7 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
struct kfd_process_device *pdd;
struct dma_fence *mfence = NULL;
struct migrate_vma migrate = { 0 };
+ bool is_private_device = false;
unsigned long cpages = 0;
unsigned long mpages = 0;
dma_addr_t *scratch;
@@ -488,11 +585,17 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
if (svm_range_window_resident(node, prange, start, end))
return 0;
+ is_private_device = svm_is_private_zone(adev);
+
memset(&migrate, 0, sizeof(migrate));
migrate.vma = vma;
migrate.start = start;
migrate.end = end;
migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
+
+ if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
+ migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
+
migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
buf = kvcalloc(npages,
@@ -686,6 +789,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
u64 i = 0, j;
u64 addr;
int r = 0;
+ u64 l, k;
pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
prange->last);
@@ -694,8 +798,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
src = (u64 *)(scratch + npages);
dst = scratch;
-
- for (i = 0, j = 0; i < npages; i++, addr += PAGE_SIZE) {
+ for (i = 0, j = 0, k = 1; i < npages; i += k, addr += k*PAGE_SIZE, k = 1) {
struct page *spage;
spage = migrate_pfn_to_page(migrate->src[i]);
@@ -711,6 +814,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
goto out_oom;
j = 0;
}
+
continue;
}
src[i] = svm_migrate_addr(adev, spage);
@@ -724,7 +828,19 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
j = 0;
}
- dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
+ if (IS_ALIGNED(page_to_pfn(spage), HPAGE_PMD_NR) &&
+ (addr + HPAGE_PMD_SIZE) <= migrate->end &&
+ IS_ALIGNED(addr, HPAGE_PMD_SIZE) &&
+ migrate->src[i] & MIGRATE_PFN_COMPOUND) {
+
+ k = HPAGE_PMD_NR;
+ dpage = svm_migrate_get_sys_page(migrate->vma, addr,
+ HPAGE_PMD_ORDER);
+ } else {
+ k = 1;
+ dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
+ }
+
if (!dpage) {
pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
prange->svms, prange->start, prange->last);
@@ -732,7 +848,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
goto out_oom;
}
- dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+ dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE*k, DMA_BIDIRECTIONAL);
r = dma_mapping_error(dev, dst[i]);
if (r) {
dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r);
@@ -740,13 +856,47 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
goto out_oom;
}
- pr_debug_ratelimited("dma mapping dst to 0x%llx, pfn 0x%lx\n",
- dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
-
migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
- dpage = NULL;
- j++;
+ if (k > 1) {
+ migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
+ /* migrate previous accumulated pages */
+ if (j) {
+ r = svm_migrate_copy_memory_gart(adev, dst + i - j,
+ src + i - j, j, FROM_VRAM_TO_RAM, mfence);
+ if (r)
+ goto out_oom;
+ j = 0;
+ }
+
+ for (l = 1; l < k; l++) {
+
+ src[i + l] = src[i] + l*PAGE_SIZE;
+ dst[i + l] = dst[i] + l*PAGE_SIZE;
+ migrate->dst[i + l] = 0;
+ }
+
+ /* migrate the HPAGE_PMD_NR pages above
+ * the migration is for 2MB THP
+ */
+ r = svm_migrate_copy_memory_gart(
+ adev, dst + i,
+ src + i, HPAGE_PMD_NR,
+ FROM_VRAM_TO_RAM,
+ mfence);
+
+ /* mark head page dma mapping as THP, tail pages dma addr
+ * are set to 0 for following dma_unmap
+ */
+ dst[i] |= SVM_RANGE_DMA_THP;
+ for (l = 1; l < k; l++)
+ dst[i + l] = 0;
+
+ if (r)
+ goto out_oom;
+
+ } else
+ j++;
}
if (j > 0)
@@ -765,12 +915,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
/* release previous allocated sys pages and unmap dma address */
while (i--) {
- if (dst[i]) {
- dma_unmap_page(dev, dst[i], PAGE_SIZE,
- DMA_BIDIRECTIONAL);
- dst[i] = 0;
- }
-
+ /* follwing svm_range_dma_unmap_dev will do dma unmap anyway
+ * not need do dma unmap here
+ */
dpage = migrate_pfn_to_page(migrate->dst[i]);
if (!dpage)
continue;
@@ -811,6 +958,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
unsigned long cpages = 0;
unsigned long mpages = 0;
struct amdgpu_device *adev = node->adev;
+ bool is_private_device = false;
struct kfd_process_device *pdd;
struct dma_fence *mfence = NULL;
struct migrate_vma migrate = { 0 };
@@ -818,6 +966,8 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
void *buf;
int r = -ENOMEM;
+ is_private_device = svm_is_private_zone(adev);
+
memset(&migrate, 0, sizeof(migrate));
migrate.vma = vma;
migrate.start = start;
@@ -828,6 +978,9 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
else
migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
+ if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
+ migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
+
buf = kvcalloc(npages,
2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),
GFP_KERNEL);
--
2.34.1
next prev parent reply other threads:[~2026-10-19 13:38 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-19 13:38 [PATCH v3 0/3] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
2026-10-19 13:38 ` [PATCH v3 1/3] drm/amdkfd: Add awareness of THP of device and system RAM " Xiaogang.Chen
2026-11-13 23:28 ` Philip Yang
2026-11-16 15:54 ` Chen, Xiaogang
2026-10-19 13:38 ` [PATCH v3 2/3] drm/amdkfd: Do not migrate partial THP memory at CPU/GPU page fault handler Xiaogang.Chen
2026-10-19 13:38 ` Xiaogang.Chen [this message]
2026-11-13 19:29 ` [PATCH v3 3/3] drm/amdkfd: Enable HMM device-private THP migration in KFD SVM driver Philip Yang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261019133845.166916-4-xiaogang.chen@amd.com \
--to=xiaogang.chen@amd.com \
--cc=amd-gfx@lists.freedesktop.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.