* [PATCH v2 1/4] drm/amdkfd: Add awareness of THP of device and system RAM in kfd svm driver
2026-09-04 19:54 [PATCH v2 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
@ 2026-09-04 19:54 ` Xiaogang.Chen
2026-10-06 21:40 ` Felix Kuehling
2026-09-04 19:54 ` [PATCH v2 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size Xiaogang.Chen
` (2 subsequent siblings)
3 siblings, 1 reply; 11+ messages in thread
From: Xiaogang.Chen @ 2026-09-04 19:54 UTC (permalink / raw)
To: amd-gfx; +Cc: Xiaogang Chen
From: Xiaogang Chen <xiaogang.chen@amd.com>
Extend kfd/svm function to allocate HPAGE_PMD_SIZE based device memory by buddy
allocator, each drm_buddy_block is HPAGE_PMD_SIZE aligned and to allocate THP
system ram by vma_alloc_folio.
Introduce SVM_RANGE_DMA_THP flag that indicates dma map of THP. THP dma
addresss will use this flag.
Add dev_pagemap_ops->folio_split callback that is called by folio_split when
core MM splits device memory folio.
These are preparations for following support for (THP) migration of zone
device-private memory, no function change.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 50 +++++++++++++++++++-----
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 15 +++++--
drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 1 +
3 files changed, 54 insertions(+), 12 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 253365a8257e..813f3c1d29dc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -217,14 +217,19 @@ svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr)
}
static void
-svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
+svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn,
+ int order)
{
struct page *page;
+ struct folio *folio;
page = pfn_to_page(pfn);
+ folio = page_folio(page);
+
+ zone_device_folio_init(folio, folio->pgmap, order);
+
+ folio_set_zone_device_data(folio, prange->svm_bo);
svm_range_bo_ref(prange->svm_bo);
- page->zone_device_data = prange->svm_bo;
- zone_device_page_init(page, page_pgmap(page), 0);
}
static void
@@ -247,11 +252,17 @@ svm_migrate_addr(struct amdgpu_device *adev, struct page *page)
}
static struct page *
-svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr)
+svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr,
+ unsigned long order)
{
struct page *page;
- page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+ if (order)
+ page = folio_page(vma_alloc_folio(GFP_HIGHUSER,
+ order, vma, addr), 0);
+ else
+ page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+
if (page)
lock_page(page);
@@ -265,8 +276,12 @@ static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)
for (i = 0; i < migrate->npages; i++) {
if (migrate->dst[i] & MIGRATE_PFN_VALID &&
- migrate->src[i] & MIGRATE_PFN_MIGRATE)
- mpages++;
+ migrate->src[i] & MIGRATE_PFN_MIGRATE) {
+ if (migrate->dst[i] & MIGRATE_PFN_COMPOUND)
+ mpages += HPAGE_PMD_NR;
+ else
+ mpages++;
+ }
}
return mpages;
}
@@ -300,7 +315,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
dst[i] = cursor.start + (j << PAGE_SHIFT);
migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
- svm_migrate_get_vram_page(prange, migrate->dst[i]);
+ svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
migrate->dst[i] = migrate_pfn(migrate->dst[i]);
mpages++;
}
@@ -568,6 +583,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
return r < 0 ? r : 0;
}
+/* folio can be compound folio or single page */
static void svm_migrate_folio_free(struct folio *folio)
{
struct page *page = &folio->page;
@@ -630,7 +646,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
j = 0;
}
- dpage = svm_migrate_get_sys_page(migrate->vma, addr);
+ dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
if (!dpage) {
pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
prange->svms, prange->start, prange->last);
@@ -1033,9 +1049,25 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
return r ? VM_FAULT_SIGBUS : 0;
}
+static void svm_migrate_folio_split(struct folio *head, struct folio *tail)
+{
+ struct svm_range_bo *svm_bo;
+
+ if (tail == NULL)
+ return;
+
+ tail->pgmap = head->pgmap;
+ tail->mapping = head->mapping;
+
+ svm_bo = folio_zone_device_data(head);
+ folio_set_zone_device_data(tail, svm_bo);
+ svm_range_bo_ref(svm_bo);
+}
+
static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
.folio_free = svm_migrate_folio_free,
.migrate_to_ram = svm_migrate_to_ram,
+ .folio_split = svm_migrate_folio_split,
};
/* Each VRAM page uses sizeof(struct page) on system memory */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index fa4054d51f60..6b783d12bce4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -245,7 +245,16 @@ void svm_range_dma_unmap_dev(struct device *dev, dma_addr_t *dma_addr,
if (!svm_is_valid_dma_mapping_addr(dev, dma_addr[i]))
continue;
pr_debug_ratelimited("unmap 0x%llx\n", dma_addr[i] >> PAGE_SHIFT);
- dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
+
+ /* dma unmap of THP */
+ if (dma_addr[i] & SVM_RANGE_DMA_THP) {
+
+ dma_addr[i] &= ~SVM_RANGE_DMA_THP;
+ dma_unmap_page(dev, dma_addr[i], PAGE_SIZE*HPAGE_PMD_NR,
+ DMA_BIDIRECTIONAL);
+ } else
+ dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
+
dma_addr[i] = 0;
}
}
@@ -578,10 +587,10 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
}
memset(&bp, 0, sizeof(bp));
- bp.size = prange->npages * PAGE_SIZE;
+ bp.size = ALIGN(prange->npages * PAGE_SIZE, HPAGE_PMD_SIZE);
bp.bo_ptr_size = sizeof(struct svm_range_bo);
bp.destroy = svm_range_bo_destroy;
- bp.byte_align = PAGE_SIZE;
+ bp.byte_align = HPAGE_PMD_SIZE;
bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index c7d7adae4476..f2b3a05cd8cf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -35,6 +35,7 @@
#include "kfd_priv.h"
#define SVM_RANGE_VRAM_DOMAIN (1UL << 0)
+#define SVM_RANGE_DMA_THP (1UL << 1)
#define SVM_ADEV_PGMAP_OWNER(adev)\
((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
--
2.34.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH v2 1/4] drm/amdkfd: Add awareness of THP of device and system RAM in kfd svm driver
2026-09-04 19:54 ` [PATCH v2 1/4] drm/amdkfd: Add awareness of THP of device and system RAM " Xiaogang.Chen
@ 2026-10-06 21:40 ` Felix Kuehling
2026-10-06 22:49 ` Chen, Xiaogang
0 siblings, 1 reply; 11+ messages in thread
From: Felix Kuehling @ 2026-10-06 21:40 UTC (permalink / raw)
To: Xiaogang.Chen, amd-gfx
On 2026-09-04 15:54, Xiaogang.Chen wrote:
> From: Xiaogang Chen <xiaogang.chen@amd.com>
>
> Extend kfd/svm function to allocate HPAGE_PMD_SIZE based device memory by buddy
> allocator, each drm_buddy_block is HPAGE_PMD_SIZE aligned and to allocate THP
> system ram by vma_alloc_folio.
>
> Introduce SVM_RANGE_DMA_THP flag that indicates dma map of THP. THP dma
> addresss will use this flag.
>
> Add dev_pagemap_ops->folio_split callback that is called by folio_split when
> core MM splits device memory folio.
>
> These are preparations for following support for (THP) migration of zone
> device-private memory, no function change.
>
> Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
> ---
> drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 50 +++++++++++++++++++-----
> drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 15 +++++--
> drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 1 +
> 3 files changed, 54 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> index 253365a8257e..813f3c1d29dc 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> @@ -217,14 +217,19 @@ svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr)
> }
>
> static void
> -svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
> +svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn,
> + int order)
> {
> struct page *page;
> + struct folio *folio;
>
> page = pfn_to_page(pfn);
> + folio = page_folio(page);
> +
> + zone_device_folio_init(folio, folio->pgmap, order);
> +
> + folio_set_zone_device_data(folio, prange->svm_bo);
I see that folio_set_zone_device_data checks that folio is
device_private. This will fail for device_coherent pages. We can't use
this function without breaking MI200 A+A.
> svm_range_bo_ref(prange->svm_bo);
> - page->zone_device_data = prange->svm_bo;
> - zone_device_page_init(page, page_pgmap(page), 0);
> }
>
> static void
> @@ -247,11 +252,17 @@ svm_migrate_addr(struct amdgpu_device *adev, struct page *page)
> }
>
> static struct page *
> -svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr)
> +svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr,
> + unsigned long order)
> {
> struct page *page;
>
> - page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
> + if (order)
> + page = folio_page(vma_alloc_folio(GFP_HIGHUSER,
> + order, vma, addr), 0);
> + else
> + page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
> +
> if (page)
> lock_page(page);
>
> @@ -265,8 +276,12 @@ static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)
>
> for (i = 0; i < migrate->npages; i++) {
> if (migrate->dst[i] & MIGRATE_PFN_VALID &&
> - migrate->src[i] & MIGRATE_PFN_MIGRATE)
> - mpages++;
> + migrate->src[i] & MIGRATE_PFN_MIGRATE) {
> + if (migrate->dst[i] & MIGRATE_PFN_COMPOUND)
> + mpages += HPAGE_PMD_NR;
> + else
> + mpages++;
> + }
Please fix the indentation. This is one tab too deep. The alignment of
the condition should remain unchanged.
> }
> return mpages;
> }
> @@ -300,7 +315,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
> if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
> dst[i] = cursor.start + (j << PAGE_SHIFT);
> migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
> - svm_migrate_get_vram_page(prange, migrate->dst[i]);
> + svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
> migrate->dst[i] = migrate_pfn(migrate->dst[i]);
> mpages++;
> }
> @@ -568,6 +583,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
> return r < 0 ? r : 0;
> }
>
> +/* folio can be compound folio or single page */
> static void svm_migrate_folio_free(struct folio *folio)
> {
> struct page *page = &folio->page;
> @@ -630,7 +646,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
> j = 0;
> }
>
> - dpage = svm_migrate_get_sys_page(migrate->vma, addr);
> + dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
> if (!dpage) {
> pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
> prange->svms, prange->start, prange->last);
> @@ -1033,9 +1049,25 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
> return r ? VM_FAULT_SIGBUS : 0;
> }
>
> +static void svm_migrate_folio_split(struct folio *head, struct folio *tail)
> +{
> + struct svm_range_bo *svm_bo;
> +
> + if (tail == NULL)
> + return;
> +
> + tail->pgmap = head->pgmap;
> + tail->mapping = head->mapping;
> +
> + svm_bo = folio_zone_device_data(head);
> + folio_set_zone_device_data(tail, svm_bo);
> + svm_range_bo_ref(svm_bo);
> +}
> +
> static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
> .folio_free = svm_migrate_folio_free,
> .migrate_to_ram = svm_migrate_to_ram,
> + .folio_split = svm_migrate_folio_split,
> };
>
> /* Each VRAM page uses sizeof(struct page) on system memory */
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> index fa4054d51f60..6b783d12bce4 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> @@ -245,7 +245,16 @@ void svm_range_dma_unmap_dev(struct device *dev, dma_addr_t *dma_addr,
> if (!svm_is_valid_dma_mapping_addr(dev, dma_addr[i]))
> continue;
> pr_debug_ratelimited("unmap 0x%llx\n", dma_addr[i] >> PAGE_SHIFT);
> - dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
> +
> + /* dma unmap of THP */
> + if (dma_addr[i] & SVM_RANGE_DMA_THP) {
> +
Unnecessary empty line.
> + dma_addr[i] &= ~SVM_RANGE_DMA_THP;
> + dma_unmap_page(dev, dma_addr[i], PAGE_SIZE*HPAGE_PMD_NR,
> + DMA_BIDIRECTIONAL);
> + } else
> + dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
The else-branch should also use {} braces if the if-branch does.
> +
> dma_addr[i] = 0;
> }
> }
> @@ -578,10 +587,10 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
> }
>
> memset(&bp, 0, sizeof(bp));
> - bp.size = prange->npages * PAGE_SIZE;
> + bp.size = ALIGN(prange->npages * PAGE_SIZE, HPAGE_PMD_SIZE);
This wastes memory for small ranges. Maybe guard this so it only applies
to ranges that are larger than HPAGE_PMD_NR pages.
> bp.bo_ptr_size = sizeof(struct svm_range_bo);
> bp.destroy = svm_range_bo_destroy;
> - bp.byte_align = PAGE_SIZE;
> + bp.byte_align = HPAGE_PMD_SIZE;
Same as above. There is no need to align smaller allocations.
Regards,
Felix
> bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
> bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
> bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> index c7d7adae4476..f2b3a05cd8cf 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> @@ -35,6 +35,7 @@
> #include "kfd_priv.h"
>
> #define SVM_RANGE_VRAM_DOMAIN (1UL << 0)
> +#define SVM_RANGE_DMA_THP (1UL << 1)
> #define SVM_ADEV_PGMAP_OWNER(adev)\
> ((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
>
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH v2 1/4] drm/amdkfd: Add awareness of THP of device and system RAM in kfd svm driver
2026-10-06 21:40 ` Felix Kuehling
@ 2026-10-06 22:49 ` Chen, Xiaogang
2026-10-06 23:01 ` Felix Kuehling
0 siblings, 1 reply; 11+ messages in thread
From: Chen, Xiaogang @ 2026-10-06 22:49 UTC (permalink / raw)
To: Felix Kuehling, amd-gfx
On 10/6/2026 4:40 PM, Felix Kuehling wrote:
> On 2026-09-04 15:54, Xiaogang.Chen wrote:
>> From: Xiaogang Chen <xiaogang.chen@amd.com>
>>
>> Extend kfd/svm function to allocate HPAGE_PMD_SIZE based device
>> memory by buddy
>> allocator, each drm_buddy_block is HPAGE_PMD_SIZE aligned and to
>> allocate THP
>> system ram by vma_alloc_folio.
>>
>> Introduce SVM_RANGE_DMA_THP flag that indicates dma map of THP. THP dma
>> addresss will use this flag.
>>
>> Add dev_pagemap_ops->folio_split callback that is called by
>> folio_split when
>> core MM splits device memory folio.
>>
>> These are preparations for following support for (THP) migration of zone
>> device-private memory, no function change.
>>
>> Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 50 +++++++++++++++++++-----
>> drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 15 +++++--
>> drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 1 +
>> 3 files changed, 54 insertions(+), 12 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>> b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>> index 253365a8257e..813f3c1d29dc 100644
>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>> @@ -217,14 +217,19 @@ svm_migrate_addr_to_pfn(struct amdgpu_device
>> *adev, unsigned long addr)
>> }
>> static void
>> -svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
>> +svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn,
>> + int order)
>> {
>> struct page *page;
>> + struct folio *folio;
>> page = pfn_to_page(pfn);
>> + folio = page_folio(page);
>> +
>> + zone_device_folio_init(folio, folio->pgmap, order);
>> +
>> + folio_set_zone_device_data(folio, prange->svm_bo);
>
> I see that folio_set_zone_device_data checks that folio is
> device_private. This will fail for device_coherent pages. We can't use
> this function without breaking MI200 A+A.
Yes, I was looking if this work has changes that can affect
device_coherent. It is one of them. This work is for GPU with device
private memory by now. I will change this part to:
if (folio_is_device_private(folio))
folio_set_zone_device_data(folio, prange->svm_bo);
else
folio->page.zone_device_data = prange->svm_bo;
>
>
>> svm_range_bo_ref(prange->svm_bo);
>> - page->zone_device_data = prange->svm_bo;
>> - zone_device_page_init(page, page_pgmap(page), 0);
>> }
>> static void
>> @@ -247,11 +252,17 @@ svm_migrate_addr(struct amdgpu_device *adev,
>> struct page *page)
>> }
>> static struct page *
>> -svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long
>> addr)
>> +svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long
>> addr,
>> + unsigned long order)
>> {
>> struct page *page;
>> - page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
>> + if (order)
>> + page = folio_page(vma_alloc_folio(GFP_HIGHUSER,
>> + order, vma, addr), 0);
>> + else
>> + page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
>> +
>> if (page)
>> lock_page(page);
>> @@ -265,8 +276,12 @@ static unsigned long
>> svm_migrate_successful_pages(struct migrate_vma *migrate)
>> for (i = 0; i < migrate->npages; i++) {
>> if (migrate->dst[i] & MIGRATE_PFN_VALID &&
>> - migrate->src[i] & MIGRATE_PFN_MIGRATE)
>> - mpages++;
>> + migrate->src[i] & MIGRATE_PFN_MIGRATE) {
>> + if (migrate->dst[i] & MIGRATE_PFN_COMPOUND)
>> + mpages += HPAGE_PMD_NR;
>> + else
>> + mpages++;
>> + }
>
> Please fix the indentation. This is one tab too deep. The alignment
> of the condition should remain unchanged.
ok
>
>
>> }
>> return mpages;
>> }
>> @@ -300,7 +315,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node,
>> struct svm_range *prange,
>> if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
>> dst[i] = cursor.start + (j << PAGE_SHIFT);
>> migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
>> - svm_migrate_get_vram_page(prange, migrate->dst[i]);
>> + svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
>> migrate->dst[i] = migrate_pfn(migrate->dst[i]);
>> mpages++;
>> }
>> @@ -568,6 +583,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange,
>> uint32_t best_loc,
>> return r < 0 ? r : 0;
>> }
>> +/* folio can be compound folio or single page */
>> static void svm_migrate_folio_free(struct folio *folio)
>> {
>> struct page *page = &folio->page;
>> @@ -630,7 +646,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device
>> *adev, struct svm_range *prange,
>> j = 0;
>> }
>> - dpage = svm_migrate_get_sys_page(migrate->vma, addr);
>> + dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
>> if (!dpage) {
>> pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
>> prange->svms, prange->start, prange->last);
>> @@ -1033,9 +1049,25 @@ static vm_fault_t svm_migrate_to_ram(struct
>> vm_fault *vmf)
>> return r ? VM_FAULT_SIGBUS : 0;
>> }
>> +static void svm_migrate_folio_split(struct folio *head, struct
>> folio *tail)
>> +{
>> + struct svm_range_bo *svm_bo;
>> +
>> + if (tail == NULL)
>> + return;
>> +
>> + tail->pgmap = head->pgmap;
>> + tail->mapping = head->mapping;
>> +
>> + svm_bo = folio_zone_device_data(head);
>> + folio_set_zone_device_data(tail, svm_bo);
>> + svm_range_bo_ref(svm_bo);
>> +}
>> +
>> static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
>> .folio_free = svm_migrate_folio_free,
>> .migrate_to_ram = svm_migrate_to_ram,
>> + .folio_split = svm_migrate_folio_split,
>> };
>> /* Each VRAM page uses sizeof(struct page) on system memory */
>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> index fa4054d51f60..6b783d12bce4 100644
>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>> @@ -245,7 +245,16 @@ void svm_range_dma_unmap_dev(struct device *dev,
>> dma_addr_t *dma_addr,
>> if (!svm_is_valid_dma_mapping_addr(dev, dma_addr[i]))
>> continue;
>> pr_debug_ratelimited("unmap 0x%llx\n", dma_addr[i] >>
>> PAGE_SHIFT);
>> - dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
>> +
>> + /* dma unmap of THP */
>> + if (dma_addr[i] & SVM_RANGE_DMA_THP) {
>> +
>
> Unnecessary empty line.
>
>
ok
>> + dma_addr[i] &= ~SVM_RANGE_DMA_THP;
>> + dma_unmap_page(dev, dma_addr[i], PAGE_SIZE*HPAGE_PMD_NR,
>> + DMA_BIDIRECTIONAL);
>> + } else
>> + dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
>
> The else-branch should also use {} braces if the if-branch does.
ok
>
>
>> +
>> dma_addr[i] = 0;
>> }
>> }
>> @@ -578,10 +587,10 @@ svm_range_vram_node_new(struct kfd_node *node,
>> struct svm_range *prange,
>> }
>> memset(&bp, 0, sizeof(bp));
>> - bp.size = prange->npages * PAGE_SIZE;
>> + bp.size = ALIGN(prange->npages * PAGE_SIZE, HPAGE_PMD_SIZE);
>
> This wastes memory for small ranges. Maybe guard this so it only
> applies to ranges that are larger than HPAGE_PMD_NR pages.
I am think this too. The waste is 2MB at most instead of current 4KB. I
will change allocation size/aliment according to request size.
Regards
Xiaogang
>
>
>> bp.bo_ptr_size = sizeof(struct svm_range_bo);
>> bp.destroy = svm_range_bo_destroy;
>> - bp.byte_align = PAGE_SIZE;
>> + bp.byte_align = HPAGE_PMD_SIZE;
>
> Same as above. There is no need to align smaller allocations.
>
> Regards,
> Felix
>
>
>> bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
>> bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
>> bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
>> b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
>> index c7d7adae4476..f2b3a05cd8cf 100644
>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
>> @@ -35,6 +35,7 @@
>> #include "kfd_priv.h"
>> #define SVM_RANGE_VRAM_DOMAIN (1UL << 0)
>> +#define SVM_RANGE_DMA_THP (1UL << 1)
>> #define SVM_ADEV_PGMAP_OWNER(adev)\
>> ((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH v2 1/4] drm/amdkfd: Add awareness of THP of device and system RAM in kfd svm driver
2026-10-06 22:49 ` Chen, Xiaogang
@ 2026-10-06 23:01 ` Felix Kuehling
0 siblings, 0 replies; 11+ messages in thread
From: Felix Kuehling @ 2026-10-06 23:01 UTC (permalink / raw)
To: Chen, Xiaogang, amd-gfx
[-- Attachment #1: Type: text/plain, Size: 10183 bytes --]
On 2026-10-06 18:49, Chen, Xiaogang wrote:
>
> On 10/6/2026 4:40 PM, Felix Kuehling wrote:
>> On 2026-09-04 15:54, Xiaogang.Chen wrote:
>>> From: Xiaogang Chen <xiaogang.chen@amd.com>
>>>
>>> Extend kfd/svm function to allocate HPAGE_PMD_SIZE based device
>>> memory by buddy
>>> allocator, each drm_buddy_block is HPAGE_PMD_SIZE aligned and to
>>> allocate THP
>>> system ram by vma_alloc_folio.
>>>
>>> Introduce SVM_RANGE_DMA_THP flag that indicates dma map of THP. THP dma
>>> addresss will use this flag.
>>>
>>> Add dev_pagemap_ops->folio_split callback that is called by
>>> folio_split when
>>> core MM splits device memory folio.
>>>
>>> These are preparations for following support for (THP) migration of
>>> zone
>>> device-private memory, no function change.
>>>
>>> Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
>>> ---
>>> drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 50
>>> +++++++++++++++++++-----
>>> drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 15 +++++--
>>> drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 1 +
>>> 3 files changed, 54 insertions(+), 12 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>>> index 253365a8257e..813f3c1d29dc 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>>> @@ -217,14 +217,19 @@ svm_migrate_addr_to_pfn(struct amdgpu_device
>>> *adev, unsigned long addr)
>>> }
>>> static void
>>> -svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
>>> +svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn,
>>> + int order)
>>> {
>>> struct page *page;
>>> + struct folio *folio;
>>> page = pfn_to_page(pfn);
>>> + folio = page_folio(page);
>>> +
>>> + zone_device_folio_init(folio, folio->pgmap, order);
>>> +
>>> + folio_set_zone_device_data(folio, prange->svm_bo);
>>
>> I see that folio_set_zone_device_data checks that folio is
>> device_private. This will fail for device_coherent pages. We can't
>> use this function without breaking MI200 A+A.
>
> Yes, I was looking if this work has changes that can affect
> device_coherent. It is one of them. This work is for GPU with device
> private memory by now. I will change this part to:
>
> if (folio_is_device_private(folio))
> folio_set_zone_device_data(folio, prange->svm_bo);
> else
> folio->page.zone_device_data = prange->svm_bo;
Then you're effectively just duplicating the check for device_private,
and then bypassing it if it fails. I think we should just use
folio->page.zone_device_data = prange->svm_bo;
If we want a check, we can do
if (folio_is_device_private(folio) || folio_is_device_coherent(folio))
folio->page.zone_device_data = prange->svm_bo;
Ideally we'd fix folio_set_zone_device_data to do just that, but we'd
need to keep this hack in our DKMS code to work with older kernels.
Regards,
Felix
>
>>
>>
>>> svm_range_bo_ref(prange->svm_bo);
>>> - page->zone_device_data = prange->svm_bo;
>>> - zone_device_page_init(page, page_pgmap(page), 0);
>>> }
>>> static void
>>> @@ -247,11 +252,17 @@ svm_migrate_addr(struct amdgpu_device *adev,
>>> struct page *page)
>>> }
>>> static struct page *
>>> -svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long
>>> addr)
>>> +svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long
>>> addr,
>>> + unsigned long order)
>>> {
>>> struct page *page;
>>> - page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
>>> + if (order)
>>> + page = folio_page(vma_alloc_folio(GFP_HIGHUSER,
>>> + order, vma, addr), 0);
>>> + else
>>> + page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
>>> +
>>> if (page)
>>> lock_page(page);
>>> @@ -265,8 +276,12 @@ static unsigned long
>>> svm_migrate_successful_pages(struct migrate_vma *migrate)
>>> for (i = 0; i < migrate->npages; i++) {
>>> if (migrate->dst[i] & MIGRATE_PFN_VALID &&
>>> - migrate->src[i] & MIGRATE_PFN_MIGRATE)
>>> - mpages++;
>>> + migrate->src[i] & MIGRATE_PFN_MIGRATE) {
>>> + if (migrate->dst[i] & MIGRATE_PFN_COMPOUND)
>>> + mpages += HPAGE_PMD_NR;
>>> + else
>>> + mpages++;
>>> + }
>>
>> Please fix the indentation. This is one tab too deep. The alignment
>> of the condition should remain unchanged.
> ok
>>
>>
>>> }
>>> return mpages;
>>> }
>>> @@ -300,7 +315,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node,
>>> struct svm_range *prange,
>>> if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
>>> dst[i] = cursor.start + (j << PAGE_SHIFT);
>>> migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
>>> - svm_migrate_get_vram_page(prange, migrate->dst[i]);
>>> + svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
>>> migrate->dst[i] = migrate_pfn(migrate->dst[i]);
>>> mpages++;
>>> }
>>> @@ -568,6 +583,7 @@ svm_migrate_ram_to_vram(struct svm_range
>>> *prange, uint32_t best_loc,
>>> return r < 0 ? r : 0;
>>> }
>>> +/* folio can be compound folio or single page */
>>> static void svm_migrate_folio_free(struct folio *folio)
>>> {
>>> struct page *page = &folio->page;
>>> @@ -630,7 +646,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device
>>> *adev, struct svm_range *prange,
>>> j = 0;
>>> }
>>> - dpage = svm_migrate_get_sys_page(migrate->vma, addr);
>>> + dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
>>> if (!dpage) {
>>> pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
>>> prange->svms, prange->start, prange->last);
>>> @@ -1033,9 +1049,25 @@ static vm_fault_t svm_migrate_to_ram(struct
>>> vm_fault *vmf)
>>> return r ? VM_FAULT_SIGBUS : 0;
>>> }
>>> +static void svm_migrate_folio_split(struct folio *head, struct
>>> folio *tail)
>>> +{
>>> + struct svm_range_bo *svm_bo;
>>> +
>>> + if (tail == NULL)
>>> + return;
>>> +
>>> + tail->pgmap = head->pgmap;
>>> + tail->mapping = head->mapping;
>>> +
>>> + svm_bo = folio_zone_device_data(head);
>>> + folio_set_zone_device_data(tail, svm_bo);
>>> + svm_range_bo_ref(svm_bo);
>>> +}
>>> +
>>> static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
>>> .folio_free = svm_migrate_folio_free,
>>> .migrate_to_ram = svm_migrate_to_ram,
>>> + .folio_split = svm_migrate_folio_split,
>>> };
>>> /* Each VRAM page uses sizeof(struct page) on system memory */
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>>> index fa4054d51f60..6b783d12bce4 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
>>> @@ -245,7 +245,16 @@ void svm_range_dma_unmap_dev(struct device
>>> *dev, dma_addr_t *dma_addr,
>>> if (!svm_is_valid_dma_mapping_addr(dev, dma_addr[i]))
>>> continue;
>>> pr_debug_ratelimited("unmap 0x%llx\n", dma_addr[i] >>
>>> PAGE_SHIFT);
>>> - dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
>>> +
>>> + /* dma unmap of THP */
>>> + if (dma_addr[i] & SVM_RANGE_DMA_THP) {
>>> +
>>
>> Unnecessary empty line.
>>
>>
> ok
>>> + dma_addr[i] &= ~SVM_RANGE_DMA_THP;
>>> + dma_unmap_page(dev, dma_addr[i], PAGE_SIZE*HPAGE_PMD_NR,
>>> + DMA_BIDIRECTIONAL);
>>> + } else
>>> + dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
>>
>> The else-branch should also use {} braces if the if-branch does.
> ok
>>
>>
>>> +
>>> dma_addr[i] = 0;
>>> }
>>> }
>>> @@ -578,10 +587,10 @@ svm_range_vram_node_new(struct kfd_node *node,
>>> struct svm_range *prange,
>>> }
>>> memset(&bp, 0, sizeof(bp));
>>> - bp.size = prange->npages * PAGE_SIZE;
>>> + bp.size = ALIGN(prange->npages * PAGE_SIZE, HPAGE_PMD_SIZE);
>>
>> This wastes memory for small ranges. Maybe guard this so it only
>> applies to ranges that are larger than HPAGE_PMD_NR pages.
>
> I am think this too. The waste is 2MB at most instead of current 4KB.
> I will change allocation size/aliment according to request size.
>
> Regards
>
> Xiaogang
>
>>
>>
>>> bp.bo_ptr_size = sizeof(struct svm_range_bo);
>>> bp.destroy = svm_range_bo_destroy;
>>> - bp.byte_align = PAGE_SIZE;
>>> + bp.byte_align = HPAGE_PMD_SIZE;
>>
>> Same as above. There is no need to align smaller allocations.
>>
>> Regards,
>> Felix
>>
>>
>>> bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
>>> bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
>>> bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
>>> b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
>>> index c7d7adae4476..f2b3a05cd8cf 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
>>> @@ -35,6 +35,7 @@
>>> #include "kfd_priv.h"
>>> #define SVM_RANGE_VRAM_DOMAIN (1UL << 0)
>>> +#define SVM_RANGE_DMA_THP (1UL << 1)
>>> #define SVM_ADEV_PGMAP_OWNER(adev)\
>>> ((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
[-- Attachment #2: Type: text/html, Size: 19205 bytes --]
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v2 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size
2026-09-04 19:54 [PATCH v2 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
2026-09-04 19:54 ` [PATCH v2 1/4] drm/amdkfd: Add awareness of THP of device and system RAM " Xiaogang.Chen
@ 2026-09-04 19:54 ` Xiaogang.Chen
2026-10-06 21:43 ` Felix Kuehling
2026-09-04 19:54 ` [PATCH v2 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver Xiaogang.Chen
2026-09-04 19:54 ` [PATCH v2 4/4] drm/amdkfd: Apply AMDGPU_PTE_FRAG to pte of gart page table for THP mapping Xiaogang.Chen
3 siblings, 1 reply; 11+ messages in thread
From: Xiaogang.Chen @ 2026-09-04 19:54 UTC (permalink / raw)
To: amd-gfx; +Cc: Xiaogang Chen
From: Xiaogang Chen <xiaogang.chen@amd.com>
When use HPAGE_PMD_SIZE based device private pages during migration core HMM
treats device private memory in HPAGE_PMD_SIZE compound folio if possible.
Current kfd driver uses prange->granularity that can be changed by user. Need
have migration size in CPU and GPU page fault handler in HPAGE_PMD_SIZE based.
For AMD GPU that exposes private device memory choose HPAGE_PMD_SIZE as
minimums migration size in CPU and GPU page fault handler. For x86 it is
same as default prange->granularity.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 6 ++++--
drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 13 +++++++++++--
drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 12 ++++++++++++
3 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 813f3c1d29dc..bbf0fefd5722 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -1026,8 +1026,10 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
if (!prange->actual_loc)
goto out_unlock_prange;
- /* Align migration range start and size to granularity size */
- size = 1UL << prange->granularity;
+ /* Align migration range start and size to max of
+ * THP with HPAGE_PMD_ORDER and granularity size
+ */
+ size = 1UL << max(prange->granularity, HPAGE_PMD_ORDER);
start = max(ALIGN_DOWN(addr, size), prange->start);
last = min(ALIGN(addr + 1, size) - 1, prange->last);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 6b783d12bce4..482cd4e7eee5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -3071,6 +3071,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
struct kfd_node *node;
int32_t best_loc;
int32_t gpuid, gpuidx = MAX_GPU_INSTANCE;
+ bool is_private_device = false;
bool write_locked = false;
struct vm_area_struct *vma;
bool migration = false;
@@ -3087,6 +3088,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
return 0;
}
svms = &p->svms;
+ is_private_device = svm_is_private_zone(adev);
pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
@@ -3224,8 +3226,15 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
kfd_smi_event_page_fault_start(node, p->lead_thread, addr,
write_fault, timestamp);
- /* Align migration range start and size to granularity size */
- size = 1UL << prange->granularity;
+ if (is_private_device)
+ /* Align migration range start and size to max of
+ * THP and granularity size
+ */
+ size = 1UL << max(prange->granularity, HPAGE_PMD_ORDER);
+ else
+ /* Align migration range start and size to granularity size */
+ size = 1UL << prange->granularity;
+
start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);
last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);
if (prange->actual_loc != 0 || best_loc != 0) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index f2b3a05cd8cf..e78ee94ba33c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -215,6 +215,13 @@ void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
void svm_range_set_max_pages(struct amdgpu_device *adev);
int svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled);
+/* check adev has device private zone memory */
+static inline bool svm_is_private_zone(struct amdgpu_device *adev)
+{
+ struct amdgpu_kfd_dev *kfddev = &adev->kfd;
+ return (kfddev->pgmap.type == MEMORY_DEVICE_PRIVATE);
+}
+
#else
struct kfd_process;
@@ -277,6 +284,11 @@ static inline void svm_range_set_max_pages(struct amdgpu_device *adev)
{
}
+static inline bool svm_is_private_zone(struct amdgpu_device *adev)
+{
+ return false;
+}
+
#define KFD_IS_SVM_API_SUPPORTED(dev) false
#endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
--
2.34.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH v2 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size
2026-09-04 19:54 ` [PATCH v2 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size Xiaogang.Chen
@ 2026-10-06 21:43 ` Felix Kuehling
0 siblings, 0 replies; 11+ messages in thread
From: Felix Kuehling @ 2026-10-06 21:43 UTC (permalink / raw)
To: Xiaogang.Chen, amd-gfx
On 2026-09-04 15:54, Xiaogang.Chen wrote:
> From: Xiaogang Chen <xiaogang.chen@amd.com>
>
> When use HPAGE_PMD_SIZE based device private pages during migration core HMM
> treats device private memory in HPAGE_PMD_SIZE compound folio if possible.
> Current kfd driver uses prange->granularity that can be changed by user. Need
> have migration size in CPU and GPU page fault handler in HPAGE_PMD_SIZE based.
>
> For AMD GPU that exposes private device memory choose HPAGE_PMD_SIZE as
> minimums migration size in CPU and GPU page fault handler. For x86 it is
> same as default prange->granularity.
As we discussed before, this breaks the API semantics. The default
setting is fine for allowing THP. But we want applications to be able to
use smaller granularity for use cases that access or distribute data
between multiple devices at finer granularity. In those cases, the
additional memory management and TLB overhead is offset by reduced
migrations or thrashing.
Please just drop this patch.
Regards,
Felix
>
> Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
> ---
> drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 6 ++++--
> drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 13 +++++++++++--
> drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 12 ++++++++++++
> 3 files changed, 27 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> index 813f3c1d29dc..bbf0fefd5722 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> @@ -1026,8 +1026,10 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
> if (!prange->actual_loc)
> goto out_unlock_prange;
>
> - /* Align migration range start and size to granularity size */
> - size = 1UL << prange->granularity;
> + /* Align migration range start and size to max of
> + * THP with HPAGE_PMD_ORDER and granularity size
> + */
> + size = 1UL << max(prange->granularity, HPAGE_PMD_ORDER);
> start = max(ALIGN_DOWN(addr, size), prange->start);
> last = min(ALIGN(addr + 1, size) - 1, prange->last);
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> index 6b783d12bce4..482cd4e7eee5 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
> @@ -3071,6 +3071,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
> struct kfd_node *node;
> int32_t best_loc;
> int32_t gpuid, gpuidx = MAX_GPU_INSTANCE;
> + bool is_private_device = false;
> bool write_locked = false;
> struct vm_area_struct *vma;
> bool migration = false;
> @@ -3087,6 +3088,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
> return 0;
> }
> svms = &p->svms;
> + is_private_device = svm_is_private_zone(adev);
>
> pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
>
> @@ -3224,8 +3226,15 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
> kfd_smi_event_page_fault_start(node, p->lead_thread, addr,
> write_fault, timestamp);
>
> - /* Align migration range start and size to granularity size */
> - size = 1UL << prange->granularity;
> + if (is_private_device)
> + /* Align migration range start and size to max of
> + * THP and granularity size
> + */
> + size = 1UL << max(prange->granularity, HPAGE_PMD_ORDER);
> + else
> + /* Align migration range start and size to granularity size */
> + size = 1UL << prange->granularity;
> +
> start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);
> last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);
> if (prange->actual_loc != 0 || best_loc != 0) {
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> index f2b3a05cd8cf..e78ee94ba33c 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
> @@ -215,6 +215,13 @@ void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
> void svm_range_set_max_pages(struct amdgpu_device *adev);
> int svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled);
>
> +/* check adev has device private zone memory */
> +static inline bool svm_is_private_zone(struct amdgpu_device *adev)
> +{
> + struct amdgpu_kfd_dev *kfddev = &adev->kfd;
> + return (kfddev->pgmap.type == MEMORY_DEVICE_PRIVATE);
> +}
> +
> #else
>
> struct kfd_process;
> @@ -277,6 +284,11 @@ static inline void svm_range_set_max_pages(struct amdgpu_device *adev)
> {
> }
>
> +static inline bool svm_is_private_zone(struct amdgpu_device *adev)
> +{
> + return false;
> +}
> +
> #define KFD_IS_SVM_API_SUPPORTED(dev) false
>
> #endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v2 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver
2026-09-04 19:54 [PATCH v2 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
2026-09-04 19:54 ` [PATCH v2 1/4] drm/amdkfd: Add awareness of THP of device and system RAM " Xiaogang.Chen
2026-09-04 19:54 ` [PATCH v2 2/4] drm/amdkfd: Change migration size in CPU/GPU page fault handler to THP size Xiaogang.Chen
@ 2026-09-04 19:54 ` Xiaogang.Chen
2026-10-06 22:44 ` Felix Kuehling
2026-09-04 19:54 ` [PATCH v2 4/4] drm/amdkfd: Apply AMDGPU_PTE_FRAG to pte of gart page table for THP mapping Xiaogang.Chen
3 siblings, 1 reply; 11+ messages in thread
From: Xiaogang.Chen @ 2026-09-04 19:54 UTC (permalink / raw)
To: amd-gfx; +Cc: Xiaogang Chen
From: Xiaogang Chen <xiaogang.chen@amd.com>
Update kfd svm driver to migrate device-private THP introduced from HMM core
migration function. Select this function by flag MIGRATE_VMA_SELECT_COMPOUND
when call migrate_vma_setup. kfd migration procedure is updated according to
collected page type that can be either compound folio(physical continuous) or
normal size page.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 271 ++++++++++++++++++-----
1 file changed, 219 insertions(+), 52 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index bbf0fefd5722..af39e547c1fa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -298,7 +298,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
u64 mpages = 0;
dma_addr_t *src;
u64 *dst;
- u64 i, j;
+ u64 i, j, k, l, m;
int r = 0;
pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,
@@ -309,59 +309,158 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
amdgpu_res_first(prange->ttm_res, ttm_res_offset,
npages << PAGE_SHIFT, &cursor);
- for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {
+ for (i = j = m = 0; (i < npages) && (mpages < migrate->cpages);) {
struct page *spage;
+ unsigned long cur_dst_pfn;
+ bool is_large = false;
- if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
- dst[i] = cursor.start + (j << PAGE_SHIFT);
- migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
- svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
- migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+ cur_dst_pfn = svm_migrate_addr_to_pfn(adev,
+ cursor.start + (m << PAGE_SHIFT));
+
+ /* when migrate->src[i] has MIGRATE_PFN_COMPOUND set the src page
+ * is compound THP; its vm address is HPAGE_PMD_SIZE aligned and
+ * its MIGRATE_PFN_MIGRATE is set
+ */
+ if ((m + HPAGE_PMD_NR) <= (cursor.size >> PAGE_SHIFT) &&
+ (i + HPAGE_PMD_NR) <= npages &&
+ (migrate->src[i] & MIGRATE_PFN_COMPOUND) &&
+ IS_ALIGNED(cur_dst_pfn, HPAGE_PMD_NR)) {
+
+ is_large = true;
+ k = HPAGE_PMD_NR;
+ } else
+ k = 1;
+
+ /* for THP src[0] alwas MIGRATE_PFN_MIGRATE
+ * just the first migrate->dst need be setup, others are zero
+ */
+ if (is_large) {
+
+ dst[i] = cursor.start + (m << PAGE_SHIFT);
+ svm_migrate_get_vram_page(prange, cur_dst_pfn,
+ HPAGE_PMD_ORDER);
+
+ migrate->dst[i] = migrate_pfn(cur_dst_pfn);
+ migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
+
+ for (l=1; l < k; l++)
+ migrate->dst[i+l] = 0;
+
+ mpages++;
+
+ } else if ((migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
+ dst[i] = cursor.start + (m << PAGE_SHIFT);
+ svm_migrate_get_vram_page(prange, cur_dst_pfn, 0);
+ migrate->dst[i] = migrate_pfn(cur_dst_pfn);
mpages++;
}
- spage = migrate_pfn_to_page(migrate->src[i]);
- if (spage && !is_zone_device_page(spage)) {
- src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
- DMA_BIDIRECTIONAL);
- r = dma_mapping_error(dev, src[i]);
- if (r) {
- src[i] = 0;
- dev_err(dev, "%s: fail %d dma_map_page\n",
- __func__, r);
- goto out_free_vram_pages;
+
+ if (is_large) {
+ if (j) {
+ /* migrate previous accumulated pages */
+ r = svm_migrate_copy_memory_gart(
+ adev, src + i - j,
+ dst + i - j, j,
+ FROM_RAM_TO_VRAM,
+ mfence);
+
+ if (r)
+ goto out_free_vram_pages;
+
+ j = 0;
+ }
+
+ /* for THP check if the first src page is valid
+ * if not valid skip following HPAGE_PMD_NR - 1 pages
+ */
+ spage = migrate_pfn_to_page(migrate->src[i]);
+ if (spage && !is_zone_device_page(spage)) {
+ /* dma_map continuous HPAGE_PMD_NR sys ram pages */
+ src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE*HPAGE_PMD_NR,
+ DMA_BIDIRECTIONAL);
+
+ r = dma_mapping_error(dev, src[i]);
+ if (r) {
+ dev_err(dev, "%s: fail %d dma_map_page\n",
+ __func__, r);
+ goto out_free_vram_pages;
+ }
+
+ /* get dma address for following HPAGE_PMD_NR-1 pages
+ * since src pages are continuous their dma addresses
+ * are continuous too.
+ */
+ for (l=1; l < k; l++)
+ src[i + l] = src[i] + l*PAGE_SIZE;
+
+ /* migrate the HPAGE_PMD_NR pages above */
+ r = svm_migrate_copy_memory_gart(
+ adev, src + i,
+ dst + i, HPAGE_PMD_NR,
+ FROM_RAM_TO_VRAM,
+ mfence);
+
+ /* mark head page dma mapping as THP, tail pages dma addr
+ * are set to 0 for following dma_unmap
+ */
+ src[i] |= SVM_RANGE_DMA_THP;
+ for (l = 1; l < k; l++)
+ src[i + l] = 0;
+
+ if (r)
+ goto out_free_vram_pages;
+
+ j = 0;
}
} else {
- if (j) {
+ /* single normal page case */
+ spage = migrate_pfn_to_page(migrate->src[i]);
+ if (spage && !is_zone_device_page(spage)) {
+ src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+
+ r = dma_mapping_error(dev, src[i]);
+
+ if (r) {
+ dev_err(dev, "%s: fail %d dma_map_page\n",
+ __func__, r);
+ goto out_free_vram_pages;
+ }
+ j += 1;
+
+ } else if (j) {
r = svm_migrate_copy_memory_gart(
adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
mfence);
+
if (r)
goto out_free_vram_pages;
- amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
+
j = 0;
- } else {
- amdgpu_res_next(&cursor, PAGE_SIZE);
}
- continue;
}
- pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
- src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+ pr_debug_ratelimited("dma mapping %lld pages, src to 0x%llx, pfn 0x%lx\n",
+ k, src[i] >> PAGE_SHIFT, migrate->src[i] >> MIGRATE_PFN_SHIFT);
+ i += k;
+ m += k;
+
+ if (m >= (cursor.size >> PAGE_SHIFT)) {
+ if (j > 0) {
+ r = svm_migrate_copy_memory_gart(adev, src + i - j,
+ dst + i - j, j,
+ FROM_RAM_TO_VRAM,
+ mfence);
+ if (r)
+ goto out_free_vram_pages;
+ }
+
+ amdgpu_res_next(&cursor, m*PAGE_SIZE);
- /* accumulated j + 1 pages reach end of current drm_buddy_block */
- if (j + 1 >= (cursor.size >> PAGE_SHIFT)) {
- r = svm_migrate_copy_memory_gart(adev, src + i - j,
- dst + i - j, j + 1,
- FROM_RAM_TO_VRAM,
- mfence);
- if (r)
- goto out_free_vram_pages;
- amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
j = 0;
- } else {
- j++;
+ m = 0;
}
}
@@ -410,17 +509,24 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
struct kfd_process_device *pdd;
struct dma_fence *mfence = NULL;
struct migrate_vma migrate = { 0 };
+ bool is_private_device = false;
unsigned long cpages = 0;
unsigned long mpages = 0;
dma_addr_t *scratch;
void *buf;
int r = -ENOMEM;
+ is_private_device = svm_is_private_zone(adev);
+
memset(&migrate, 0, sizeof(migrate));
migrate.vma = vma;
migrate.start = start;
migrate.end = end;
migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
+
+ if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
+ migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
+
migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
buf = kvcalloc(npages,
@@ -609,6 +715,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
u64 addr;
int r = 0;
+ u64 l, k;
+ bool is_large = false;
+
pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
prange->last);
@@ -616,8 +725,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
src = (u64 *)(scratch + npages);
dst = scratch;
-
- for (i = 0, j = 0; i < npages; i++, addr += PAGE_SIZE) {
+ for (i = 0, j = 0; i < npages;) {
struct page *spage;
spage = migrate_pfn_to_page(migrate->src[i]);
@@ -633,6 +741,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
goto out_oom;
j = 0;
}
+
+ addr += PAGE_SIZE;
+ i++;
continue;
}
src[i] = svm_migrate_addr(adev, spage);
@@ -646,7 +757,22 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
j = 0;
}
- dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
+ if(IS_ALIGNED(page_to_pfn(spage), HPAGE_PMD_NR) &&
+ (addr + HPAGE_PMD_SIZE) <= migrate->end &&
+ IS_ALIGNED (addr, HPAGE_PMD_SIZE) &&
+ migrate->src[i] & MIGRATE_PFN_COMPOUND) {
+
+ is_large = true;
+ k = HPAGE_PMD_NR;
+
+ dpage = svm_migrate_get_sys_page(migrate->vma, addr,
+ HPAGE_PMD_ORDER);
+ } else {
+ k = 1;
+ is_large = false;
+ dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
+ }
+
if (!dpage) {
pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
prange->svms, prange->start, prange->last);
@@ -654,21 +780,59 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
goto out_oom;
}
- dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+ dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE*k, DMA_BIDIRECTIONAL);
r = dma_mapping_error(dev, dst[i]);
if (r) {
dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r);
- dst[i] = 0;
goto out_oom;
}
- pr_debug_ratelimited("dma mapping dst to 0x%llx, pfn 0x%lx\n",
- dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
-
migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
+ if (is_large)
+ migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
- dpage = NULL;
- j++;
+ if (is_large) {
+ /* migrate previous accumulated pages */
+ if(j) {
+ r = svm_migrate_copy_memory_gart(adev, dst + i - j,
+ src + i - j, j, FROM_VRAM_TO_RAM, mfence);
+ if (r)
+ goto out_oom;
+ j = 0;
+ }
+
+ for (l = 1; l < k; l++) {
+
+ src[i + l] = src[i] + l*PAGE_SIZE;
+ dst[i + l] = dst[i] + l*PAGE_SIZE;
+ migrate->dst[i + l] = 0;
+ }
+
+ /* migrate the HPAGE_PMD_NR pages above */
+ /* svm_migrate_copy_memory_gart will add a paramter to indicate
+ * the migration is for 2MB THP
+ */
+ r = svm_migrate_copy_memory_gart(
+ adev, dst + i,
+ src + i, HPAGE_PMD_NR,
+ FROM_VRAM_TO_RAM,
+ mfence);
+
+ /* mark head page dma mapping as THP, tail pages dma addr
+ * are set to 0 for following dma_unmap
+ */
+ dst[i] |= SVM_RANGE_DMA_THP;
+ for (l = 1; l < k; l++)
+ dst[i + l] = 0;
+
+ if (r)
+ goto out_oom;
+
+ } else
+ j++;
+
+ addr += PAGE_SIZE*k;
+ i += k;
}
if (j > 0)
@@ -687,12 +851,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
/* release previous allocated sys pages and unmap dma address */
while (i--) {
- if (dst[i]) {
- dma_unmap_page(dev, dst[i], PAGE_SIZE,
- DMA_BIDIRECTIONAL);
- dst[i] = 0;
- }
-
+ /* follwing svm_range_dma_unmap_dev will do dma unmap anyway
+ * not need do dma unmap here
+ */
dpage = migrate_pfn_to_page(migrate->dst[i]);
if (!dpage)
continue;
@@ -733,6 +894,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
unsigned long cpages = 0;
unsigned long mpages = 0;
struct amdgpu_device *adev = node->adev;
+ bool is_private_device = false;
struct kfd_process_device *pdd;
struct dma_fence *mfence = NULL;
struct migrate_vma migrate = { 0 };
@@ -740,6 +902,8 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
void *buf;
int r = -ENOMEM;
+ is_private_device = svm_is_private_zone(adev);
+
memset(&migrate, 0, sizeof(migrate));
migrate.vma = vma;
migrate.start = start;
@@ -750,6 +914,9 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
else
migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
+ if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
+ migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
+
buf = kvcalloc(npages,
2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),
GFP_KERNEL);
@@ -1132,7 +1299,7 @@ int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
- pr_info("HMM registered %ldMB device memory\n", size >> 20);
+ pr_info("---XCHEN 3.2 HMM registered %ldMB device memory\n", size >> 20);
return 0;
}
--
2.34.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH v2 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver
2026-09-04 19:54 ` [PATCH v2 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver Xiaogang.Chen
@ 2026-10-06 22:44 ` Felix Kuehling
2026-10-07 18:57 ` Chen, Xiaogang
0 siblings, 1 reply; 11+ messages in thread
From: Felix Kuehling @ 2026-10-06 22:44 UTC (permalink / raw)
To: Xiaogang.Chen, amd-gfx
[-- Attachment #1: Type: text/plain, Size: 14961 bytes --]
On 2026-09-04 15:54, Xiaogang.Chen wrote:
> From: Xiaogang Chen<xiaogang.chen@amd.com>
>
> Update kfd svm driver to migrate device-private THP introduced from HMM core
> migration function. Select this function by flag MIGRATE_VMA_SELECT_COMPOUND
> when call migrate_vma_setup. kfd migration procedure is updated according to
> collected page type that can be either compound folio(physical continuous) or
> normal size page.
>
> Signed-off-by: Xiaogang Chen<xiaogang.chen@amd.com>
> ---
> drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 271 ++++++++++++++++++-----
> 1 file changed, 219 insertions(+), 52 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> index bbf0fefd5722..af39e547c1fa 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> @@ -298,7 +298,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
> u64 mpages = 0;
> dma_addr_t *src;
> u64 *dst;
> - u64 i, j;
> + u64 i, j, k, l, m;
> int r = 0;
>
> pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,
> @@ -309,59 +309,158 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
>
> amdgpu_res_first(prange->ttm_res, ttm_res_offset,
> npages << PAGE_SHIFT, &cursor);
> - for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {
> + for (i = j = m = 0; (i < npages) && (mpages < migrate->cpages);) {
> struct page *spage;
> + unsigned long cur_dst_pfn;
> + bool is_large = false;
>
> - if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
> - dst[i] = cursor.start + (j << PAGE_SHIFT);
> - migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
> - svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
> - migrate->dst[i] = migrate_pfn(migrate->dst[i]);
> + cur_dst_pfn = svm_migrate_addr_to_pfn(adev,
> + cursor.start + (m << PAGE_SHIFT));
> +
> + /* when migrate->src[i] has MIGRATE_PFN_COMPOUND set the src page
> + * is compound THP; its vm address is HPAGE_PMD_SIZE aligned and
> + * its MIGRATE_PFN_MIGRATE is set
> + */
> + if ((m + HPAGE_PMD_NR) <= (cursor.size >> PAGE_SHIFT) &&
> + (i + HPAGE_PMD_NR) <= npages &&
> + (migrate->src[i] & MIGRATE_PFN_COMPOUND) &&
> + IS_ALIGNED(cur_dst_pfn, HPAGE_PMD_NR)) {
> +
Unnecessary empty line.
> + is_large = true;
> + k = HPAGE_PMD_NR;
is_large is redundant. Just use (k > 1).
> + } else
> + k = 1;
Bad indentation.
> +
> + /* for THP src[0] alwas MIGRATE_PFN_MIGRATE
> + * just the first migrate->dst need be setup, others are zero
> + */
> + if (is_large) {
> +
> + dst[i] = cursor.start + (m << PAGE_SHIFT);
> + svm_migrate_get_vram_page(prange, cur_dst_pfn,
> + HPAGE_PMD_ORDER);
> +
> + migrate->dst[i] = migrate_pfn(cur_dst_pfn);
> + migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
> +
> + for (l=1; l < k; l++)
> + migrate->dst[i+l] = 0;
> +
> + mpages++;
> +
> + } else if ((migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
> + dst[i] = cursor.start + (m << PAGE_SHIFT);
> + svm_migrate_get_vram_page(prange, cur_dst_pfn, 0);
> + migrate->dst[i] = migrate_pfn(cur_dst_pfn);
> mpages++;
> }
> - spage = migrate_pfn_to_page(migrate->src[i]);
> - if (spage && !is_zone_device_page(spage)) {
> - src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
> - DMA_BIDIRECTIONAL);
> - r = dma_mapping_error(dev, src[i]);
> - if (r) {
> - src[i] = 0;
> - dev_err(dev, "%s: fail %d dma_map_page\n",
> - __func__, r);
> - goto out_free_vram_pages;
> +
> + if (is_large) {
> + if (j) {
> + /* migrate previous accumulated pages */
> + r = svm_migrate_copy_memory_gart(
> + adev, src + i - j,
> + dst + i - j, j,
> + FROM_RAM_TO_VRAM,
> + mfence);
> +
> + if (r)
> + goto out_free_vram_pages;
> +
> + j = 0;
> + }
> +
> + /* for THP check if the first src page is valid
> + * if not valid skip following HPAGE_PMD_NR - 1 pages
> + */
> + spage = migrate_pfn_to_page(migrate->src[i]);
> + if (spage && !is_zone_device_page(spage)) {
> + /* dma_map continuous HPAGE_PMD_NR sys ram pages */
> + src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE*HPAGE_PMD_NR,
> + DMA_BIDIRECTIONAL);
> +
> + r = dma_mapping_error(dev, src[i]);
> + if (r) {
> + dev_err(dev, "%s: fail %d dma_map_page\n",
> + __func__, r);
> + goto out_free_vram_pages;
> + }
> +
> + /* get dma address for following HPAGE_PMD_NR-1 pages
> + * since src pages are continuous their dma addresses
> + * are continuous too.
> + */
> + for (l=1; l < k; l++)
> + src[i + l] = src[i] + l*PAGE_SIZE;
> +
> + /* migrate the HPAGE_PMD_NR pages above */
> + r = svm_migrate_copy_memory_gart(
> + adev, src + i,
> + dst + i, HPAGE_PMD_NR,
> + FROM_RAM_TO_VRAM,
> + mfence);
> +
> + /* mark head page dma mapping as THP, tail pages dma addr
> + * are set to 0 for following dma_unmap
> + */
> + src[i] |= SVM_RANGE_DMA_THP;
> + for (l = 1; l < k; l++)
> + src[i + l] = 0;
I hope this doesn't break partial mapping or unmapping. We'd need to be
sure that code always aligns addresses to huge-page boundaries and gets
the whole huge page. A safer alternative would be to use a different
flag for the second and subsequent compound pages. So you'd still have
the DMA addresses, but you could ignore them for DMA unmapping.
> +
> + if (r)
> + goto out_free_vram_pages;
> +
> + j = 0;
> }
> } else {
> - if (j) {
> + /* single normal page case */
> + spage = migrate_pfn_to_page(migrate->src[i]);
> + if (spage && !is_zone_device_page(spage)) {
> + src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
> + DMA_BIDIRECTIONAL);
> +
> + r = dma_mapping_error(dev, src[i]);
> +
> + if (r) {
> + dev_err(dev, "%s: fail %d dma_map_page\n",
> + __func__, r);
> + goto out_free_vram_pages;
> + }
> + j += 1;
> +
> + } else if (j) {
> r = svm_migrate_copy_memory_gart(
> adev, src + i - j,
> dst + i - j, j,
> FROM_RAM_TO_VRAM,
> mfence);
> +
> if (r)
> goto out_free_vram_pages;
> - amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
> +
> j = 0;
> - } else {
> - amdgpu_res_next(&cursor, PAGE_SIZE);
> }
> - continue;
> }
>
> - pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
> - src[i] >> PAGE_SHIFT, page_to_pfn(spage));
> + pr_debug_ratelimited("dma mapping %lld pages, src to 0x%llx, pfn 0x%lx\n",
> + k, src[i] >> PAGE_SHIFT, migrate->src[i] >> MIGRATE_PFN_SHIFT);
> + i += k;
> + m += k;
> +
> + if (m >= (cursor.size >> PAGE_SHIFT)) {
> + if (j > 0) {
> + r = svm_migrate_copy_memory_gart(adev, src + i - j,
> + dst + i - j, j,
> + FROM_RAM_TO_VRAM,
> + mfence);
Are you sure this is correct? The old code incremented i after this copy
was done. Your new code does it before. I think that will mess up your
address calculations.
> + if (r)
> + goto out_free_vram_pages;
> + }
> +
> + amdgpu_res_next(&cursor, m*PAGE_SIZE);
>
> - /* accumulated j + 1 pages reach end of current drm_buddy_block */
> - if (j + 1 >= (cursor.size >> PAGE_SHIFT)) {
> - r = svm_migrate_copy_memory_gart(adev, src + i - j,
> - dst + i - j, j + 1,
> - FROM_RAM_TO_VRAM,
> - mfence);
> - if (r)
> - goto out_free_vram_pages;
> - amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
> j = 0;
> - } else {
> - j++;
> + m = 0;
> }
> }
>
> @@ -410,17 +509,24 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
> struct kfd_process_device *pdd;
> struct dma_fence *mfence = NULL;
> struct migrate_vma migrate = { 0 };
> + bool is_private_device = false;
> unsigned long cpages = 0;
> unsigned long mpages = 0;
> dma_addr_t *scratch;
> void *buf;
> int r = -ENOMEM;
>
> + is_private_device = svm_is_private_zone(adev);
> +
> memset(&migrate, 0, sizeof(migrate));
> migrate.vma = vma;
> migrate.start = start;
> migrate.end = end;
> migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
> +
> + if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
> + migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
> +
Why do you apply this only to device_private memory. This should work
just as well for device_coherent on MI200 A+A.
> migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
>
> buf = kvcalloc(npages,
> @@ -609,6 +715,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
> u64 addr;
> int r = 0;
>
> + u64 l, k;
> + bool is_large = false;
> +
> pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
> prange->last);
>
> @@ -616,8 +725,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
>
> src = (u64 *)(scratch + npages);
> dst = scratch;
> -
> - for (i = 0, j = 0; i < npages; i++, addr += PAGE_SIZE) {
> + for (i = 0, j = 0; i < npages;) {
If you reset k = 1, you could keep the increment in the loop header.
Just update it to
for (i = 0, j = 0, k = 1; i < npages; i += k, addr += k*PAGE_SIZE, k = 1)
> struct page *spage;
>
> spage = migrate_pfn_to_page(migrate->src[i]);
> @@ -633,6 +741,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
> goto out_oom;
> j = 0;
> }
> +
> + addr += PAGE_SIZE;
> + i++;
> continue;
> }
> src[i] = svm_migrate_addr(adev, spage);
> @@ -646,7 +757,22 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
> j = 0;
> }
>
> - dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
> + if(IS_ALIGNED(page_to_pfn(spage), HPAGE_PMD_NR) &&
There should be a space after "if". I see a few more coding style issues
below. Please run check_patch.pl to check for common coding style issues.
> + (addr + HPAGE_PMD_SIZE) <= migrate->end &&
> + IS_ALIGNED (addr, HPAGE_PMD_SIZE) &&
> + migrate->src[i] & MIGRATE_PFN_COMPOUND) {
> +
> + is_large = true;
> + k = HPAGE_PMD_NR;
is_large is redundant. You could just use (k > 1).
> +
> + dpage = svm_migrate_get_sys_page(migrate->vma, addr,
> + HPAGE_PMD_ORDER);
Do we need a fallback to small pages if if huge-page allocation fails?
> + } else {
> + k = 1;
> + is_large = false;
> + dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
> + }
> +
> if (!dpage) {
> pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
> prange->svms, prange->start, prange->last);
> @@ -654,21 +780,59 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
> goto out_oom;
> }
>
> - dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
> + dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE*k, DMA_BIDIRECTIONAL);
> r = dma_mapping_error(dev, dst[i]);
> if (r) {
> dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r);
> - dst[i] = 0;
Why did you remove this?
> goto out_oom;
> }
>
> - pr_debug_ratelimited("dma mapping dst to 0x%llx, pfn 0x%lx\n",
> - dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
> -
> migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
> + if (is_large)
> + migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
Looks like you can merge that into the next if-block just below.
>
> - dpage = NULL;
> - j++;
> + if (is_large) {
> + /* migrate previous accumulated pages */
> + if(j) {
> + r = svm_migrate_copy_memory_gart(adev, dst + i - j,
> + src + i - j, j, FROM_VRAM_TO_RAM, mfence);
> + if (r)
> + goto out_oom;
> + j = 0;
> + }
> +
> + for (l = 1; l < k; l++) {
> +
> + src[i + l] = src[i] + l*PAGE_SIZE;
> + dst[i + l] = dst[i] + l*PAGE_SIZE;
> + migrate->dst[i + l] = 0;
> + }
> +
> + /* migrate the HPAGE_PMD_NR pages above */
> + /* svm_migrate_copy_memory_gart will add a paramter to indicate
> + * the migration is for 2MB THP
> + */
> + r = svm_migrate_copy_memory_gart(
> + adev, dst + i,
> + src + i, HPAGE_PMD_NR,
> + FROM_VRAM_TO_RAM,
> + mfence);
> +
> + /* mark head page dma mapping as THP, tail pages dma addr
> + * are set to 0 for following dma_unmap
> + */
> + dst[i] |= SVM_RANGE_DMA_THP;
> + for (l = 1; l < k; l++)
> + dst[i + l] = 0;
I hope this doesn't break partial mapping or unmapping. We'd need to be
sure that code always aligns addresses to huge-page boundaries and gets
the whole huge page.
> +
> + if (r)
> + goto out_oom;
> +
> + } else
> + j++;
> +
> + addr += PAGE_SIZE*k;
> + i += k;
> }
>
> if (j > 0)
> @@ -687,12 +851,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
> /* release previous allocated sys pages and unmap dma address */
> while (i--) {
>
> - if (dst[i]) {
> - dma_unmap_page(dev, dst[i], PAGE_SIZE,
> - DMA_BIDIRECTIONAL);
> - dst[i] = 0;
> - }
> -
> + /* follwing svm_range_dma_unmap_dev will do dma unmap anyway
> + * not need do dma unmap here
> + */
> dpage = migrate_pfn_to_page(migrate->dst[i]);
> if (!dpage)
> continue;
> @@ -733,6 +894,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
> unsigned long cpages = 0;
> unsigned long mpages = 0;
> struct amdgpu_device *adev = node->adev;
> + bool is_private_device = false;
> struct kfd_process_device *pdd;
> struct dma_fence *mfence = NULL;
> struct migrate_vma migrate = { 0 };
> @@ -740,6 +902,8 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
> void *buf;
> int r = -ENOMEM;
>
> + is_private_device = svm_is_private_zone(adev);
> +
> memset(&migrate, 0, sizeof(migrate));
> migrate.vma = vma;
> migrate.start = start;
> @@ -750,6 +914,9 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
> else
> migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
>
> + if (is_private_device && ((end - start) >> PAGE_SHIFT) >= HPAGE_PMD_NR)
> + migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
Why do you apply this only to device_private memory. This should work
just as well for device_coherent on MI200 A+A.
> +
> buf = kvcalloc(npages,
> 2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),
> GFP_KERNEL);
> @@ -1132,7 +1299,7 @@ int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
>
> amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
>
> - pr_info("HMM registered %ldMB device memory\n", size >> 20);
> + pr_info("---XCHEN 3.2 HMM registered %ldMB device memory\n", size >> 20);
This looks like it's not meant to be submitted.
Regards,
Felix
>
> return 0;
> }
[-- Attachment #2: Type: text/html, Size: 17907 bytes --]
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH v2 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver
2026-10-06 22:44 ` Felix Kuehling
@ 2026-10-07 18:57 ` Chen, Xiaogang
0 siblings, 0 replies; 11+ messages in thread
From: Chen, Xiaogang @ 2026-10-07 18:57 UTC (permalink / raw)
To: Felix Kuehling, amd-gfx
On 10/6/2026 5:44 PM, Felix Kuehling wrote:
> On 2026-09-04 15:54, Xiaogang.Chen wrote:
>> From: Xiaogang Chen<xiaogang.chen@amd.com>
>>
>> Update kfd svm driver to migrate device-private THP introduced from
>> HMM core
>> migration function. Select this function by flag
>> MIGRATE_VMA_SELECT_COMPOUND
>> when call migrate_vma_setup. kfd migration procedure is updated
>> according to
>> collected page type that can be either compound folio(physical
>> continuous) or
>> normal size page.
>>
>> Signed-off-by: Xiaogang Chen<xiaogang.chen@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 271 ++++++++++++++++++-----
>> 1 file changed, 219 insertions(+), 52 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>> b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>> index bbf0fefd5722..af39e547c1fa 100644
>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
>> @@ -298,7 +298,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node,
>> struct svm_range *prange,
>> u64 mpages = 0;
>> dma_addr_t *src;
>> u64 *dst;
>> - u64 i, j;
>> + u64 i, j, k, l, m;
>> int r = 0;
>> pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms,
>> prange->start,
>> @@ -309,59 +309,158 @@ svm_migrate_copy_to_vram(struct kfd_node
>> *node, struct svm_range *prange,
>> amdgpu_res_first(prange->ttm_res, ttm_res_offset,
>> npages << PAGE_SHIFT, &cursor);
>> - for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {
>> + for (i = j = m = 0; (i < npages) && (mpages < migrate->cpages);) {
>> struct page *spage;
>> + unsigned long cur_dst_pfn;
>> + bool is_large = false;
>> - if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
>> - dst[i] = cursor.start + (j << PAGE_SHIFT);
>> - migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
>> - svm_migrate_get_vram_page(prange, migrate->dst[i], 0);
>> - migrate->dst[i] = migrate_pfn(migrate->dst[i]);
>> + cur_dst_pfn = svm_migrate_addr_to_pfn(adev,
>> + cursor.start + (m << PAGE_SHIFT));
>> +
>> + /* when migrate->src[i] has MIGRATE_PFN_COMPOUND set the src
>> page
>> + * is compound THP; its vm address is HPAGE_PMD_SIZE aligned
>> and
>> + * its MIGRATE_PFN_MIGRATE is set
>> + */
>> + if ((m + HPAGE_PMD_NR) <= (cursor.size >> PAGE_SHIFT) &&
>> + (i + HPAGE_PMD_NR) <= npages &&
>> + (migrate->src[i] & MIGRATE_PFN_COMPOUND) &&
>> + IS_ALIGNED(cur_dst_pfn, HPAGE_PMD_NR)) {
>> +
>
> Unnecessary empty line.
>
ok
>
>> + is_large = true;
>> + k = HPAGE_PMD_NR;
>
> is_large is redundant. Just use (k > 1).
>
>
>> + } else
>> + k = 1;
>
> Bad indentation.
ok
>
>
>> +
>> + /* for THP src[0] alwas MIGRATE_PFN_MIGRATE
>> + * just the first migrate->dst need be setup, others are zero
>> + */
>> + if (is_large) {
>> +
>> + dst[i] = cursor.start + (m << PAGE_SHIFT);
>> + svm_migrate_get_vram_page(prange, cur_dst_pfn,
>> + HPAGE_PMD_ORDER);
>> +
>> + migrate->dst[i] = migrate_pfn(cur_dst_pfn);
>> + migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
>> +
>> + for (l=1; l < k; l++)
>> + migrate->dst[i+l] = 0;
>> +
>> + mpages++;
>> +
>> + } else if ((migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
>> + dst[i] = cursor.start + (m << PAGE_SHIFT);
>> + svm_migrate_get_vram_page(prange, cur_dst_pfn, 0);
>> + migrate->dst[i] = migrate_pfn(cur_dst_pfn);
>> mpages++;
>> }
>> - spage = migrate_pfn_to_page(migrate->src[i]);
>> - if (spage && !is_zone_device_page(spage)) {
>> - src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
>> - DMA_BIDIRECTIONAL);
>> - r = dma_mapping_error(dev, src[i]);
>> - if (r) {
>> - src[i] = 0;
>> - dev_err(dev, "%s: fail %d dma_map_page\n",
>> - __func__, r);
>> - goto out_free_vram_pages;
>> +
>> + if (is_large) {
>> + if (j) {
>> + /* migrate previous accumulated pages */
>> + r = svm_migrate_copy_memory_gart(
>> + adev, src + i - j,
>> + dst + i - j, j,
>> + FROM_RAM_TO_VRAM,
>> + mfence);
>> +
>> + if (r)
>> + goto out_free_vram_pages;
>> +
>> + j = 0;
>> + }
>> +
>> + /* for THP check if the first src page is valid
>> + * if not valid skip following HPAGE_PMD_NR - 1 pages
>> + */
>> + spage = migrate_pfn_to_page(migrate->src[i]);
>> + if (spage && !is_zone_device_page(spage)) {
>> + /* dma_map continuous HPAGE_PMD_NR sys ram pages */
>> + src[i] = dma_map_page(dev, spage, 0,
>> PAGE_SIZE*HPAGE_PMD_NR,
>> + DMA_BIDIRECTIONAL);
>> +
>> + r = dma_mapping_error(dev, src[i]);
>> + if (r) {
>> + dev_err(dev, "%s: fail %d dma_map_page\n",
>> + __func__, r);
>> + goto out_free_vram_pages;
>> + }
>> +
>> + /* get dma address for following HPAGE_PMD_NR-1 pages
>> + * since src pages are continuous their dma addresses
>> + * are continuous too.
>> + */
>> + for (l=1; l < k; l++)
>> + src[i + l] = src[i] + l*PAGE_SIZE;
>> +
>> + /* migrate the HPAGE_PMD_NR pages above */
>> + r = svm_migrate_copy_memory_gart(
>> + adev, src + i,
>> + dst + i, HPAGE_PMD_NR,
>> + FROM_RAM_TO_VRAM,
>> + mfence);
>> +
>> + /* mark head page dma mapping as THP, tail pages dma
>> addr
>> + * are set to 0 for following dma_unmap
>> + */
>> + src[i] |= SVM_RANGE_DMA_THP;
>> + for (l = 1; l < k; l++)
>> + src[i + l] = 0;
>
> I hope this doesn't break partial mapping or unmapping. We'd need to
> be sure that code always aligns addresses to huge-page boundaries and
> gets the whole huge page. A safer alternative would be to use a
> different flag for the second and subsequent compound pages. So you'd
> still have the DMA addresses, but you could ignore them for DMA
> unmapping.
For THP system memory core HMM provides
spage=migrate_pfn_to_page(migrate->src[i]) is 2MB aligned. dma adress
src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE*HPAGE_PMD_NR,..), is also
2MB aligned since spage is 2MB aliged and size is 2MB.
I put SVM_RANGE_DMA_THP bit just at dma addr of head page. The following
511 dma address is put 0. At svm_range_dma_unmap_dev I should have
jumped i by 512 when hit SVM_RANGE_DMA_THP:
if (dma_addr[i] & SVM_RANGE_DMA_THP) {
dma_addr[i] &= ~SVM_RANGE_DMA_THP;
dma_unmap_page(dev, dma_addr[i], PAGE_SIZE*HPAGE_PMD_NR,
DMA_BIDIRECTIONAL);
} else
dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
>
>> +
>> + if (r)
>> + goto out_free_vram_pages;
>> +
>> + j = 0;
>> }
>> } else {
>> - if (j) {
>> + /* single normal page case */
>> + spage = migrate_pfn_to_page(migrate->src[i]);
>> + if (spage && !is_zone_device_page(spage)) {
>> + src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
>> + DMA_BIDIRECTIONAL);
>> +
>> + r = dma_mapping_error(dev, src[i]);
>> +
>> + if (r) {
>> + dev_err(dev, "%s: fail %d dma_map_page\n",
>> + __func__, r);
>> + goto out_free_vram_pages;
>> + }
>> + j += 1;
>> +
>> + } else if (j) {
>> r = svm_migrate_copy_memory_gart(
>> adev, src + i - j,
>> dst + i - j, j,
>> FROM_RAM_TO_VRAM,
>> mfence);
>> +
>> if (r)
>> goto out_free_vram_pages;
>> - amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
>> +
>> j = 0;
>> - } else {
>> - amdgpu_res_next(&cursor, PAGE_SIZE);
>> }
>> - continue;
>> }
>> - pr_debug_ratelimited("dma mapping src to 0x%llx, pfn
>> 0x%lx\n",
>> - src[i] >> PAGE_SHIFT, page_to_pfn(spage));
>> + pr_debug_ratelimited("dma mapping %lld pages, src to 0x%llx,
>> pfn 0x%lx\n",
>> + k, src[i] >> PAGE_SHIFT, migrate->src[i] >>
>> MIGRATE_PFN_SHIFT);
>> + i += k;
>> + m += k;
>> +
>> + if (m >= (cursor.size >> PAGE_SHIFT)) {
>> + if (j > 0) {
>> + r = svm_migrate_copy_memory_gart(adev, src + i - j,
>> + dst + i - j, j,
>> + FROM_RAM_TO_VRAM,
>> + mfence);
>
> Are you sure this is correct? The old code incremented i after this
> copy was done. Your new code does it before. I think that will mess up
> your address calculations.
The old code changes i at for-loop: for (i = j = 0; i < npages; i++) .
Here I update i after handle page(either one 4k page or one THP), then
increase i(either 1 or 512). When reach current vram segment boundary,
migrate previous accumulated 4k pages. THP has been migrated before.
Here the code handles the 4k pages that was accumulated when reach to
current vram segment boundary.
>
>
>> + if (r)
>> + goto out_free_vram_pages;
>> + }
>> +
>> + amdgpu_res_next(&cursor, m*PAGE_SIZE);
>> - /* accumulated j + 1 pages reach end of current
>> drm_buddy_block */
>> - if (j + 1 >= (cursor.size >> PAGE_SHIFT)) {
>> - r = svm_migrate_copy_memory_gart(adev, src + i - j,
>> - dst + i - j, j + 1,
>> - FROM_RAM_TO_VRAM,
>> - mfence);
>> - if (r)
>> - goto out_free_vram_pages;
>> - amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
>> j = 0;
>> - } else {
>> - j++;
>> + m = 0;
>> }
>> }
>> @@ -410,17 +509,24 @@ svm_migrate_vma_to_vram(struct kfd_node
>> *node, struct svm_range *prange,
>> struct kfd_process_device *pdd;
>> struct dma_fence *mfence = NULL;
>> struct migrate_vma migrate = { 0 };
>> + bool is_private_device = false;
>> unsigned long cpages = 0;
>> unsigned long mpages = 0;
>> dma_addr_t *scratch;
>> void *buf;
>> int r = -ENOMEM;
>> + is_private_device = svm_is_private_zone(adev);
>> +
>> memset(&migrate, 0, sizeof(migrate));
>> migrate.vma = vma;
>> migrate.start = start;
>> migrate.end = end;
>> migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
>> +
>> + if (is_private_device && ((end - start) >> PAGE_SHIFT) >=
>> HPAGE_PMD_NR)
>> + migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
>> +
>
> Why do you apply this only to device_private memory. This should work
> just as well for device_coherent on MI200 A+A.
Current work is for private device memory only. Need update core HMM
code for coherent device memory to support THP.
Driver's work to enable device THP actually has no real difference
between private and coherence device memory.
>
>
>> migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
>> buf = kvcalloc(npages,
>> @@ -609,6 +715,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device
>> *adev, struct svm_range *prange,
>> u64 addr;
>> int r = 0;
>> + u64 l, k;
>> + bool is_large = false;
>> +
>> pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
>> prange->last);
>> @@ -616,8 +725,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device
>> *adev, struct svm_range *prange,
>> src = (u64 *)(scratch + npages);
>> dst = scratch;
>> -
>> - for (i = 0, j = 0; i < npages; i++, addr += PAGE_SIZE) {
>> + for (i = 0, j = 0; i < npages;) {
>
> If you reset k = 1, you could keep the increment in the loop header.
> Just update it to
>
> for (i = 0, j = 0, k = 1; i < npages; i += k, addr += k*PAGE_SIZE,
> k = 1)
ok, that makes code more concise.
>
>
>> struct page *spage;
>> spage = migrate_pfn_to_page(migrate->src[i]);
>> @@ -633,6 +741,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device
>> *adev, struct svm_range *prange,
>> goto out_oom;
>> j = 0;
>> }
>> +
>> + addr += PAGE_SIZE;
>> + i++;
>> continue;
>> }
>> src[i] = svm_migrate_addr(adev, spage);
>> @@ -646,7 +757,22 @@ svm_migrate_copy_to_ram(struct amdgpu_device
>> *adev, struct svm_range *prange,
>> j = 0;
>> }
>> - dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
>> + if(IS_ALIGNED(page_to_pfn(spage), HPAGE_PMD_NR) &&
>
> There should be a space after "if". I see a few more coding style
> issues below. Please run check_patch.pl to check for common coding
> style issues.
ok, will check code style next submit.
>
>
>> + (addr + HPAGE_PMD_SIZE) <= migrate->end &&
>> + IS_ALIGNED (addr, HPAGE_PMD_SIZE) &&
>> + migrate->src[i] & MIGRATE_PFN_COMPOUND) {
>> +
>> + is_large = true;
>> + k = HPAGE_PMD_NR;
>
> is_large is redundant. You could just use (k > 1).
ok
>
>
>> +
>> + dpage = svm_migrate_get_sys_page(migrate->vma, addr,
>> + HPAGE_PMD_ORDER);
>
> Do we need a fallback to small pages if if huge-page allocation fails?
I thought about that for system page allocation: if kernel cannot
provide THP system memory allocate in regular page base, if still
cannot, fail svm_migrate_get_sys_page.
Same for device memory allocation when allocate THP from ttm. If fail,
fallback to 4k page based allocation.
Before migration check driver allocated page is THP or not, then use
different procedure.
>
>
>> + } else {
>> + k = 1;
>> + is_large = false;
>> + dpage = svm_migrate_get_sys_page(migrate->vma, addr, 0);
>> + }
>> +
>> if (!dpage) {
>> pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
>> prange->svms, prange->start, prange->last);
>> @@ -654,21 +780,59 @@ svm_migrate_copy_to_ram(struct amdgpu_device
>> *adev, struct svm_range *prange,
>> goto out_oom;
>> }
>> - dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE,
>> DMA_BIDIRECTIONAL);
>> + dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE*k,
>> DMA_BIDIRECTIONAL);
>> r = dma_mapping_error(dev, dst[i]);
>> if (r) {
>> dev_err(adev->dev, "%s: fail %d dma_map_page\n",
>> __func__, r);
>> - dst[i] = 0;
>
> Why did you remove this?
ok, dst[i] is still used at svm_range_dma_unmap_dev for error handling
case.
>
>
>> goto out_oom;
>> }
>> - pr_debug_ratelimited("dma mapping dst to 0x%llx, pfn
>> 0x%lx\n",
>> - dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
>> -
>> migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
>> + if (is_large)
>> + migrate->dst[i] |= MIGRATE_PFN_COMPOUND;
>
> Looks like you can merge that into the next if-block just below.
Yes, I do not know what I thought when did it. Maybe forgot cleaning
code after removed some debug code here.
>
>
>> - dpage = NULL;
>> - j++;
>> + if (is_large) {
>> + /* migrate previous accumulated pages */
>> + if(j) {
>> + r = svm_migrate_copy_memory_gart(adev, dst + i - j,
>> + src + i - j, j, FROM_VRAM_TO_RAM,
>> mfence);
>> + if (r)
>> + goto out_oom;
>> + j = 0;
>> + }
>> +
>> + for (l = 1; l < k; l++) {
>> +
>> + src[i + l] = src[i] + l*PAGE_SIZE;
>> + dst[i + l] = dst[i] + l*PAGE_SIZE;
>> + migrate->dst[i + l] = 0;
>> + }
>> +
>> + /* migrate the HPAGE_PMD_NR pages above */
>> + /* svm_migrate_copy_memory_gart will add a paramter to
>> indicate
>> + * the migration is for 2MB THP
>> + */
>> + r = svm_migrate_copy_memory_gart(
>> + adev, dst + i,
>> + src + i, HPAGE_PMD_NR,
>> + FROM_VRAM_TO_RAM,
>> + mfence);
>> +
>> + /* mark head page dma mapping as THP, tail pages dma addr
>> + * are set to 0 for following dma_unmap
>> + */
>> + dst[i] |= SVM_RANGE_DMA_THP;
>> + for (l = 1; l < k; l++)
>> + dst[i + l] = 0;
>
> I hope this doesn't break partial mapping or unmapping. We'd need to
> be sure that code always aligns addresses to huge-page boundaries and
> gets the whole huge page.
Same as migration from sys to device. dst[i] is dma address for THP
system page. It is 2MB aligned and the mapping size is
PAGE_SIZE*k(k=512) for THP.
>
>
>> +
>> + if (r)
>> + goto out_oom;
>> +
>> + } else
>> + j++;
>> +
>> + addr += PAGE_SIZE*k;
>> + i += k;
>> }
>> if (j > 0)
>> @@ -687,12 +851,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device
>> *adev, struct svm_range *prange,
>> /* release previous allocated sys pages and unmap dma
>> address */
>> while (i--) {
>> - if (dst[i]) {
>> - dma_unmap_page(dev, dst[i], PAGE_SIZE,
>> - DMA_BIDIRECTIONAL);
>> - dst[i] = 0;
>> - }
>> -
>> + /* follwing svm_range_dma_unmap_dev will do dma unmap
>> anyway
>> + * not need do dma unmap here
>> + */
>> dpage = migrate_pfn_to_page(migrate->dst[i]);
>> if (!dpage)
>> continue;
>> @@ -733,6 +894,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node,
>> struct svm_range *prange,
>> unsigned long cpages = 0;
>> unsigned long mpages = 0;
>> struct amdgpu_device *adev = node->adev;
>> + bool is_private_device = false;
>> struct kfd_process_device *pdd;
>> struct dma_fence *mfence = NULL;
>> struct migrate_vma migrate = { 0 };
>> @@ -740,6 +902,8 @@ svm_migrate_vma_to_ram(struct kfd_node *node,
>> struct svm_range *prange,
>> void *buf;
>> int r = -ENOMEM;
>> + is_private_device = svm_is_private_zone(adev);
>> +
>> memset(&migrate, 0, sizeof(migrate));
>> migrate.vma = vma;
>> migrate.start = start;
>> @@ -750,6 +914,9 @@ svm_migrate_vma_to_ram(struct kfd_node *node,
>> struct svm_range *prange,
>> else
>> migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
>> + if (is_private_device && ((end - start) >> PAGE_SHIFT) >=
>> HPAGE_PMD_NR)
>> + migrate.flags = migrate.flags | MIGRATE_VMA_SELECT_COMPOUND;
>
> Why do you apply this only to device_private memory. This should work
> just as well for device_coherent on MI200 A+A.
Current work is for private device only. Need update core HMM code for
coherent device memory to support THP.
Driver's work to enable device THP actually has no real difference
between private and coherence device memory since driver uses same
migration path.
>
>
>> +
>> buf = kvcalloc(npages,
>> 2 * sizeof(*migrate.src) + sizeof(u64) +
>> sizeof(dma_addr_t),
>> GFP_KERNEL);
>> @@ -1132,7 +1299,7 @@ int kgd2kfd_init_zone_device(struct
>> amdgpu_device *adev)
>> amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
>> - pr_info("HMM registered %ldMB device memory\n", size >> 20);
>> + pr_info("---XCHEN 3.2 HMM registered %ldMB device memory\n",
>> size >> 20);
>
> This looks like it's not meant to be submitted.
Sorry about that. I should have removed debug stuff put during triage
before submit.
Thanks
Xiaogang
>
> Regards,
> Felix
>
>
>> return 0;
>> }
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v2 4/4] drm/amdkfd: Apply AMDGPU_PTE_FRAG to pte of gart page table for THP mapping
2026-09-04 19:54 [PATCH v2 0/4] drm/amdkfd: Enable device private memory THP support in kfd svm driver Xiaogang.Chen
` (2 preceding siblings ...)
2026-09-04 19:54 ` [PATCH v2 3/4] drm/amdkfd: Apply HMM THP zone device-private memory migration in kfd driver Xiaogang.Chen
@ 2026-09-04 19:54 ` Xiaogang.Chen
3 siblings, 0 replies; 11+ messages in thread
From: Xiaogang.Chen @ 2026-09-04 19:54 UTC (permalink / raw)
To: amd-gfx; +Cc: Xiaogang Chen
From: Xiaogang Chen <xiaogang.chen@amd.com>
When both sys ram and vram are physical continuous HPAGE_PMD_NR pages during
migration set AMDGPU_PTE_FRAG(HPAGE_PMD_ORDER) at pte in gart page table to
let hardware know the migrating pages are HPAGE_PMD_NR size THP. That will
reduce hardware TLB pressure or increase TLB hit ration.
Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 45 ++++++++++++++----------
1 file changed, 27 insertions(+), 18 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index af39e547c1fa..8a2fbe3a7613 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -48,7 +48,7 @@ static int
svm_migrate_gart_map(struct amdgpu_ring *ring,
struct amdgpu_ttm_buffer_entity *entity,
u64 npages,
- dma_addr_t *addr, u64 *gart_addr, u64 flags)
+ dma_addr_t *addr, u64 *gart_addr, u64 flags, bool is_thp)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_job *job;
@@ -90,6 +90,9 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
pte_flags |= AMDGPU_PTE_WRITEABLE;
pte_flags |= adev->gart.gart_pte_flags;
+ if (is_thp)
+ pte_flags |= AMDGPU_PTE_FRAG(HPAGE_PMD_ORDER);
+
cpu_addr = &job->ibs[0].ptr[num_dw];
amdgpu_gart_map(adev, 0, npages, addr, pte_flags, cpu_addr);
@@ -108,6 +111,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
* @npages: number of pages to copy
* @direction: enum MIGRATION_COPY_DIR
* @mfence: output, sdma fence to signal after sdma is done
+ * @is_thp: both sys and vram are physical continuous HPAGE_PMD_NR pages
*
* ram address uses GART table continuous entries mapping to ram pages,
* vram address uses direct mapping of vram pages, which must have npages
@@ -126,7 +130,7 @@ static int
svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
u64 *vram, u64 npages,
enum MIGRATION_COPY_DIR direction,
- struct dma_fence **mfence)
+ struct dma_fence **mfence, bool is_thp)
{
const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT);
struct amdgpu_ring *ring;
@@ -136,6 +140,12 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
u64 size;
int r = 0;
+ if (is_thp && npages != HPAGE_PMD_NR) {
+ dev_warn(adev->dev, "THP migration should have %d pages\n",
+ HPAGE_PMD_NR);
+ is_thp = false;
+ }
+
ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
entity = &adev->mman.move_entities[0];
@@ -146,11 +156,12 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
if (direction == FROM_VRAM_TO_RAM) {
gart_s = svm_migrate_direct_mapping_addr(adev, *vram);
- r = svm_migrate_gart_map(ring, entity, size, sys, &gart_d, 0);
+ r = svm_migrate_gart_map(ring, entity, size, sys, &gart_d, 0,
+ is_thp);
} else if (direction == FROM_RAM_TO_VRAM) {
r = svm_migrate_gart_map(ring, entity, size, sys, &gart_s,
- KFD_IOCTL_SVM_FLAG_GPU_RO);
+ KFD_IOCTL_SVM_FLAG_GPU_RO, is_thp);
gart_d = svm_migrate_direct_mapping_addr(adev, *vram);
}
if (r) {
@@ -362,7 +373,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
- mfence);
+ mfence, false);
if (r)
goto out_free_vram_pages;
@@ -398,7 +409,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
adev, src + i,
dst + i, HPAGE_PMD_NR,
FROM_RAM_TO_VRAM,
- mfence);
+ mfence, true);
/* mark head page dma mapping as THP, tail pages dma addr
* are set to 0 for following dma_unmap
@@ -433,7 +444,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
- mfence);
+ mfence, false);
if (r)
goto out_free_vram_pages;
@@ -452,7 +463,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
r = svm_migrate_copy_memory_gart(adev, src + i - j,
dst + i - j, j,
FROM_RAM_TO_VRAM,
- mfence);
+ mfence, false);
if (r)
goto out_free_vram_pages;
}
@@ -466,7 +477,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
if (j > 0)
r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
- FROM_RAM_TO_VRAM, mfence);
+ FROM_RAM_TO_VRAM, mfence, false);
out_free_vram_pages:
if (r) {
@@ -736,7 +747,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
r = svm_migrate_copy_memory_gart(adev, dst + i - j,
src + i - j, j,
FROM_VRAM_TO_RAM,
- mfence);
+ mfence, false);
if (r)
goto out_oom;
j = 0;
@@ -751,7 +762,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
r = svm_migrate_copy_memory_gart(adev, dst + i - j,
src + i - j, j,
FROM_VRAM_TO_RAM,
- mfence);
+ mfence, false);
if (r)
goto out_oom;
j = 0;
@@ -795,7 +806,8 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
/* migrate previous accumulated pages */
if(j) {
r = svm_migrate_copy_memory_gart(adev, dst + i - j,
- src + i - j, j, FROM_VRAM_TO_RAM, mfence);
+ src + i - j, j, FROM_VRAM_TO_RAM,
+ mfence, false);
if (r)
goto out_oom;
j = 0;
@@ -809,14 +821,11 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
}
/* migrate the HPAGE_PMD_NR pages above */
- /* svm_migrate_copy_memory_gart will add a paramter to indicate
- * the migration is for 2MB THP
- */
r = svm_migrate_copy_memory_gart(
adev, dst + i,
src + i, HPAGE_PMD_NR,
FROM_VRAM_TO_RAM,
- mfence);
+ mfence, true);
/* mark head page dma mapping as THP, tail pages dma addr
* are set to 0 for following dma_unmap
@@ -837,7 +846,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
if (j > 0)
r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
- FROM_VRAM_TO_RAM, mfence);
+ FROM_VRAM_TO_RAM, mfence, false);
out_oom:
if (r) {
pr_debug("failed %d copy to ram\n", r);
@@ -1299,7 +1308,7 @@ int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
- pr_info("---XCHEN 3.2 HMM registered %ldMB device memory\n", size >> 20);
+ pr_info("HMM registered %ldMB device memory\n", size >> 20);
return 0;
}
--
2.34.1
^ permalink raw reply related [flat|nested] 11+ messages in thread