[PATCH] mm/damon/vaddr: attempt per-vma lock during page table walk

[PATCH] mm/damon/vaddr: attempt per-vma lock during page table walk

[Kefeng Wang]

Currently, DAMON virtual address operations use mmap_read_lock
during page table walks, which can cause unnecessary contention
under high concurrency.

Introduce damon_va_walk_page_range() to first attempt acquiring a
per-vma lock. If the VMA is found and the range is fully contained
within it, the page table walk proceeds with the per-vma lock
instead of mmap_read_lock.

This optimization is particularly effective for damon_va_young()
and damon_va_mkold(), which are frequently called and typically
operate within a single VMA.

Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
---
 mm/damon/vaddr.c | 66 +++++++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 26 deletions(-)

diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index dd5f2d7027ac..cd6c0c5f3655 100644
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -237,6 +237,32 @@ static void damon_va_update(struct damon_ctx *ctx)
 	}
 }
 
+static void damon_va_walk_page_range(struct mm_struct *mm, unsigned long start,
+		unsigned long end, struct mm_walk_ops *ops, void *private)
+{
+	struct vm_area_struct *vma;
+
+	vma = lock_vma_under_rcu(mm, start);
+	if (!vma)
+		goto lock_mmap;
+
+	if (end > vma->vm_end) {
+		vma_end_read(vma);
+		goto lock_mmap;
+	}
+
+	ops->walk_lock = PGWALK_VMA_RDLOCK_VERIFY;
+	walk_page_range_vma(vma, start, end, ops, private);
+	vma_end_read(vma);
+	return;
+
+lock_mmap:
+	mmap_read_lock(mm);
+	ops->walk_lock = PGWALK_RDLOCK;
+	walk_page_range(mm, start, end, ops, private);
+	mmap_read_unlock(mm);
+}
+
 static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
 		unsigned long next, struct mm_walk *walk)
 {
@@ -315,17 +341,14 @@ static int damon_mkold_hugetlb_entry(pte_t *pte, unsigned long hmask,
 #define damon_mkold_hugetlb_entry NULL
 #endif /* CONFIG_HUGETLB_PAGE */
 
-static const struct mm_walk_ops damon_mkold_ops = {
-	.pmd_entry = damon_mkold_pmd_entry,
-	.hugetlb_entry = damon_mkold_hugetlb_entry,
-	.walk_lock = PGWALK_RDLOCK,
-};
-
 static void damon_va_mkold(struct mm_struct *mm, unsigned long addr)
 {
-	mmap_read_lock(mm);
-	walk_page_range(mm, addr, addr + 1, &damon_mkold_ops, NULL);
-	mmap_read_unlock(mm);
+	struct mm_walk_ops damon_mkold_ops = {
+		.pmd_entry = damon_mkold_pmd_entry,
+		.hugetlb_entry = damon_mkold_hugetlb_entry,
+	};
+
+	damon_va_walk_page_range(mm, addr, addr + 1, &damon_mkold_ops, NULL);
 }
 
 /*
@@ -444,12 +467,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask,
 #define damon_young_hugetlb_entry NULL
 #endif /* CONFIG_HUGETLB_PAGE */
 
-static const struct mm_walk_ops damon_young_ops = {
-	.pmd_entry = damon_young_pmd_entry,
-	.hugetlb_entry = damon_young_hugetlb_entry,
-	.walk_lock = PGWALK_RDLOCK,
-};
-
 static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
 		unsigned long *folio_sz)
 {
@@ -458,9 +475,12 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
 		.young = false,
 	};
 
-	mmap_read_lock(mm);
-	walk_page_range(mm, addr, addr + 1, &damon_young_ops, &arg);
-	mmap_read_unlock(mm);
+	struct mm_walk_ops damon_young_ops = {
+		.pmd_entry = damon_young_pmd_entry,
+		.hugetlb_entry = damon_young_hugetlb_entry,
+	};
+
+	damon_va_walk_page_range(mm, addr, addr + 1, &damon_young_ops, &arg);
 	return arg.young;
 }
 
@@ -749,7 +769,6 @@ static unsigned long damos_va_migrate(struct damon_target *target,
 	struct mm_walk_ops walk_ops = {
 		.pmd_entry = damos_va_migrate_pmd_entry,
 		.pte_entry = NULL,
-		.walk_lock = PGWALK_RDLOCK,
 	};
 
 	use_target_nid = dests->nr_dests == 0;
@@ -767,9 +786,7 @@ static unsigned long damos_va_migrate(struct damon_target *target,
 	if (!mm)
 		goto free_lists;
 
-	mmap_read_lock(mm);
-	walk_page_range(mm, r->ar.start, r->ar.end, &walk_ops, &priv);
-	mmap_read_unlock(mm);
+	damon_va_walk_page_range(mm, r->ar.start, r->ar.end, &walk_ops, &priv);
 	mmput(mm);
 
 	for (int i = 0; i < nr_dests; i++) {
@@ -861,7 +878,6 @@ static unsigned long damos_va_stat(struct damon_target *target,
 	struct mm_struct *mm;
 	struct mm_walk_ops walk_ops = {
 		.pmd_entry = damos_va_stat_pmd_entry,
-		.walk_lock = PGWALK_RDLOCK,
 	};
 
 	priv.scheme = s;
@@ -874,9 +890,7 @@ static unsigned long damos_va_stat(struct damon_target *target,
 	if (!mm)
 		return 0;
 
-	mmap_read_lock(mm);
-	walk_page_range(mm, r->ar.start, r->ar.end, &walk_ops, &priv);
-	mmap_read_unlock(mm);
+	damon_va_walk_page_range(mm, r->ar.start, r->ar.end, &walk_ops, &priv);
 	mmput(mm);
 	return 0;
 }
-- 
2.27.0

Re: [PATCH] mm/damon/vaddr: attempt per-vma lock during page table walk

[SeongJae Park]

On Mon, 11 May 2026 21:25:46 +0800 Kefeng Wang <wangkefeng.wang@huawei.com> wrote:

> Currently, DAMON virtual address operations use mmap_read_lock
> during page table walks, which can cause unnecessary contention
> under high concurrency.
> 
> Introduce damon_va_walk_page_range() to first attempt acquiring a
> per-vma lock. If the VMA is found and the range is fully contained
> within it, the page table walk proceeds with the per-vma lock
> instead of mmap_read_lock.
> 
> This optimization is particularly effective for damon_va_young()
> and damon_va_mkold(), which are frequently called and typically
> operate within a single VMA.

Makes sense.  Do you have some measurements?

> 
> Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>

Looks good to me.  Nonetheless, because I'm not familiar with per-vma locking,
I'd like to wait for Sashiko review.


Thanks,
SJ

[...]

Re: [PATCH] mm/damon/vaddr: attempt per-vma lock during page table walk

[Kefeng Wang]

On 5/12/2026 9:31 AM, SeongJae Park wrote:
> On Mon, 11 May 2026 21:25:46 +0800 Kefeng Wang <wangkefeng.wang@huawei.com> wrote:
> 
>> Currently, DAMON virtual address operations use mmap_read_lock
>> during page table walks, which can cause unnecessary contention
>> under high concurrency.
>>
>> Introduce damon_va_walk_page_range() to first attempt acquiring a
>> per-vma lock. If the VMA is found and the range is fully contained
>> within it, the page table walk proceeds with the per-vma lock
>> instead of mmap_read_lock.
>>
>> This optimization is particularly effective for damon_va_young()
>> and damon_va_mkold(), which are frequently called and typically
>> operate within a single VMA.
> 
> Makes sense.  Do you have some measurements?

In fact, I do not have performance-related tests.

> 
>>
>> Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
> 
> Looks good to me.  Nonetheless, because I'm not familiar with per-vma locking,
> I'd like to wait for Sashiko review.
> 

Sashiko review reports a issue about handling VMAs with the VM_PFNMAP
flag inconsistently[1]，We indeed do not need to handle the vma of
VM_PFNMAP for damon, so a quick fix is as follows,

diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index eed2ae132a58..d27147603564 100644
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -251,8 +251,11 @@ static void damon_va_walk_page_range(struct 
mm_struct *mm, unsigned long start,
                 goto lock_mmap;
         }

-       ops->walk_lock = PGWALK_VMA_RDLOCK_VERIFY;
-       walk_page_range_vma(vma, start, end, ops, private);
+       if (!(vma->vm_flags & VM_PFNMAP)) {
+               ops->walk_lock = PGWALK_VMA_RDLOCK_VERIFY;
+               walk_page_range_vma(vma, start, end, ops, private);
+       }
+
         vma_end_read(vma);
         return;

Any more comments?

Thanks.

[1] 
https://sashiko.dev/#/patchset/20260511132546.1973270-1-wangkefeng.wang@huawei.com?part=1
> 
> Thanks,
> SJ
> 
> [...]
> 

Re: [PATCH] mm/damon/vaddr: attempt per-vma lock during page table walk

[SeongJae Park]

On Tue, 12 May 2026 21:59:26 +0800 Kefeng Wang <wangkefeng.wang@huawei.com> wrote:

> 
> 
> On 5/12/2026 9:31 AM, SeongJae Park wrote:
> > On Mon, 11 May 2026 21:25:46 +0800 Kefeng Wang <wangkefeng.wang@huawei.com> wrote:
> > 
> >> Currently, DAMON virtual address operations use mmap_read_lock
> >> during page table walks, which can cause unnecessary contention
> >> under high concurrency.
> >>
> >> Introduce damon_va_walk_page_range() to first attempt acquiring a
> >> per-vma lock. If the VMA is found and the range is fully contained
> >> within it, the page table walk proceeds with the per-vma lock
> >> instead of mmap_read_lock.
> >>
> >> This optimization is particularly effective for damon_va_young()
> >> and damon_va_mkold(), which are frequently called and typically
> >> operate within a single VMA.
> > 
> > Makes sense.  Do you have some measurements?
> 
> In fact, I do not have performance-related tests.
> 
> > 
> >>
> >> Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
> > 
> > Looks good to me.  Nonetheless, because I'm not familiar with per-vma locking,
> > I'd like to wait for Sashiko review.
> > 
> 
> Sashiko review reports a issue about handling VMAs with the VM_PFNMAP
> flag inconsistently[1]，

For other readers who may want to read the Sashiko review and might want to
commeent, it is also available [1] at lore.

> We indeed do not need to handle the vma of
> VM_PFNMAP for damon, so a quick fix is as follows,
> 
> diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
> index eed2ae132a58..d27147603564 100644
> --- a/mm/damon/vaddr.c
> +++ b/mm/damon/vaddr.c
> @@ -251,8 +251,11 @@ static void damon_va_walk_page_range(struct 
> mm_struct *mm, unsigned long start,
>                  goto lock_mmap;
>          }
> 
> -       ops->walk_lock = PGWALK_VMA_RDLOCK_VERIFY;
> -       walk_page_range_vma(vma, start, end, ops, private);
> +       if (!(vma->vm_flags & VM_PFNMAP)) {
> +               ops->walk_lock = PGWALK_VMA_RDLOCK_VERIFY;
> +               walk_page_range_vma(vma, start, end, ops, private);
> +       }
> +
>          vma_end_read(vma);
>          return;
> 
> Any more comments?

Looks good to me, thank you!  Could you send v2 with this change?

> 
> Thanks.
> 
> [1] 
> https://sashiko.dev/#/patchset/20260511132546.1973270-1-wangkefeng.wang@huawei.com?part=1

[1] https://lore.kernel.org/20260512052628.83798C2BCC7@smtp.kernel.org


Thanks,
SJ

[...]