* [PATCH v2] userfaultfd: opportunistic TLB-flush batching for present pages in MOVE
@ 2025-08-05 12:14 Lokesh Gidra
2025-08-06 17:18 ` Lorenzo Stoakes
0 siblings, 1 reply; 4+ messages in thread
From: Lokesh Gidra @ 2025-08-05 12:14 UTC (permalink / raw)
To: akpm
Cc: aarcange, linux-mm, linux-kernel, 21cnbao, ngeoffray,
Lokesh Gidra, Suren Baghdasaryan, Kalesh Singh, Barry Song,
David Hildenbrand, Peter Xu
MOVE ioctl's runtime is dominated by TLB-flush cost, which is required
for moving present pages. Mitigate this cost by opportunistically
batching present contiguous pages for TLB flushing.
Without batching, in our testing on an arm64 Android device with UFFD GC,
which uses MOVE ioctl for compaction, we observed that out of the total
time spent in move_pages_pte(), over 40% is in ptep_clear_flush(), and
~20% in vm_normal_folio().
With batching, the proportion of vm_normal_folio() increases to over
70% of move_pages_pte() without any changes to vm_normal_folio().
Furthermore, time spent within move_pages_pte() is only ~20%, which
includes TLB-flush overhead.
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Kalesh Singh <kaleshsingh@google.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Peter Xu <peterx@redhat.com>
Signed-off-by: Lokesh Gidra <lokeshgidra@google.com>
---
Changes since v1 [1]
- Removed flush_tlb_batched_pending(), per Barry Song
- Unified single and multi page case, per Barry Song
[1] https://lore.kernel.org/all/20250731104726.103071-1-lokeshgidra@google.com/
mm/userfaultfd.c | 171 +++++++++++++++++++++++++++++++++--------------
1 file changed, 119 insertions(+), 52 deletions(-)
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index cbed91b09640..0ab51bcf264c 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -1026,18 +1026,61 @@ static inline bool is_pte_pages_stable(pte_t *dst_pte, pte_t *src_pte,
pmd_same(dst_pmdval, pmdp_get_lockless(dst_pmd));
}
-static int move_present_pte(struct mm_struct *mm,
- struct vm_area_struct *dst_vma,
- struct vm_area_struct *src_vma,
- unsigned long dst_addr, unsigned long src_addr,
- pte_t *dst_pte, pte_t *src_pte,
- pte_t orig_dst_pte, pte_t orig_src_pte,
- pmd_t *dst_pmd, pmd_t dst_pmdval,
- spinlock_t *dst_ptl, spinlock_t *src_ptl,
- struct folio *src_folio)
+/*
+ * Checks if the two ptes and the corresponding folio are eligible for batched
+ * move. If so, then returns pointer to the folio, after locking it. Otherwise,
+ * returns NULL.
+ */
+static struct folio *check_ptes_for_batched_move(struct vm_area_struct *src_vma,
+ unsigned long src_addr,
+ pte_t *src_pte, pte_t *dst_pte)
+{
+ pte_t orig_dst_pte, orig_src_pte;
+ struct folio *folio;
+
+ orig_dst_pte = ptep_get(dst_pte);
+ if (!pte_none(orig_dst_pte))
+ return NULL;
+
+ orig_src_pte = ptep_get(src_pte);
+ if (pte_none(orig_src_pte) || !pte_present(orig_src_pte) ||
+ is_zero_pfn(pte_pfn(orig_src_pte)))
+ return NULL;
+
+ folio = vm_normal_folio(src_vma, src_addr, orig_src_pte);
+ if (!folio || !folio_trylock(folio))
+ return NULL;
+ if (!PageAnonExclusive(&folio->page) || folio_test_large(folio)) {
+ folio_unlock(folio);
+ return NULL;
+ }
+ return folio;
+}
+
+static long move_present_ptes(struct mm_struct *mm,
+ struct vm_area_struct *dst_vma,
+ struct vm_area_struct *src_vma,
+ unsigned long dst_addr, unsigned long src_addr,
+ pte_t *dst_pte, pte_t *src_pte,
+ pte_t orig_dst_pte, pte_t orig_src_pte,
+ pmd_t *dst_pmd, pmd_t dst_pmdval,
+ spinlock_t *dst_ptl, spinlock_t *src_ptl,
+ struct folio *src_folio, unsigned long len)
{
int err = 0;
+ unsigned long src_start = src_addr;
+ unsigned long addr_end;
+ if (len > PAGE_SIZE) {
+ addr_end = (dst_addr + PMD_SIZE) & PMD_MASK;
+ if (dst_addr + len > addr_end)
+ len = addr_end - dst_addr;
+
+ addr_end = (src_addr + PMD_SIZE) & PMD_MASK;
+ if (src_addr + len > addr_end)
+ len = addr_end - src_addr;
+ }
+ flush_cache_range(src_vma, src_addr, src_addr + len);
double_pt_lock(dst_ptl, src_ptl);
if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst_pte, orig_src_pte,
@@ -1051,31 +1094,53 @@ static int move_present_pte(struct mm_struct *mm,
err = -EBUSY;
goto out;
}
+ arch_enter_lazy_mmu_mode();
+
+ addr_end = src_start + len;
+ while (true) {
+ orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte);
+ /* Folio got pinned from under us. Put it back and fail the move. */
+ if (folio_maybe_dma_pinned(src_folio)) {
+ set_pte_at(mm, src_addr, src_pte, orig_src_pte);
+ err = -EBUSY;
+ break;
+ }
- orig_src_pte = ptep_clear_flush(src_vma, src_addr, src_pte);
- /* Folio got pinned from under us. Put it back and fail the move. */
- if (folio_maybe_dma_pinned(src_folio)) {
- set_pte_at(mm, src_addr, src_pte, orig_src_pte);
- err = -EBUSY;
- goto out;
- }
-
- folio_move_anon_rmap(src_folio, dst_vma);
- src_folio->index = linear_page_index(dst_vma, dst_addr);
+ folio_move_anon_rmap(src_folio, dst_vma);
+ src_folio->index = linear_page_index(dst_vma, dst_addr);
- orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot);
- /* Set soft dirty bit so userspace can notice the pte was moved */
+ orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot);
+ /* Set soft dirty bit so userspace can notice the pte was moved */
#ifdef CONFIG_MEM_SOFT_DIRTY
- orig_dst_pte = pte_mksoft_dirty(orig_dst_pte);
+ orig_dst_pte = pte_mksoft_dirty(orig_dst_pte);
#endif
- if (pte_dirty(orig_src_pte))
- orig_dst_pte = pte_mkdirty(orig_dst_pte);
- orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma);
+ if (pte_dirty(orig_src_pte))
+ orig_dst_pte = pte_mkdirty(orig_dst_pte);
+ orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma);
+ set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte);
+
+ src_addr += PAGE_SIZE;
+ if (src_addr == addr_end)
+ break;
+ src_pte++;
+ dst_pte++;
+
+ folio_unlock(src_folio);
+ src_folio = check_ptes_for_batched_move(src_vma, src_addr, src_pte, dst_pte);
+ if (!src_folio)
+ break;
+ dst_addr += PAGE_SIZE;
+ }
+
+ arch_leave_lazy_mmu_mode();
+ if (src_addr > src_start)
+ flush_tlb_range(src_vma, src_start, src_addr);
- set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte);
out:
double_pt_unlock(dst_ptl, src_ptl);
- return err;
+ if (src_folio)
+ folio_unlock(src_folio);
+ return src_addr > src_start ? src_addr - src_start : err;
}
static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma,
@@ -1140,7 +1205,7 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma,
set_pte_at(mm, dst_addr, dst_pte, orig_src_pte);
double_pt_unlock(dst_ptl, src_ptl);
- return 0;
+ return PAGE_SIZE;
}
static int move_zeropage_pte(struct mm_struct *mm,
@@ -1154,6 +1219,7 @@ static int move_zeropage_pte(struct mm_struct *mm,
{
pte_t zero_pte;
+ flush_cache_range(src_vma, src_addr, src_addr + PAGE_SIZE);
double_pt_lock(dst_ptl, src_ptl);
if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst_pte, orig_src_pte,
dst_pmd, dst_pmdval)) {
@@ -1167,20 +1233,19 @@ static int move_zeropage_pte(struct mm_struct *mm,
set_pte_at(mm, dst_addr, dst_pte, zero_pte);
double_pt_unlock(dst_ptl, src_ptl);
- return 0;
+ return PAGE_SIZE;
}
/*
- * The mmap_lock for reading is held by the caller. Just move the page
- * from src_pmd to dst_pmd if possible, and return true if succeeded
- * in moving the page.
+ * The mmap_lock for reading is held by the caller. Just move the page(s)
+ * from src_pmd to dst_pmd if possible, and return number of bytes moved.
*/
-static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd,
- struct vm_area_struct *dst_vma,
- struct vm_area_struct *src_vma,
- unsigned long dst_addr, unsigned long src_addr,
- __u64 mode)
+static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd,
+ struct vm_area_struct *dst_vma,
+ struct vm_area_struct *src_vma,
+ unsigned long dst_addr, unsigned long src_addr,
+ unsigned long len, __u64 mode)
{
swp_entry_t entry;
struct swap_info_struct *si = NULL;
@@ -1196,9 +1261,8 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd,
struct mmu_notifier_range range;
int err = 0;
- flush_cache_range(src_vma, src_addr, src_addr + PAGE_SIZE);
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
- src_addr, src_addr + PAGE_SIZE);
+ src_addr, src_addr + len);
mmu_notifier_invalidate_range_start(&range);
retry:
/*
@@ -1257,7 +1321,7 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd,
if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES))
err = -ENOENT;
else /* nothing to do to move a hole */
- err = 0;
+ err = PAGE_SIZE;
goto out;
}
@@ -1375,10 +1439,13 @@ static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd,
}
}
- err = move_present_pte(mm, dst_vma, src_vma,
- dst_addr, src_addr, dst_pte, src_pte,
- orig_dst_pte, orig_src_pte, dst_pmd,
- dst_pmdval, dst_ptl, src_ptl, src_folio);
+ err = move_present_ptes(mm, dst_vma, src_vma,
+ dst_addr, src_addr, dst_pte, src_pte,
+ orig_dst_pte, orig_src_pte, dst_pmd,
+ dst_pmdval, dst_ptl, src_ptl, src_folio, len);
+ /* folio is already unlocked by move_present_ptes() */
+ folio_put(src_folio);
+ src_folio = NULL;
} else {
struct folio *folio = NULL;
@@ -1732,7 +1799,7 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start,
{
struct mm_struct *mm = ctx->mm;
struct vm_area_struct *src_vma, *dst_vma;
- unsigned long src_addr, dst_addr;
+ unsigned long src_addr, dst_addr, src_end;
pmd_t *src_pmd, *dst_pmd;
long err = -EINVAL;
ssize_t moved = 0;
@@ -1775,8 +1842,8 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start,
if (err)
goto out_unlock;
- for (src_addr = src_start, dst_addr = dst_start;
- src_addr < src_start + len;) {
+ for (src_addr = src_start, dst_addr = dst_start, src_end = src_start + len;
+ src_addr < src_end;) {
spinlock_t *ptl;
pmd_t dst_pmdval;
unsigned long step_size;
@@ -1857,10 +1924,10 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start,
break;
}
- err = move_pages_pte(mm, dst_pmd, src_pmd,
- dst_vma, src_vma,
- dst_addr, src_addr, mode);
- step_size = PAGE_SIZE;
+ err = move_pages_ptes(mm, dst_pmd, src_pmd,
+ dst_vma, src_vma, dst_addr,
+ src_addr, src_end - src_addr, mode);
+ step_size = err;
}
cond_resched();
@@ -1872,7 +1939,7 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start,
break;
}
- if (err) {
+ if (err < 0) {
if (err == -EAGAIN)
continue;
break;
base-commit: 7e161a991ea71e6ec526abc8f40c6852ebe3d946
--
2.50.1.565.gc32cd1483b-goog
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH v2] userfaultfd: opportunistic TLB-flush batching for present pages in MOVE
2025-08-05 12:14 [PATCH v2] userfaultfd: opportunistic TLB-flush batching for present pages in MOVE Lokesh Gidra
@ 2025-08-06 17:18 ` Lorenzo Stoakes
2025-08-06 17:30 ` Lokesh Gidra
0 siblings, 1 reply; 4+ messages in thread
From: Lorenzo Stoakes @ 2025-08-06 17:18 UTC (permalink / raw)
To: Lokesh Gidra
Cc: akpm, aarcange, linux-mm, linux-kernel, 21cnbao, ngeoffray,
Suren Baghdasaryan, Kalesh Singh, Barry Song, David Hildenbrand,
Peter Xu
Andrew - Could we drop this for now, please, it's splatting and has broken
mm-new.
Lokesh - could you make sure to run the mm self tests with CONFIG_DBUG_VM
set before you submit please? As this is splat is occurring immediately on
uffd-unit-tests.
On Tue, Aug 05, 2025 at 05:14:10AM -0700, Lokesh Gidra wrote:
> MOVE ioctl's runtime is dominated by TLB-flush cost, which is required
> for moving present pages. Mitigate this cost by opportunistically
> batching present contiguous pages for TLB flushing.
>
> Without batching, in our testing on an arm64 Android device with UFFD GC,
> which uses MOVE ioctl for compaction, we observed that out of the total
> time spent in move_pages_pte(), over 40% is in ptep_clear_flush(), and
> ~20% in vm_normal_folio().
>
> With batching, the proportion of vm_normal_folio() increases to over
> 70% of move_pages_pte() without any changes to vm_normal_folio().
> Furthermore, time spent within move_pages_pte() is only ~20%, which
> includes TLB-flush overhead.
>
> Cc: Suren Baghdasaryan <surenb@google.com>
> Cc: Kalesh Singh <kaleshsingh@google.com>
> Cc: Barry Song <v-songbaohua@oppo.com>
> Cc: David Hildenbrand <david@redhat.com>
> Cc: Peter Xu <peterx@redhat.com>
> Signed-off-by: Lokesh Gidra <lokeshgidra@google.com>
> ---
> Changes since v1 [1]
> - Removed flush_tlb_batched_pending(), per Barry Song
> - Unified single and multi page case, per Barry Song
Splat, decoded via scripts/decode_stacktrace.sh:
$ sudo ./uffd-unit-tests
Testing UFFDIO_API (with syscall)... done
Testing UFFDIO_API (with /dev/userfaultfd)... done
Testing register-ioctls on anon... done
Testing register-ioctls on shmem... done
Testing register-ioctls on shmem-private... done
Testing register-ioctls on hugetlb... skipped [reason: memory allocation failed]
Testing register-ioctls on hugetlb-private... skipped [reason: memory allocation failed]
Testing zeropage on anon... done
Testing zeropage on shmem... done
Testing zeropage on shmem-private... done
Testing zeropage on hugetlb... skipped [reason: memory allocation failed]
Testing zeropage on hugetlb-private... skipped [reason: memory allocation failed]
Testing move on anon... [ 12.230740] Kernel panic - not syncing: kernel: panic_on_warn set ...
[ 12.231322] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Arch Linux 1.17.0-1-1 04/01/2014
[ 12.231655] Call Trace:
[ 12.231748] <TASK>
[ 12.231830] dump_stack_lvl (lib/dump_stack.c:122)
[ 12.231963] vpanic (kernel/panic.c:448)
[ 12.232088] panic (kernel/panic.c:312 kernel/panic.c:303)
[ 12.232199] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
Appears to be:
VM_WARN_ON_ONCE(err > 0);
[ 12.232341] check_panic_on_warn.cold (kernel/panic.c:327)
[ 12.232502] __warn.cold (kernel/panic.c:839)
[ 12.232628] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
[ 12.232764] report_bug (lib/bug.c:176 lib/bug.c:215)
[ 12.232891] handle_bug (arch/x86/kernel/traps.c:338 (discriminator 1))
[ 12.233034] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
[ 12.233174] exc_invalid_op (arch/x86/kernel/traps.c:392 (discriminator 3))
[ 12.233312] asm_exc_invalid_op (./arch/x86/include/asm/idtentry.h:621)
[ 12.233460] RIP: 0010:move_pages (mm/userfaultfd.c:1964 (discriminator 2))
[ 12.233615] Code: 5e 41 5f c3 cc cc cc cc 49 89 c5 e9 e1 fe ff ff eb c4 e9 6d ff ff ff 90 0f 0b 90 45 31 ff eb cf 90 0f 0b 90 48 85 d2 7e c6 90 <0f> 0b 90 eb b9 90 0f 0b 90 f7 c1 ff 0f 00 00 0f 84 4e fe ff ff 90
All code
========
0: 5e pop %rsi
1: 41 5f pop %r15
3: c3 ret
4: cc int3
5: cc int3
6: cc int3
7: cc int3
8: 49 89 c5 mov %rax,%r13
b: e9 e1 fe ff ff jmp 0xfffffffffffffef1
10: eb c4 jmp 0xffffffffffffffd6
12: e9 6d ff ff ff jmp 0xffffffffffffff84
17: 90 nop
18: 0f 0b ud2
1a: 90 nop
1b: 45 31 ff xor %r15d,%r15d
1e: eb cf jmp 0xffffffffffffffef
20: 90 nop
21: 0f 0b ud2
23: 90 nop
24: 48 85 d2 test %rdx,%rdx
27: 7e c6 jle 0xffffffffffffffef
29: 90 nop
2a:* 0f 0b ud2 <-- trapping instruction
2c: 90 nop
2d: eb b9 jmp 0xffffffffffffffe8
2f: 90 nop
30: 0f 0b ud2
32: 90 nop
33: f7 c1 ff 0f 00 00 test $0xfff,%ecx
39: 0f 84 4e fe ff ff je 0xfffffffffffffe8d
3f: 90 nop
Code starting with the faulting instruction
===========================================
0: 0f 0b ud2
2: 90 nop
3: eb b9 jmp 0xffffffffffffffbe
5: 90 nop
6: 0f 0b ud2
8: 90 nop
9: f7 c1 ff 0f 00 00 test $0xfff,%ecx
f: 0f 84 4e fe ff ff je 0xfffffffffffffe63
15: 90 nop
[ 12.234294] RSP: 0018:ffffafeb00483d70 EFLAGS: 00010206
[ 12.234484] RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000002
[ 12.234738] RDX: 0000000000001000 RSI: ffff90afbb433078 RDI: ffff90afbc7f0080
[ 12.234997] RBP: ffff90afbc7f0000 R08: ffff90b037cae540 R09: 0000000000000001
[ 12.235255] R10: ffffffffffffffff R11: 0000000000000003 R12: ffff90af01acb980
[ 12.235508] R13: ffff90afbc7f0240 R14: ffffefc386e80b00 R15: 0000000000001000
[ 12.235764] userfaultfd_ioctl (fs/userfaultfd.c:1925 fs/userfaultfd.c:2046)
[ 12.235917] __x64_sys_ioctl (fs/ioctl.c:51 fs/ioctl.c:598 fs/ioctl.c:584 fs/ioctl.c:584)
[ 12.236065] ? ksys_read (./include/linux/file.h:63 ./include/linux/file.h:80 ./include/linux/file.h:85 fs/read_write.c:706)
[ 12.236202] do_syscall_64 (arch/x86/entry/syscall_64.c:63 (discriminator 1) arch/x86/entry/syscall_64.c:94 (discriminator 1))
[ 12.236345] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130)
[ 12.236524] RIP: 0033:0x7f457600fecd
[ 12.236658] Code: 04 25 28 00 00 00 48 89 45 c8 31 c0 48 8d 45 10 c7 45 b0 10 00 00 00 48 89 45 b8 48 8d 45 d0 48 89 45 c0 b8 10 00 00 00 0f 05 <89> c2 3d 00 f0 ff ff 77 1a 48 8b 45 c8 64 48 2b 04 25 28 00 00 00
All code
========
0: 04 25 add $0x25,%al
2: 28 00 sub %al,(%rax)
4: 00 00 add %al,(%rax)
6: 48 89 45 c8 mov %rax,-0x38(%rbp)
a: 31 c0 xor %eax,%eax
c: 48 8d 45 10 lea 0x10(%rbp),%rax
10: c7 45 b0 10 00 00 00 movl $0x10,-0x50(%rbp)
17: 48 89 45 b8 mov %rax,-0x48(%rbp)
1b: 48 8d 45 d0 lea -0x30(%rbp),%rax
1f: 48 89 45 c0 mov %rax,-0x40(%rbp)
23: b8 10 00 00 00 mov $0x10,%eax
28: 0f 05 syscall
2a:* 89 c2 mov %eax,%edx <-- trapping instruction
2c: 3d 00 f0 ff ff cmp $0xfffff000,%eax
31: 77 1a ja 0x4d
33: 48 8b 45 c8 mov -0x38(%rbp),%rax
37: 64 48 2b 04 25 28 00 sub %fs:0x28,%rax
3e: 00 00
Code starting with the faulting instruction
===========================================
0: 89 c2 mov %eax,%edx
2: 3d 00 f0 ff ff cmp $0xfffff000,%eax
7: 77 1a ja 0x23
9: 48 8b 45 c8 mov -0x38(%rbp),%rax
d: 64 48 2b 04 25 28 00 sub %fs:0x28,%rax
14: 00 00
[ 12.237333] RSP: 002b:00007f4569dfed20 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
[ 12.237599] RAX: ffffffffffffffda RBX: 0000000000001000 RCX: 00007f457600fecd
[ 12.237854] RDX: 00007f4569dfeda0 RSI: 00000000c028aa05 RDI: 0000000000000009
[ 12.238129] RBP: 00007f4569dfed70 R08: 0000000000000000 R09: 0000000000000000
[ 12.238384] R10: 0000000000000000 R11: 0000000000000246 R12: 00007fffb974ba60
[ 12.238636] R13: 00007fffb974b830 R14: 00007f4569dffcdc R15: 00007fffb974b937
[ 12.238890] </TASK>
[ 12.239094] Kernel Offset: 0x33000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
[ 12.239480] ---[ end Kernel panic - not syncing: kernel: panic_on_warn set ... ]---
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH v2] userfaultfd: opportunistic TLB-flush batching for present pages in MOVE
2025-08-06 17:18 ` Lorenzo Stoakes
@ 2025-08-06 17:30 ` Lokesh Gidra
2025-08-07 10:42 ` Lokesh Gidra
0 siblings, 1 reply; 4+ messages in thread
From: Lokesh Gidra @ 2025-08-06 17:30 UTC (permalink / raw)
To: Lorenzo Stoakes
Cc: akpm, aarcange, linux-mm, linux-kernel, 21cnbao, ngeoffray,
Suren Baghdasaryan, Kalesh Singh, Barry Song, David Hildenbrand,
Peter Xu
On Wed, Aug 6, 2025 at 10:18 AM Lorenzo Stoakes
<lorenzo.stoakes@oracle.com> wrote:
>
> Andrew - Could we drop this for now, please, it's splatting and has broken
> mm-new.
>
> Lokesh - could you make sure to run the mm self tests with CONFIG_DBUG_VM
> set before you submit please? As this is splat is occurring immediately on
> uffd-unit-tests.
Sincere apologies. Will be extra careful from now on.
>
> On Tue, Aug 05, 2025 at 05:14:10AM -0700, Lokesh Gidra wrote:
> > MOVE ioctl's runtime is dominated by TLB-flush cost, which is required
> > for moving present pages. Mitigate this cost by opportunistically
> > batching present contiguous pages for TLB flushing.
> >
> > Without batching, in our testing on an arm64 Android device with UFFD GC,
> > which uses MOVE ioctl for compaction, we observed that out of the total
> > time spent in move_pages_pte(), over 40% is in ptep_clear_flush(), and
> > ~20% in vm_normal_folio().
> >
> > With batching, the proportion of vm_normal_folio() increases to over
> > 70% of move_pages_pte() without any changes to vm_normal_folio().
> > Furthermore, time spent within move_pages_pte() is only ~20%, which
> > includes TLB-flush overhead.
> >
> > Cc: Suren Baghdasaryan <surenb@google.com>
> > Cc: Kalesh Singh <kaleshsingh@google.com>
> > Cc: Barry Song <v-songbaohua@oppo.com>
> > Cc: David Hildenbrand <david@redhat.com>
> > Cc: Peter Xu <peterx@redhat.com>
> > Signed-off-by: Lokesh Gidra <lokeshgidra@google.com>
> > ---
> > Changes since v1 [1]
> > - Removed flush_tlb_batched_pending(), per Barry Song
> > - Unified single and multi page case, per Barry Song
>
> Splat, decoded via scripts/decode_stacktrace.sh:
>
> $ sudo ./uffd-unit-tests
> Testing UFFDIO_API (with syscall)... done
> Testing UFFDIO_API (with /dev/userfaultfd)... done
> Testing register-ioctls on anon... done
> Testing register-ioctls on shmem... done
> Testing register-ioctls on shmem-private... done
> Testing register-ioctls on hugetlb... skipped [reason: memory allocation failed]
> Testing register-ioctls on hugetlb-private... skipped [reason: memory allocation failed]
> Testing zeropage on anon... done
> Testing zeropage on shmem... done
> Testing zeropage on shmem-private... done
> Testing zeropage on hugetlb... skipped [reason: memory allocation failed]
> Testing zeropage on hugetlb-private... skipped [reason: memory allocation failed]
> Testing move on anon... [ 12.230740] Kernel panic - not syncing: kernel: panic_on_warn set ...
> [ 12.231322] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Arch Linux 1.17.0-1-1 04/01/2014
> [ 12.231655] Call Trace:
> [ 12.231748] <TASK>
> [ 12.231830] dump_stack_lvl (lib/dump_stack.c:122)
> [ 12.231963] vpanic (kernel/panic.c:448)
> [ 12.232088] panic (kernel/panic.c:312 kernel/panic.c:303)
> [ 12.232199] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
>
> Appears to be:
>
> VM_WARN_ON_ONCE(err > 0);
>
> [ 12.232341] check_panic_on_warn.cold (kernel/panic.c:327)
> [ 12.232502] __warn.cold (kernel/panic.c:839)
> [ 12.232628] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
> [ 12.232764] report_bug (lib/bug.c:176 lib/bug.c:215)
> [ 12.232891] handle_bug (arch/x86/kernel/traps.c:338 (discriminator 1))
> [ 12.233034] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
> [ 12.233174] exc_invalid_op (arch/x86/kernel/traps.c:392 (discriminator 3))
> [ 12.233312] asm_exc_invalid_op (./arch/x86/include/asm/idtentry.h:621)
> [ 12.233460] RIP: 0010:move_pages (mm/userfaultfd.c:1964 (discriminator 2))
> [ 12.233615] Code: 5e 41 5f c3 cc cc cc cc 49 89 c5 e9 e1 fe ff ff eb c4 e9 6d ff ff ff 90 0f 0b 90 45 31 ff eb cf 90 0f 0b 90 48 85 d2 7e c6 90 <0f> 0b 90 eb b9 90 0f 0b 90 f7 c1 ff 0f 00 00 0f 84 4e fe ff ff 90
> All code
> ========
> 0: 5e pop %rsi
> 1: 41 5f pop %r15
> 3: c3 ret
> 4: cc int3
> 5: cc int3
> 6: cc int3
> 7: cc int3
> 8: 49 89 c5 mov %rax,%r13
> b: e9 e1 fe ff ff jmp 0xfffffffffffffef1
> 10: eb c4 jmp 0xffffffffffffffd6
> 12: e9 6d ff ff ff jmp 0xffffffffffffff84
> 17: 90 nop
> 18: 0f 0b ud2
> 1a: 90 nop
> 1b: 45 31 ff xor %r15d,%r15d
> 1e: eb cf jmp 0xffffffffffffffef
> 20: 90 nop
> 21: 0f 0b ud2
> 23: 90 nop
> 24: 48 85 d2 test %rdx,%rdx
> 27: 7e c6 jle 0xffffffffffffffef
> 29: 90 nop
> 2a:* 0f 0b ud2 <-- trapping instruction
> 2c: 90 nop
> 2d: eb b9 jmp 0xffffffffffffffe8
> 2f: 90 nop
> 30: 0f 0b ud2
> 32: 90 nop
> 33: f7 c1 ff 0f 00 00 test $0xfff,%ecx
> 39: 0f 84 4e fe ff ff je 0xfffffffffffffe8d
> 3f: 90 nop
>
> Code starting with the faulting instruction
> ===========================================
> 0: 0f 0b ud2
> 2: 90 nop
> 3: eb b9 jmp 0xffffffffffffffbe
> 5: 90 nop
> 6: 0f 0b ud2
> 8: 90 nop
> 9: f7 c1 ff 0f 00 00 test $0xfff,%ecx
> f: 0f 84 4e fe ff ff je 0xfffffffffffffe63
> 15: 90 nop
> [ 12.234294] RSP: 0018:ffffafeb00483d70 EFLAGS: 00010206
> [ 12.234484] RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000002
> [ 12.234738] RDX: 0000000000001000 RSI: ffff90afbb433078 RDI: ffff90afbc7f0080
> [ 12.234997] RBP: ffff90afbc7f0000 R08: ffff90b037cae540 R09: 0000000000000001
> [ 12.235255] R10: ffffffffffffffff R11: 0000000000000003 R12: ffff90af01acb980
> [ 12.235508] R13: ffff90afbc7f0240 R14: ffffefc386e80b00 R15: 0000000000001000
> [ 12.235764] userfaultfd_ioctl (fs/userfaultfd.c:1925 fs/userfaultfd.c:2046)
> [ 12.235917] __x64_sys_ioctl (fs/ioctl.c:51 fs/ioctl.c:598 fs/ioctl.c:584 fs/ioctl.c:584)
> [ 12.236065] ? ksys_read (./include/linux/file.h:63 ./include/linux/file.h:80 ./include/linux/file.h:85 fs/read_write.c:706)
> [ 12.236202] do_syscall_64 (arch/x86/entry/syscall_64.c:63 (discriminator 1) arch/x86/entry/syscall_64.c:94 (discriminator 1))
> [ 12.236345] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130)
> [ 12.236524] RIP: 0033:0x7f457600fecd
> [ 12.236658] Code: 04 25 28 00 00 00 48 89 45 c8 31 c0 48 8d 45 10 c7 45 b0 10 00 00 00 48 89 45 b8 48 8d 45 d0 48 89 45 c0 b8 10 00 00 00 0f 05 <89> c2 3d 00 f0 ff ff 77 1a 48 8b 45 c8 64 48 2b 04 25 28 00 00 00
> All code
> ========
> 0: 04 25 add $0x25,%al
> 2: 28 00 sub %al,(%rax)
> 4: 00 00 add %al,(%rax)
> 6: 48 89 45 c8 mov %rax,-0x38(%rbp)
> a: 31 c0 xor %eax,%eax
> c: 48 8d 45 10 lea 0x10(%rbp),%rax
> 10: c7 45 b0 10 00 00 00 movl $0x10,-0x50(%rbp)
> 17: 48 89 45 b8 mov %rax,-0x48(%rbp)
> 1b: 48 8d 45 d0 lea -0x30(%rbp),%rax
> 1f: 48 89 45 c0 mov %rax,-0x40(%rbp)
> 23: b8 10 00 00 00 mov $0x10,%eax
> 28: 0f 05 syscall
> 2a:* 89 c2 mov %eax,%edx <-- trapping instruction
> 2c: 3d 00 f0 ff ff cmp $0xfffff000,%eax
> 31: 77 1a ja 0x4d
> 33: 48 8b 45 c8 mov -0x38(%rbp),%rax
> 37: 64 48 2b 04 25 28 00 sub %fs:0x28,%rax
> 3e: 00 00
>
> Code starting with the faulting instruction
> ===========================================
> 0: 89 c2 mov %eax,%edx
> 2: 3d 00 f0 ff ff cmp $0xfffff000,%eax
> 7: 77 1a ja 0x23
> 9: 48 8b 45 c8 mov -0x38(%rbp),%rax
> d: 64 48 2b 04 25 28 00 sub %fs:0x28,%rax
> 14: 00 00
> [ 12.237333] RSP: 002b:00007f4569dfed20 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
> [ 12.237599] RAX: ffffffffffffffda RBX: 0000000000001000 RCX: 00007f457600fecd
> [ 12.237854] RDX: 00007f4569dfeda0 RSI: 00000000c028aa05 RDI: 0000000000000009
> [ 12.238129] RBP: 00007f4569dfed70 R08: 0000000000000000 R09: 0000000000000000
> [ 12.238384] R10: 0000000000000000 R11: 0000000000000246 R12: 00007fffb974ba60
> [ 12.238636] R13: 00007fffb974b830 R14: 00007f4569dffcdc R15: 00007fffb974b937
> [ 12.238890] </TASK>
> [ 12.239094] Kernel Offset: 0x33000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
> [ 12.239480] ---[ end Kernel panic - not syncing: kernel: panic_on_warn set ... ]---
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH v2] userfaultfd: opportunistic TLB-flush batching for present pages in MOVE
2025-08-06 17:30 ` Lokesh Gidra
@ 2025-08-07 10:42 ` Lokesh Gidra
0 siblings, 0 replies; 4+ messages in thread
From: Lokesh Gidra @ 2025-08-07 10:42 UTC (permalink / raw)
To: akpm, Barry Song
Cc: aarcange, linux-mm, linux-kernel, Lorenzo Stoakes, 21cnbao,
ngeoffray, Suren Baghdasaryan, Kalesh Singh, David Hildenbrand,
Peter Xu
Posted v3 at https://lore.kernel.org/all/20250807103902.2242717-1-lokeshgidra@google.com/
Thanks!
On Wed, Aug 6, 2025 at 10:30 AM Lokesh Gidra <lokeshgidra@google.com> wrote:
>
> On Wed, Aug 6, 2025 at 10:18 AM Lorenzo Stoakes
> <lorenzo.stoakes@oracle.com> wrote:
> >
> > Andrew - Could we drop this for now, please, it's splatting and has broken
> > mm-new.
> >
> > Lokesh - could you make sure to run the mm self tests with CONFIG_DBUG_VM
> > set before you submit please? As this is splat is occurring immediately on
> > uffd-unit-tests.
>
> Sincere apologies. Will be extra careful from now on.
> >
> > On Tue, Aug 05, 2025 at 05:14:10AM -0700, Lokesh Gidra wrote:
> > > MOVE ioctl's runtime is dominated by TLB-flush cost, which is required
> > > for moving present pages. Mitigate this cost by opportunistically
> > > batching present contiguous pages for TLB flushing.
> > >
> > > Without batching, in our testing on an arm64 Android device with UFFD GC,
> > > which uses MOVE ioctl for compaction, we observed that out of the total
> > > time spent in move_pages_pte(), over 40% is in ptep_clear_flush(), and
> > > ~20% in vm_normal_folio().
> > >
> > > With batching, the proportion of vm_normal_folio() increases to over
> > > 70% of move_pages_pte() without any changes to vm_normal_folio().
> > > Furthermore, time spent within move_pages_pte() is only ~20%, which
> > > includes TLB-flush overhead.
> > >
> > > Cc: Suren Baghdasaryan <surenb@google.com>
> > > Cc: Kalesh Singh <kaleshsingh@google.com>
> > > Cc: Barry Song <v-songbaohua@oppo.com>
> > > Cc: David Hildenbrand <david@redhat.com>
> > > Cc: Peter Xu <peterx@redhat.com>
> > > Signed-off-by: Lokesh Gidra <lokeshgidra@google.com>
> > > ---
> > > Changes since v1 [1]
> > > - Removed flush_tlb_batched_pending(), per Barry Song
> > > - Unified single and multi page case, per Barry Song
> >
> > Splat, decoded via scripts/decode_stacktrace.sh:
> >
> > $ sudo ./uffd-unit-tests
> > Testing UFFDIO_API (with syscall)... done
> > Testing UFFDIO_API (with /dev/userfaultfd)... done
> > Testing register-ioctls on anon... done
> > Testing register-ioctls on shmem... done
> > Testing register-ioctls on shmem-private... done
> > Testing register-ioctls on hugetlb... skipped [reason: memory allocation failed]
> > Testing register-ioctls on hugetlb-private... skipped [reason: memory allocation failed]
> > Testing zeropage on anon... done
> > Testing zeropage on shmem... done
> > Testing zeropage on shmem-private... done
> > Testing zeropage on hugetlb... skipped [reason: memory allocation failed]
> > Testing zeropage on hugetlb-private... skipped [reason: memory allocation failed]
> > Testing move on anon... [ 12.230740] Kernel panic - not syncing: kernel: panic_on_warn set ...
> > [ 12.231322] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Arch Linux 1.17.0-1-1 04/01/2014
> > [ 12.231655] Call Trace:
> > [ 12.231748] <TASK>
> > [ 12.231830] dump_stack_lvl (lib/dump_stack.c:122)
> > [ 12.231963] vpanic (kernel/panic.c:448)
> > [ 12.232088] panic (kernel/panic.c:312 kernel/panic.c:303)
> > [ 12.232199] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
> >
> > Appears to be:
> >
> > VM_WARN_ON_ONCE(err > 0);
> >
> > [ 12.232341] check_panic_on_warn.cold (kernel/panic.c:327)
> > [ 12.232502] __warn.cold (kernel/panic.c:839)
> > [ 12.232628] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
> > [ 12.232764] report_bug (lib/bug.c:176 lib/bug.c:215)
> > [ 12.232891] handle_bug (arch/x86/kernel/traps.c:338 (discriminator 1))
> > [ 12.233034] ? move_pages (mm/userfaultfd.c:1964 (discriminator 2))
> > [ 12.233174] exc_invalid_op (arch/x86/kernel/traps.c:392 (discriminator 3))
> > [ 12.233312] asm_exc_invalid_op (./arch/x86/include/asm/idtentry.h:621)
> > [ 12.233460] RIP: 0010:move_pages (mm/userfaultfd.c:1964 (discriminator 2))
> > [ 12.233615] Code: 5e 41 5f c3 cc cc cc cc 49 89 c5 e9 e1 fe ff ff eb c4 e9 6d ff ff ff 90 0f 0b 90 45 31 ff eb cf 90 0f 0b 90 48 85 d2 7e c6 90 <0f> 0b 90 eb b9 90 0f 0b 90 f7 c1 ff 0f 00 00 0f 84 4e fe ff ff 90
> > All code
> > ========
> > 0: 5e pop %rsi
> > 1: 41 5f pop %r15
> > 3: c3 ret
> > 4: cc int3
> > 5: cc int3
> > 6: cc int3
> > 7: cc int3
> > 8: 49 89 c5 mov %rax,%r13
> > b: e9 e1 fe ff ff jmp 0xfffffffffffffef1
> > 10: eb c4 jmp 0xffffffffffffffd6
> > 12: e9 6d ff ff ff jmp 0xffffffffffffff84
> > 17: 90 nop
> > 18: 0f 0b ud2
> > 1a: 90 nop
> > 1b: 45 31 ff xor %r15d,%r15d
> > 1e: eb cf jmp 0xffffffffffffffef
> > 20: 90 nop
> > 21: 0f 0b ud2
> > 23: 90 nop
> > 24: 48 85 d2 test %rdx,%rdx
> > 27: 7e c6 jle 0xffffffffffffffef
> > 29: 90 nop
> > 2a:* 0f 0b ud2 <-- trapping instruction
> > 2c: 90 nop
> > 2d: eb b9 jmp 0xffffffffffffffe8
> > 2f: 90 nop
> > 30: 0f 0b ud2
> > 32: 90 nop
> > 33: f7 c1 ff 0f 00 00 test $0xfff,%ecx
> > 39: 0f 84 4e fe ff ff je 0xfffffffffffffe8d
> > 3f: 90 nop
> >
> > Code starting with the faulting instruction
> > ===========================================
> > 0: 0f 0b ud2
> > 2: 90 nop
> > 3: eb b9 jmp 0xffffffffffffffbe
> > 5: 90 nop
> > 6: 0f 0b ud2
> > 8: 90 nop
> > 9: f7 c1 ff 0f 00 00 test $0xfff,%ecx
> > f: 0f 84 4e fe ff ff je 0xfffffffffffffe63
> > 15: 90 nop
> > [ 12.234294] RSP: 0018:ffffafeb00483d70 EFLAGS: 00010206
> > [ 12.234484] RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000002
> > [ 12.234738] RDX: 0000000000001000 RSI: ffff90afbb433078 RDI: ffff90afbc7f0080
> > [ 12.234997] RBP: ffff90afbc7f0000 R08: ffff90b037cae540 R09: 0000000000000001
> > [ 12.235255] R10: ffffffffffffffff R11: 0000000000000003 R12: ffff90af01acb980
> > [ 12.235508] R13: ffff90afbc7f0240 R14: ffffefc386e80b00 R15: 0000000000001000
> > [ 12.235764] userfaultfd_ioctl (fs/userfaultfd.c:1925 fs/userfaultfd.c:2046)
> > [ 12.235917] __x64_sys_ioctl (fs/ioctl.c:51 fs/ioctl.c:598 fs/ioctl.c:584 fs/ioctl.c:584)
> > [ 12.236065] ? ksys_read (./include/linux/file.h:63 ./include/linux/file.h:80 ./include/linux/file.h:85 fs/read_write.c:706)
> > [ 12.236202] do_syscall_64 (arch/x86/entry/syscall_64.c:63 (discriminator 1) arch/x86/entry/syscall_64.c:94 (discriminator 1))
> > [ 12.236345] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130)
> > [ 12.236524] RIP: 0033:0x7f457600fecd
> > [ 12.236658] Code: 04 25 28 00 00 00 48 89 45 c8 31 c0 48 8d 45 10 c7 45 b0 10 00 00 00 48 89 45 b8 48 8d 45 d0 48 89 45 c0 b8 10 00 00 00 0f 05 <89> c2 3d 00 f0 ff ff 77 1a 48 8b 45 c8 64 48 2b 04 25 28 00 00 00
> > All code
> > ========
> > 0: 04 25 add $0x25,%al
> > 2: 28 00 sub %al,(%rax)
> > 4: 00 00 add %al,(%rax)
> > 6: 48 89 45 c8 mov %rax,-0x38(%rbp)
> > a: 31 c0 xor %eax,%eax
> > c: 48 8d 45 10 lea 0x10(%rbp),%rax
> > 10: c7 45 b0 10 00 00 00 movl $0x10,-0x50(%rbp)
> > 17: 48 89 45 b8 mov %rax,-0x48(%rbp)
> > 1b: 48 8d 45 d0 lea -0x30(%rbp),%rax
> > 1f: 48 89 45 c0 mov %rax,-0x40(%rbp)
> > 23: b8 10 00 00 00 mov $0x10,%eax
> > 28: 0f 05 syscall
> > 2a:* 89 c2 mov %eax,%edx <-- trapping instruction
> > 2c: 3d 00 f0 ff ff cmp $0xfffff000,%eax
> > 31: 77 1a ja 0x4d
> > 33: 48 8b 45 c8 mov -0x38(%rbp),%rax
> > 37: 64 48 2b 04 25 28 00 sub %fs:0x28,%rax
> > 3e: 00 00
> >
> > Code starting with the faulting instruction
> > ===========================================
> > 0: 89 c2 mov %eax,%edx
> > 2: 3d 00 f0 ff ff cmp $0xfffff000,%eax
> > 7: 77 1a ja 0x23
> > 9: 48 8b 45 c8 mov -0x38(%rbp),%rax
> > d: 64 48 2b 04 25 28 00 sub %fs:0x28,%rax
> > 14: 00 00
> > [ 12.237333] RSP: 002b:00007f4569dfed20 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
> > [ 12.237599] RAX: ffffffffffffffda RBX: 0000000000001000 RCX: 00007f457600fecd
> > [ 12.237854] RDX: 00007f4569dfeda0 RSI: 00000000c028aa05 RDI: 0000000000000009
> > [ 12.238129] RBP: 00007f4569dfed70 R08: 0000000000000000 R09: 0000000000000000
> > [ 12.238384] R10: 0000000000000000 R11: 0000000000000246 R12: 00007fffb974ba60
> > [ 12.238636] R13: 00007fffb974b830 R14: 00007f4569dffcdc R15: 00007fffb974b937
> > [ 12.238890] </TASK>
> > [ 12.239094] Kernel Offset: 0x33000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
> > [ 12.239480] ---[ end Kernel panic - not syncing: kernel: panic_on_warn set ... ]---
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2025-08-07 10:42 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2025-08-05 12:14 [PATCH v2] userfaultfd: opportunistic TLB-flush batching for present pages in MOVE Lokesh Gidra
2025-08-06 17:18 ` Lorenzo Stoakes
2025-08-06 17:30 ` Lokesh Gidra
2025-08-07 10:42 ` Lokesh Gidra
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).