Re: [PATCH v2 0/2] mm/mprotect: micro-optimization work

["David Hildenbrand (Arm)" <david@kernel.org>]

On 3/30/26 22:06, Andrew Morton wrote:
> On Mon, 30 Mar 2026 15:55:51 -0400 Luke Yang <luyang@redhat.com> wrote:
> 
>> Thanks for working on this. I just wanted to share that we've created a
>> test kernel with your patches and tested on the following CPUs:
>>
>> --- aarch64 ---
>> Ampere Altra
>> Ampere Altra Max
>>
>> --- x86_64 ---
>> AMD EPYC 7713
>> AMD EPYC 7351
>> AMD EPYC 7542
>> AMD EPYC 7573X
>> AMD EPYC 7702
>> AMD EPYC 9754
>> Intel Xeon Gold 6126
>> Into Xeon Gold 6330
>> Intel Xeon Gold 6530
>> Intel Xeon Platinum 8351N
>> Intel Core i7-6820HQ
>>
>> --- ppc64le ---
>> IBM Power 10
>>
>> On average, we see improvements ranging from a minimum of 5% to a
>> maximum of 55%, with most improvements showing around a 25% speed up in
>> the libmicro/mprot_tw4m micro benchmark.
> 
> Thanks, that's nice.  I've added some of the above into the changelog
> and I took the liberty of adding your Tested-by: to both patches.
> 
> fyi, regarding [2/2]: it's unclear to me whether the discussion with
> David will result in any alterations.  If there's something I need to
> it always helps to lmk ;)

I think we want to get a better understanding of which exact __always_inline
is really helpful in patch #2, and where to apply the nr_ptes==1 forced
optimization.

I updated my microbenchmark I use for fork+unmap etc to measure
mprotect as well

	https://gitlab.com/davidhildenbrand/scratchspace/-/raw/main/pte-mapped-folio-benchmarks.c?ref_type=heads

Running some simple tests with order-0 on 1 GiB of memory:


Upstream Linus:

./pte-mapped-folio-benchmarks 0 write-protect 5
0.005779
...
./pte-mapped-folio-benchmarks 0 write-unprotect 5
0.009113
...


With Pedro's patch #2:
$ ./pte-mapped-folio-benchmarks 0 write-protect 5
0.003941
...
$ ./pte-mapped-folio-benchmarks 0 write-unprotect 5
0.006163
...


With the patch below:

$ ./pte-mapped-folio-benchmarks 0 write-protect 5
0.003364

$ ./pte-mapped-folio-benchmarks 0 write-unprotect 5
0.005729


So patch #2 might be improved. And the forced inlining of
mprotect_folio_pte_batch() should likely not go into the same patch.


---

From cf1a2a4a6ef95ed541947f2fd9d8351bef664426 Mon Sep 17 00:00:00 2001
From: "David Hildenbrand (Arm)" <david@kernel.org>
Date: Wed, 1 Apr 2026 08:15:44 +0000
Subject: [PATCH] tmp

Signed-off-by: David Hildenbrand (Arm) <david@kernel.org>
---
 mm/mprotect.c | 79 +++++++++++++++++++++++++++++++--------------------
 1 file changed, 48 insertions(+), 31 deletions(-)

diff --git a/mm/mprotect.c b/mm/mprotect.c
index c0571445bef7..8d14c05a11a2 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -117,7 +117,7 @@ static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep,
 }
 
 /* Set nr_ptes number of ptes, starting from idx */
-static void prot_commit_flush_ptes(struct vm_area_struct *vma, unsigned long addr,
+static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, unsigned long addr,
                pte_t *ptep, pte_t oldpte, pte_t ptent, int nr_ptes,
                int idx, bool set_write, struct mmu_gather *tlb)
 {
@@ -143,7 +143,7 @@ static void prot_commit_flush_ptes(struct vm_area_struct *vma, unsigned long add
  * !PageAnonExclusive() pages, starting from start_idx. Caller must enforce
  * that the ptes point to consecutive pages of the same anon large folio.
  */
-static int page_anon_exclusive_sub_batch(int start_idx, int max_len,
+static __always_inline int page_anon_exclusive_sub_batch(int start_idx, int max_len,
                struct page *first_page, bool expected_anon_exclusive)
 {
        int idx;
@@ -169,7 +169,7 @@ static int page_anon_exclusive_sub_batch(int start_idx, int max_len,
  * pte of the batch. Therefore, we must individually check all pages and
  * retrieve sub-batches.
  */
-static void commit_anon_folio_batch(struct vm_area_struct *vma,
+static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma,
                struct folio *folio, struct page *first_page, unsigned long addr, pte_t *ptep,
                pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
 {
@@ -188,7 +188,7 @@ static void commit_anon_folio_batch(struct vm_area_struct *vma,
        }
 }
 
-static void set_write_prot_commit_flush_ptes(struct vm_area_struct *vma,
+static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_struct *vma,
                struct folio *folio, struct page *page, unsigned long addr, pte_t *ptep,
                pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
 {
@@ -211,6 +211,41 @@ static void set_write_prot_commit_flush_ptes(struct vm_area_struct *vma,
        commit_anon_folio_batch(vma, folio, page, addr, ptep, oldpte, ptent, nr_ptes, tlb);
 }
 
+static __always_inline void change_present_ptes(struct mmu_gather *tlb,
+               struct vm_area_struct *vma, unsigned long addr,
+               pgprot_t newprot, unsigned long cp_flags,
+               struct folio *folio, struct page *page, pte_t *pte,
+               unsigned int nr_ptes)
+{
+       bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
+       bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+       pte_t oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
+       pte_t ptent = pte_modify(oldpte, newprot);
+
+       if (uffd_wp)
+               ptent = pte_mkuffd_wp(ptent);
+       else if (uffd_wp_resolve)
+               ptent = pte_clear_uffd_wp(ptent);
+
+       /*
+        * In some writable, shared mappings, we might want to catch actual
+        * write access -- see vma_wants_writenotify().
+        *
+        * In all writable, private mappings, we have to properly handle COW.
+        *
+        * In both cases, we can sometimes still change PTEs writable and avoid
+        * the write-fault handler, for example, if a PTE is already dirty and
+        * no other COW or special handling is required.
+        */
+       if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
+            !pte_write(ptent))
+               set_write_prot_commit_flush_ptes(vma, folio, page, addr, pte,
+                                                oldpte, ptent, nr_ptes, tlb);
+       else
+               prot_commit_flush_ptes(vma, addr, pte, oldpte, ptent,
+                       nr_ptes, /* idx = */ 0, /* set_write = */ false, tlb);
+}
+
 static long change_pte_range(struct mmu_gather *tlb,
                struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
                unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -242,7 +277,6 @@ static long change_pte_range(struct mmu_gather *tlb,
                        int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
                        struct folio *folio = NULL;
                        struct page *page;
-                       pte_t ptent;
 
                        /* Already in the desired state. */
                        if (prot_numa && pte_protnone(oldpte))
@@ -268,34 +302,17 @@ static long change_pte_range(struct mmu_gather *tlb,
 
                        nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags);
 
-                       oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
-                       ptent = pte_modify(oldpte, newprot);
-
-                       if (uffd_wp)
-                               ptent = pte_mkuffd_wp(ptent);
-                       else if (uffd_wp_resolve)
-                               ptent = pte_clear_uffd_wp(ptent);
-
                        /*
-                        * In some writable, shared mappings, we might want
-                        * to catch actual write access -- see
-                        * vma_wants_writenotify().
-                        *
-                        * In all writable, private mappings, we have to
-                        * properly handle COW.
-                        *
-                        * In both cases, we can sometimes still change PTEs
-                        * writable and avoid the write-fault handler, for
-                        * example, if a PTE is already dirty and no other
-                        * COW or special handling is required.
+                        * Optimize for order-0 folios by optimizing out all
+                        * loops.
                         */
-                       if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-                            !pte_write(ptent))
-                               set_write_prot_commit_flush_ptes(vma, folio, page,
-                               addr, pte, oldpte, ptent, nr_ptes, tlb);
-                       else
-                               prot_commit_flush_ptes(vma, addr, pte, oldpte, ptent,
-                                       nr_ptes, /* idx = */ 0, /* set_write = */ false, tlb);
+                       if (nr_ptes == 1) {
+                               change_present_ptes(tlb, vma, addr, newprot,
+                                                   cp_flags, folio, page, pte, 1);
+                       } else {
+                               change_present_ptes(tlb, vma, addr, newprot,
+                                                   cp_flags, folio, page, pte, nr_ptes);
+                       }
                        pages += nr_ptes;
                } else if (pte_none(oldpte)) {
                        /*
-- 
2.53.0


-- 
Cheers,

David