[PATCH] ARM: mm: LPAE: Change THP helpers to comply with generic MM semantics

[PATCH] ARM: mm: LPAE: Change THP helpers to comply with generic MM semantics

[Karl Mehltretter]

pmd_mkinvalid() clears L_PMD_SECT_VALID from a huge PMD. A PROT_NONE
huge mapping also lacks that bit. On LPAE, pmd_present() therefore
reports both entries as non-present even though they still describe a
huge page.

Generic THP teardown then takes the non-present path and decodes the PMD
as a softleaf entry. zap_huge_pmd() cannot find the folio and leaves the
huge page and its deposited page table behind. An unprivileged process
can leak 2 MiB on each mprotect(PROT_NONE) and munmap cycle.

Reserve an LPAE software bit for an invalidated-but-still-present
section, and make pmd_present() recognize it and L_PMD_SECT_NONE. An
invalidated PMD no longer contains PMD_TYPE_SECT, so make pmd_leaf() use
the resulting present state rather than the bare type bits.

Bit 56 is the only unused bit in LPAE's four-bit software field: the PMD
definitions already use bits 55, 57, and 58 for dirty, PROT_NONE, and
read-only state.

Since pmd_present() no longer indicates hardware accessibility, provide
pmd_access_permitted(), which checks the valid and user bits and the
read-only bit for writes. This keeps GUP-fast from accessing PROT_NONE
and invalidated huge PMDs. Do not retain the transition-only bit in
pmd_modify().

This mirrors arm64 commit b65399f6111b ("arm64/mm: Change THP helpers
to comply with generic MM semantics").

Fixes: 624531886987 ("ARM: 8578/1: mm: ensure pmd_present only checks the valid bit")
Cc: <stable@vger.kernel.org>
Assisted-by: LLM
Signed-off-by: Karl Mehltretter <kmehltretter@gmail.com>
---
Testing: In an unpatched 1 GiB QEMU guest, 400 cycles reduced MemFree
from 981 MiB to 160 MiB. AnonHugePages remained at 819200 kB after exit,
and drop_caches recovered nothing. MADV_HUGEPAGE was unnecessary with
THP set to always.

The patch passed the 64-cycle reproducer and both CONFIG_GUP_TEST
fast-GUP checks on QEMU and Pi 400. It also passed DEBUG_VM_PGTABLE and
a four-vCPU CONFIG_PREEMPT=y test with 20,000 protection changes.

Full zImage builds passed for LPAE with and without THP and for non-LPAE.
LPAE+THP also built at v7.2-12555-g4352b8aee980.

 arch/arm/include/asm/pgtable-3level.h | 30 ++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
index 7b71a3d41..b2ca7408b 100644
--- a/arch/arm/include/asm/pgtable-3level.h
+++ b/arch/arm/include/asm/pgtable-3level.h
@@ -81,6 +81,7 @@
 
 #define L_PMD_SECT_VALID	(_AT(pmdval_t, 1) << 0)
 #define L_PMD_SECT_DIRTY	(_AT(pmdval_t, 1) << 55)
+#define L_PMD_SECT_PRESENT_INVALID	(_AT(pmdval_t, 1) << 56)
 #define L_PMD_SECT_NONE		(_AT(pmdval_t, 1) << 57)
 #define L_PMD_SECT_RDONLY	(_AT(pteval_t, 1) << 58)
 
@@ -116,9 +117,7 @@
 #define pud_present(pud)	(pud_val(pud))
 #define pmd_table(pmd)		((pmd_val(pmd) & PMD_TYPE_MASK) == \
 						 PMD_TYPE_TABLE)
-#define pmd_sect(pmd)		((pmd_val(pmd) & PMD_TYPE_MASK) == \
-						 PMD_TYPE_SECT)
-#define pmd_leaf(pmd)		pmd_sect(pmd)
+#define pmd_leaf(pmd)		(pmd_present(pmd) && !pmd_table(pmd))
 
 #define pud_clear(pudp)			\
 	do {				\
@@ -177,7 +176,9 @@ static inline pmd_t *pud_pgtable(pud_t pud)
 						: !!(pmd_val(pmd) & (val)))
 #define pmd_isclear(pmd, val)	(!(pmd_val(pmd) & (val)))
 
-#define pmd_present(pmd)	(pmd_isset((pmd), L_PMD_SECT_VALID))
+#define pmd_present(pmd)	(pmd_isset((pmd), L_PMD_SECT_VALID |	\
+					   L_PMD_SECT_NONE |		\
+					   L_PMD_SECT_PRESENT_INVALID))
 #define pmd_young(pmd)		(pmd_isset((pmd), PMD_SECT_AF))
 #define pte_special(pte)	(pte_isset((pte), L_PTE_SPECIAL))
 static inline pte_t pte_mkspecial(pte_t pte)
@@ -189,6 +190,19 @@ static inline pte_t pte_mkspecial(pte_t pte)
 #define pmd_write(pmd)		(pmd_isclear((pmd), L_PMD_SECT_RDONLY))
 #define pmd_dirty(pmd)		(pmd_isset((pmd), L_PMD_SECT_DIRTY))
 
+static inline bool pmd_access_permitted(pmd_t pmd, bool write)
+{
+	pmdval_t mask = L_PMD_SECT_VALID | PMD_SECT_USER;
+	pmdval_t needed = mask;
+
+	if (write)
+		mask |= L_PMD_SECT_RDONLY;
+
+	return (pmd_val(pmd) & mask) == needed;
+}
+
+#define pmd_access_permitted pmd_access_permitted
+
 #define pmd_hugewillfault(pmd)	(!pmd_young(pmd) || !pmd_write(pmd))
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -213,16 +227,18 @@ PMD_BIT_FUNC(mkyoung,   |= PMD_SECT_AF);
 /* No hardware dirty/accessed bits -- generic_pmdp_establish() fits */
 #define pmdp_establish generic_pmdp_establish
 
-/* represent a notpresent pmd by faulting entry, this is used by pmdp_invalidate */
+/* Create a faulting but software-present PMD for pmdp_invalidate(). */
 static inline pmd_t pmd_mkinvalid(pmd_t pmd)
 {
-	return __pmd(pmd_val(pmd) & ~L_PMD_SECT_VALID);
+	return __pmd((pmd_val(pmd) & ~L_PMD_SECT_VALID) |
+		     L_PMD_SECT_PRESENT_INVALID);
 }
 
 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
 {
 	const pmdval_t mask = PMD_SECT_USER | PMD_SECT_XN | L_PMD_SECT_RDONLY |
-				L_PMD_SECT_VALID | L_PMD_SECT_NONE;
+				L_PMD_SECT_VALID | L_PMD_SECT_NONE |
+				L_PMD_SECT_PRESENT_INVALID;
 	pmd_val(pmd) = (pmd_val(pmd) & ~mask) | (pgprot_val(newprot) & mask);
 	return pmd;
 }
-- 
2.39.5 (Apple Git-154)