[PATCH 1/2] sparc64: flush only the aliased page, not its whole folio

[PATCH 1/2] sparc64: flush only the aliased page, not its whole folio

[Imre Kaloz]

Since the conversion to folios, move_pte() and tlb_batch_add() flush the
D-cache of every page in the folio a single PTE maps.  Both run once per
PTE, so moving or unmapping a range backed by a large folio flushes each
of its pages once for every PTE of that folio, and each page flush is a
cross-call to all other online CPUs.

Only the page the PTE maps can hold lines at the old colour, so flush
just that page, as before the conversion.  flush_dcache_folio_all() has
no other users and becomes flush_dcache_page_all() again.

On SMP QEMU guests, apt rebuilding its caches through mremap() triggers
RCU stalls and soft lockups in move_pte().

Fixes: 1a10a44dfc1d ("sparc64: implement the new page table range API")
Cc: stable@vger.kernel.org
Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---
 arch/sparc/include/asm/cacheflush_64.h |  3 +--
 arch/sparc/include/asm/pgtable_64.h    |  7 +++++--
 arch/sparc/kernel/smp_64.c             | 24 +++++++++++++-----------
 arch/sparc/mm/init_64.c                | 22 ++++++++++++++++++++++
 arch/sparc/mm/tlb.c                    |  2 +-
 5 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h
index 06092572c045..02c969417e7b 100644
--- a/arch/sparc/include/asm/cacheflush_64.h
+++ b/arch/sparc/include/asm/cacheflush_64.h
@@ -38,11 +38,10 @@ void __flush_dcache_page(void *addr, int flush_icache);
 void flush_dcache_folio_impl(struct folio *folio);
 #ifdef CONFIG_SMP
 void smp_flush_dcache_folio_impl(struct folio *folio, int cpu);
-void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio);
 #else
 #define smp_flush_dcache_folio_impl(folio, cpu) flush_dcache_folio_impl(folio)
-#define flush_dcache_folio_all(mm, folio) flush_dcache_folio_impl(folio)
 #endif
+void flush_dcache_page_all(struct mm_struct *mm, struct page *page);
 
 void __flush_dcache_range(unsigned long start, unsigned long end);
 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 0837ebbc5dce..6712eac4baf1 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -946,6 +946,9 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
 	set_pte_at((mm), (addr), (ptep), __pte(0UL))
 
 #ifdef DCACHE_ALIASING_POSSIBLE
+/* Without pte_batch_hint(), move_ptes() calls this once per PTE, so
+ * only the page this PTE maps can have lines at the old colour.
+ */
 #define __HAVE_ARCH_MOVE_PTE
 #define move_pte(pte, old_addr, new_addr)				\
 ({									\
@@ -955,8 +958,8 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
 									\
 		if (pfn_valid(this_pfn) &&				\
 		    (((old_addr) ^ (new_addr)) & (1 << 13)))		\
-			flush_dcache_folio_all(current->mm,		\
-				page_folio(pfn_to_page(this_pfn)));	\
+			flush_dcache_page_all(current->mm,		\
+					      pfn_to_page(this_pfn));	\
 	}								\
 	newpte;								\
 })
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index 371460e34484..18b6145da591 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -982,8 +982,9 @@ void smp_flush_dcache_folio_impl(struct folio *folio, int cpu)
 	put_cpu();
 }
 
-void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio)
+void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
 {
+	struct folio *folio = page_folio(page);
 	void *pg_addr;
 	u64 data0;
 
@@ -996,7 +997,7 @@ void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio)
 	atomic_inc(&dcpage_flushes);
 #endif
 	data0 = 0;
-	pg_addr = folio_address(folio);
+	pg_addr = page_address(page);
 	if (tlb_type == spitfire) {
 		data0 = ((u64)&xcall_flush_dcache_page_spitfire);
 		if (folio_flush_mapping(folio) != NULL)
@@ -1007,18 +1008,19 @@ void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio)
 #endif
 	}
 	if (data0) {
-		unsigned int i, nr = folio_nr_pages(folio);
-
-		for (i = 0; i < nr; i++) {
-			xcall_deliver(data0, __pa(pg_addr),
-				      (u64) pg_addr, cpu_online_mask);
+		xcall_deliver(data0, __pa(pg_addr),
+			      (u64)pg_addr, cpu_online_mask);
 #ifdef CONFIG_DEBUG_DCFLUSH
-			atomic_inc(&dcpage_flushes_xcall);
+		atomic_inc(&dcpage_flushes_xcall);
 #endif
-			pg_addr += PAGE_SIZE;
-		}
 	}
-	__local_flush_dcache_folio(folio);
+#ifdef DCACHE_ALIASING_POSSIBLE
+	__flush_dcache_page(pg_addr,
+			    tlb_type == spitfire && folio_flush_mapping(folio));
+#else
+	if (tlb_type == spitfire && folio_flush_mapping(folio))
+		__flush_icache_page(__pa(pg_addr));
+#endif
 
 	preempt_enable();
 }
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 103db4683b16..8792e5d92517 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -214,6 +214,28 @@ inline void flush_dcache_folio_impl(struct folio *folio)
 #endif
 }
 
+#ifndef CONFIG_SMP
+void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
+{
+	struct folio *folio = page_folio(page);
+
+	if (tlb_type == hypervisor)
+		return;
+
+#ifdef CONFIG_DEBUG_DCFLUSH
+	atomic_inc(&dcpage_flushes);
+#endif
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+	__flush_dcache_page(page_address(page),
+			    tlb_type == spitfire && folio_flush_mapping(folio));
+#else
+	if (tlb_type == spitfire && folio_flush_mapping(folio))
+		__flush_icache_page(page_to_phys(page));
+#endif
+}
+#endif
+
 #define PG_dcache_dirty		PG_arch_1
 #define PG_dcache_cpu_shift	32UL
 #define PG_dcache_cpu_mask	\
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 6d9dd5eb1328..1221814ca0e1 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -144,7 +144,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
 
 		paddr = (unsigned long) page_address(page);
 		if ((paddr ^ vaddr) & (1 << 13))
-			flush_dcache_folio_all(mm, folio);
+			flush_dcache_page_all(mm, page);
 	}
 
 no_cache_flush:
-- 
2.47.3

Re: [PATCH 1/2] sparc64: flush only the aliased page, not its whole folio

[Stian Halseth]

[-- Attachment #1: Type: text/plain, Size: 1198 bytes --]

Hi Imre,

On Sun, 2026-09-27 at 12:55 +0200, Imre Kaloz wrote:
> Since the conversion to folios, move_pte() and tlb_batch_add() flush
> the
> D-cache of every page in the folio a single PTE maps.  Both run once
> per
> PTE, so moving or unmapping a range backed by a large folio flushes
> each
> of its pages once for every PTE of that folio, and each page flush is
> a
> cross-call to all other online CPUs.
> 
> Only the page the PTE maps can hold lines at the old colour, so flush
> just that page, as before the conversion.  flush_dcache_folio_all()
> has
> no other users and becomes flush_dcache_page_all() again.
> 
> On SMP QEMU guests, apt rebuilding its caches through mremap()
> triggers
> RCU stalls and soft lockups in move_pte().

Confirmed on a Sun Fire V240 (UltraSPARC IIIi, SMP, THP madvise). 
The unpatched kernel exhibits the quadratic slowdown patch 1 fixes: a
same-colour 2-page move takes 0.4 ms, but a colour-flipping 1-page move
takes 7293 ms (roughly a million redundant page flushes across 1024
PTEs). Thanks for the fix!

No regressions found on sun4v (SPARC T4-1).

Tested-by: Stian Halseth <stian@itx.no> # Sun Fire V240, SPARC T4-1



[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

[PATCH 2/2] sparc64: flush vmalloc ranges from the D-cache on map and unmap

[Imre Kaloz]

The L1 D-cache of Spitfire and Cheetah is indexed with VA bit 13, but
flush_cache_vmap() and flush_cache_vunmap() are no-ops. Lines loaded
through a vmalloc address survive the unmap, and once the page is
written through the other colour, e.g. zeroed through the linear
mapping, a later mapping of the old colour reads stale data. Anonymous
user pages arrive the same way, as tlb_batch_add() does not flush them.
On UltraSPARC III this corrupts BPF programs and module data.

Flush each page on every CPU on map and unmap, or the whole D-cache in
one cross call once the range exceeds its size.

Link: https://github.com/sparclinux/issues/issues/29#issuecomment-5041121375
Closes: https://github.com/sparclinux/issues/issues/29
Suggested-by: nsafran1217 <54966414+nsafran1217@users.noreply.github.com>
Cc: stable@vger.kernel.org
Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---

Notes:
    No Fixes: tag: flush_cache_vmap/vunmap have been no-ops on
    sparc64 since the file was introduced, before the git
    history.

 arch/sparc/include/asm/cacheflush_64.h |  6 ++--
 arch/sparc/kernel/smp_64.c             | 18 +++++++++++
 arch/sparc/mm/init_64.c                | 42 ++++++++++++++++++++++++++
 arch/sparc/mm/ultra.S                  | 25 +++++++++++++++
 4 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h
index 02c969417e7b..54a2d37008e6 100644
--- a/arch/sparc/include/asm/cacheflush_64.h
+++ b/arch/sparc/include/asm/cacheflush_64.h
@@ -42,6 +42,8 @@ void smp_flush_dcache_folio_impl(struct folio *folio, int cpu);
 #define smp_flush_dcache_folio_impl(folio, cpu) flush_dcache_folio_impl(folio)
 #endif
 void flush_dcache_page_all(struct mm_struct *mm, struct page *page);
+void __flush_dcache_all(unsigned long size, unsigned long line_size);
+void flush_dcache_all(void);
 
 void __flush_dcache_range(unsigned long start, unsigned long end);
 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
@@ -73,9 +75,9 @@ void flush_ptrace_access(struct vm_area_struct *, struct page *,
 #define flush_dcache_mmap_lock(mapping)		do { } while (0)
 #define flush_dcache_mmap_unlock(mapping)	do { } while (0)
 
-#define flush_cache_vmap(start, end)		do { } while (0)
+void flush_cache_vmap(unsigned long start, unsigned long end);
 #define flush_cache_vmap_early(start, end)	do { } while (0)
-#define flush_cache_vunmap(start, end)		do { } while (0)
+#define flush_cache_vunmap(start, end)		flush_cache_vmap(start, end)
 
 #endif /* !__ASSEMBLER__ */
 
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index 18b6145da591..bebfc44241b3 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -915,6 +915,7 @@ extern unsigned long xcall_kgdb_capture;
 
 #ifdef DCACHE_ALIASING_POSSIBLE
 extern unsigned long xcall_flush_dcache_page_cheetah;
+extern unsigned long xcall_flush_dcache_all;
 #endif
 extern unsigned long xcall_flush_dcache_page_spitfire;
 
@@ -1025,6 +1026,23 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
 	preempt_enable();
 }
 
+#ifdef DCACHE_ALIASING_POSSIBLE
+void flush_dcache_all(void)
+{
+	unsigned long size, line_size;
+
+	if (tlb_type == hypervisor)
+		return;
+
+	preempt_disable();
+	size = local_cpu_data().dcache_size;
+	line_size = local_cpu_data().dcache_line_size;
+	smp_cross_call(&xcall_flush_dcache_all, 0, size, line_size);
+	__flush_dcache_all(size, line_size);
+	preempt_enable();
+}
+#endif
+
 #ifdef CONFIG_KGDB
 void kgdb_roundup_cpus(void)
 {
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 8792e5d92517..462338d875cd 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -27,6 +27,7 @@
 #include <linux/percpu.h>
 #include <linux/mmzone.h>
 #include <linux/gfp.h>
+#include <linux/vmalloc.h>
 
 #include <asm/head.h>
 #include <asm/page.h>
@@ -234,6 +235,17 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
 		__flush_icache_page(page_to_phys(page));
 #endif
 }
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+void flush_dcache_all(void)
+{
+	if (tlb_type == hypervisor)
+		return;
+
+	__flush_dcache_all(local_cpu_data().dcache_size,
+			   local_cpu_data().dcache_line_size);
+}
+#endif
 #endif
 
 #define PG_dcache_dirty		PG_arch_1
@@ -3072,6 +3084,36 @@ arch_initcall(report_memory);
 #define do_flush_tlb_kernel_range	__flush_tlb_kernel_range
 #endif
 
+/*
+ * The L1 D-cache is indexed with VA bit 13, so lines loaded through a
+ * vmalloc address outlive the mapping.  Stores through a mapping of the
+ * other colour do not update them, and a later load through their colour
+ * returns stale data.  Flush at unmap so vmalloc's lines do not follow
+ * the page back to the allocator, and at map time since anonymous user
+ * pages are freed without a flush (see tlb_batch_add()).  Past the
+ * D-cache size, one whole-cache flush is cheaper than a cross call per
+ * page.
+ */
+void flush_cache_vmap(unsigned long start, unsigned long end)
+{
+#ifdef DCACHE_ALIASING_POSSIBLE
+	if (tlb_type == hypervisor)
+		return;
+
+	if (end - start > cpu_data(raw_smp_processor_id()).dcache_size) {
+		flush_dcache_all();
+		return;
+	}
+
+	for (; start < end; start += PAGE_SIZE) {
+		struct page *page = vmalloc_to_page((void *)start);
+
+		if (page)
+			flush_dcache_page_all(NULL, page);
+	}
+#endif
+}
+
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
 	if (start < HI_OBP_ADDRESS && end > LOW_OBP_ADDRESS) {
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index 70e658d107e0..ff190670a235 100644
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -242,6 +242,20 @@ __flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
 	retl
 	 nop
 
+	/* Zeroing every D-cache tag invalidates the whole cache on
+	 * Spitfire and Cheetah alike.
+	 */
+	.align		32
+	.globl		__flush_dcache_all
+__flush_dcache_all:	/* %o0=D-cache size, %o1=D-cache line size */
+1:	subcc		%o0, %o1, %o0
+	stxa		%g0, [%o0] ASI_DCACHE_TAG
+	membar		#Sync
+	bne,pt		%icc, 1b
+	 nop
+	retl
+	 nop
+
 #endif /* DCACHE_ALIASING_POSSIBLE */
 
 	.previous
@@ -801,6 +815,17 @@ xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
 	 nop
 	retry
 	nop
+
+	.align		32
+	.globl		xcall_flush_dcache_all
+xcall_flush_dcache_all: /* %g1 == D-cache size, %g7 == D-cache line size */
+1:	subcc		%g1, %g7, %g1
+	stxa		%g0, [%g1] ASI_DCACHE_TAG
+	membar		#Sync
+	bne,pt		%icc, 1b
+	 nop
+	retry
+	nop
 #endif /* DCACHE_ALIASING_POSSIBLE */
 
 	.globl		xcall_flush_dcache_page_spitfire
-- 
2.47.3

Re: [PATCH 2/2] sparc64: flush vmalloc ranges from the D-cache on map and unmap

[Stian Halseth]

[-- Attachment #1: Type: text/plain, Size: 1637 bytes --]

Hi Imre,

On Sun, 2026-09-27 at 12:55 +0200, Imre Kaloz wrote:
> +void flush_cache_vmap(unsigned long start, unsigned long end)
> +{
> +#ifdef DCACHE_ALIASING_POSSIBLE
> +	if (tlb_type == hypervisor)
> +		return;

I found that this also runs early in boot, on the vmemmap.
__populate_section_memmap() calls it from sparse_init(). That is before
init_IRQ() has set up the cross-call mondo blocks, so xcall_deliver()
writes its three mondo words through __va(0), to physical address 0.

On SMP, cpu_data() is not filled in yet either, so dcache_size is 0 and
the flush itself does nothing useful.

Physical address 0 is ordinary RAM on the Sun Fire V240. In a test it
held zeros before the first of these cross-calls, and the three mondo
words after. The kernel uses that page later in boot. This happens
once per memory section, four times here.

Since the vmemmap has no aliases to flush, I suggest we skip anything
outside vmalloc and module space, like this:

        if (tlb_type == hypervisor ||
            !is_vmalloc_or_module_addr((void *)start))
                return;

With that, the V240 makes no cross-calls before init_IRQ().

Tested on a Sun Fire V240 with a module that reads a page through a
vmalloc address, rewrites it through the linear map on the same CPU and
maps it again at the same address: stale data in nearly every iteration
before, none after. Our V240s never showed #29 (no bpf() syscall
there), so that is a synthetic test. No change on a T4-1 with the
series as posted.

With the vmemmap check:
Tested-by: Stian Halseth <stian@itx.no> # Sun Fire V240, SPARC T4-1


[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 228 bytes --]

Re: [PATCH 2/2] sparc64: flush vmalloc ranges from the D-cache on map and unmap

[Imre Kaloz]

Hi Stian,

> Since the vmemmap has no aliases to flush, I suggest we skip
> anything outside vmalloc and module space, like this:

Thanks for chasing this down. Confirmed the path: dcache_size is
still 0 there, so the old code took the "exceeds cache size" branch
into flush_dcache_all() every time, and the mondo block PA is 0
until init_IRQ() runs, so the cross-call landed on physical address
0 instead of a real mondo queue. Skipping non-vmalloc/module
addresses is the right fix.

v2 with your check and your Tested-by follows.


Best,
Imre

Re: [PATCH 2/2] sparc64: flush vmalloc ranges from the D-cache on map and unmap

[Stian Halseth]

[-- Attachment #1: Type: text/plain, Size: 663 bytes --]

Hi Imre,

> 
> Thanks for chasing this down. Confirmed the path: dcache_size is
> still 0 there, so the old code took the "exceeds cache size" branch
> into flush_dcache_all() every time, and the mondo block PA is 0
> until init_IRQ() runs, so the cross-call landed on physical address
> 0 instead of a real mondo queue. Skipping non-vmalloc/module
> addresses is the right fix.
> 
> v2 with your check and your Tested-by follows.
> 

ofc, and thanks for the patch!

Makes me happy to see so many SPARC fixes.

Hoping to more of them in mainline / stable backports soon. 

Poor Andreas will have his hands full now ;-)

Best regards,
Stian

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 228 bytes --]