[RFC PATCH 2/6] alpha: implement simple mm_cpumask TLB flush filter

[Nicholas Piggin <npiggin@gmail.com>]

Implement a sticky mm_cpumask and use it to filter TLB flush IPIs.
Sticky meaning that when an mm runs on a CPU, it gets set in the mask
and never goes away. This is the "base" mm_cpumask implementation that
comes with the least complexity.

This reduces IPIs booting into a small rootfs by about 10-15% on 4CPU
system.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
 arch/alpha/include/asm/mmu_context.h | 19 +++++++++++++++++++
 arch/alpha/kernel/smp.c              | 16 ++++++++++++++--
 arch/alpha/mm/init.c                 |  2 ++
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/arch/alpha/include/asm/mmu_context.h b/arch/alpha/include/asm/mmu_context.h
index 8ce89350e4b3..9c9e9a8c01a4 100644
--- a/arch/alpha/include/asm/mmu_context.h
+++ b/arch/alpha/include/asm/mmu_context.h
@@ -135,6 +135,21 @@ ev5_switch_mm(struct mm_struct *prev_mm, struct mm_struct *next_mm,
 #ifdef CONFIG_SMP
 	cpu_data[cpu].asn_lock = 1;
 	barrier();
+
+	if (!cpumask_test_cpu(cpu, mm_cpumask(next_mm))) {
+		cpumask_set_cpu(cpu, mm_cpumask(next_mm));
+		/*
+		 * Store to mm_cpumask must be visible to CPUs performing
+		 * TLB flushes before memory accesses that could bring in
+		 * new TLB entries. This orders the store above with the
+		 * load of the new context and subsequent loads of PTEs
+		 * that can then be cached in the TLB.
+		 *
+		 * The other side is in the mm_cpumask testing in TLB
+		 * flush.
+		 */
+		smp_mb();
+	}
 #endif
 	asn = cpu_last_asn(cpu);
 	mmc = next_mm->context[cpu];
@@ -151,6 +166,8 @@ ev5_switch_mm(struct mm_struct *prev_mm, struct mm_struct *next_mm,
 	   a new mm->context (via flush_tlb_mm) without the ASN serial
 	   number wrapping.  We have no way to detect when this is needed.  */
 	task_thread_info(next)->pcb.asn = mmc & HARDWARE_ASN_MASK;
+
+	WARN_ON(!cpumask_test_cpu(cpu, mm_cpumask(prev_mm)));
 }
 
 static inline void
@@ -195,12 +212,14 @@ do {								\
 static inline void
 ev5_activate_mm(struct mm_struct *prev_mm, struct mm_struct *next_mm)
 {
+	cpumask_set_cpu(smp_processor_id(), mm_cpumask(next_mm));
 	__load_new_mm_context(next_mm);
 }
 
 static inline void
 ev4_activate_mm(struct mm_struct *prev_mm, struct mm_struct *next_mm)
 {
+	cpumask_set_cpu(smp_processor_id(), mm_cpumask(next_mm));
 	__load_new_mm_context(next_mm);
 	tbiap();
 }
diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index 7439b2377df5..b702372fbaba 100644
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -145,6 +145,7 @@ smp_callin(void)
 	/* All kernel threads share the same mm context.  */
 	mmgrab(&init_mm);
 	current->active_mm = &init_mm;
+	cpumask_set_cpu(smp_processor_id(), mm_cpumask(&init_mm));
 
 	/* inform the notifiers about the new cpu */
 	notify_cpu_starting(cpuid);
@@ -655,7 +656,17 @@ flush_tlb_mm(struct mm_struct *mm)
 		}
 	}
 
-	smp_call_function(ipi_flush_tlb_mm, mm, 1);
+	/*
+	 * TLB flush IPIs will be sent to all CPUs with mm_cpumask set. The
+	 * problem of ordering the load of mm_cpumask vs a CPU switching to
+	 * the mm and caching a translation from a PTE being invalidated and
+	 * flushed here means we must have a memory barrier. This orders the
+	 * prior stores to invalidate the PTEs from the load of mm_cpumask.
+	 *
+	 * The other side is switch_mm.
+	 */
+	smp_mb();
+	smp_call_function_many(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
 
 	preempt_enable();
 }
@@ -706,7 +717,8 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 	data.mm = mm;
 	data.addr = addr;
 
-	smp_call_function(ipi_flush_tlb_page, &data, 1);
+	smp_mb(); /* see flush_tlb_mm */
+	smp_call_function_many(mm_cpumask(mm), ipi_flush_tlb_page, &data, 1);
 
 	preempt_enable();
 }
diff --git a/arch/alpha/mm/init.c b/arch/alpha/mm/init.c
index a155180d7a83..33f4c0abd2c8 100644
--- a/arch/alpha/mm/init.c
+++ b/arch/alpha/mm/init.c
@@ -254,6 +254,8 @@ void __init paging_init(void)
 
 	/* Initialize the kernel's ZERO_PGE. */
 	memset(absolute_pointer(ZERO_PGE), 0, PAGE_SIZE);
+
+	cpumask_set_cpu(raw_smp_processor_id(), mm_cpumask(&init_mm));
 }
 
 #if defined(CONFIG_ALPHA_GENERIC) || defined(CONFIG_ALPHA_SRM)
-- 
2.40.1