[PATCH v5 0/7] tlb flush optimization for x86

[PATCH v5 0/7] tlb flush optimization for x86

[Alex Shi]

This the 5th version patchset. New changes are:
1, tlb_flushall_shift knob move to debugfs.
2, include model 58 IVB mobile CPU tlb_flushall_shift setting
3, many small code changes response for most all of comments for v4.

Any more comments are appreciated!

Alex Shi

[PATCH v5 1/7] x86/tlb: unify TLB_FLUSH_ALL definition

[Alex Shi]

32bits kernel is using ILP32 model. 64 bits kernel is using LP64 model.
So, here TLB_FLUSH_ALL can be defined as '-1UL', that fit in both
x86_32/x86_64 mode.

Thanks for Rob Landley and Peter Avin's infos

http://www.unix.org/whitepapers/64bit.html
http://www.unix.org/version2/whatsnew/lp64_wp.html

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/include/asm/tlbflush.h |    6 +-----
 1 files changed, 1 insertions(+), 5 deletions(-)

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c0e108e..7e8a096 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -62,11 +62,7 @@ static inline void __flush_tlb_one(unsigned long addr)
 		__flush_tlb();
 }
 
-#ifdef CONFIG_X86_32
-# define TLB_FLUSH_ALL	0xffffffff
-#else
-# define TLB_FLUSH_ALL	-1ULL
-#endif
+#define TLB_FLUSH_ALL	-1UL
 
 /*
  * TLB flushing:
-- 
1.7.5.4

[PATCH v5 2/7] x86/tlb_info: get last level TLB entry number of CPU

[Alex Shi]

For 4KB pages, x86 CPU has 2 or 1 level TLB, first level is data TLB and
instruction TLB, second level is shared TLB for both data and instructions.

For hupe page TLB, usually there is just one level and seperated by 2MB/4MB
and 1GB.

Although each levels TLB size is important for performance tuning, but for
genernal and rude optimizing, last level TLB entry number is suitable. And
in fact, last level TLB always has the biggest entry number.

This patch will get the biggest TLB entry number and use it in furture TLB
optimizing.

Accroding Borislav's suggestion, except tlb_ll[i/d]_* array, other
function and data will be released after system boot up.

For all kinds of x86 vendor friendly, vendor specific code was moved to its
specific files.

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/include/asm/processor.h |   11 +++
 arch/x86/kernel/cpu/common.c     |   21 ++++++
 arch/x86/kernel/cpu/cpu.h        |    9 +++
 arch/x86/kernel/cpu/intel.c      |  141 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 182 insertions(+), 0 deletions(-)

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 4fa7dcc..797faca 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -61,6 +61,17 @@ static inline void *current_text_addr(void)
 # define ARCH_MIN_MMSTRUCT_ALIGN	0
 #endif
 
+enum tlb_infos {
+	ENTRIES,
+	NR_INFO
+};
+
+extern u16 __read_mostly tlb_lli_4k[NR_INFO];
+extern u16 __read_mostly tlb_lli_2m[NR_INFO];
+extern u16 __read_mostly tlb_lli_4m[NR_INFO];
+extern u16 __read_mostly tlb_lld_4k[NR_INFO];
+extern u16 __read_mostly tlb_lld_2m[NR_INFO];
+extern u16 __read_mostly tlb_lld_4m[NR_INFO];
 /*
  *  CPU type and hardware bug flags. Kept separately for each CPU.
  *  Members of this structure are referenced in head.S, so think twice
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index cf79302..0152082 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -452,6 +452,25 @@ void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
 	c->x86_cache_size = l2size;
 }
 
+u16 __read_mostly tlb_lli_4k[NR_INFO];
+u16 __read_mostly tlb_lli_2m[NR_INFO];
+u16 __read_mostly tlb_lli_4m[NR_INFO];
+u16 __read_mostly tlb_lld_4k[NR_INFO];
+u16 __read_mostly tlb_lld_2m[NR_INFO];
+u16 __read_mostly tlb_lld_4m[NR_INFO];
+
+void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
+{
+	if (c->x86_vendor == X86_VENDOR_INTEL)
+		intel_cpu_detect_tlb(c);
+
+	printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
+		"Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
+		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
+		tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
+		tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES]);
+}
+
 void __cpuinit detect_ht(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_X86_HT
@@ -911,6 +930,8 @@ void __init identify_boot_cpu(void)
 #else
 	vgetcpu_set_mode();
 #endif
+	if (boot_cpu_data.cpuid_level >= 2)
+		cpu_detect_tlb(&boot_cpu_data);
 }
 
 void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index 8bacc78..78db1d9 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -24,6 +24,14 @@ struct cpu_dev {
 	int		c_x86_vendor;
 };
 
+struct _tlb_table {
+	unsigned char descriptor;
+	char tlb_type;
+	unsigned int entries;
+	/* unsigned int ways; */
+	char info[128];
+};
+
 #define cpu_dev_register(cpu_devX) \
 	static const struct cpu_dev *const __cpu_dev_##cpu_devX __used \
 	__attribute__((__section__(".x86_cpu_dev.init"))) = \
@@ -34,4 +42,5 @@ extern const struct cpu_dev *const __x86_cpu_dev_start[],
 
 extern void get_cpu_cap(struct cpuinfo_x86 *c);
 extern void cpu_detect_cache_sizes(struct cpuinfo_x86 *c);
+extern void intel_cpu_detect_tlb(struct cpuinfo_x86 *c) __cpuinit;
 #endif /* ARCH_X86_CPU_H */
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 3e6ff6c..28ecd1b 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -491,6 +491,147 @@ static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned i
 }
 #endif
 
+#define TLB_INST_4K	0x01
+#define TLB_INST_4M	0x02
+#define TLB_INST_2M_4M	0x03
+
+#define TLB_INST_ALL	0x05
+#define TLB_INST_1G	0x06
+
+#define TLB_DATA_4K	0x11
+#define TLB_DATA_4M	0x12
+#define TLB_DATA_2M_4M	0x13
+#define TLB_DATA_4K_4M	0x14
+
+#define TLB_DATA_1G	0x16
+
+#define TLB_DATA0_4K	0x21
+#define TLB_DATA0_4M	0x22
+#define TLB_DATA0_2M_4M	0x23
+
+#define STLB_4K		0x41
+
+static const struct _tlb_table intel_tlb_table[] __cpuinitconst = {
+	{ 0x01, TLB_INST_4K,		32,	" TLB_INST 4 KByte pages, 4-way set associative" },
+	{ 0x02, TLB_INST_4M,		2,	" TLB_INST 4 MByte pages, full associative" },
+	{ 0x03, TLB_DATA_4K,		64,	" TLB_DATA 4 KByte pages, 4-way set associative" },
+	{ 0x04, TLB_DATA_4M,		8,	" TLB_DATA 4 MByte pages, 4-way set associative" },
+	{ 0x05, TLB_DATA_4M,		32,	" TLB_DATA 4 MByte pages, 4-way set associative" },
+	{ 0x0b, TLB_INST_4M,		4,	" TLB_INST 4 MByte pages, 4-way set associative" },
+	{ 0x4f, TLB_INST_4K,		32,	" TLB_INST 4 KByte pages */" },
+	{ 0x50, TLB_INST_ALL,		64,	" TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+	{ 0x51, TLB_INST_ALL,		128,	" TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+	{ 0x52, TLB_INST_ALL,		256,	" TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+	{ 0x55, TLB_INST_2M_4M,		7,	" TLB_INST 2-MByte or 4-MByte pages, fully associative" },
+	{ 0x56, TLB_DATA0_4M,		16,	" TLB_DATA0 4 MByte pages, 4-way set associative" },
+	{ 0x57, TLB_DATA0_4K,		16,	" TLB_DATA0 4 KByte pages, 4-way associative" },
+	{ 0x59, TLB_DATA0_4K,		16,	" TLB_DATA0 4 KByte pages, fully associative" },
+	{ 0x5a, TLB_DATA0_2M_4M,	32,	" TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative" },
+	{ 0x5b, TLB_DATA_4K_4M,		64,	" TLB_DATA 4 KByte and 4 MByte pages" },
+	{ 0x5c, TLB_DATA_4K_4M,		128,	" TLB_DATA 4 KByte and 4 MByte pages" },
+	{ 0x5d, TLB_DATA_4K_4M,		256,	" TLB_DATA 4 KByte and 4 MByte pages" },
+	{ 0xb0, TLB_INST_4K,		128,	" TLB_INST 4 KByte pages, 4-way set associative" },
+	{ 0xb1, TLB_INST_2M_4M,		4,	" TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries" },
+	{ 0xb2, TLB_INST_4K,		64,	" TLB_INST 4KByte pages, 4-way set associative" },
+	{ 0xb3, TLB_DATA_4K,		128,	" TLB_DATA 4 KByte pages, 4-way set associative" },
+	{ 0xb4, TLB_DATA_4K,		256,	" TLB_DATA 4 KByte pages, 4-way associative" },
+	{ 0xba, TLB_DATA_4K,		64,	" TLB_DATA 4 KByte pages, 4-way associative" },
+	{ 0xc0, TLB_DATA_4K_4M,		8,	" TLB_DATA 4 KByte and 4 MByte pages, 4-way associative" },
+	{ 0xca, STLB_4K,		512,	" STLB 4 KByte pages, 4-way associative" },
+	{ 0x00, 0, 0 }
+};
+
+static void __cpuinit intel_tlb_lookup(const unsigned char desc)
+{
+	unsigned char k;
+	if (desc == 0)
+		return;
+
+	/* look up this descriptor in the table */
+	for (k = 0; intel_tlb_table[k].descriptor != desc && \
+			intel_tlb_table[k].descriptor != 0; k++)
+		;
+
+	if (intel_tlb_table[k].tlb_type == 0)
+		return;
+
+	switch (intel_tlb_table[k].tlb_type) {
+	case STLB_4K:
+		if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+		if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	case TLB_INST_ALL:
+		if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+		if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
+		if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	case TLB_INST_4K:
+		if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	case TLB_INST_4M:
+		if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	case TLB_INST_2M_4M:
+		if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
+		if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	case TLB_DATA_4K:
+	case TLB_DATA0_4K:
+		if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	case TLB_DATA_4M:
+	case TLB_DATA0_4M:
+		if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	case TLB_DATA_2M_4M:
+	case TLB_DATA0_2M_4M:
+		if (tlb_lld_2m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lld_2m[ENTRIES] = intel_tlb_table[k].entries;
+		if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	case TLB_DATA_4K_4M:
+		if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+		if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+			tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+		break;
+	}
+}
+
+void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
+{
+	int i, j, n;
+	unsigned int regs[4];
+	unsigned char *desc = (unsigned char *)regs;
+	/* Number of times to iterate */
+	n = cpuid_eax(2) & 0xFF;
+
+	for (i = 0 ; i < n ; i++) {
+		cpuid(2, &regs[0], &regs[1], &regs[2], &regs[3]);
+
+		/* If bit 31 is set, this is an unknown format */
+		for (j = 0 ; j < 3 ; j++)
+			if (regs[j] & (1 << 31))
+				regs[j] = 0;
+
+		/* Byte 0 is level count, not a descriptor */
+		for (j = 1 ; j < 16 ; j++)
+			intel_tlb_lookup(desc[j]);
+	}
+}
+
 static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
 	.c_vendor	= "Intel",
 	.c_ident	= { "GenuineIntel" },
-- 
1.7.5.4

[PATCH v5 3/7] x86/flush_tlb: try flush_tlb_single one by one in flush_tlb_range

[Alex Shi]

x86 has no flush_tlb_range support in instruction level. Currently the
flush_tlb_range just implemented by flushing all page table. That is not
the best solution for all scenarios. In fact, if we just use 'invlpg' to
flush few lines from TLB, we can get the performance gain from later
remain TLB lines accessing.

But the 'invlpg' instruction costs much of time. Its execution time can
compete with cr3 rewriting, and even a bit more on SNB CPU.

So, on a 512 4KB TLB entries CPU, the balance points is at:
	(512 - X) * 100ns(assumed TLB refill cost) =
		X(TLB flush entries) * 100ns(assumed invlpg cost)

Here, X is 256, that is 1/2 of 512 entries.

But with the mysterious CPU pre-fetcher and page miss handler Unit, the
assumed TLB refill cost is far lower then 100ns in sequential access. And
2 HT siblings in one core makes the memory access more faster if they are
accessing the same memory. So, in the patch, I just do the change when
the target entries is less than 1/16 of whole active tlb entries.
Actually, I have no data support for the percentage '1/16', so any
suggestions are welcomed.

As to hugetlb, guess due to smaller page table, and smaller active TLB
entries, I didn't see benefit via my benchmark, so no optimizing now.

My macro benchmark show in ideal scenarios, the performance improves 70
percent in reading. And in worst scenario, the reading/writing
performance is similar with unpatched 3.4-rc4 kernel.

Here is the reading data on my 2P * 4cores *HT NHM EP machine, with THP
'always':

multi thread testing, '-t' paramter is thread number:
	       	        with patch   unpatched 3.4-rc4
./mprotect -t 1           14ns		24ns
./mprotect -t 2           13ns		22ns
./mprotect -t 4           12ns		19ns
./mprotect -t 8           14ns		16ns
./mprotect -t 16          28ns		26ns
./mprotect -t 32          54ns		51ns
./mprotect -t 128         200ns		199ns

Single process with sequencial flushing and memory accessing:

		       	with patch   unpatched 3.4-rc4
./mprotect		    7ns			11ns
./mprotect -p 4096  -l 8 -n 10240
			    21ns		21ns

I also tried other benchmarks on Intel core2/NHM/SNB EP and NHM EX machine.
No clear performance change on specjbb2005 with openjdk, and oltp reading.

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/include/asm/paravirt.h       |    5 +-
 arch/x86/include/asm/paravirt_types.h |    3 +-
 arch/x86/include/asm/tlbflush.h       |   23 +++-----
 arch/x86/include/asm/uv/uv.h          |    5 +-
 arch/x86/mm/tlb.c                     |   97 +++++++++++++++++++++++++++------
 arch/x86/platform/uv/tlb_uv.c         |    6 +-
 arch/x86/xen/mmu.c                    |    9 ++--
 include/trace/events/xen.h            |   12 +++--
 8 files changed, 113 insertions(+), 47 deletions(-)

diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index aa0f913..03da4ab 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -397,9 +397,10 @@ static inline void __flush_tlb_single(unsigned long addr)
 
 static inline void flush_tlb_others(const struct cpumask *cpumask,
 				    struct mm_struct *mm,
-				    unsigned long va)
+				    unsigned long start,
+				    unsigned long end)
 {
-	PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
+	PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
 }
 
 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 8e8b9a4..600a5fcac9 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -250,7 +250,8 @@ struct pv_mmu_ops {
 	void (*flush_tlb_single)(unsigned long addr);
 	void (*flush_tlb_others)(const struct cpumask *cpus,
 				 struct mm_struct *mm,
-				 unsigned long va);
+				 unsigned long start,
+				 unsigned long end);
 
 	/* Hooks for allocating and freeing a pagetable top-level */
 	int  (*pgd_alloc)(struct mm_struct *mm);
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 7e8a096..c39c94e 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -73,14 +73,10 @@ static inline void __flush_tlb_one(unsigned long addr)
  *  - flush_tlb_page(vma, vmaddr) flushes one page
  *  - flush_tlb_range(vma, start, end) flushes a range of pages
  *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
- *  - flush_tlb_others(cpumask, mm, va) flushes TLBs on other cpus
+ *  - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
  *
  * ..but the i386 has somewhat limited tlb flushing capabilities,
  * and page-granular flushes are available only on i486 and up.
- *
- * x86-64 can only flush individual pages or full VMs. For a range flush
- * we always do the full VM. Might be worth trying if for a small
- * range a few INVLPGs in a row are a win.
  */
 
 #ifndef CONFIG_SMP
@@ -111,7 +107,8 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
 
 static inline void native_flush_tlb_others(const struct cpumask *cpumask,
 					   struct mm_struct *mm,
-					   unsigned long va)
+					   unsigned long start,
+					   unsigned long end)
 {
 }
 
@@ -129,17 +126,14 @@ extern void flush_tlb_all(void);
 extern void flush_tlb_current_task(void);
 extern void flush_tlb_mm(struct mm_struct *);
 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
+extern void flush_tlb_range(struct vm_area_struct *vma,
+				   unsigned long start, unsigned long end);
 
 #define flush_tlb()	flush_tlb_current_task()
 
-static inline void flush_tlb_range(struct vm_area_struct *vma,
-				   unsigned long start, unsigned long end)
-{
-	flush_tlb_mm(vma->vm_mm);
-}
-
 void native_flush_tlb_others(const struct cpumask *cpumask,
-			     struct mm_struct *mm, unsigned long va);
+				struct mm_struct *mm,
+				unsigned long start, unsigned long end);
 
 #define TLBSTATE_OK	1
 #define TLBSTATE_LAZY	2
@@ -159,7 +153,8 @@ static inline void reset_lazy_tlbstate(void)
 #endif	/* SMP */
 
 #ifndef CONFIG_PARAVIRT
-#define flush_tlb_others(mask, mm, va)	native_flush_tlb_others(mask, mm, va)
+#define flush_tlb_others(mask, mm, start, end)	\
+	native_flush_tlb_others(mask, mm, start, end)
 #endif
 
 static inline void flush_tlb_kernel_range(unsigned long start,
diff --git a/arch/x86/include/asm/uv/uv.h b/arch/x86/include/asm/uv/uv.h
index 3bb9491..b47c2a8 100644
--- a/arch/x86/include/asm/uv/uv.h
+++ b/arch/x86/include/asm/uv/uv.h
@@ -15,7 +15,8 @@ extern void uv_nmi_init(void);
 extern void uv_system_init(void);
 extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 						 struct mm_struct *mm,
-						 unsigned long va,
+						 unsigned long start,
+						 unsigned end,
 						 unsigned int cpu);
 
 #else	/* X86_UV */
@@ -26,7 +27,7 @@ static inline void uv_cpu_init(void)	{ }
 static inline void uv_system_init(void)	{ }
 static inline const struct cpumask *
 uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm,
-		    unsigned long va, unsigned int cpu)
+		    unsigned long start, unsigned long end, unsigned int cpu)
 { return cpumask; }
 
 #endif	/* X86_UV */
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..7d92079 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -41,7 +41,8 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
 union smp_flush_state {
 	struct {
 		struct mm_struct *flush_mm;
-		unsigned long flush_va;
+		unsigned long flush_start;
+		unsigned long flush_end;
 		raw_spinlock_t tlbstate_lock;
 		DECLARE_BITMAP(flush_cpumask, NR_CPUS);
 	};
@@ -154,10 +155,19 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
 
 	if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
 		if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
-			if (f->flush_va == TLB_FLUSH_ALL)
+			if (f->flush_end == TLB_FLUSH_ALL
+					|| !cpu_has_invlpg)
 				local_flush_tlb();
-			else
-				__flush_tlb_one(f->flush_va);
+			else if (!f->flush_end)
+				__flush_tlb_single(f->flush_start);
+			else {
+				unsigned long addr;
+				addr = f->flush_start;
+				while (addr <= f->flush_end) {
+					__flush_tlb_single(addr);
+					addr += PAGE_SIZE;
+				}
+			}
 		} else
 			leave_mm(cpu);
 	}
@@ -170,7 +180,8 @@ out:
 }
 
 static void flush_tlb_others_ipi(const struct cpumask *cpumask,
-				 struct mm_struct *mm, unsigned long va)
+				 struct mm_struct *mm, unsigned long start,
+				 unsigned long end)
 {
 	unsigned int sender;
 	union smp_flush_state *f;
@@ -183,7 +194,8 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask,
 		raw_spin_lock(&f->tlbstate_lock);
 
 	f->flush_mm = mm;
-	f->flush_va = va;
+	f->flush_start = start;
+	f->flush_end = end;
 	if (cpumask_andnot(to_cpumask(f->flush_cpumask), cpumask, cpumask_of(smp_processor_id()))) {
 		/*
 		 * We have to send the IPI only to
@@ -197,24 +209,26 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask,
 	}
 
 	f->flush_mm = NULL;
-	f->flush_va = 0;
+	f->flush_start = 0;
+	f->flush_end = 0;
 	if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS)
 		raw_spin_unlock(&f->tlbstate_lock);
 }
 
 void native_flush_tlb_others(const struct cpumask *cpumask,
-			     struct mm_struct *mm, unsigned long va)
+				 struct mm_struct *mm, unsigned long start,
+				 unsigned long end)
 {
 	if (is_uv_system()) {
 		unsigned int cpu;
 
 		cpu = smp_processor_id();
-		cpumask = uv_flush_tlb_others(cpumask, mm, va, cpu);
+		cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu);
 		if (cpumask)
-			flush_tlb_others_ipi(cpumask, mm, va);
+			flush_tlb_others_ipi(cpumask, mm, start, end);
 		return;
 	}
-	flush_tlb_others_ipi(cpumask, mm, va);
+	flush_tlb_others_ipi(cpumask, mm, start, end);
 }
 
 static void __cpuinit calculate_tlb_offset(void)
@@ -280,7 +294,7 @@ void flush_tlb_current_task(void)
 
 	local_flush_tlb();
 	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
-		flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
+		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
 	preempt_enable();
 }
 
@@ -295,12 +309,63 @@ void flush_tlb_mm(struct mm_struct *mm)
 			leave_mm(smp_processor_id());
 	}
 	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
-		flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
+		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+
+	preempt_enable();
+}
+
+#define FLUSHALL_BAR	16
+
+void flush_tlb_range(struct vm_area_struct *vma,
+				   unsigned long start, unsigned long end)
+{
+	struct mm_struct *mm;
+
+	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) {
+		flush_tlb_mm(vma->vm_mm);
+		return;
+	}
+
+	preempt_disable();
+	mm = vma->vm_mm;
+	if (current->active_mm == mm) {
+		if (current->mm) {
+			unsigned long addr, vmflag = vma->vm_flags;
+			unsigned act_entries, tlb_entries = 0;
+
+			if (vmflag & VM_EXEC)
+				tlb_entries = tlb_lli_4k[ENTRIES];
+			else
+				tlb_entries = tlb_lld_4k[ENTRIES];
+
+			act_entries = tlb_entries > mm->total_vm ?
+					mm->total_vm : tlb_entries;
 
+			if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR)
+				local_flush_tlb();
+			else {
+				for (addr = start; addr <= end;
+						addr += PAGE_SIZE)
+					__flush_tlb_single(addr);
+
+				if (cpumask_any_but(mm_cpumask(mm),
+					smp_processor_id()) < nr_cpu_ids)
+					flush_tlb_others(mm_cpumask(mm), mm,
+								start, end);
+				preempt_enable();
+				return;
+			}
+		} else {
+			leave_mm(smp_processor_id());
+		}
+	}
+	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
 	preempt_enable();
 }
 
-void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
+
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
 {
 	struct mm_struct *mm = vma->vm_mm;
 
@@ -308,13 +373,13 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
 
 	if (current->active_mm == mm) {
 		if (current->mm)
-			__flush_tlb_one(va);
+			__flush_tlb_one(start);
 		else
 			leave_mm(smp_processor_id());
 	}
 
 	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
-		flush_tlb_others(mm_cpumask(mm), mm, va);
+		flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
 
 	preempt_enable();
 }
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 3ae0e61..0df5ad2 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1068,8 +1068,8 @@ static int set_distrib_bits(struct cpumask *flush_mask, struct bau_control *bcp,
  * done.  The returned pointer is valid till preemption is re-enabled.
  */
 const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
-				struct mm_struct *mm, unsigned long va,
-				unsigned int cpu)
+				struct mm_struct *mm, unsigned long start,
+				unsigned end, unsigned int cpu)
 {
 	int locals = 0;
 	int remotes = 0;
@@ -1112,7 +1112,7 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
 
 	record_send_statistics(stat, locals, hubs, remotes, bau_desc);
 
-	bau_desc->payload.address = va;
+	bau_desc->payload.address = start;
 	bau_desc->payload.sending_cpu = cpu;
 	/*
 	 * uv_flush_send_and_wait returns 0 if all cpu's were messaged,
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index b8e2794..75bab52 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1239,7 +1239,8 @@ static void xen_flush_tlb_single(unsigned long addr)
 }
 
 static void xen_flush_tlb_others(const struct cpumask *cpus,
-				 struct mm_struct *mm, unsigned long va)
+				 struct mm_struct *mm, unsigned long start,
+				 unsigned long end)
 {
 	struct {
 		struct mmuext_op op;
@@ -1251,7 +1252,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
 	} *args;
 	struct multicall_space mcs;
 
-	trace_xen_mmu_flush_tlb_others(cpus, mm, va);
+	trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
 
 	if (cpumask_empty(cpus))
 		return;		/* nothing to do */
@@ -1264,11 +1265,11 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
 	cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
 	cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
 
-	if (va == TLB_FLUSH_ALL) {
+	if (start == TLB_FLUSH_ALL) {
 		args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
 	} else {
 		args->op.cmd = MMUEXT_INVLPG_MULTI;
-		args->op.arg1.linear_addr = va;
+		args->op.arg1.linear_addr = start;
 	}
 
 	MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
diff --git a/include/trace/events/xen.h b/include/trace/events/xen.h
index 92f1a79..15ba03b 100644
--- a/include/trace/events/xen.h
+++ b/include/trace/events/xen.h
@@ -397,18 +397,20 @@ TRACE_EVENT(xen_mmu_flush_tlb_single,
 
 TRACE_EVENT(xen_mmu_flush_tlb_others,
 	    TP_PROTO(const struct cpumask *cpus, struct mm_struct *mm,
-		     unsigned long addr),
-	    TP_ARGS(cpus, mm, addr),
+		     unsigned long addr, unsigned long end),
+	    TP_ARGS(cpus, mm, addr, end),
 	    TP_STRUCT__entry(
 		    __field(unsigned, ncpus)
 		    __field(struct mm_struct *, mm)
 		    __field(unsigned long, addr)
+		    __field(unsigned long, end)
 		    ),
 	    TP_fast_assign(__entry->ncpus = cpumask_weight(cpus);
 			   __entry->mm = mm;
-			   __entry->addr = addr),
-	    TP_printk("ncpus %d mm %p addr %lx",
-		      __entry->ncpus, __entry->mm, __entry->addr)
+			   __entry->addr = addr,
+			   __entry->end = end),
+	    TP_printk("ncpus %d mm %p addr %lx, end %lx",
+		      __entry->ncpus, __entry->mm, __entry->addr, __entry->end)
 	);
 
 TRACE_EVENT(xen_mmu_write_cr3,
-- 
1.7.5.4

[PATCH v5 4/7] x86/tlb: fall back to flush all when meet a THP large page

[Alex Shi]

We don't need to flush large pages by PAGE_SIZE step, that just waste
time. and actually, large page don't need 'invlpg' optimizing according
to our macro benchmark. So, just flush whole TLB is enough for them.

The following result is tested on a 2CPU * 4cores * 2HT NHM EP machine,
with THP 'always' setting.

Multi-thread testing, '-t' paramter is thread number:
                       without this patch 	with this patch
./mprotect -t 1         14ns                       13ns
./mprotect -t 2         13ns                       13ns
./mprotect -t 4         12ns                       11ns
./mprotect -t 8         14ns                       10ns
./mprotect -t 16        28ns                       28ns
./mprotect -t 32        54ns                       52ns
./mprotect -t 128       200ns                      200ns

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/mm/tlb.c |   34 ++++++++++++++++++++++++++++++++++
 1 files changed, 34 insertions(+), 0 deletions(-)

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 7d92079..22e5bb1 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -316,12 +316,42 @@ void flush_tlb_mm(struct mm_struct *mm)
 
 #define FLUSHALL_BAR	16
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline int has_large_page(struct mm_struct *mm,
+				 unsigned long start, unsigned long end)
+{
+	pgd_t *pgd;
+	pud_t *pud;
+	pmd_t *pmd;
+	unsigned long addr = ALIGN(start, HPAGE_SIZE);
+	for (; addr < end; addr += HPAGE_SIZE) {
+		pgd = pgd_offset(mm, addr);
+		if (likely(!pgd_none(*pgd))) {
+			pud = pud_offset(pgd, addr);
+			if (likely(!pud_none(*pud))) {
+				pmd = pmd_offset(pud, addr);
+				if (likely(!pmd_none(*pmd)))
+					if (pmd_large(*pmd))
+						return 1;
+			}
+		}
+	}
+	return 0;
+}
+#else
+static inline int has_large_page(struct mm_struct *mm,
+				 unsigned long start, unsigned long end)
+{
+	return 0;
+}
+#endif
 void flush_tlb_range(struct vm_area_struct *vma,
 				   unsigned long start, unsigned long end)
 {
 	struct mm_struct *mm;
 
 	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) {
+flush_all:
 		flush_tlb_mm(vma->vm_mm);
 		return;
 	}
@@ -344,6 +374,10 @@ void flush_tlb_range(struct vm_area_struct *vma,
 			if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR)
 				local_flush_tlb();
 			else {
+				if (has_large_page(mm, start, end)) {
+					preempt_enable();
+					goto flush_all;
+				}
 				for (addr = start; addr <= end;
 						addr += PAGE_SIZE)
 					__flush_tlb_single(addr);
-- 
1.7.5.4

[PATCH v5 5/7] x86/tlb: add tlb_flushall_shift for specific CPU

[Alex Shi]

Testing show different CPU type(micro architectures and NUMA mode) has
different balance points between the TLB flush all and multiple invlpg.
And there also has cases the tlb flush change has no any help.

This patch give a interface to let x86 vendor developers have a chance
to set different shift for different CPU type.

like some machine in my hands, balance points is 16 entries on
Romely-EP; while it is at 8 entries on Bloomfield NHM-EP; but on model
15 core2 Xeon using invlpg has nothing help.

For untested machine, do a conservative optimization, same as NHM CPU.

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/include/asm/processor.h |    2 ++
 arch/x86/kernel/cpu/common.c     |   14 ++++++++++++--
 arch/x86/kernel/cpu/intel.c      |   34 ++++++++++++++++++++++++++++++++++
 arch/x86/mm/tlb.c                |    8 ++++----
 include/asm-generic/tlb.h        |    3 ++-
 5 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 797faca..3c0cc3d 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -72,6 +72,8 @@ extern u16 __read_mostly tlb_lli_4m[NR_INFO];
 extern u16 __read_mostly tlb_lld_4k[NR_INFO];
 extern u16 __read_mostly tlb_lld_2m[NR_INFO];
 extern u16 __read_mostly tlb_lld_4m[NR_INFO];
+extern u16 __read_mostly tlb_flushall_shift;
+
 /*
  *  CPU type and hardware bug flags. Kept separately for each CPU.
  *  Members of this structure are referenced in head.S, so think twice
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 0152082..24255ca 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -459,16 +459,26 @@ u16 __read_mostly tlb_lld_4k[NR_INFO];
 u16 __read_mostly tlb_lld_2m[NR_INFO];
 u16 __read_mostly tlb_lld_4m[NR_INFO];
 
+/*
+ * tlb_flushall_shift shows the balance point in replacing cr3 write
+ * with multiple 'invlpg'. It will do this replacement when
+ *   flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
+ * If tlb_flushall_shift is -1, means the replacement will be disabled.
+ */
+u16 __read_mostly tlb_flushall_shift;
+
 void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
 {
 	if (c->x86_vendor == X86_VENDOR_INTEL)
 		intel_cpu_detect_tlb(c);
 
 	printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
-		"Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
+		"Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n"	     \
+		"tlb_flushall_shift is 0x%x\n",
 		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
 		tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
-		tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES]);
+		tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
+		tlb_flushall_shift);
 }
 
 void __cpuinit detect_ht(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 28ecd1b..a51ab82 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -610,6 +610,39 @@ static void __cpuinit intel_tlb_lookup(const unsigned char desc)
 	}
 }
 
+static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c)
+{
+	if (!cpu_has_invlpg) {
+		tlb_flushall_shift = -1;
+		return;
+	}
+	switch ((c->x86 << 8) + c->x86_model) {
+	case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
+	case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
+	case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
+	case 0x61d: /* six-core 45 nm xeon "Dunnington" */
+		tlb_flushall_shift = -1;
+		break;
+	case 0x61a: /* 45 nm nehalem, "Bloomfield" */
+	case 0x61e: /* 45 nm nehalem, "Lynnfield" */
+	case 0x625: /* 32 nm nehalem, "Clarkdale" */
+	case 0x62c: /* 32 nm nehalem, "Gulftown" */
+	case 0x62e: /* 45 nm nehalem-ex, "Beckton" */
+	case 0x62f: /* 32 nm Xeon E7 */
+		tlb_flushall_shift = 6;
+		break;
+	case 0x62a: /* SandyBridge */
+	case 0x62d: /* SandyBridge, "Romely-EP" */
+		tlb_flushall_shift = 5;
+		break;
+	case 0x63a: /* Ivybridge */
+		tlb_flushall_shift = 0;
+		break;
+	default:
+		tlb_flushall_shift = 6;
+	}
+}
+
 void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
 {
 	int i, j, n;
@@ -630,6 +663,7 @@ void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
 		for (j = 1 ; j < 16 ; j++)
 			intel_tlb_lookup(desc[j]);
 	}
+	intel_tlb_flushall_shift_set(c);
 }
 
 static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 22e5bb1..5bf4e85 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -314,8 +314,6 @@ void flush_tlb_mm(struct mm_struct *mm)
 	preempt_enable();
 }
 
-#define FLUSHALL_BAR	16
-
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline int has_large_page(struct mm_struct *mm,
 				 unsigned long start, unsigned long end)
@@ -350,7 +348,8 @@ void flush_tlb_range(struct vm_area_struct *vma,
 {
 	struct mm_struct *mm;
 
-	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) {
+	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
+			|| tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
 flush_all:
 		flush_tlb_mm(vma->vm_mm);
 		return;
@@ -371,7 +370,8 @@ flush_all:
 			act_entries = tlb_entries > mm->total_vm ?
 					mm->total_vm : tlb_entries;
 
-			if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR)
+			if ((end - start) >> PAGE_SHIFT >
+					act_entries >> tlb_flushall_shift)
 				local_flush_tlb();
 			else {
 				if (has_large_page(mm, start, end)) {
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index f96a5b5..75e888b 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -113,7 +113,8 @@ static inline int tlb_fast_mode(struct mmu_gather *tlb)
 
 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
 void tlb_flush_mmu(struct mmu_gather *tlb);
-void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end);
+void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
+							unsigned long end);
 int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
 
 /* tlb_remove_page
-- 
1.7.5.4

Re: [PATCH v5 5/7] x86/tlb: add tlb_flushall_shift for specific CPU

[Alex Shi]

On 05/15/2012 04:55 PM, Alex Shi wrote:

update this patch for a little change commit log and conservative 
balance point on IVB mobile cpu.
----

>From 0a3d618e5a8b29efc6296e44a56bc5c87b506659 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Mon, 14 May 2012 09:13:28 +0800
Subject: [PATCH 5/7] x86/tlb: add tlb_flushall_shift for specific CPU

Testing show different CPU type(micro architectures and NUMA mode) has
different balance points between the TLB flush all and multiple invlpg.
And there also has cases the tlb flush change has no any help.

This patch give a interface to let x86 vendor developers have a chance
to set different shift for different CPU type.

like some machine in my hands, balance points is 16 entries on
Romely-EP; while it is at 8 entries on Bloomfield NHM-EP; and is 256 on
IVB mobile CPU. but on model 15 core2 Xeon using invlpg has nothing
help.

For untested machine, do a conservative optimization, same as NHM CPU.

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/include/asm/processor.h |    2 ++
 arch/x86/kernel/cpu/common.c     |   14 ++++++++++++--
 arch/x86/kernel/cpu/intel.c      |   34 ++++++++++++++++++++++++++++++++++
 arch/x86/mm/tlb.c                |    8 ++++----
 include/asm-generic/tlb.h        |    3 ++-
 5 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 797faca..3c0cc3d 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -72,6 +72,8 @@ extern u16 __read_mostly tlb_lli_4m[NR_INFO];
 extern u16 __read_mostly tlb_lld_4k[NR_INFO];
 extern u16 __read_mostly tlb_lld_2m[NR_INFO];
 extern u16 __read_mostly tlb_lld_4m[NR_INFO];
+extern u16 __read_mostly tlb_flushall_shift;
+
 /*
  *  CPU type and hardware bug flags. Kept separately for each CPU.
  *  Members of this structure are referenced in head.S, so think twice
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 0152082..24255ca 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -459,16 +459,26 @@ u16 __read_mostly tlb_lld_4k[NR_INFO];
 u16 __read_mostly tlb_lld_2m[NR_INFO];
 u16 __read_mostly tlb_lld_4m[NR_INFO];
 
+/*
+ * tlb_flushall_shift shows the balance point in replacing cr3 write
+ * with multiple 'invlpg'. It will do this replacement when
+ *   flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
+ * If tlb_flushall_shift is -1, means the replacement will be disabled.
+ */
+u16 __read_mostly tlb_flushall_shift;
+
 void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
 {
 	if (c->x86_vendor == X86_VENDOR_INTEL)
 		intel_cpu_detect_tlb(c);
 
 	printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
-		"Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
+		"Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n"	     \
+		"tlb_flushall_shift is 0x%x\n",
 		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
 		tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
-		tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES]);
+		tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
+		tlb_flushall_shift);
 }
 
 void __cpuinit detect_ht(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 28ecd1b..bb90754 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -610,6 +610,39 @@ static void __cpuinit intel_tlb_lookup(const unsigned char desc)
 	}
 }
 
+static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c)
+{
+	if (!cpu_has_invlpg) {
+		tlb_flushall_shift = -1;
+		return;
+	}
+	switch ((c->x86 << 8) + c->x86_model) {
+	case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
+	case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
+	case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
+	case 0x61d: /* six-core 45 nm xeon "Dunnington" */
+		tlb_flushall_shift = -1;
+		break;
+	case 0x61a: /* 45 nm nehalem, "Bloomfield" */
+	case 0x61e: /* 45 nm nehalem, "Lynnfield" */
+	case 0x625: /* 32 nm nehalem, "Clarkdale" */
+	case 0x62c: /* 32 nm nehalem, "Gulftown" */
+	case 0x62e: /* 45 nm nehalem-ex, "Beckton" */
+	case 0x62f: /* 32 nm Xeon E7 */
+		tlb_flushall_shift = 6;
+		break;
+	case 0x62a: /* SandyBridge */
+	case 0x62d: /* SandyBridge, "Romely-EP" */
+		tlb_flushall_shift = 5;
+		break;
+	case 0x63a: /* Ivybridge */
+		tlb_flushall_shift = 1;
+		break;
+	default:
+		tlb_flushall_shift = 6;
+	}
+}
+
 void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
 {
 	int i, j, n;
@@ -630,6 +663,7 @@ void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
 		for (j = 1 ; j < 16 ; j++)
 			intel_tlb_lookup(desc[j]);
 	}
+	intel_tlb_flushall_shift_set(c);
 }
 
 static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 22e5bb1..5bf4e85 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -314,8 +314,6 @@ void flush_tlb_mm(struct mm_struct *mm)
 	preempt_enable();
 }
 
-#define FLUSHALL_BAR	16
-
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline int has_large_page(struct mm_struct *mm,
 				 unsigned long start, unsigned long end)
@@ -350,7 +348,8 @@ void flush_tlb_range(struct vm_area_struct *vma,
 {
 	struct mm_struct *mm;
 
-	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) {
+	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
+			|| tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
 flush_all:
 		flush_tlb_mm(vma->vm_mm);
 		return;
@@ -371,7 +370,8 @@ flush_all:
 			act_entries = tlb_entries > mm->total_vm ?
 					mm->total_vm : tlb_entries;
 
-			if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR)
+			if ((end - start) >> PAGE_SHIFT >
+					act_entries >> tlb_flushall_shift)
 				local_flush_tlb();
 			else {
 				if (has_large_page(mm, start, end)) {
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index f96a5b5..75e888b 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -113,7 +113,8 @@ static inline int tlb_fast_mode(struct mmu_gather *tlb)
 
 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
 void tlb_flush_mmu(struct mmu_gather *tlb);
-void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end);
+void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
+							unsigned long end);
 int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
 
 /* tlb_remove_page
-- 
1.7.5.4

Re: [PATCH v5 5/7] x86/tlb: add tlb_flushall_shift for specific CPU

[H. Peter Anvin]

On 05/15/2012 11:49 PM, Alex Shi wrote:
> +	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
> +			|| tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {

I asked for you to fold the cpu_has_invlpg test into tlb_flushall_shift.
 Also, (u16)TLB_FLUSH_ALL really is hit up with the ugly stick... why on
earth is this u16?  It's a bit shift, so it can't be bigger than 6 bits
anyway... and if you want to be able to use -1 as a sentinel you might
as well make it a signed value and use < 0 or == -1 as the test.

Am I missing something really fundamental here?

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH v5 5/7] x86/tlb: add tlb_flushall_shift for specific CPU

[Alex Shi]

On 05/17/2012 01:55 AM, H. Peter Anvin wrote:

> On 05/15/2012 11:49 PM, Alex Shi wrote:
>> +	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
>> +			|| tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
> 
> I asked for you to fold the cpu_has_invlpg test into tlb_flushall_shift.
>  Also, (u16)TLB_FLUSH_ALL really is hit up with the ugly stick... why on
> earth is this u16?  It's a bit shift, so it can't be bigger than 6 bits
> anyway... and if you want to be able to use -1 as a sentinel you might
> as well make it a signed value and use < 0 or == -1 as the test.


!cpu_has_invlpg be moved in next patch, but surely, it is better to be
removed here too.

Sure, u16 make code looks ugly. So, s8 is better and worth to add value
check on later knob patch. Since this change cause later patch
modification. I am going to re-update whole patches.
>

> Am I missing something really fundamental here?


No.

> 
> 	-hpa
> 

[PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

Not every flush_tlb_mm execution moment is really need to evacuate all
TLB entries, like in munmap, just few 'invlpg' is better for whole
process performance, since it leaves most of TLB entries for later
accessing.

This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
in cases.

The performance balance points checking is left in __flush_tlb_range()

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/include/asm/tlb.h      |    2 +-
 arch/x86/include/asm/tlbflush.h |    5 +-
 arch/x86/mm/pgtable.c           |    2 +-
 arch/x86/mm/tlb.c               |  114 +++++++++++++++++++-------------------
 fs/proc/task_mmu.c              |    2 +-
 include/asm-generic/tlb.h       |    5 +-
 include/asm-generic/tlbflush.h  |    3 +-
 kernel/fork.c                   |    2 +-
 mm/memory.c                     |    9 ++--
 9 files changed, 74 insertions(+), 70 deletions(-)

diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215f..505fdfe 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -4,7 +4,7 @@
 #define tlb_start_vma(tlb, vma) do { } while (0)
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+#define tlb_flush(tlb, start, end) flush_tlb_mm((tlb)->mm, start, end)
 
 #include <asm-generic/tlb.h>
 
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c39c94e..1d07cf1 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -85,7 +85,8 @@ static inline void __flush_tlb_one(unsigned long addr)
 #define flush_tlb_all() __flush_tlb_all()
 #define local_flush_tlb() __flush_tlb()
 
-static inline void flush_tlb_mm(struct mm_struct *mm)
+static inline void flush_tlb_mm(struct mm_struct *mm,
+			unsigned long start, unsigned long end)
 {
 	if (mm == current->active_mm)
 		__flush_tlb();
@@ -124,7 +125,7 @@ static inline void reset_lazy_tlbstate(void)
 
 extern void flush_tlb_all(void);
 extern void flush_tlb_current_task(void);
-extern void flush_tlb_mm(struct mm_struct *);
+extern void flush_tlb_mm(struct mm_struct *, unsigned long, unsigned long);
 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
 extern void flush_tlb_range(struct vm_area_struct *vma,
 				   unsigned long start, unsigned long end);
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 8573b83..5aea5b0 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -168,7 +168,7 @@ void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
 	 * section 8.1: in PAE mode we explicitly have to flush the
 	 * TLB via cr3 if the top-level pgd is changed...
 	 */
-	flush_tlb_mm(mm);
+	flush_tlb_mm(mm, 0UL, -1UL);
 }
 #else  /* !CONFIG_X86_PAE */
 
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5bf4e85..b2e25e8 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -298,22 +298,6 @@ void flush_tlb_current_task(void)
 	preempt_enable();
 }
 
-void flush_tlb_mm(struct mm_struct *mm)
-{
-	preempt_disable();
-
-	if (current->active_mm == mm) {
-		if (current->mm)
-			local_flush_tlb();
-		else
-			leave_mm(smp_processor_id());
-	}
-	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
-		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
-
-	preempt_enable();
-}
-
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline int has_large_page(struct mm_struct *mm,
 				 unsigned long start, unsigned long end)
@@ -343,61 +327,77 @@ static inline int has_large_page(struct mm_struct *mm,
 	return 0;
 }
 #endif
-void flush_tlb_range(struct vm_area_struct *vma,
-				   unsigned long start, unsigned long end)
-{
-	struct mm_struct *mm;
 
-	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
-			|| tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
-flush_all:
-		flush_tlb_mm(vma->vm_mm);
-		return;
-	}
+void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
+				unsigned long end, unsigned long vmflag)
+{
+	unsigned long addr;
+	unsigned act_entries, tlb_entries = 0;
 
 	preempt_disable();
-	mm = vma->vm_mm;
-	if (current->active_mm == mm) {
-		if (current->mm) {
-			unsigned long addr, vmflag = vma->vm_flags;
-			unsigned act_entries, tlb_entries = 0;
+	if (current->active_mm != mm)
+		goto flush_all;
 
-			if (vmflag & VM_EXEC)
-				tlb_entries = tlb_lli_4k[ENTRIES];
-			else
-				tlb_entries = tlb_lld_4k[ENTRIES];
+	if (!current->mm) {
+		leave_mm(smp_processor_id());
+		goto flush_all;
+	}
 
-			act_entries = tlb_entries > mm->total_vm ?
-					mm->total_vm : tlb_entries;
+	if (end == TLB_FLUSH_ALL ||
+			tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
+		local_flush_tlb();
+		goto flush_all;
+	}
 
-			if ((end - start) >> PAGE_SHIFT >
-					act_entries >> tlb_flushall_shift)
-				local_flush_tlb();
-			else {
-				if (has_large_page(mm, start, end)) {
-					preempt_enable();
-					goto flush_all;
-				}
-				for (addr = start; addr <= end;
-						addr += PAGE_SIZE)
-					__flush_tlb_single(addr);
+	if (vmflag & VM_EXEC)
+		tlb_entries = tlb_lli_4k[ENTRIES];
+	else
+		tlb_entries = tlb_lld_4k[ENTRIES];
+	act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;
 
-				if (cpumask_any_but(mm_cpumask(mm),
-					smp_processor_id()) < nr_cpu_ids)
-					flush_tlb_others(mm_cpumask(mm), mm,
-								start, end);
-				preempt_enable();
-				return;
-			}
-		} else {
-			leave_mm(smp_processor_id());
+	if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
+		local_flush_tlb();
+	else {
+		if (has_large_page(mm, start, end)) {
+			local_flush_tlb();
+			goto flush_all;
 		}
+		for (addr = start; addr <= end;	addr += PAGE_SIZE)
+			__flush_tlb_single(addr);
+
+		if (cpumask_any_but(mm_cpumask(mm),
+				smp_processor_id()) < nr_cpu_ids)
+			flush_tlb_others(mm_cpumask(mm), mm, start, end);
+		preempt_enable();
+		return;
 	}
+
+flush_all:
 	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
 		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
 	preempt_enable();
 }
 
+void flush_tlb_mm(struct mm_struct *mm, unsigned long start, unsigned long end)
+{
+	if (!cpu_has_invlpg)
+		__flush_tlb_range(mm, 0UL, TLB_FLUSH_ALL, 0UL);
+	else
+		__flush_tlb_range(mm, start, end, 0UL);
+}
+
+void flush_tlb_range(struct vm_area_struct *vma,
+				   unsigned long start, unsigned long end)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	unsigned long vmflag = vma->vm_flags;
+
+	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB)
+		__flush_tlb_range(mm, 0UL, TLB_FLUSH_ALL, 0UL);
+	else
+		__flush_tlb_range(mm, start, end, vmflag);
+}
+
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
 {
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 2d60492..5728c8f 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -660,7 +660,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
 			walk_page_range(vma->vm_start, vma->vm_end,
 					&clear_refs_walk);
 		}
-		flush_tlb_mm(mm);
+		flush_tlb_mm(mm, 0UL, -1UL);
 		up_read(&mm->mmap_sem);
 		mmput(mm);
 	}
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 75e888b..3a24c97 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -112,7 +112,8 @@ static inline int tlb_fast_mode(struct mmu_gather *tlb)
 }
 
 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
-void tlb_flush_mmu(struct mmu_gather *tlb);
+void tlb_flush_mmu(struct mmu_gather *tlb, unsigned long start,
+							unsigned long end);
 void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
 							unsigned long end);
 int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
@@ -124,7 +125,7 @@ int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
 	if (!__tlb_remove_page(tlb, page))
-		tlb_flush_mmu(tlb);
+		tlb_flush_mmu(tlb, 0UL, -1UL);
 }
 
 /**
diff --git a/include/asm-generic/tlbflush.h b/include/asm-generic/tlbflush.h
index d6d0a88..db1d4bb 100644
--- a/include/asm-generic/tlbflush.h
+++ b/include/asm-generic/tlbflush.h
@@ -11,7 +11,8 @@
 
 #include <linux/bug.h>
 
-static inline void flush_tlb_mm(struct mm_struct *mm)
+static inline void flush_tlb_mm(struct mm_struct *mm,
+			unsigned long start, unsigned long end)
 {
 	BUG();
 }
diff --git a/kernel/fork.c b/kernel/fork.c
index b9372a0..a4f0c64 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -427,7 +427,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
 	retval = 0;
 out:
 	up_write(&mm->mmap_sem);
-	flush_tlb_mm(oldmm);
+	flush_tlb_mm(oldmm, 0UL, -1UL);
 	up_write(&oldmm->mmap_sem);
 	return retval;
 fail_nomem_anon_vma_fork:
diff --git a/mm/memory.c b/mm/memory.c
index 6105f47..05e2c2e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -218,14 +218,15 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
 #endif
 }
 
-void tlb_flush_mmu(struct mmu_gather *tlb)
+void tlb_flush_mmu(struct mmu_gather *tlb,
+			unsigned long start, unsigned long end)
 {
 	struct mmu_gather_batch *batch;
 
 	if (!tlb->need_flush)
 		return;
 	tlb->need_flush = 0;
-	tlb_flush(tlb);
+	tlb_flush(tlb, start, end);
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	tlb_table_flush(tlb);
 #endif
@@ -248,7 +249,7 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
 {
 	struct mmu_gather_batch *batch, *next;
 
-	tlb_flush_mmu(tlb);
+	tlb_flush_mmu(tlb, start, end);
 
 	/* keep the page table cache within bounds */
 	check_pgt_cache();
@@ -1204,7 +1205,7 @@ again:
 	 */
 	if (force_flush) {
 		force_flush = 0;
-		tlb_flush_mmu(tlb);
+		tlb_flush_mmu(tlb, addr, end);
 		if (addr != end)
 			goto again;
 	}
-- 
1.7.5.4

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Tue, 2012-05-15 at 16:55 +0800, Alex Shi wrote:
> Not every flush_tlb_mm execution moment is really need to evacuate all
> TLB entries, like in munmap, just few 'invlpg' is better for whole
> process performance, since it leaves most of TLB entries for later
> accessing.
> 
> This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start,
> end)
> in cases. 

Ok, so you're not getting it.

NACKED-BY: Peter Zijlstra <a.p.zijlstra@chello.nl>

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Nick Piggin]

So this should go to linux-arch...

On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
> Not every flush_tlb_mm execution moment is really need to evacuate all
> TLB entries, like in munmap, just few 'invlpg' is better for whole
> process performance, since it leaves most of TLB entries for later
> accessing.
>
> This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
> in cases.

What happened with Peter's comment about using flush_tlb_range for this?

flush_tlb_mm() API should just stay unchanged AFAIKS.

Then you need to work out the best way to give range info to the tlb/mmu gather
API. Possibly passing in the rage for that guy is OK, which x86 can
then implement
as flush range.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Nick Piggin]

On 15 May 2012 19:15, Nick Piggin <npiggin@gmail.com> wrote:
> So this should go to linux-arch...
>
> On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
>> Not every flush_tlb_mm execution moment is really need to evacuate all
>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>> process performance, since it leaves most of TLB entries for later
>> accessing.

Did you have microbenchmarks for this like your mprotect numbers,
by the way? Test munmap numbers and see how that looks. Also,
does it show up on any macro-benchmarks like specjbb?

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Luming Yu]

On Tue, May 15, 2012 at 5:17 PM, Nick Piggin <npiggin@gmail.com> wrote:
> On 15 May 2012 19:15, Nick Piggin <npiggin@gmail.com> wrote:
>> So this should go to linux-arch...
>>
>> On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
>>> Not every flush_tlb_mm execution moment is really need to evacuate all
>>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>>> process performance, since it leaves most of TLB entries for later
>>> accessing.
>
> Did you have microbenchmarks for this like your mprotect numbers,
> by the way? Test munmap numbers and see how that looks. Also,

Might be off topic, but I just spent few minutes to test out the difference
between write CR3 vs. invlpg on a pretty old but still reliable P4 desktop
with my simple hardware latency and bandwidth test tool I posted for
RFC several weeks ago on LKML.

Both __native_flush_tlb() and __native_flush_tlb_single(...)
introduced roughly 1 ns  latency to tsc sampling executed in
stop_machine_context in two logical CPUs

Just to fuel the discussion. :-)

Cheers,
/l

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
> 
> 
> Both __native_flush_tlb() and __native_flush_tlb_single(...)
> introduced roughly 1 ns  latency to tsc sampling executed in
> stop_machine_context in two logical CPUs 

But you have to weight that against the cost of re-population, and
that's the difficult bit, since we have no clue how many tlb entries are
in use by the current cr3.

It might be possible for intel to give us this information, I've asked
for something similar for cachelines.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Luming Yu]

On Tue, May 15, 2012 at 9:06 PM, Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
>>
>>
>> Both __native_flush_tlb() and __native_flush_tlb_single(...)
>> introduced roughly 1 ns  latency to tsc sampling executed in

Fix typo, I just observed 1us with current tool, I would check if I
can push the accuracy to nanoseconds level.

>> stop_machine_context in two logical CPUs
>
> But you have to weight that against the cost of re-population, and

Right, it's hard to detect, but I will try if I can get measurement
done in a simple test tool to help people measure
this kind of stuff in few minutes.

> that's the difficult bit, since we have no clue how many tlb entries are
> in use by the current cr3.
>
> It might be possible for intel to give us this information, I've asked
> for something similar for cachelines.

This is the official document
http://www.intel.com/content/dam/doc/manual/64-ia-32-architectures-optimization-manual.pdf

Let me know if it can answer your question.

>

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/15/2012 09:27 PM, Luming Yu wrote:

> On Tue, May 15, 2012 at 9:06 PM, Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
>> On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
>>>
>>>
>>> Both __native_flush_tlb() and __native_flush_tlb_single(...)
>>> introduced roughly 1 ns  latency to tsc sampling executed in
> 
> Fix typo, I just observed 1us with current tool, I would check if I
> can push the accuracy to nanoseconds level.
> 
>>> stop_machine_context in two logical CPUs
>>
>> But you have to weight that against the cost of re-population, and
> 
> Right, it's hard to detect, but I will try if I can get measurement
> done in a simple test tool to help people measure
> this kind of stuff in few minutes.
> 
>> that's the difficult bit, since we have no clue how many tlb entries are
>> in use by the current cr3.
>>
>> It might be possible for intel to give us this information, I've asked
>> for something similar for cachelines.
> 
> This is the official document
> http://www.intel.com/content/dam/doc/manual/64-ia-32-architectures-optimization-manual.pdf
> 


Please, such huge documents! and it also has no such info.

> Let me know if it can answer your question.
> 
>>

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/15/2012 09:06 PM, Peter Zijlstra wrote:

> On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
>>
>>
>> Both __native_flush_tlb() and __native_flush_tlb_single(...)
>> introduced roughly 1 ns  latency to tsc sampling executed in
>> stop_machine_context in two logical CPUs 
> 
> But you have to weight that against the cost of re-population, and
> that's the difficult bit, since we have no clue how many tlb entries are
> in use by the current cr3.
> 
> It might be possible for intel to give us this information, I've asked
> for something similar for cachelines.
> 


I don't know if such info exist in cpu. Maybe US engineer know more.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Steven Rostedt]

On Tue, 2012-05-15 at 15:06 +0200, Peter Zijlstra wrote:
> On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
> > 
> > 
> > Both __native_flush_tlb() and __native_flush_tlb_single(...)
> > introduced roughly 1 ns  latency to tsc sampling executed in
> > stop_machine_context in two logical CPUs 
> 
> But you have to weight that against the cost of re-population, and
> that's the difficult bit, since we have no clue how many tlb entries are
> in use by the current cr3.
> 
> It might be possible for intel to give us this information, I've asked
> for something similar for cachelines.

What information? The # of tlb entries in use?

-- Steve

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Borislav Petkov]

On Tue, May 15, 2012 at 09:39:05AM -0400, Steven Rostedt wrote:
> > But you have to weight that against the cost of re-population, and
> > that's the difficult bit, since we have no clue how many tlb entries are
> > in use by the current cr3.
> > 
> > It might be possible for intel to give us this information, I've asked
> > for something similar for cachelines.
> 
> What information? The # of tlb entries in use?

... by the current %cr3, yes.

And also, before we delve into details, we still don't have a
representative benchmark where this shows any improvement.

-- 
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
GM: Alberto Bozzo
Reg: Dornach, Landkreis Muenchen
HRB Nr. 43632 WEEE Registernr: 129 19551

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Luming Yu]

On Tue, May 15, 2012 at 8:58 PM, Luming Yu <luming.yu@gmail.com> wrote:
> On Tue, May 15, 2012 at 5:17 PM, Nick Piggin <npiggin@gmail.com> wrote:
>> On 15 May 2012 19:15, Nick Piggin <npiggin@gmail.com> wrote:
>>> So this should go to linux-arch...
>>>
>>> On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
>>>> Not every flush_tlb_mm execution moment is really need to evacuate all
>>>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>>>> process performance, since it leaves most of TLB entries for later
>>>> accessing.
>>
>> Did you have microbenchmarks for this like your mprotect numbers,
>> by the way? Test munmap numbers and see how that looks. Also,
>
> Might be off topic, but I just spent few minutes to test out the difference
> between write CR3 vs. invlpg on a pretty old but still reliable P4 desktop
> with my simple hardware latency and bandwidth test tool I posted for
> RFC several weeks ago on LKML.
>
> Both __native_flush_tlb() and __native_flush_tlb_single(...)
> introduced roughly 1 ns  latency to tsc sampling executed in

sorry, typo, 1us.. but I should capture nanosecond data. :-(

> stop_machine_context in two logical CPUs
>
> Just to fuel the discussion. :-)
>
> Cheers,
> /l

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/15/2012 05:17 PM, Nick Piggin wrote:

> On 15 May 2012 19:15, Nick Piggin <npiggin@gmail.com> wrote:
>> So this should go to linux-arch...
>>
>> On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
>>> Not every flush_tlb_mm execution moment is really need to evacuate all
>>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>>> process performance, since it leaves most of TLB entries for later
>>> accessing.
> 
> Did you have microbenchmarks for this like your mprotect numbers,
> by the way? Test munmap numbers and see how that looks. Also,
> does it show up on any macro-benchmarks like specjbb?


Yongjie has tested the patchset and get some positive data on Virtual
machine.
Yongjie, could you like to share your data?

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Tue, 2012-05-15 at 19:15 +1000, Nick Piggin wrote:
> So this should go to linux-arch...
> 
> On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
> > Not every flush_tlb_mm execution moment is really need to evacuate all
> > TLB entries, like in munmap, just few 'invlpg' is better for whole
> > process performance, since it leaves most of TLB entries for later
> > accessing.
> >
> > This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
> > in cases.
> 
> What happened with Peter's comment about using flush_tlb_range for this?
> 
> flush_tlb_mm() API should just stay unchanged AFAIKS.
> 
> Then you need to work out the best way to give range info to the tlb/mmu gather
> API. Possibly passing in the rage for that guy is OK, which x86 can
> then implement
> as flush range.

Right, most archs that have tlb_flush_range() do range tracking in
mmu_gather. Our TLB ops fully support that, there's absolutely no need
to go change the interface for thos.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Nick Piggin]

On 15 May 2012 19:18, Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> On Tue, 2012-05-15 at 19:15 +1000, Nick Piggin wrote:
>> So this should go to linux-arch...
>>
>> On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
>> > Not every flush_tlb_mm execution moment is really need to evacuate all
>> > TLB entries, like in munmap, just few 'invlpg' is better for whole
>> > process performance, since it leaves most of TLB entries for later
>> > accessing.
>> >
>> > This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
>> > in cases.
>>
>> What happened with Peter's comment about using flush_tlb_range for this?
>>
>> flush_tlb_mm() API should just stay unchanged AFAIKS.
>>
>> Then you need to work out the best way to give range info to the tlb/mmu gather
>> API. Possibly passing in the rage for that guy is OK, which x86 can
>> then implement
>> as flush range.
>
> Right, most archs that have tlb_flush_range() do range tracking in
> mmu_gather. Our TLB ops fully support that, there's absolutely no need
> to go change the interface for thos.

It could be warranted to change tlb_flush_mmu to a range API to
avoid doing the per-entry tracking which those architectures do?

The callers have range available easily, so ignoring those could be
noop for generic helpers.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Tue, 2012-05-15 at 19:52 +1000, Nick Piggin wrote:
> 
> It could be warranted to change tlb_flush_mmu to a range API to
> avoid doing the per-entry tracking which those architectures do? 

The per-entry could result in a much smaller range, there's no point in
flushing tlbs for unpopulated pages.

Anyway, I don't think even think we'd need to change the API for that,
you could track the entire range through tlb_start_vma() if you wanted
(although nobody does that IIRC).

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Nick Piggin]

On 15 May 2012 20:00, Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> On Tue, 2012-05-15 at 19:52 +1000, Nick Piggin wrote:
>>
>> It could be warranted to change tlb_flush_mmu to a range API to
>> avoid doing the per-entry tracking which those architectures do?
>
> The per-entry could result in a much smaller range, there's no point in
> flushing tlbs for unpopulated pages.

Well common case for small ranges hopefully would be quite dense
I think. It could be not worth the extra work (although maybe it would
be).

>
> Anyway, I don't think even think we'd need to change the API for that,
> you could track the entire range through tlb_start_vma() if you wanted
> (although nobody does that IIRC).

I'm not sure if you can do that very well, because the tlb might have to
be flushed part way through a vma when we fill up the gather, so you
don't want to flush the full range each time.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Tue, 2012-05-15 at 20:06 +1000, Nick Piggin wrote:
> On 15 May 2012 20:00, Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> > On Tue, 2012-05-15 at 19:52 +1000, Nick Piggin wrote:
> >>
> >> It could be warranted to change tlb_flush_mmu to a range API to
> >> avoid doing the per-entry tracking which those architectures do?
> >
> > The per-entry could result in a much smaller range, there's no point in
> > flushing tlbs for unpopulated pages.
> 
> Well common case for small ranges hopefully would be quite dense
> I think. It could be not worth the extra work (although maybe it would
> be).
> 
> >
> > Anyway, I don't think even think we'd need to change the API for that,
> > you could track the entire range through tlb_start_vma() if you wanted
> > (although nobody does that IIRC).
> 
> I'm not sure if you can do that very well, because the tlb might have to
> be flushed part way through a vma when we fill up the gather, so you
> don't want to flush the full range each time.


Fair enough. But that's still an entirely unrelated optimization and
should go with proper benchmarking and preferably across all archs that
have flush_tlb_range() :-)

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/15/2012 05:18 PM, Peter Zijlstra wrote:

> On Tue, 2012-05-15 at 19:15 +1000, Nick Piggin wrote:
>> So this should go to linux-arch...
>>
>> On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
>>> Not every flush_tlb_mm execution moment is really need to evacuate all
>>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>>> process performance, since it leaves most of TLB entries for later
>>> accessing.
>>>
>>> This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
>>> in cases.
>>
>> What happened with Peter's comment about using flush_tlb_range for this?
>>
>> flush_tlb_mm() API should just stay unchanged AFAIKS.
>>
>> Then you need to work out the best way to give range info to the tlb/mmu gather
>> API. Possibly passing in the rage for that guy is OK, which x86 can
>> then implement
>> as flush range.
> 
> Right, most archs that have tlb_flush_range() do range tracking in
> mmu_gather. Our TLB ops fully support that, there's absolutely no need
> to go change the interface for thos.



Ok. this should be your wanted, 
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)

If no objection, I will modify the patch accordingly. 

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/15/2012 05:15 PM, Nick Piggin wrote:

> So this should go to linux-arch...
> 
> On 15 May 2012 18:55, Alex Shi <alex.shi@intel.com> wrote:
>> Not every flush_tlb_mm execution moment is really need to evacuate all
>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>> process performance, since it leaves most of TLB entries for later
>> accessing.
>>
>> This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
>> in cases.
> 
> What happened with Peter's comment about using flush_tlb_range for this?
> 
> flush_tlb_mm() API should just stay unchanged AFAIKS.
> 
> Then you need to work out the best way to give range info to the tlb/mmu gather
> API. Possibly passing in the rage for that guy is OK, which x86 can
> then implement
> as flush range.


Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally. 

So, Peter, the correct change should like following, am I right?

-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Tue, 2012-05-15 at 21:24 +0800, Alex Shi wrote:

> Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally. 
> 
> So, Peter, the correct change should like following, am I right?
> 
> -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
> +#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)

No.. the correct change is to do range tracking like the other archs
that support flush_tlb_range() do.

You do not modify the tlb interface.

Again, see: http://marc.info/?l=linux-arch&m=129952026504268&w=2

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Tue, 2012-05-15 at 16:36 +0200, Peter Zijlstra wrote:
> On Tue, 2012-05-15 at 21:24 +0800, Alex Shi wrote:
> 
> > Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally. 
> > 
> > So, Peter, the correct change should like following, am I right?
> > 
> > -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
> > +#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
> 
> No.. the correct change is to do range tracking like the other archs
> that support flush_tlb_range() do.
> 
> You do not modify the tlb interface.
> 
> Again, see: http://marc.info/?l=linux-arch&m=129952026504268&w=2

Just to be _very_ clear, you do not modify:

 mm/memory.c                     |    9 ++--
 kernel/fork.c                   |    2 +-   
 fs/proc/task_mmu.c              |    2 +-

As it stands your patch breaks compilation on a whole bunch of
architectures.

If you touch the TLB interface, you get to touch _ALL_ architectures.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/15/2012 10:57 PM, Peter Zijlstra wrote:

> On Tue, 2012-05-15 at 16:36 +0200, Peter Zijlstra wrote:
>> On Tue, 2012-05-15 at 21:24 +0800, Alex Shi wrote:
>>
>>> Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally. 
>>>
>>> So, Peter, the correct change should like following, am I right?
>>>
>>> -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
>>> +#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
>>
>> No.. the correct change is to do range tracking like the other archs
>> that support flush_tlb_range() do.
>>
>> You do not modify the tlb interface.
>>
>> Again, see: http://marc.info/?l=linux-arch&m=129952026504268&w=2


this code is for multiple architecture, but x86 still need implement
'flush tlb range' with 'invlpg'.

> 
> Just to be _very_ clear, you do not modify:
> 
>  mm/memory.c                     |    9 ++--
>  kernel/fork.c                   |    2 +-   
>  fs/proc/task_mmu.c              |    2 +-
> 


Thanks a lot. I see.

> As it stands your patch breaks compilation on a whole bunch of
> architectures.
> 
> If you touch the TLB interface, you get to touch _ALL_ architectures.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/15/2012 10:57 PM, Peter Zijlstra wrote:

> On Tue, 2012-05-15 at 16:36 +0200, Peter Zijlstra wrote:
>> On Tue, 2012-05-15 at 21:24 +0800, Alex Shi wrote:
>>
>>> Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally. 
>>>
>>> So, Peter, the correct change should like following, am I right?
>>>
>>> -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
>>> +#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
>>
>> No.. the correct change is to do range tracking like the other archs
>> that support flush_tlb_range() do.
>>
>> You do not modify the tlb interface.
>>
>> Again, see: http://marc.info/?l=linux-arch&m=129952026504268&w=2
> 
> Just to be _very_ clear, you do not modify:
> 
>  mm/memory.c                     |    9 ++--
>  kernel/fork.c                   |    2 +-   
>  fs/proc/task_mmu.c              |    2 +-
> 
> As it stands your patch breaks compilation on a whole bunch of
> architectures.
> 
> If you touch the TLB interface, you get to touch _ALL_ architectures.



Thanks for Nick and Peter's comments. I rewrite the patch according to
your opinions. Is this met your expectation? 

----
>From a01864af75d8c86668f4fa73d6ca18ebe5835b18 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Mon, 14 May 2012 09:17:03 +0800
Subject: [PATCH 6/7] x86/tlb: optimizing tlb_finish_mmu on x86

Not every tlb_flush execution moment is really need to evacuate all
TLB entries, like in munmap, just few 'invlpg' is better for whole
process performance, since it leaves most of TLB entries for later
accessing.

Thanks for Peter Zijlstra reminder, tlb interfaces in mm/memory.c
are for all architectures. So, I keep current interfaces, just
reimplement x86 specific 'tlb_flush' only. Some of ideas also are
picked up from Peter's old patch, thanks!

This patch also rewrite flush_tlb_range for 2 purposes:
1, split it out to get flush_blt_mm_range function.
2, clean up to reduce line breaking, thanks for Borislav's input.

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/include/asm/tlb.h      |    9 +++-
 arch/x86/include/asm/tlbflush.h |    2 +
 arch/x86/mm/tlb.c               |  120 +++++++++++++++++++++------------------
 include/asm-generic/tlb.h       |    2 +
 mm/memory.c                     |    6 ++
 5 files changed, 82 insertions(+), 57 deletions(-)

diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215f..4fef207 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -4,7 +4,14 @@
 #define tlb_start_vma(tlb, vma) do { } while (0)
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+
+#define tlb_flush(tlb)							\
+{									\
+	if (tlb->fullmm == 0)						\
+		flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL);	\
+	else								\
+		flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL);	\
+}
 
 #include <asm-generic/tlb.h>
 
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c39c94e..0107f3c 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -128,6 +128,8 @@ extern void flush_tlb_mm(struct mm_struct *);
 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
 extern void flush_tlb_range(struct vm_area_struct *vma,
 				   unsigned long start, unsigned long end);
+extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+				unsigned long end, unsigned long vmflag);
 
 #define flush_tlb()	flush_tlb_current_task()
 
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5bf4e85..52f6a5a 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -298,22 +298,6 @@ void flush_tlb_current_task(void)
 	preempt_enable();
 }
 
-void flush_tlb_mm(struct mm_struct *mm)
-{
-	preempt_disable();
-
-	if (current->active_mm == mm) {
-		if (current->mm)
-			local_flush_tlb();
-		else
-			leave_mm(smp_processor_id());
-	}
-	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
-		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
-
-	preempt_enable();
-}
-
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline int has_large_page(struct mm_struct *mm,
 				 unsigned long start, unsigned long end)
@@ -343,61 +327,85 @@ static inline int has_large_page(struct mm_struct *mm,
 	return 0;
 }
 #endif
-void flush_tlb_range(struct vm_area_struct *vma,
-				   unsigned long start, unsigned long end)
+
+void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+				unsigned long end, unsigned long vmflag)
 {
-	struct mm_struct *mm;
+	unsigned long addr;
+	unsigned act_entries, tlb_entries = 0;
 
-	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
-			|| tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
-flush_all:
-		flush_tlb_mm(vma->vm_mm);
-		return;
+	preempt_disable();
+	if (current->active_mm != mm)
+		goto flush_all;
+
+	if (!current->mm) {
+		leave_mm(smp_processor_id());
+		goto flush_all;
 	}
 
-	preempt_disable();
-	mm = vma->vm_mm;
-	if (current->active_mm == mm) {
-		if (current->mm) {
-			unsigned long addr, vmflag = vma->vm_flags;
-			unsigned act_entries, tlb_entries = 0;
+	if (end == TLB_FLUSH_ALL ||
+			tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
+		local_flush_tlb();
+		goto flush_all;
+	}
 
-			if (vmflag & VM_EXEC)
-				tlb_entries = tlb_lli_4k[ENTRIES];
-			else
-				tlb_entries = tlb_lld_4k[ENTRIES];
+	if (vmflag & VM_EXEC)
+		tlb_entries = tlb_lli_4k[ENTRIES];
+	else
+		tlb_entries = tlb_lld_4k[ENTRIES];
+	act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;
 
-			act_entries = tlb_entries > mm->total_vm ?
-					mm->total_vm : tlb_entries;
+	if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
+		local_flush_tlb();
+	else {
+		if (has_large_page(mm, start, end)) {
+			local_flush_tlb();
+			goto flush_all;
+		}
+		for (addr = start; addr <= end;	addr += PAGE_SIZE)
+			__flush_tlb_single(addr);
 
-			if ((end - start) >> PAGE_SHIFT >
-					act_entries >> tlb_flushall_shift)
-				local_flush_tlb();
-			else {
-				if (has_large_page(mm, start, end)) {
-					preempt_enable();
-					goto flush_all;
-				}
-				for (addr = start; addr <= end;
-						addr += PAGE_SIZE)
-					__flush_tlb_single(addr);
+		if (cpumask_any_but(mm_cpumask(mm),
+				smp_processor_id()) < nr_cpu_ids)
+			flush_tlb_others(mm_cpumask(mm), mm, start, end);
+		preempt_enable();
+		return;
+	}
 
-				if (cpumask_any_but(mm_cpumask(mm),
-					smp_processor_id()) < nr_cpu_ids)
-					flush_tlb_others(mm_cpumask(mm), mm,
-								start, end);
-				preempt_enable();
-				return;
-			}
-		} else {
+flush_all:
+	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+	preempt_enable();
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+	preempt_disable();
+
+	if (current->active_mm == mm) {
+		if (current->mm)
+			local_flush_tlb();
+		else
 			leave_mm(smp_processor_id());
-		}
 	}
 	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
 		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+
 	preempt_enable();
 }
 
+void flush_tlb_range(struct vm_area_struct *vma,
+				   unsigned long start, unsigned long end)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	unsigned long vmflag = vma->vm_flags;
+
+	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB)
+		flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL);
+	else
+		flush_tlb_mm_range(mm, start, end, vmflag);
+}
+
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
 {
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 75e888b..ed6642a 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -86,6 +86,8 @@ struct mmu_gather {
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	struct mmu_table_batch	*batch;
 #endif
+	unsigned long		start;
+	unsigned long		end;
 	unsigned int		need_flush : 1,	/* Did free PTEs */
 				fast_mode  : 1; /* No batching   */
 
diff --git a/mm/memory.c b/mm/memory.c
index 6105f47..b176172 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
 	tlb->mm = mm;
 
 	tlb->fullmm     = fullmm;
+	tlb->start	= -1UL;
+	tlb->end	= 0;
 	tlb->need_flush = 0;
 	tlb->fast_mode  = (num_possible_cpus() == 1);
 	tlb->local.next = NULL;
@@ -248,6 +250,8 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
 {
 	struct mmu_gather_batch *batch, *next;
 
+	tlb->start = start;
+	tlb->end   = end;
 	tlb_flush_mmu(tlb);
 
 	/* keep the page table cache within bounds */
@@ -1204,6 +1208,8 @@ again:
 	 */
 	if (force_flush) {
 		force_flush = 0;
+		tlb->start = addr;
+		tlb->end = end;
 		tlb_flush_mmu(tlb);
 		if (addr != end)
 			goto again;
-- 
1.7.5.4

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Wed, 2012-05-16 at 14:46 +0800, Alex Shi wrote:
> diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
> index 75e888b..ed6642a 100644
> --- a/include/asm-generic/tlb.h
> +++ b/include/asm-generic/tlb.h
> @@ -86,6 +86,8 @@ struct mmu_gather {
>  #ifdef CONFIG_HAVE_RCU_TABLE_FREE
>         struct mmu_table_batch  *batch;
>  #endif
> +       unsigned long           start;
> +       unsigned long           end;
>         unsigned int            need_flush : 1, /* Did free PTEs */
>                                 fast_mode  : 1; /* No batching   */
>  
> diff --git a/mm/memory.c b/mm/memory.c
> index 6105f47..b176172 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
>         tlb->mm = mm;
>  
>         tlb->fullmm     = fullmm;
> +       tlb->start      = -1UL;
> +       tlb->end        = 0;
>         tlb->need_flush = 0;
>         tlb->fast_mode  = (num_possible_cpus() == 1);
>         tlb->local.next = NULL;
> @@ -248,6 +250,8 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
>  {
>         struct mmu_gather_batch *batch, *next;
>  
> +       tlb->start = start;
> +       tlb->end   = end;
>         tlb_flush_mmu(tlb);
>  
>         /* keep the page table cache within bounds */
> @@ -1204,6 +1208,8 @@ again:
>          */
>         if (force_flush) {
>                 force_flush = 0;
> +               tlb->start = addr;
> +               tlb->end = end;
>                 tlb_flush_mmu(tlb);
>                 if (addr != end)
>                         goto again; 


ARGH.. no. What bit about you don't need to modify the generic code
don't you get?

Both ARM and IA64 (and possible others) already do range tracking, you
don't need to modify mm/memory.c _AT_ALL_.

Also, if you modify include/asm-generic/tlb.h to include the ranges it
would be very nice to make that optional, most archs using it won't use
this.

Now IF you're going to change the tlb interface like this, you're going
to get to do it for all architectures, along with a sane benchmark to
show its beneficial to track ranges like this.

But as it stands, people are still questioning the validity of your
mprotect micro-bench, so no, you don't get to change the tlb interface.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Wed, 2012-05-16 at 10:00 +0200, Peter Zijlstra wrote:
> On Wed, 2012-05-16 at 14:46 +0800, Alex Shi wrote:
> > diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
> > index 75e888b..ed6642a 100644
> > --- a/include/asm-generic/tlb.h
> > +++ b/include/asm-generic/tlb.h
> > @@ -86,6 +86,8 @@ struct mmu_gather {
> >  #ifdef CONFIG_HAVE_RCU_TABLE_FREE
> >         struct mmu_table_batch  *batch;
> >  #endif
> > +       unsigned long           start;
> > +       unsigned long           end;
> >         unsigned int            need_flush : 1, /* Did free PTEs */
> >                                 fast_mode  : 1; /* No batching   */
> >  
> > diff --git a/mm/memory.c b/mm/memory.c
> > index 6105f47..b176172 100644
> > --- a/mm/memory.c
> > +++ b/mm/memory.c
> > @@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
> >         tlb->mm = mm;
> >  
> >         tlb->fullmm     = fullmm;
> > +       tlb->start      = -1UL;
> > +       tlb->end        = 0;
> >         tlb->need_flush = 0;
> >         tlb->fast_mode  = (num_possible_cpus() == 1);
> >         tlb->local.next = NULL;

Also, you just broke compilation on a bunch of archs.. again.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/16/2012 04:04 PM, Peter Zijlstra wrote:

> On Wed, 2012-05-16 at 10:00 +0200, Peter Zijlstra wrote:
>> On Wed, 2012-05-16 at 14:46 +0800, Alex Shi wrote:
>>> diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
>>> index 75e888b..ed6642a 100644
>>> --- a/include/asm-generic/tlb.h
>>> +++ b/include/asm-generic/tlb.h
>>> @@ -86,6 +86,8 @@ struct mmu_gather {
>>>  #ifdef CONFIG_HAVE_RCU_TABLE_FREE
>>>         struct mmu_table_batch  *batch;
>>>  #endif
>>> +       unsigned long           start;
>>> +       unsigned long           end;
>>>         unsigned int            need_flush : 1, /* Did free PTEs */
>>>                                 fast_mode  : 1; /* No batching   */
>>>  
>>> diff --git a/mm/memory.c b/mm/memory.c
>>> index 6105f47..b176172 100644
>>> --- a/mm/memory.c
>>> +++ b/mm/memory.c
>>> @@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
>>>         tlb->mm = mm;
>>>  
>>>         tlb->fullmm     = fullmm;
>>> +       tlb->start      = -1UL;
>>> +       tlb->end        = 0;
>>>         tlb->need_flush = 0;
>>>         tlb->fast_mode  = (num_possible_cpus() == 1);
>>>         tlb->local.next = NULL;
> 
> Also, you just broke compilation on a bunch of archs.. again.


Sorry. Do you mean not every archs use 'include/asm-generic/tlb.h', so
the assignment of tlb->start in tlb_gather_mmu make trouble?

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Wed, 2012-05-16 at 16:53 +0800, Alex Shi wrote:
> 
> Sorry. Do you mean not every archs use 'include/asm-generic/tlb.h', so
> the assignment of tlb->start in tlb_gather_mmu make trouble?
> 
Yes exactly. I know you work for Intel, but surely its not forbidden by
contract to look outside of arch/x86/ ? I know!, look at arch/ia64/
that's still Intel.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On Wed, May 16, 2012 at 4:58 PM, Peter Zijlstra <peterz@infradead.org> wrote:
> On Wed, 2012-05-16 at 16:53 +0800, Alex Shi wrote:
>>
>> Sorry. Do you mean not every archs use 'include/asm-generic/tlb.h', so
>> the assignment of tlb->start in tlb_gather_mmu make trouble?
>>
> Yes exactly. I know you work for Intel, but surely its not forbidden by
> contract to look outside of arch/x86/ ? I know!, look at arch/ia64/
> that's still Intel.

:) It is my fault. 'make cscope' just help me focus insight into x86.

But frankly speaking, it looks mess that every arch implement similar fields of
 mmu_gather. If someone can unify the common fields of mmu_gather, and
left private field for specific arch. it will be great and much helpful.

> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Wed, 2012-05-16 at 18:58 +0800, Alex Shi wrote:
> If someone can unify the common fields of mmu_gather, and
> left private field for specific arch. it will be great and much
> helpful. 

I've send you a link to a patch-set that does exactly that twice now.

  http://marc.info/?l=linux-mm&m=129952019004146&w=2

There, 3rd time, now go read all 15 patches ;-)


Getting that merged is still on the todo list, I just got preempted by
other stuff :/

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/16/2012 07:04 PM, Peter Zijlstra wrote:

> On Wed, 2012-05-16 at 18:58 +0800, Alex Shi wrote:
>> If someone can unify the common fields of mmu_gather, and
>> left private field for specific arch. it will be great and much
>> helpful. 
> 
> I've send you a link to a patch-set that does exactly that twice now.
> 
>   http://marc.info/?l=linux-mm&m=129952019004146&w=2


Thanks for resend. Actually, I had read many times above patch, but it's
still hard to catch some lines, maybe due to it can not apply on current
mm/memory.c that is due to your bit newer code 9547d01b on 2011-05-24.
or maybe I am too stupid. :)

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/16/2012 04:00 PM, Peter Zijlstra wrote:

> On Wed, 2012-05-16 at 14:46 +0800, Alex Shi wrote:
>> diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
>> index 75e888b..ed6642a 100644
>> --- a/include/asm-generic/tlb.h
>> +++ b/include/asm-generic/tlb.h
>> @@ -86,6 +86,8 @@ struct mmu_gather {
>>  #ifdef CONFIG_HAVE_RCU_TABLE_FREE
>>         struct mmu_table_batch  *batch;
>>  #endif
>> +       unsigned long           start;
>> +       unsigned long           end;
>>         unsigned int            need_flush : 1, /* Did free PTEs */
>>                                 fast_mode  : 1; /* No batching   */
>>  
>> diff --git a/mm/memory.c b/mm/memory.c
>> index 6105f47..b176172 100644
>> --- a/mm/memory.c
>> +++ b/mm/memory.c
>> @@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
>>         tlb->mm = mm;
>>  
>>         tlb->fullmm     = fullmm;
>> +       tlb->start      = -1UL;
>> +       tlb->end        = 0;
>>         tlb->need_flush = 0;
>>         tlb->fast_mode  = (num_possible_cpus() == 1);
>>         tlb->local.next = NULL;
>> @@ -248,6 +250,8 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
>>  {
>>         struct mmu_gather_batch *batch, *next;
>>  
>> +       tlb->start = start;
>> +       tlb->end   = end;
>>         tlb_flush_mmu(tlb);
>>  
>>         /* keep the page table cache within bounds */
>> @@ -1204,6 +1208,8 @@ again:
>>          */
>>         if (force_flush) {
>>                 force_flush = 0;
>> +               tlb->start = addr;
>> +               tlb->end = end;
>>                 tlb_flush_mmu(tlb);
>>                 if (addr != end)
>>                         goto again; 
> 
> 
> ARGH.. no. What bit about you don't need to modify the generic code
> don't you get?
> 
> Both ARM and IA64 (and possible others) already do range tracking, you
> don't need to modify mm/memory.c _AT_ALL_.



Thanks for time and time remdiner. (shame for me)

In my code checking, the other archs can use self mmu_gather struct
since they code are excluded by HAVE_GENERIC_MMU_GATHER. In another word
if the code protected by HAVE_GENERIC_MMU_GATHER, it is safe for others
That is why tlb_flush_mmu/tlb_finish_mmu enabled both in mm/memory.c and
 other archs.

So, if the minimum change of tlb->start/end can be protected by
HAVE_GENERIC_MMU_GATHER, it is safe and harmless, am I right?

If so, the following patch should work on any condition.

---

>From ca29d791c3524887c1776136e9274d10d2114624 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Mon, 14 May 2012 09:17:03 +0800
Subject: [PATCH 6/7] x86/tlb: optimizing tlb_finish_mmu on x86

Not every tlb_flush execution moment is really need to evacuate all
TLB entries, like in munmap, just few 'invlpg' is better for whole
process performance, since it leaves most of TLB entries for later
accessing.

Since all of tlb interfaces in mm/memory.c is reused by all
architecture CPU, except few of them which protected under
HAVE_GENERIC_MMU_GATHER, I keeps global interfaces, just
re-implement x86 specific 'tlb_flush' only. and put the minimum
change under HAVE_GENERIC_MMU_GATHER too.

This patch also rewrite flush_tlb_range for 2 purposes:
1, split it out to get flush_blt_mm_range function.
2, clean up to reduce line breaking, thanks for Borislav's input.

Thanks for Peter Zijlstra time and time reminder for multiple
architecture code safe!

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/include/asm/tlb.h      |    9 +++-
 arch/x86/include/asm/tlbflush.h |    2 +
 arch/x86/mm/tlb.c               |  120 +++++++++++++++++++++------------------
 include/asm-generic/tlb.h       |    2 +
 mm/memory.c                     |    9 +++
 5 files changed, 85 insertions(+), 57 deletions(-)

diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215f..4fef207 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -4,7 +4,14 @@
 #define tlb_start_vma(tlb, vma) do { } while (0)
 #define tlb_end_vma(tlb, vma) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+
+#define tlb_flush(tlb)							\
+{									\
+	if (tlb->fullmm == 0)						\
+		flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL);	\
+	else								\
+		flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL);	\
+}
 
 #include <asm-generic/tlb.h>
 
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c39c94e..0107f3c 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -128,6 +128,8 @@ extern void flush_tlb_mm(struct mm_struct *);
 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
 extern void flush_tlb_range(struct vm_area_struct *vma,
 				   unsigned long start, unsigned long end);
+extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+				unsigned long end, unsigned long vmflag);
 
 #define flush_tlb()	flush_tlb_current_task()
 
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5bf4e85..52f6a5a 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -298,22 +298,6 @@ void flush_tlb_current_task(void)
 	preempt_enable();
 }
 
-void flush_tlb_mm(struct mm_struct *mm)
-{
-	preempt_disable();
-
-	if (current->active_mm == mm) {
-		if (current->mm)
-			local_flush_tlb();
-		else
-			leave_mm(smp_processor_id());
-	}
-	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
-		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
-
-	preempt_enable();
-}
-
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline int has_large_page(struct mm_struct *mm,
 				 unsigned long start, unsigned long end)
@@ -343,61 +327,85 @@ static inline int has_large_page(struct mm_struct *mm,
 	return 0;
 }
 #endif
-void flush_tlb_range(struct vm_area_struct *vma,
-				   unsigned long start, unsigned long end)
+
+void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+				unsigned long end, unsigned long vmflag)
 {
-	struct mm_struct *mm;
+	unsigned long addr;
+	unsigned act_entries, tlb_entries = 0;
 
-	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
-			|| tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
-flush_all:
-		flush_tlb_mm(vma->vm_mm);
-		return;
+	preempt_disable();
+	if (current->active_mm != mm)
+		goto flush_all;
+
+	if (!current->mm) {
+		leave_mm(smp_processor_id());
+		goto flush_all;
 	}
 
-	preempt_disable();
-	mm = vma->vm_mm;
-	if (current->active_mm == mm) {
-		if (current->mm) {
-			unsigned long addr, vmflag = vma->vm_flags;
-			unsigned act_entries, tlb_entries = 0;
+	if (end == TLB_FLUSH_ALL ||
+			tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
+		local_flush_tlb();
+		goto flush_all;
+	}
 
-			if (vmflag & VM_EXEC)
-				tlb_entries = tlb_lli_4k[ENTRIES];
-			else
-				tlb_entries = tlb_lld_4k[ENTRIES];
+	if (vmflag & VM_EXEC)
+		tlb_entries = tlb_lli_4k[ENTRIES];
+	else
+		tlb_entries = tlb_lld_4k[ENTRIES];
+	act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;
 
-			act_entries = tlb_entries > mm->total_vm ?
-					mm->total_vm : tlb_entries;
+	if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
+		local_flush_tlb();
+	else {
+		if (has_large_page(mm, start, end)) {
+			local_flush_tlb();
+			goto flush_all;
+		}
+		for (addr = start; addr <= end;	addr += PAGE_SIZE)
+			__flush_tlb_single(addr);
 
-			if ((end - start) >> PAGE_SHIFT >
-					act_entries >> tlb_flushall_shift)
-				local_flush_tlb();
-			else {
-				if (has_large_page(mm, start, end)) {
-					preempt_enable();
-					goto flush_all;
-				}
-				for (addr = start; addr <= end;
-						addr += PAGE_SIZE)
-					__flush_tlb_single(addr);
+		if (cpumask_any_but(mm_cpumask(mm),
+				smp_processor_id()) < nr_cpu_ids)
+			flush_tlb_others(mm_cpumask(mm), mm, start, end);
+		preempt_enable();
+		return;
+	}
 
-				if (cpumask_any_but(mm_cpumask(mm),
-					smp_processor_id()) < nr_cpu_ids)
-					flush_tlb_others(mm_cpumask(mm), mm,
-								start, end);
-				preempt_enable();
-				return;
-			}
-		} else {
+flush_all:
+	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+	preempt_enable();
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+	preempt_disable();
+
+	if (current->active_mm == mm) {
+		if (current->mm)
+			local_flush_tlb();
+		else
 			leave_mm(smp_processor_id());
-		}
 	}
 	if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
 		flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+
 	preempt_enable();
 }
 
+void flush_tlb_range(struct vm_area_struct *vma,
+				   unsigned long start, unsigned long end)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	unsigned long vmflag = vma->vm_flags;
+
+	if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB)
+		flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL);
+	else
+		flush_tlb_mm_range(mm, start, end, vmflag);
+}
+
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
 {
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 75e888b..ed6642a 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -86,6 +86,8 @@ struct mmu_gather {
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	struct mmu_table_batch	*batch;
 #endif
+	unsigned long		start;
+	unsigned long		end;
 	unsigned int		need_flush : 1,	/* Did free PTEs */
 				fast_mode  : 1; /* No batching   */
 
diff --git a/mm/memory.c b/mm/memory.c
index 6105f47..a1078af 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
 	tlb->mm = mm;
 
 	tlb->fullmm     = fullmm;
+	tlb->start	= -1UL;
+	tlb->end	= 0;
 	tlb->need_flush = 0;
 	tlb->fast_mode  = (num_possible_cpus() == 1);
 	tlb->local.next = NULL;
@@ -248,6 +250,8 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
 {
 	struct mmu_gather_batch *batch, *next;
 
+	tlb->start = start;
+	tlb->end   = end;
 	tlb_flush_mmu(tlb);
 
 	/* keep the page table cache within bounds */
@@ -1204,6 +1208,11 @@ again:
 	 */
 	if (force_flush) {
 		force_flush = 0;
+
+#ifdef HAVE_GENERIC_MMU_GATHER
+		tlb->start = addr;
+		tlb->end = end;
+#endif
 		tlb_flush_mmu(tlb);
 		if (addr != end)
 			goto again;
-- 
1.7.5.4

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Peter Zijlstra]

On Wed, 2012-05-16 at 21:34 +0800, Alex Shi wrote:
> 
> So, if the minimum change of tlb->start/end can be protected by
> HAVE_GENERIC_MMU_GATHER, it is safe and harmless, am I right?
> 
safe yes, but not entirely harmless. A quick look seems to suggest you
fail for VM_HUGETLB. If your mmu_gather spans a vma with VM_HUGETLB
you'll do a regular range flush not a full mm flush like the other paths
do.

Anyway, I did a quick refresh of my series on a recent -tip tree:

 git://git.kernel.org/pub/scm/linux/kernel/git/peterz/mmu.git tlb-unify

With that all you need is to "select HAVE_MMU_GATHER_RANGE" for x86 and
implement a useful flush_tlb_range().

In particular, see:
http://git.kernel.org/?p=linux/kernel/git/peterz/mmu.git;a=commitdiff;h=05e53144177e6242fda404045f50f48114bcf185;hp=2cd7dc710652127522392f4b7ecb5fa6e954941e

I've slightly changed the code to address an open issue with the
vm_flags tracking. We now force flush the mmu_gather whenever VM_HUGETLB
flips because most (all?) archs that look at that flag expect pure huge
pages and not a mixture.

I've seem to have misplaced my cross-compiler set, so I've only compiled
x86-64 for now.

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/17/2012 05:09 AM, Peter Zijlstra wrote:

> On Wed, 2012-05-16 at 21:34 +0800, Alex Shi wrote:
>>
>> So, if the minimum change of tlb->start/end can be protected by
>> HAVE_GENERIC_MMU_GATHER, it is safe and harmless, am I right?
>>
> safe yes, but not entirely harmless. A quick look seems to suggest you
> fail for VM_HUGETLB. If your mmu_gather spans a vma with VM_HUGETLB
> you'll do a regular range flush not a full mm flush like the other paths
> do.


Thanks!
Uh, HUGETLB will be found by has_large_page() if THP enabled now. And I
will remove THP cost, then HUGETLB will be deal well. Since the max
number of TLB entries is 512, has_large_page just need execute once just
when the start address is align at HPAGE_SIZE.

IMHO, this patch enabled generic mmu range flush support with just teen
lines. and it is supported well by low level x86 architecture. I like it. :)

> 
> Anyway, I did a quick refresh of my series on a recent -tip tree:
> 
>  git://git.kernel.org/pub/scm/linux/kernel/git/peterz/mmu.git tlb-unify
> 
> With that all you need is to "select HAVE_MMU_GATHER_RANGE" for x86 and
> implement a useful flush_tlb_range().

> 
> In particular, see:
> http://git.kernel.org/?p=linux/kernel/git/peterz/mmu.git;a=commitdiff;h=05e53144177e6242fda404045f50f48114bcf185;hp=2cd7dc710652127522392f4b7ecb5fa6e954941e
> 
> I've slightly changed the code to address an open issue with the
> vm_flags tracking. We now force flush the mmu_gather whenever VM_HUGETLB
> flips because most (all?) archs that look at that flag expect pure huge
> pages and not a mixture.
> 
> I've seem to have misplaced my cross-compiler set, so I've only compiled
> x86-64 for now.


Oh, I also need a cross-compiler for other archs. Thanks reminder!

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Steven Rostedt]

[-- Attachment #1: Type: text/plain, Size: 606 bytes --]

On Thu, 2012-05-17 at 08:43 +0800, Alex Shi wrote:

> > I've seem to have misplaced my cross-compiler set, so I've only compiled
> > x86-64 for now.
> 
> 
> Oh, I also need a cross-compiler for other archs. Thanks reminder!

Here:

http://kernel.org/pub/tools/crosstool/

Oh, and if you want to automate this. I attached a ktest.pl config that
does it for you. I'll be pushing this config and others into a examples
directory come the next merge window.

Ktest is located in the Linux tree under tools/testing/ktest/

You can run a bunch of cross compiles by doing:

ktest.pl crosstests.conf

-- Steve





[-- Attachment #2: crosstests.conf --]
[-- Type: text/plain, Size: 6921 bytes --]

#
# Example config for cross compiling
#
# In this config, it is expected that the tool chains from:
#
#   http://kernel.org/pub/tools/crosstool/files/bin/x86_64/
#
# running on a x86_64 system have been downloaded and installed into:
#
#   /usr/local/
#
# such that the compiler binaries are something like:
#
#   /usr/local/gcc-4.5.2-nolibc/mips-linux/bin/mips-linux-gcc
#
# Some of the archs will use gcc-4.5.1 instead of gcc-4.5.2
# this config uses variables to differentiate them.
# 
# Comments will be described for some areas, but the opitons are
# all documented in the samples.conf file.

# ${PWD} is defined by ktest.pl to be the directory that the user
# was in when they executed ktest.pl. It may be better to hardcode the
# path name here. THIS_DIR is the variable used through out the config file
# in case you want to change it.

THIS_DIR := ${PWD}

# Update the BUILD_DIR option to the location of your git repo you want to test.
BUILD_DIR = ${THIS_DIR}/linux.git

# The build will go into this directory. It will be created when you run the test.
OUTPUT_DIR = ${THIS_DIR}/cross-compile

# The build will be compiled with -j8
BUILD_OPTIONS = -j8

# The test will not stop when it hits a failure.
DIE_ON_FAILURE = 0

# If you want to have ktest.pl store the failure somewhere, uncomment this option
# and change the directory where ktest should store the failures.
#STORE_FAILURES = ${THIS_DIR}/failures

# The log file is stored in the OUTPUT_DIR called cross.log
# If you enable this, you need to create the OUTPUT_DIR. It wont be created for you.
#LOG_FILE = ${OUTPUT_DIR}/cross.log

# The log file will be cleared each time you run ktest.
CLEAR_LOG = 1

# As some archs do not build with the defconfig, they have been marked
# to be ignored. If you want to test them anyway, change DO_FAILED to one.
# If a test that has been marked as DO_FAILED passes, then you should change
# that test to be DO_DEFAULT

DO_FAILED := 0
DO_DEFAULT := 1

# By setting both DO_FAILED and DO_DEFAULT to zero, you can pick a single
# arch that you want to test. (uncomment RUN and chose your arch)
#RUN := m32r

# At the bottom of the config file exists a bisect test. You can update that
# test and set DO_FAILED and DO_DEFAULT to zero, and uncomment this variable
# to run the bisect on the arch.
#RUN := bisect

# By default all tests will be running gcc 4.5.2. Some tests are using 4.5.1
# and they select that in the test.
# Note: GCC_VER is declared as on option and not a variable ('=' istead of ':=')
# This is important. A variable is used only in the config file and if it is set
# it stays that way for the rest of the config file until it is change again.
# Here we want GCC_VER to remain persistent throughout the tests, as it is used in
# the MAKE_CMD. By using '=' instead of ':=' we achieve our goal.

GCC_VER = 4.5.2
MAKE_CMD = PATH=/usr/local/gcc-${GCC_VER}-nolibc/${CROSS}/bin:$PATH CROSS_COMPILE=${CROSS}- make ARCH=${ARCH}

# all tests are only doing builds.
TEST_TYPE = build

# If you want to add configs on top of the defconfig, you can add those configs into
# the add-config file and uncomment this option. This is useful if you want to test
# all cross compiles with PREEMPT set, or TRACING on, etc.
#ADD_CONFIG = ${THIS_DIR}/add-config

# All tests are using defconfig
BUILD_TYPE = defconfig

# The test names will have the arch and cross compiler used. This will be shown in
# the results.
TEST_NAME = ${ARCH} ${CROSS}

# alpha
TEST_START IF ${RUN} == alpha || ${DO_DEFAULT}
CROSS = alpha-linux
ARCH = alpha

# arm
TEST_START IF ${RUN} == arm || ${DO_DEFAULT}
CROSS = arm-unknown-linux-gnueabi
ARCH = arm

# black fin
TEST_START IF ${RUN} == bfin || ${DO_DEFAULT}
CROSS = bfin-uclinux
ARCH = blackfin
BUILD_OPTIONS = -j8 vmlinux

# cris - FAILS?
TEST_START IF ${RUN} == cris || ${RUN} == cris64 || ${DO_FAILED}
CROSS = cris-linux
ARCH = cris

# cris32 - not right arch?
TEST_START IF ${RUN} == cris || ${RUN} == cris32 || ${DO_FAILED}
CROSS = crisv32-linux
ARCH = cris

# ia64
TEST_START IF ${RUN} == ia64 || ${DO_DEFAULT}
CROSS = ia64-linux
ARCH = ia64

# frv
TEST_START IF ${RUN} == frv || ${DO_FAILED}
CROSS = frv-linux
ARCH = frv
GCC_VER = 4.5.1

# h8300 - failed make defconfig??
TEST_START IF ${RUN} == h8300 || ${DO_FAILED}
CROSS = h8300-elf
ARCH = h8300
GCC_VER = 4.5.1

# m68k fails with error?
TEST_START IF ${RUN} == m68k || ${DO_DEFAULT}
CROSS = m68k-linux
ARCH = m68k

# mips64
TEST_START IF ${RUN} == mips || ${RUN} == mips64 || ${DO_DEFAULT}
CROSS = mips64-linux
ARCH = mips

# mips32
TEST_START IF ${RUN} == mips || ${RUN} == mips32 || ${DO_DEFAULT}
CROSS = mips-linux
ARCH = mips

# m32r
TEST_START IF ${RUN} == m32r || ${DO_FAILED}
CROSS = m32r-linux
ARCH = m32r
GCC_VER = 4.5.1
BUILD_OPTIONS = -j8 vmlinux

# parisc64 failed?
TEST_START IF ${RUN} == hppa || ${RUN} == hppa64 || ${DO_FAILED}
CROSS = hppa64-linux
ARCH = parisc

# parisc
TEST_START IF ${RUN} == hppa || ${RUN} == hppa32 || ${DO_FAILED}
CROSS = hppa-linux
ARCH = parisc

# ppc
TEST_START IF ${RUN} == ppc || ${RUN} == ppc32 || ${DO_DEFAULT}
CROSS = powerpc-linux
ARCH = powerpc

# ppc64
TEST_START IF ${RUN} == ppc || ${RUN} == ppc64 || ${DO_DEFAULT}
CROSS = powerpc64-linux
ARCH = powerpc

# s390
TEST_START IF ${RUN} == s390 || ${DO_DEFAULT}
CROSS = s390x-linux
ARCH = s390

# sh
TEST_START IF ${RUN} == sh || ${DO_DEFAULT}
CROSS = sh4-linux
ARCH = sh

# sparc64
TEST_START IF ${RUN} == sparc || ${RUN} == sparc64 || ${DO_DEFAULT}
CROSS = sparc64-linux
ARCH = sparc64

# sparc
TEST_START IF ${RUN} == sparc || ${RUN} == sparc32 || ${DO_DEFAULT}
CROSS = sparc-linux
ARCH = sparc

# xtensa failed
TEST_START IF ${RUN} == xtensa || ${DO_FAILED}
CROSS = xtensa-linux
ARCH = xtensa

# UML
TEST_START IF ${RUN} == uml || ${DO_DEFAULT}
MAKE_CMD = make ARCH=um SUBARCH=x86_64
ARCH = uml
CROSS =

TEST_START IF ${RUN} == x86 || ${RUN} == i386 || ${DO_DEFAULT}
MAKE_CMD = make ARCH=i386
ARCH = i386
CROSS = 

TEST_START IF ${RUN} == x86 || ${RUN} == x86_64 || ${DO_DEFAULT}
MAKE_CMD = make ARCH=x86_64
ARCH = x86_64
CROSS = 

#################################

# This is a bisect if needed. You need to give it a MIN_CONFIG that
# will be the config file it uses. Basically, just copy the created defconfig
# for the arch someplace and point MIN_CONFIG to it.
TEST_START IF ${RUN} == bisect
MIN_CONFIG = ${THIS_DIR}/min-config
CROSS = s390x-linux
ARCH = s390
TEST_TYPE = bisect
BISECT_TYPE = build
BISECT_GOOD = v3.1
BISECT_BAD = v3.2
CHECKOUT = v3.2

#################################

# These defaults are needed to keep ktest.pl from complaining. They are
# ignored because the test does not go pass the build. No install or
# booting of the target images.

DEFAULTS
MACHINE = crosstest
SSH_USER = root
BUILD_TARGET = cross
TARGET_IMAGE = image
POWER_CYCLE = cycle
CONSOLE = console
LOCALVERSION = version
GRUB_MENU = grub

REBOOT_ON_ERROR = 0
POWEROFF_ON_ERROR = 0
POWEROFF_ON_SUCCESS = 0
REBOOT_ON_SUCCESS = 0

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

On 05/17/2012 10:07 AM, Steven Rostedt wrote:

> On Thu, 2012-05-17 at 08:43 +0800, Alex Shi wrote:
> 
>>> I've seem to have misplaced my cross-compiler set, so I've only compiled
>>> x86-64 for now.
>>
>>
>> Oh, I also need a cross-compiler for other archs. Thanks reminder!
> 
> Here:
> 
> http://kernel.org/pub/tools/crosstool/
> 
> Oh, and if you want to automate this. I attached a ktest.pl config that
> does it for you. I'll be pushing this config and others into a examples
> directory come the next merge window.
> 
> Ktest is located in the Linux tree under tools/testing/ktest/
> 
> You can run a bunch of cross compiles by doing:
> 
> ktest.pl crosstests.conf


It works fine. :)
But does ktest only do one kind of config testing?
How it do randconfig testing?

> 
> -- Steve
> 
> 
> 
> 

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Paul Mundt]

On Wed, May 16, 2012 at 11:09:29PM +0200, Peter Zijlstra wrote:
> On Wed, 2012-05-16 at 21:34 +0800, Alex Shi wrote:
> > 
> > So, if the minimum change of tlb->start/end can be protected by
> > HAVE_GENERIC_MMU_GATHER, it is safe and harmless, am I right?
> > 
> safe yes, but not entirely harmless. A quick look seems to suggest you
> fail for VM_HUGETLB. If your mmu_gather spans a vma with VM_HUGETLB
> you'll do a regular range flush not a full mm flush like the other paths
> do.
> 
> Anyway, I did a quick refresh of my series on a recent -tip tree:
> 
>  git://git.kernel.org/pub/scm/linux/kernel/git/peterz/mmu.git tlb-unify
> 
> With that all you need is to "select HAVE_MMU_GATHER_RANGE" for x86 and
> implement a useful flush_tlb_range().
> 
> In particular, see:
> http://git.kernel.org/?p=linux/kernel/git/peterz/mmu.git;a=commitdiff;h=05e53144177e6242fda404045f50f48114bcf185;hp=2cd7dc710652127522392f4b7ecb5fa6e954941e
> 
> I've slightly changed the code to address an open issue with the
> vm_flags tracking. We now force flush the mmu_gather whenever VM_HUGETLB
> flips because most (all?) archs that look at that flag expect pure huge
> pages and not a mixture.
> 
> I've seem to have misplaced my cross-compiler set, so I've only compiled
> x86-64 for now.

It was on my list to test when you sent out the series initially, but
seems to have slipped my mind until I saw this thread. Here's a patch on
top of your tlb-unify branch that gets sh working (tested on all of
2-level, 3-level, and nommu).

I opted to shove the asm/cacheflush.h include in to tlb.h directly since
it is calling flush_cache_range() openly now, and the rest of the
architectures are just getting at it through various whimsical means. sh
was getting it through pagemap.h -> highmem.h, while ARM presently can't
seem to make up its mind and includes pagemap.h for nommu only as well as
cacheflush.h explicitly.

With the reworked interface we don't seem to actually need to stub out
the interface for the nommu case anymore anyways, all of the users are
insular to mm/memory.c which we don't build for nommu.

Signed-off-by: Paul Mundt <lethal@linux-sh.org>

---

diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h
index 8c00785..bedc2ed 100644
--- a/arch/sh/include/asm/pgalloc.h
+++ b/arch/sh/include/asm/pgalloc.h
@@ -13,6 +13,8 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
 extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address);
 extern void pmd_free(struct mm_struct *mm, pmd_t *pmd);
+
+#define __pmd_free_tlb(tlb, pmdp, addr)	pmd_free((tlb)->mm, pmdp)
 #endif
 
 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h
index 45e5925..71af915 100644
--- a/arch/sh/include/asm/tlb.h
+++ b/arch/sh/include/asm/tlb.h
@@ -6,18 +6,7 @@
 #endif
 
 #ifndef __ASSEMBLY__
-#include <linux/pagemap.h>
-
 #ifdef CONFIG_MMU
-#include <linux/swap.h>
-
-#define __tlb_remove_tlb_entry(tlb, ptep, addr) do { } while (0)
-
-#define __pte_free_tlb(tlb, ptep, addr)	pte_free((tlb)->mm, ptep)
-#define __pmd_free_tlb(tlb, pmdp, addr)	pmd_free((tlb)->mm, pmdp)
-#define __pud_free_tlb(tlb, pudp, addr)	pud_free((tlb)->mm, pudp)
-
-#include <asm-generic/tlb.h>
 
 #if defined(CONFIG_CPU_SH4) || defined(CONFIG_SUPERH64)
 extern void tlb_wire_entry(struct vm_area_struct *, unsigned long, pte_t);
@@ -35,8 +24,6 @@ static inline void tlb_unwire_entry(void)
 }
 #endif
 
-#else /* CONFIG_MMU */
-
 #define __tlb_remove_tlb_entry(tlb, pte, address)	do { } while (0)
 
 #include <asm-generic/tlb.h>
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 90a725c..571e2cf 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -18,6 +18,7 @@
 #include <linux/swap.h>
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
 
 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
 

Re: [PATCH v5 6/7] x86/tlb: optimizing flush_tlb_mm

[Alex Shi]

> Now IF you're going to change the tlb interface like this, you're going
> to get to do it for all architectures, along with a sane benchmark to
> show its beneficial to track ranges like this.
> 
> But as it stands, people are still questioning the validity of your
> mprotect micro-bench, so no, you don't get to change the tlb interface.


Yes, sure. You are definitely right!

[PATCH v5 7/7] x86/tlb: add tlb_flushall_shift knob into debugfs

[Alex Shi]

kernel will replace cr3 rewrite with invlpg when
  tlb_flush_entries <= active_tlb_entries / 2^tlb_flushall_factor
if tlb_flushall_factor is -1, kernel won't do this replacement.

User can modify its value according to specific CPU/applications.

Thanks for Borislav providing the help message of
CONFIG_DEBUG_TLBFLUSH.

Signed-off-by: Alex Shi <alex.shi@intel.com>
---
 arch/x86/Kconfig.debug |   19 +++++++++++++++++++
 arch/x86/mm/tlb.c      |   48 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 0 deletions(-)

diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index e46c214..b322f12 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -129,6 +129,25 @@ config DOUBLEFAULT
 	  option saves about 4k and might cause you much additional grey
 	  hair.
 
+config DEBUG_TLBFLUSH
+	bool "Set upper limit of TLB entries to flush one-by-one"
+	depends on DEBUG_KERNEL && (X86_64 || X86_INVLPG)
+	---help---
+
+	X86-only for now.
+
+	This option allows the user to tune the amount of TLB entries the
+	kernel flushes one-by-one instead of doing a full TLB flush. In
+	certain situations, the former is cheaper. This is controlled by the
+	tlb_flushall_shift knob under /sys/kernel/debug/x86. If you set it
+	to -1, the code flushes the whole TLB unconditionally. Otherwise,
+	for positive values of it, the kernel will use single TLB entry
+	invalidating instructions according to the following formula:
+
+	flush_entries <= active_tlb_entries / 2^tlb_flushall_shift
+
+	If in doubt, say "N".
+
 config IOMMU_DEBUG
 	bool "Enable IOMMU debugging"
 	depends on GART_IOMMU && DEBUG_KERNEL
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index b2e25e8..b77c764 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -12,6 +12,7 @@
 #include <asm/cache.h>
 #include <asm/apic.h>
 #include <asm/uv/uv.h>
+#include <linux/debugfs.h>
 
 DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
 			= { &init_mm, 0, };
@@ -429,3 +430,50 @@ void flush_tlb_all(void)
 {
 	on_each_cpu(do_flush_tlb_all, NULL, 1);
 }
+
+#ifdef CONFIG_DEBUG_TLBFLUSH
+static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
+			     size_t count, loff_t *ppos)
+{
+	char buf[32];
+	unsigned int len;
+
+	len = sprintf(buf, "%hd\n", tlb_flushall_shift);
+	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static ssize_t tlbflush_write_file(struct file *file,
+		 const char __user *user_buf, size_t count, loff_t *ppos)
+{
+	char buf[32];
+	ssize_t len;
+	u16 shift;
+
+	len = min(count, sizeof(buf) - 1);
+	if (copy_from_user(buf, user_buf, len))
+		return -EFAULT;
+
+	buf[len] = '\0';
+	if (kstrtos16(buf, 0, &shift))
+		return -EINVAL;
+
+	tlb_flushall_shift = shift;
+	return count;
+}
+
+static const struct file_operations fops_tlbflush = {
+	.read = tlbflush_read_file,
+	.write = tlbflush_write_file,
+	.llseek = default_llseek,
+};
+
+static int __cpuinit create_tlb_flushall_shift(void)
+{
+	if (cpu_has_invlpg) {
+		debugfs_create_file("tlb_flushall_shift", S_IRUSR | S_IWUSR,
+			arch_debugfs_dir, NULL, &fops_tlbflush);
+	}
+	return 0;
+}
+late_initcall(create_tlb_flushall_shift);
+#endif
-- 
1.7.5.4