[PATCH 0/2] MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

[PATCH 0/2] MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

[David Daney]

This set of patches optimizes TLB handling in mips64r2 processors.  If
we have the mips64r2 ISA available, we can do enough bit twiddling
with out using a scratch register that it become possible and
desirable to carry the PGD pointer in c0_context rather than having to
always load it from memory.

The first patch adds some instructions to uasm that are needed by the
second patch.

The second patch contains the main changes.

I will reply with the two patches.

David Daney (2):
   MIPS: Add drotr and dins instructions to uasm.
   MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

  arch/mips/Kconfig                   |    3 +++
  arch/mips/include/asm/mmu_context.h |   29 ++++++++++++++++++++++++++++-
  arch/mips/include/asm/stackframe.h  |   20 ++++++++++----------
  arch/mips/mm/init.c                 |    2 ++
  arch/mips/mm/tlbex.c                |   28 +++++++++++++++++++++++++---
  arch/mips/mm/uasm.c                 |   16 +++++++++++++---
  arch/mips/mm/uasm.h                 |    7 +++++++
  7 files changed, 88 insertions(+), 17 deletions(-)

[PATCH 1/2] MIPS: Add drotr and dins instructions to uasm.

[David Daney]

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
---
 arch/mips/mm/uasm.c |   16 +++++++++++++---
 arch/mips/mm/uasm.h |    7 +++++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c
index f467199..0a165c5 100644
--- a/arch/mips/mm/uasm.c
+++ b/arch/mips/mm/uasm.c
@@ -60,11 +60,11 @@ enum opcode {
 	insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
 	insn_bne, insn_cache, insn_daddu, insn_daddiu, insn_dmfc0,
 	insn_dmtc0, insn_dsll, insn_dsll32, insn_dsra, insn_dsrl,
-	insn_dsrl32, insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr,
-	insn_ld, insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0,
+	insn_dsrl32, insn_drotr, insn_dsubu, insn_eret, insn_j, insn_jal,
+	insn_jr, insn_ld, insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0,
 	insn_mtc0, insn_ori, insn_pref, insn_rfe, insn_sc, insn_scd,
 	insn_sd, insn_sll, insn_sra, insn_srl, insn_subu, insn_sw,
-	insn_tlbp, insn_tlbwi, insn_tlbwr, insn_xor, insn_xori
+	insn_tlbp, insn_tlbwi, insn_tlbwr, insn_xor, insn_xori, insn_dins
 };
 
 struct insn {
@@ -104,6 +104,7 @@ static struct insn insn_table[] __cpuinitdata = {
 	{ insn_dsra, M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE },
 	{ insn_dsrl, M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE },
 	{ insn_dsrl32, M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE },
+	{ insn_drotr, M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE },
 	{ insn_dsubu, M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD },
 	{ insn_eret,  M(cop0_op, cop_op, 0, 0, 0, eret_op),  0 },
 	{ insn_j,  M(j_op, 0, 0, 0, 0, 0),  JIMM },
@@ -132,6 +133,7 @@ static struct insn insn_table[] __cpuinitdata = {
 	{ insn_tlbwr,  M(cop0_op, cop_op, 0, 0, 0, tlbwr_op),  0 },
 	{ insn_xor,  M(spec_op, 0, 0, 0, 0, xor_op),  RS | RT | RD },
 	{ insn_xori,  M(xori_op, 0, 0, 0, 0, 0),  RS | RT | UIMM },
+	{ insn_dins, M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE },
 	{ insn_invalid, 0, 0 }
 };
 
@@ -304,6 +306,12 @@ Ip_u2u1s3(op)						\
 	build_insn(buf, insn##op, b, a, c);		\
 }
 
+#define I_u2u1msbu3(op)					\
+Ip_u2u1msbu3(op)					\
+{							\
+	build_insn(buf, insn##op, b, a, c+d-1, c);	\
+}
+
 #define I_u1u2(op)					\
 Ip_u1u2(op)						\
 {							\
@@ -349,6 +357,7 @@ I_u2u1u3(_dsll32)
 I_u2u1u3(_dsra)
 I_u2u1u3(_dsrl)
 I_u2u1u3(_dsrl32)
+I_u2u1u3(_drotr)
 I_u3u1u2(_dsubu)
 I_0(_eret)
 I_u1(_j)
@@ -377,6 +386,7 @@ I_0(_tlbwi)
 I_0(_tlbwr)
 I_u3u1u2(_xor)
 I_u2u1u3(_xori)
+I_u2u1msbu3(_dins);
 
 /* Handle labels. */
 void __cpuinit uasm_build_label(struct uasm_label **lab, u32 *addr, int lid)
diff --git a/arch/mips/mm/uasm.h b/arch/mips/mm/uasm.h
index c6d1e3d..3d153ed 100644
--- a/arch/mips/mm/uasm.h
+++ b/arch/mips/mm/uasm.h
@@ -34,6 +34,11 @@ uasm_i##op(u32 **buf, unsigned int a, signed int b, unsigned int c)
 void __cpuinit								\
 uasm_i##op(u32 **buf, unsigned int a, unsigned int b, signed int c)
 
+#define Ip_u2u1msbu3(op)						\
+void __cpuinit								\
+uasm_i##op(u32 **buf, unsigned int a, unsigned int b, unsigned int c,	\
+	   unsigned int d)
+
 #define Ip_u1u2(op)							\
 void __cpuinit uasm_i##op(u32 **buf, unsigned int a, unsigned int b)
 
@@ -65,6 +70,7 @@ Ip_u2u1u3(_dsll32);
 Ip_u2u1u3(_dsra);
 Ip_u2u1u3(_dsrl);
 Ip_u2u1u3(_dsrl32);
+Ip_u2u1u3(_drotr);
 Ip_u3u1u2(_dsubu);
 Ip_0(_eret);
 Ip_u1(_j);
@@ -93,6 +99,7 @@ Ip_0(_tlbwi);
 Ip_0(_tlbwr);
 Ip_u3u1u2(_xor);
 Ip_u2u1u3(_xori);
+Ip_u2u1msbu3(_dins);
 
 /* Handle labels. */
 struct uasm_label {
-- 
1.6.0.6

[PATCH 2/2] MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

[David Daney]

Processors that support the mips64r2 ISA can in four instructions
convert a shifted PGD pointer stored in the upper bits of c0_context
into a usable pointer.  By doing this we save a memory load and
associated potential cache miss in the TLB exception handlers.

Since the upper bits of c0_context were holding the CPU number, we
move this to the upper bits of c0_xcontext which doesn't have enough
bits to hold the PGD pointer, but has plenty for the CPU number.

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
---
 arch/mips/Kconfig                   |    3 +++
 arch/mips/include/asm/mmu_context.h |   29 ++++++++++++++++++++++++++++-
 arch/mips/include/asm/stackframe.h  |   20 ++++++++++----------
 arch/mips/mm/init.c                 |    2 ++
 arch/mips/mm/tlbex.c                |   28 +++++++++++++++++++++++++---
 5 files changed, 68 insertions(+), 14 deletions(-)

diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index a3fceba..a4aea20 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -1412,6 +1412,9 @@ config CPU_SUPPORTS_64BIT_KERNEL
 	bool
 config CPU_SUPPORTS_HUGEPAGES
 	bool
+config MIPS_PGD_C0_CONTEXT
+	bool
+	default y if 64BIT && CPU_MIPSR2
 
 #
 # Set to y for ptrace access to watch registers.
diff --git a/arch/mips/include/asm/mmu_context.h b/arch/mips/include/asm/mmu_context.h
index 6083db5..145bb81 100644
--- a/arch/mips/include/asm/mmu_context.h
+++ b/arch/mips/include/asm/mmu_context.h
@@ -24,6 +24,33 @@
 #endif /* SMTC */
 #include <asm-generic/mm_hooks.h>
 
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+
+#define TLBMISS_HANDLER_SETUP_PGD(pgd)				\
+	tlbmiss_handler_setup_pgd((unsigned long)(pgd))
+
+static inline void tlbmiss_handler_setup_pgd(unsigned long pgd)
+{
+	/* Check for swapper_pg_dir and convert to physical address. */
+	if ((pgd & CKSEG3) == CKSEG0)
+		pgd = CPHYSADDR(pgd);
+	write_c0_context(pgd << 11);
+}
+
+#define TLBMISS_HANDLER_SETUP()						\
+	do {								\
+		TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir);		\
+		write_c0_xcontext((unsigned long) smp_processor_id() << 51); \
+	} while (0)
+
+
+static inline unsigned long get_current_pgd(void)
+{
+	return PHYS_TO_XKSEG_CACHED((read_c0_context() >> 11) & ~0xfffUL);
+}
+
+#else /* CONFIG_MIPS_PGD_C0_CONTEXT: using  pgd_current*/
+
 /*
  * For the fast tlb miss handlers, we keep a per cpu array of pointers
  * to the current pgd for each processor. Also, the proc. id is stuffed
@@ -46,7 +73,7 @@ extern unsigned long pgd_current[];
 	back_to_back_c0_hazard();					\
 	TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir)
 #endif
-
+#endif /* CONFIG_MIPS_PGD_C0_CONTEXT*/
 #if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
 
 #define ASID_INC	0x40
diff --git a/arch/mips/include/asm/stackframe.h b/arch/mips/include/asm/stackframe.h
index 491b3ee..156d7b0 100644
--- a/arch/mips/include/asm/stackframe.h
+++ b/arch/mips/include/asm/stackframe.h
@@ -87,15 +87,19 @@
 #ifdef CONFIG_SMP
 #ifdef CONFIG_MIPS_MT_SMTC
 #define PTEBASE_SHIFT	19	/* TCBIND */
+#define CPU_ID_REG CP0_TCBIND
+#define CPU_ID_MFC0 mfc0
+#elif defined(CONFIG_MIPS_PGD_C0_CONTEXT)
+#define PTEBASE_SHIFT	48	/* XCONTEXT */
+#define CPU_ID_REG CP0_XCONTEXT
+#define CPU_ID_MFC0 MFC0
 #else
 #define PTEBASE_SHIFT	23	/* CONTEXT */
+#define CPU_ID_REG CP0_CONTEXT
+#define CPU_ID_MFC0 MFC0
 #endif
 		.macro	get_saved_sp	/* SMP variation */
-#ifdef CONFIG_MIPS_MT_SMTC
-		mfc0	k0, CP0_TCBIND
-#else
-		MFC0	k0, CP0_CONTEXT
-#endif
+		CPU_ID_MFC0	k0, CPU_ID_REG
 #if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
 		lui	k1, %hi(kernelsp)
 #else
@@ -111,11 +115,7 @@
 		.endm
 
 		.macro	set_saved_sp stackp temp temp2
-#ifdef CONFIG_MIPS_MT_SMTC
-		mfc0	\temp, CP0_TCBIND
-#else
-		MFC0	\temp, CP0_CONTEXT
-#endif
+		CPU_ID_MFC0	\temp, CPU_ID_REG
 		LONG_SRL	\temp, PTEBASE_SHIFT
 		LONG_S	\stackp, kernelsp(\temp)
 		.endm
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 8d1f4f3..9e8d003 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -462,7 +462,9 @@ void __init_refok free_initmem(void)
 			__pa_symbol(&__init_end));
 }
 
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 unsigned long pgd_current[NR_CPUS];
+#endif
 /*
  * On 64-bit we've got three-level pagetables with a slightly
  * different layout ...
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index ebadab1..571f92f 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -160,6 +160,12 @@ static u32 tlb_handler[128] __cpuinitdata;
 static struct uasm_label labels[128] __cpuinitdata;
 static struct uasm_reloc relocs[128] __cpuinitdata;
 
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
+/*
+ * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
+ * we cannot do r3000 under these circumstances.
+ */
+
 /*
  * The R3000 TLB handler is simple.
  */
@@ -199,6 +205,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void)
 
 	dump_handler((u32 *)ebase, 32);
 }
+#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 
 /*
  * The R4000 TLB handler is much more complicated. We have two
@@ -497,8 +504,9 @@ static void __cpuinit
 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 		 unsigned int tmp, unsigned int ptr)
 {
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 	long pgdc = (long)pgd_current;
-
+#endif
 	/*
 	 * The vmalloc handling is not in the hotpath.
 	 */
@@ -506,7 +514,15 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 	uasm_il_bltz(p, r, tmp, label_vmalloc);
 	/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
 
-#ifdef CONFIG_SMP
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+	/*
+	 * &pgd << 11 stored in CONTEXT [23..63].
+	 */
+	UASM_i_MFC0(p, ptr, C0_CONTEXT);
+	uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */
+	uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0  1 0 1  << 6  xkphys cached */
+	uasm_i_drotr(p, ptr, ptr, 11);
+#elif defined(CONFIG_SMP)
 # ifdef  CONFIG_MIPS_MT_SMTC
 	/*
 	 * SMTC uses TCBind value as "CPU" index
@@ -520,7 +536,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 	 */
 	uasm_i_dmfc0(p, ptr, C0_CONTEXT);
 	uasm_i_dsrl(p, ptr, ptr, 23);
-#endif
+# endif
 	UASM_i_LA_mostly(p, tmp, pgdc);
 	uasm_i_daddu(p, ptr, ptr, tmp);
 	uasm_i_dmfc0(p, tmp, C0_BADVADDR);
@@ -1036,6 +1052,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r,
 	iPTE_LW(p, pte, ptr);
 }
 
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 /*
  * R3000 style TLB load/store/modify handlers.
  */
@@ -1187,6 +1204,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void)
 
 	dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
 }
+#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 
 /*
  * R4000 style TLB load/store/modify handlers.
@@ -1406,6 +1424,7 @@ void __cpuinit build_tlb_refill_handler(void)
 	case CPU_TX3912:
 	case CPU_TX3922:
 	case CPU_TX3927:
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 		build_r3000_tlb_refill_handler();
 		if (!run_once) {
 			build_r3000_tlb_load_handler();
@@ -1413,6 +1432,9 @@ void __cpuinit build_tlb_refill_handler(void)
 			build_r3000_tlb_modify_handler();
 			run_once++;
 		}
+#else
+		panic("No R3000 TLB refill handler");
+#endif
 		break;
 
 	case CPU_R6000:
-- 
1.6.0.6

Re: [PATCH 1/2] MIPS: Add drotr and dins instructions to uasm.

[Ralf Baechle]

On Wed, Oct 14, 2009 at 12:16:55PM -0700, David Daney wrote:

Thanks, applied.

  Ralf

Re: [PATCH 2/2] MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

[Ralf Baechle]

On Wed, Oct 14, 2009 at 12:16:56PM -0700, David Daney wrote:

and this one queued for 2.6.33 as well.

Thanks,

  Ralf

Re: [PATCH 2/2] MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

[Maciej W. Rozycki]

On Wed, 14 Oct 2009, David Daney wrote:

> @@ -1406,6 +1424,7 @@ void __cpuinit build_tlb_refill_handler(void)
>  	case CPU_TX3912:
>  	case CPU_TX3922:
>  	case CPU_TX3927:
> +#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
>  		build_r3000_tlb_refill_handler();
>  		if (!run_once) {
>  			build_r3000_tlb_load_handler();
> @@ -1413,6 +1432,9 @@ void __cpuinit build_tlb_refill_handler(void)
>  			build_r3000_tlb_modify_handler();
>  			run_once++;
>  		}
> +#else
> +		panic("No R3000 TLB refill handler");
> +#endif
>  		break;
>  
>  	case CPU_R6000:

 Shouldn't this be #error or suchlike instead?

  Maciej

Re: [PATCH 2/2] MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

[David Daney]

Maciej W. Rozycki wrote:
> On Wed, 14 Oct 2009, David Daney wrote:
> 
>> @@ -1406,6 +1424,7 @@ void __cpuinit build_tlb_refill_handler(void)
>>  	case CPU_TX3912:
>>  	case CPU_TX3922:
>>  	case CPU_TX3927:
>> +#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
>>  		build_r3000_tlb_refill_handler();
>>  		if (!run_once) {
>>  			build_r3000_tlb_load_handler();
>> @@ -1413,6 +1432,9 @@ void __cpuinit build_tlb_refill_handler(void)
>>  			build_r3000_tlb_modify_handler();
>>  			run_once++;
>>  		}
>> +#else
>> +		panic("No R3000 TLB refill handler");
>> +#endif
>>  		break;
>>  
>>  	case CPU_R6000:
> 
>  Shouldn't this be #error or suchlike instead?
> 

I don't think so.  It is a runtime check.  The kernel was configured in 
such a manner that those CPUs cannot be supported.  By the time the 
problem is detected, the preprocessor (and compiler in general) have 
already been run.

David Daney


>   Maciej
> 

Re: [PATCH 2/2] MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

[Maciej W. Rozycki]

On Thu, 15 Oct 2009, David Daney wrote:

> > > @@ -1406,6 +1424,7 @@ void __cpuinit build_tlb_refill_handler(void)
> > >  	case CPU_TX3912:
> > >  	case CPU_TX3922:
> > >  	case CPU_TX3927:
> > > +#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
> > >  		build_r3000_tlb_refill_handler();
> > >  		if (!run_once) {
> > >  			build_r3000_tlb_load_handler();
> > > @@ -1413,6 +1432,9 @@ void __cpuinit build_tlb_refill_handler(void)
> > >  			build_r3000_tlb_modify_handler();
> > >  			run_once++;
> > >  		}
> > > +#else
> > > +		panic("No R3000 TLB refill handler");
> > > +#endif
> > >  		break;
> > >   	case CPU_R6000:
> > 
> >  Shouldn't this be #error or suchlike instead?
> > 
> 
> I don't think so.  It is a runtime check.  The kernel was configured in such a
> manner that those CPUs cannot be supported.  By the time the problem is
> detected, the preprocessor (and compiler in general) have already been run.

 Hmm, you are right -- somehow I've assumed this piece of code is built 
conditionally.

 We know beforehand that CPU_R3000 precludes both of 64BIT and CPU_MIPSR2.  
And we do not support generic builds that would support multiple CPU types 
(even though there are places where it would work straight away).  So we 
can know at the build time that it is an invalid configuration; probably a 
Kconfig breakage.

 I guess this is not a problem to be solved with your change though.  And 
thanks for thinking about putting in some diagnostics at all for this 
"impossible" case. :)

  Maciej