[PATCH 4/7] powerpc/mm: 64-bit: Don't load PACA in normal TLB miss exceptions

[Scott Wood <scottwood@freescale.com>]

Load it only when needed, in recursive/linear/indirect faults,
and in the stats code.

Signed-off-by: Scott Wood <scottwood@freescale.com>
---
 arch/powerpc/include/asm/exception-64e.h |   28 +++++++++---------
 arch/powerpc/mm/tlb_low_64e.S            |   43 +++++++++++++++++------------
 2 files changed, 39 insertions(+), 32 deletions(-)

diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h
index 6921261..9b57a27 100644
--- a/arch/powerpc/include/asm/exception-64e.h
+++ b/arch/powerpc/include/asm/exception-64e.h
@@ -80,9 +80,9 @@ exc_##label##_book3e:
  *
  * This prolog handles re-entrancy (up to 3 levels supported in the PACA
  * though we currently don't test for overflow). It provides you with a
- * re-entrancy safe working space of r10...r16 and CR with r12 being used
- * as the exception area pointer in the PACA for that level of re-entrancy
- * and r13 containing the PACA pointer.
+ * re-entrancy safe working space of r10...r16 (except r13) and CR with r12
+ * being used as the exception area pointer in the PACA for that level of
+ * re-entrancy.
  *
  * SRR0 and SRR1 are saved, but DEAR and ESR are not, since they don't apply
  * as-is for instruction exceptions. It's up to the actual exception code
@@ -95,8 +95,6 @@ exc_##label##_book3e:
 	mfcr	r10;							    \
 	std	r11,EX_TLB_R11(r12);					    \
 	mfspr	r11,SPRN_SPRG_TLB_SCRATCH;				    \
-	std	r13,EX_TLB_R13(r12);					    \
-	ld	r13,EX_TLB_PACA(r12);					    \
 	std	r14,EX_TLB_R14(r12);					    \
 	addi	r14,r12,EX_TLB_SIZE;					    \
 	std	r15,EX_TLB_R15(r12);					    \
@@ -135,7 +133,6 @@ exc_##label##_book3e:
 	mtspr	SPRN_SPRG_TLB_EXFRAME,freg;				    \
 	ld	r11,EX_TLB_R11(r12);					    \
 	mtcr	r14;							    \
-	ld	r13,EX_TLB_R13(r12);					    \
 	ld	r14,EX_TLB_R14(r12);					    \
 	mtspr	SPRN_SRR0,r15;						    \
 	ld	r15,EX_TLB_R15(r12);					    \
@@ -148,11 +145,13 @@ exc_##label##_book3e:
 	TLB_MISS_RESTORE(r12)
 
 #define TLB_MISS_EPILOG_ERROR						    \
-	addi	r12,r13,PACA_EXTLB;					    \
+	ld	r10,EX_TLB_PACA(r12);					    \
+	addi	r12,r10,PACA_EXTLB;					    \
 	TLB_MISS_RESTORE(r12)
 
 #define TLB_MISS_EPILOG_ERROR_SPECIAL					    \
-	addi	r11,r13,PACA_EXTLB;					    \
+	ld	r10,EX_TLB_PACA(r12);					    \
+	addi	r11,r10,PACA_EXTLB;					    \
 	TLB_MISS_RESTORE(r11)
 
 #ifdef CONFIG_BOOK3E_MMU_TLB_STATS
@@ -160,25 +159,26 @@ exc_##label##_book3e:
 	mflr	r10;							    \
 	std	r8,EX_TLB_R8(r12);					    \
 	std	r9,EX_TLB_R9(r12);					    \
-	std	r10,EX_TLB_LR(r12);
+	std	r10,EX_TLB_LR(r12);					    \
+	ld	r9,EX_TLB_PACA(r12);
 #define TLB_MISS_RESTORE_STATS					            \
 	ld	r16,EX_TLB_LR(r12);					    \
 	ld	r9,EX_TLB_R9(r12);					    \
 	ld	r8,EX_TLB_R8(r12);					    \
 	mtlr	r16;
 #define TLB_MISS_STATS_D(name)						    \
-	addi	r9,r13,MMSTAT_DSTATS+name;				    \
+	addi	r9,r9,MMSTAT_DSTATS+name;				    \
 	bl	.tlb_stat_inc;
 #define TLB_MISS_STATS_I(name)						    \
-	addi	r9,r13,MMSTAT_ISTATS+name;				    \
+	addi	r9,r9,MMSTAT_ISTATS+name;				    \
 	bl	.tlb_stat_inc;
 #define TLB_MISS_STATS_X(name)						    \
-	ld	r8,PACA_EXTLB+EX_TLB_ESR(r13);				    \
+	ld	r8,PACA_EXTLB+EX_TLB_ESR(r9);				    \
 	cmpdi	cr2,r8,-1;						    \
 	beq	cr2,61f;						    \
-	addi	r9,r13,MMSTAT_DSTATS+name;				    \
+	addi	r9,r9,MMSTAT_DSTATS+name;				    \
 	b	62f;							    \
-61:	addi	r9,r13,MMSTAT_ISTATS+name;				    \
+61:	addi	r9,r9,MMSTAT_ISTATS+name;				    \
 62:	bl	.tlb_stat_inc;
 #define TLB_MISS_STATS_SAVE_INFO					    \
 	std	r14,EX_TLB_ESR(r12);	/* save ESR */			    \
diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S
index 17726d3..922fece 100644
--- a/arch/powerpc/mm/tlb_low_64e.S
+++ b/arch/powerpc/mm/tlb_low_64e.S
@@ -160,7 +160,6 @@
  * r16 = faulting address
  * r15 = region ID
  * r14 = crap (free to use)
- * r13 = PACA
  * r12 = TLB exception frame in PACA
  * r11 = PTE permission mask
  * r10 = crap (free to use)
@@ -299,8 +298,7 @@ normal_tlb_miss_access_fault:
  *
  * r16 = virtual page table faulting address
  * r15 = region (top 4 bits of address)
- * r14 = crap (free to use)
- * r13 = PACA
+ * r14 = crap (we load with PACA)
  * r12 = TLB exception frame in PACA
  * r11 = crap (free to use)
  * r10 = crap (free to use)
@@ -318,6 +316,8 @@ normal_tlb_miss_access_fault:
  * so we could probably optimize things a bit
  */
 virt_page_table_tlb_miss:
+	ld	r14,EX_TLB_PACA(r12)
+
 	/* Are we hitting a kernel page table ? */
 	andi.	r10,r15,0x8
 
@@ -325,7 +325,7 @@ virt_page_table_tlb_miss:
 	 * and we happen to have the swapper_pg_dir at offset 8 from the user
 	 * pgdir in the PACA :-).
 	 */
-	add	r11,r10,r13
+	add	r11,r10,r14
 
 	/* If kernel, we need to clear MAS1 TID */
 	beq	1f
@@ -417,12 +417,12 @@ virt_page_table_tlb_miss_done:
 	 * offset the return address by -4 in order to replay the tlbsrx
 	 * instruction there
 	 */
-	subf	r10,r13,r12
+	subf	r10,r14,r12
 	cmpldi	cr0,r10,PACA_EXTLB+EX_TLB_SIZE
 	bne-	1f
-	ld	r11,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13)
+	ld	r11,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r14)
 	addi	r10,r11,-4
-	std	r10,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13)
+	std	r10,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r14)
 1:
 END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
 	/* Return to caller, normal case */
@@ -449,13 +449,13 @@ virt_page_table_tlb_miss_fault:
 	 * level as well. Since we are doing that, we don't need to clear or
 	 * restore the TLB reservation neither.
 	 */
-	subf	r10,r13,r12
+	subf	r10,r14,r12
 	cmpldi	cr0,r10,PACA_EXTLB+EX_TLB_SIZE
 	bne-	virt_page_table_tlb_miss_whacko_fault
 
 	/* We dig the original DEAR and ESR from slot 0 */
-	ld	r15,EX_TLB_DEAR+PACA_EXTLB(r13)
-	ld	r16,EX_TLB_ESR+PACA_EXTLB(r13)
+	ld	r15,EX_TLB_DEAR+PACA_EXTLB(r14)
+	ld	r16,EX_TLB_ESR+PACA_EXTLB(r14)
 
 	/* We check for the "special" ESR value for instruction faults */
 	cmpdi	cr0,r16,-1
@@ -489,6 +489,8 @@ virt_page_table_tlb_miss_whacko_fault:
 	START_EXCEPTION(data_tlb_miss_htw)
 	TLB_MISS_PROLOG
 
+	ld	r15,EX_TLB_PACA(r12)
+
 	/* Now we handle the fault proper. We only save DEAR in normal
 	 * fault case since that's the only interesting values here.
 	 * We could probably also optimize by not saving SRR0/1 in the
@@ -503,8 +505,12 @@ virt_page_table_tlb_miss_whacko_fault:
 
 	/* We do the user/kernel test for the PID here along with the RW test
 	 */
+	/* The cool thing now is that r11 contains 0 for user and 8 for kernel,
+	 * and we happen to have the swapper_pg_dir at offset 8 from the user
+	 * pgdir in the PACA :-).
+	 */
 	cmpldi	cr0,r11,0		/* Check for user region */
-	ld	r15,PACAPGD(r13)	/* Load user pgdir */
+	add	r15,r15,r11
 	beq	htw_tlb_miss
 
 	/* XXX replace the RMW cycles with immediate loads + writes */
@@ -512,7 +518,6 @@ virt_page_table_tlb_miss_whacko_fault:
 	cmpldi	cr0,r11,8		/* Check for vmalloc region */
 	rlwinm	r10,r10,0,16,1		/* Clear TID */
 	mtspr	SPRN_MAS1,r10
-	ld	r15,PACA_KERNELPGD(r13)	/* Load kernel pgdir */
 	beq+	htw_tlb_miss
 
 	/* We got a crappy address, just fault with whatever DEAR and ESR
@@ -526,6 +531,8 @@ virt_page_table_tlb_miss_whacko_fault:
 	START_EXCEPTION(instruction_tlb_miss_htw)
 	TLB_MISS_PROLOG
 
+	ld	r15,EX_TLB_PACA(r12)
+
 	/* If we take a recursive fault, the second level handler may need
 	 * to know whether we are handling a data or instruction fault in
 	 * order to get to the right store fault handler. We provide that
@@ -548,7 +555,7 @@ virt_page_table_tlb_miss_whacko_fault:
 	/* We do the user/kernel test for the PID here along with the RW test
 	 */
 	cmpldi	cr0,r11,0			/* Check for user region */
-	ld	r15,PACAPGD(r13)		/* Load user pgdir */
+	add	r15,r15,r11
 	beq	htw_tlb_miss
 
 	/* XXX replace the RMW cycles with immediate loads + writes */
@@ -556,7 +563,6 @@ virt_page_table_tlb_miss_whacko_fault:
 	cmpldi	cr0,r11,8			/* Check for vmalloc region */
 	rlwinm	r10,r10,0,16,1			/* Clear TID */
 	mtspr	SPRN_MAS1,r10
-	ld	r15,PACA_KERNELPGD(r13)		/* Load kernel pgdir */
 	beq+	htw_tlb_miss
 
 	/* We got a crappy address, just fault */
@@ -570,9 +576,8 @@ virt_page_table_tlb_miss_whacko_fault:
  * misses. We are entered with:
  *
  * r16 = virtual page table faulting address
- * r15 = PGD pointer
+ * r15 = pointer to PGD pointer
  * r14 = ESR
- * r13 = PACA
  * r12 = TLB exception frame in PACA
  * r11 = crap (free to use)
  * r10 = crap (free to use)
@@ -581,6 +586,8 @@ virt_page_table_tlb_miss_whacko_fault:
  * avoid too much complication, it will save/restore things for us
  */
 htw_tlb_miss:
+	ld	r15,0(r15)
+
 	/* Search if we already have a TLB entry for that virtual address, and
 	 * if we do, bail out.
 	 *
@@ -692,7 +699,6 @@ htw_tlb_miss_fault:
  * r16 = faulting address
  * r15 = crap (free to use)
  * r14 = ESR (data) or -1 (instruction)
- * r13 = PACA
  * r12 = TLB exception frame in PACA
  * r11 = crap (free to use)
  * r10 = crap (free to use)
@@ -714,7 +720,8 @@ tlb_load_linear:
 	 * we only use 1G pages for now. That might have to be changed in a
 	 * final implementation, especially when dealing with hypervisors
 	 */
-	ld	r11,PACATOC(r13)
+	ld	r11,EX_TLB_PACA(r12)
+	ld	r11,PACATOC(r11)
 	ld	r11,linear_map_top@got(r11)
 	ld	r10,0(r11)
 	cmpld	cr0,r10,r16
-- 
1.7.4.1