* [PATCH v3 27/41] powerpc/32: Call bad_page_fault() from do_page_fault()
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
Now that non volatile registers are saved at all time, no
need to split bad_page_fault() out of do_page_fault().
Remove handle_page_fault() and use do_page_fault() directly.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 16 ----------------
arch/powerpc/kernel/head_40x.S | 4 ++--
arch/powerpc/kernel/head_8xx.S | 4 ++--
arch/powerpc/kernel/head_book3s_32.S | 4 ++--
arch/powerpc/kernel/head_booke.h | 4 ++--
arch/powerpc/kernel/head_fsl_booke.S | 2 +-
arch/powerpc/mm/fault.c | 2 +-
7 files changed, 10 insertions(+), 26 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index e538ae73394d..76e1502b3e6f 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -220,22 +220,6 @@ ret_from_kernel_thread:
li r3,0
b ret_from_syscall
-/*
- * Top-level page fault handling.
- * This is in assembler because if do_page_fault tells us that
- * it is a bad kernel page fault, we want to save the non-volatile
- * registers before calling bad_page_fault.
- */
- .globl handle_page_fault
-handle_page_fault:
- bl do_page_fault
- cmpwi r3,0
- beq+ ret_from_except
- mr r4,r3 /* err arg for bad_page_fault */
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl __bad_page_fault
- b ret_from_except_full
-
/*
* This routine switches between two different tasks. The process
* state of one is saved on its kernel stack. Then the state
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index 08563d4170c6..a65778380704 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -207,7 +207,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
*/
START_EXCEPTION(0x0300, DataStorage)
EXCEPTION_PROLOG DataStorage handle_dar_dsisr=1
- EXC_XFER_LITE(0x300, handle_page_fault)
+ EXC_XFER_LITE(0x300, do_page_fault)
/*
* 0x0400 - Instruction Storage Exception
@@ -218,7 +218,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
li r5,0
stw r5, _ESR(r11) /* Zero ESR */
stw r12, _DEAR(r11) /* SRR0 as DEAR */
- EXC_XFER_LITE(0x400, handle_page_fault)
+ EXC_XFER_LITE(0x400, do_page_fault)
/* 0x0500 - External Interrupt Exception */
EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index eb1d40a8f2c4..4078d0dc2f18 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -301,7 +301,7 @@ instruction_counter:
.Litlbie:
stw r12, _DAR(r11)
stw r5, _DSISR(r11)
- EXC_XFER_LITE(0x400, handle_page_fault)
+ EXC_XFER_LITE(0x400, do_page_fault)
/* This is the data TLB error on the MPC8xx. This could be due to
* many reasons, including a dirty update to a pte. We bail out to
@@ -322,7 +322,7 @@ DARFixed:/* Return from dcbx instruction bug workaround */
tlbie r4
.Ldtlbie:
/* 0x300 is DataAccess exception, needed by bad_page_fault() */
- EXC_XFER_LITE(0x300, handle_page_fault)
+ EXC_XFER_LITE(0x300, do_page_fault)
#ifdef CONFIG_VMAP_STACK
vmap_stack_overflow_exception
diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S
index 453f6ea959f9..e78b4a7c23af 100644
--- a/arch/powerpc/kernel/head_book3s_32.S
+++ b/arch/powerpc/kernel/head_book3s_32.S
@@ -299,7 +299,7 @@ ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
lwz r5, _DSISR(r11)
andis. r0, r5, DSISR_DABRMATCH@h
bne- 1f
- EXC_XFER_LITE(0x300, handle_page_fault)
+ EXC_XFER_LITE(0x300, do_page_fault)
1: EXC_XFER_STD(0x300, do_break)
@@ -328,7 +328,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
stw r5, _DSISR(r11)
stw r12, _DAR(r11)
- EXC_XFER_LITE(0x400, handle_page_fault)
+ EXC_XFER_LITE(0x400, do_page_fault)
/* External interrupt */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 009a56d70d76..036a69d16605 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -462,7 +462,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r5,_ESR(r11); \
mfspr r4,SPRN_DEAR; /* Grab the DEAR */ \
stw r4, _DEAR(r11); \
- EXC_XFER_LITE(0x0300, handle_page_fault)
+ EXC_XFER_LITE(0x0300, do_page_fault)
#define INSTRUCTION_STORAGE_EXCEPTION \
START_EXCEPTION(InstructionStorage) \
@@ -470,7 +470,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
stw r5,_ESR(r11); \
stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
- EXC_XFER_LITE(0x0400, handle_page_fault)
+ EXC_XFER_LITE(0x0400, do_page_fault)
#define ALIGNMENT_EXCEPTION \
START_EXCEPTION(Alignment) \
diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S
index f51c66f747ad..72e9aa45b99b 100644
--- a/arch/powerpc/kernel/head_fsl_booke.S
+++ b/arch/powerpc/kernel/head_fsl_booke.S
@@ -370,7 +370,7 @@ interrupt_base:
stw r4, _DEAR(r11)
andis. r10,r5,(ESR_ILK|ESR_DLK)@h
bne 1f
- EXC_XFER_LITE(0x0300, handle_page_fault)
+ EXC_XFER_LITE(0x0300, do_page_fault)
1:
EXC_XFER_LITE(0x0300, CacheLockingException)
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index bb368257b55c..2e54bac99a22 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -552,7 +552,7 @@ static long __do_page_fault(struct pt_regs *regs)
if (likely(entry)) {
instruction_pointer_set(regs, extable_fixup(entry));
return 0;
- } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
+ } else if (!IS_ENABLED(CONFIG_PPC_BOOK3E_64)) {
__bad_page_fault(regs, err);
return 0;
} else {
--
2.25.0
^ permalink raw reply related
* [PATCH v3 28/41] powerpc/32: Save trap number on stack in exception prolog
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
Saving the trap number into the stack goes into
the exception prolog, as EXC_XFER_xxx will soon disappear.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/head_32.h | 14 ++++-----
arch/powerpc/kernel/head_40x.S | 22 +++++++-------
arch/powerpc/kernel/head_8xx.S | 14 ++++-----
arch/powerpc/kernel/head_book3s_32.S | 14 ++++-----
arch/powerpc/kernel/head_booke.h | 44 +++++++++++++++-------------
arch/powerpc/kernel/head_fsl_booke.S | 4 +--
6 files changed, 58 insertions(+), 54 deletions(-)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index 4d638d760a96..bf4c288173ad 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -10,10 +10,10 @@
* We assume sprg3 has the physical address of the current
* task's thread_struct.
*/
-.macro EXCEPTION_PROLOG name handle_dar_dsisr=0
+.macro EXCEPTION_PROLOG trapno name handle_dar_dsisr=0
EXCEPTION_PROLOG_0 handle_dar_dsisr=\handle_dar_dsisr
EXCEPTION_PROLOG_1
- EXCEPTION_PROLOG_2 \name handle_dar_dsisr=\handle_dar_dsisr
+ EXCEPTION_PROLOG_2 \trapno \name handle_dar_dsisr=\handle_dar_dsisr
.endm
.macro EXCEPTION_PROLOG_0 handle_dar_dsisr=0
@@ -56,7 +56,7 @@
#endif
.endm
-.macro EXCEPTION_PROLOG_2 name handle_dar_dsisr=0
+.macro EXCEPTION_PROLOG_2 trapno name handle_dar_dsisr=0
#ifdef CONFIG_PPC_8xx
.if \handle_dar_dsisr
li r11, RPN_PATTERN
@@ -108,6 +108,8 @@
lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
addi r10,r10,STACK_FRAME_REGS_MARKER@l
stw r10,8(r11)
+ li r10, \trapno
+ stw r10,_TRAP(r11)
SAVE_4GPRS(3, r11)
SAVE_2GPRS(7, r11)
_ASM_NOKPROBE_SYMBOL(\name\()_virt)
@@ -182,12 +184,10 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
#define EXCEPTION(n, label, hdlr, xfer) \
START_EXCEPTION(n, label) \
- EXCEPTION_PROLOG label; \
+ EXCEPTION_PROLOG n label; \
xfer(n, hdlr)
#define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \
- li r10,trap; \
- stw r10,_TRAP(r11); \
bl tfer; \
bl hdlr; \
b ret
@@ -213,7 +213,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
#endif
lwz r1, emergency_ctx@l(r1)
addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
- EXCEPTION_PROLOG_2 vmap_stack_overflow
+ EXCEPTION_PROLOG_2 0 vmap_stack_overflow
EXC_XFER_STD(0, stack_overflow_exception)
.endm
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index a65778380704..7270caff665c 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -104,7 +104,7 @@ _ENTRY(crit_esr)
* Instead we use a couple of words of memory at low physical addresses.
* This is OK since we don't support SMP on these processors.
*/
-.macro CRITICAL_EXCEPTION_PROLOG name
+.macro CRITICAL_EXCEPTION_PROLOG trapno name
stw r10,crit_r10@l(0) /* save two registers to work with */
stw r11,crit_r11@l(0)
mfspr r10,SPRN_SRR0
@@ -161,6 +161,8 @@ _ENTRY(crit_esr)
lis r10, STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
addi r10, r10, STACK_FRAME_REGS_MARKER@l
stw r10, 8(r11)
+ li r10, \trapno + 2
+ stw r10,_TRAP(r11)
SAVE_4GPRS(3, r11)
SAVE_2GPRS(7, r11)
_ASM_NOKPROBE_SYMBOL(\name\()_virt)
@@ -184,7 +186,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
*/
#define CRITICAL_EXCEPTION(n, label, hdlr) \
START_EXCEPTION(n, label); \
- CRITICAL_EXCEPTION_PROLOG label; \
+ CRITICAL_EXCEPTION_PROLOG n label; \
EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \
crit_transfer_to_handler, ret_from_crit_exc)
@@ -206,7 +208,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
* if they can't resolve the lightweight TLB fault.
*/
START_EXCEPTION(0x0300, DataStorage)
- EXCEPTION_PROLOG DataStorage handle_dar_dsisr=1
+ EXCEPTION_PROLOG 0x300 DataStorage handle_dar_dsisr=1
EXC_XFER_LITE(0x300, do_page_fault)
/*
@@ -214,7 +216,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
* This is caused by a fetch from non-execute or guarded pages.
*/
START_EXCEPTION(0x0400, InstructionAccess)
- EXCEPTION_PROLOG InstructionAccess
+ EXCEPTION_PROLOG 0x400 InstructionAccess
li r5,0
stw r5, _ESR(r11) /* Zero ESR */
stw r12, _DEAR(r11) /* SRR0 as DEAR */
@@ -225,12 +227,12 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
/* 0x0600 - Alignment Exception */
START_EXCEPTION(0x0600, Alignment)
- EXCEPTION_PROLOG Alignment handle_dar_dsisr=1
+ EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
EXC_XFER_STD(0x600, alignment_exception)
/* 0x0700 - Program Exception */
START_EXCEPTION(0x0700, ProgramCheck)
- EXCEPTION_PROLOG ProgramCheck handle_dar_dsisr=1
+ EXCEPTION_PROLOG 0x700 ProgramCheck handle_dar_dsisr=1
EXC_XFER_STD(0x700, program_check_exception)
EXCEPTION(0x0800, Trap_08, unknown_exception, EXC_XFER_STD)
@@ -449,7 +451,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
*/
/* 0x2000 - Debug Exception */
START_EXCEPTION(0x2000, DebugTrap)
- CRITICAL_EXCEPTION_PROLOG DebugTrap
+ CRITICAL_EXCEPTION_PROLOG 0x2000 DebugTrap
/*
* If this is a single step or branch-taken exception in an
@@ -498,7 +500,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
/* Programmable Interval Timer (PIT) Exception. (from 0x1000) */
__HEAD
Decrementer:
- EXCEPTION_PROLOG Decrementer
+ EXCEPTION_PROLOG 0x1000 Decrementer
lis r0,TSR_PIS@h
mtspr SPRN_TSR,r0 /* Clear the PIT exception */
EXC_XFER_LITE(0x1000, timer_interrupt)
@@ -506,13 +508,13 @@ Decrementer:
/* Fixed Interval Timer (FIT) Exception. (from 0x1010) */
__HEAD
FITException:
- EXCEPTION_PROLOG FITException
+ EXCEPTION_PROLOG 0x1010 FITException
EXC_XFER_STD(0x1010, unknown_exception)
/* Watchdog Timer (WDT) Exception. (from 0x1020) */
__HEAD
WDTException:
- CRITICAL_EXCEPTION_PROLOG WDTException
+ CRITICAL_EXCEPTION_PROLOG 0x1020 WDTException
EXC_XFER_TEMPLATE(WatchdogException, 0x1020+2,
(MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)),
crit_transfer_to_handler, ret_from_crit_exc)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 4078d0dc2f18..c48de97f42fc 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -122,7 +122,7 @@ instruction_counter:
/* Machine check */
START_EXCEPTION(0x200, MachineCheck)
- EXCEPTION_PROLOG MachineCheck handle_dar_dsisr=1
+ EXCEPTION_PROLOG 0x200 MachineCheck handle_dar_dsisr=1
EXC_XFER_STD(0x200, machine_check_exception)
/* External interrupt */
@@ -130,7 +130,7 @@ instruction_counter:
/* Alignment exception */
START_EXCEPTION(0x600, Alignment)
- EXCEPTION_PROLOG Alignment handle_dar_dsisr=1
+ EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
EXC_XFER_STD(0x600, alignment_exception)
/* Program check exception */
@@ -292,12 +292,12 @@ instruction_counter:
* addresses. There is nothing to do but handle a big time error fault.
*/
START_EXCEPTION(0x1300, InstructionTLBError)
- EXCEPTION_PROLOG InstructionTLBError
+ /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
+ EXCEPTION_PROLOG 0x400 InstructionTLBError
andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
andis. r10,r9,SRR1_ISI_NOPT@h
beq+ .Litlbie
tlbie r12
- /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
.Litlbie:
stw r12, _DAR(r11)
stw r5, _DSISR(r11)
@@ -314,14 +314,14 @@ instruction_counter:
beq- cr1, FixupDAR /* must be a buggy dcbX, icbi insn. */
DARFixed:/* Return from dcbx instruction bug workaround */
EXCEPTION_PROLOG_1
- EXCEPTION_PROLOG_2 DataTLBError handle_dar_dsisr=1
+ /* 0x300 is DataAccess exception, needed by bad_page_fault() */
+ EXCEPTION_PROLOG_2 0x300 DataTLBError handle_dar_dsisr=1
lwz r4, _DAR(r11)
lwz r5, _DSISR(r11)
andis. r10,r5,DSISR_NOHPTE@h
beq+ .Ldtlbie
tlbie r4
.Ldtlbie:
- /* 0x300 is DataAccess exception, needed by bad_page_fault() */
EXC_XFER_LITE(0x300, do_page_fault)
#ifdef CONFIG_VMAP_STACK
@@ -345,7 +345,7 @@ DARFixed:/* Return from dcbx instruction bug workaround */
rfi
1: EXCEPTION_PROLOG_1
- EXCEPTION_PROLOG_2 DataBreakpoint handle_dar_dsisr=1
+ EXCEPTION_PROLOG_2 0x1c00 DataBreakpoint handle_dar_dsisr=1
mfspr r4,SPRN_BAR
stw r4,_DAR(r11)
EXC_XFER_STD(0x1c00, do_break)
diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S
index e78b4a7c23af..2b4875ceaea4 100644
--- a/arch/powerpc/kernel/head_book3s_32.S
+++ b/arch/powerpc/kernel/head_book3s_32.S
@@ -266,7 +266,7 @@ __secondary_hold_acknowledge:
mfspr r1, SPRN_SPRG_SCRATCH2
#endif /* CONFIG_PPC_CHRP */
EXCEPTION_PROLOG_1
-7: EXCEPTION_PROLOG_2 MachineCheck
+7: EXCEPTION_PROLOG_2 0x200 MachineCheck
#ifdef CONFIG_PPC_CHRP
beq cr1, 1f
twi 31, 0, 0
@@ -295,7 +295,7 @@ ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
#endif
1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1
EXCEPTION_PROLOG_1
- EXCEPTION_PROLOG_2 DataAccess handle_dar_dsisr=1
+ EXCEPTION_PROLOG_2 0x300 DataAccess handle_dar_dsisr=1
lwz r5, _DSISR(r11)
andis. r0, r5, DSISR_DABRMATCH@h
bne- 1f
@@ -324,7 +324,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
andi. r11, r11, MSR_PR
EXCEPTION_PROLOG_1
- EXCEPTION_PROLOG_2 InstructionAccess
+ EXCEPTION_PROLOG_2 0x400 InstructionAccess
andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
stw r5, _DSISR(r11)
stw r12, _DAR(r11)
@@ -335,7 +335,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
/* Alignment exception */
START_EXCEPTION(0x600, Alignment)
- EXCEPTION_PROLOG Alignment handle_dar_dsisr=1
+ EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
EXC_XFER_STD(0x600, alignment_exception)
/* Program check exception */
@@ -351,7 +351,7 @@ BEGIN_FTR_SECTION
*/
b ProgramCheck
END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
- EXCEPTION_PROLOG FPUnavailable
+ EXCEPTION_PROLOG 0x800 FPUnavailable
beq 1f
bl load_up_fpu /* if from user, just load it up */
b fast_exception_return
@@ -713,7 +713,7 @@ fast_hash_page_return:
__HEAD
AltiVecUnavailable:
- EXCEPTION_PROLOG AltiVecUnavailable
+ EXCEPTION_PROLOG 0xf20 AltiVecUnavailable
#ifdef CONFIG_ALTIVEC
beq 1f
bl load_up_altivec /* if from user, just load it up */
@@ -723,7 +723,7 @@ AltiVecUnavailable:
__HEAD
PerformanceMonitor:
- EXCEPTION_PROLOG PerformanceMonitor
+ EXCEPTION_PROLOG 0xf00 PerformanceMonitor
EXC_XFER_STD(0xf00, performance_monitor_exception)
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 036a69d16605..27a7358c04bb 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -44,7 +44,7 @@ END_BTB_FLUSH_SECTION
#endif
-#define NORMAL_EXCEPTION_PROLOG(intno) \
+#define NORMAL_EXCEPTION_PROLOG(trapno, intno) \
mtspr SPRN_SPRG_WSCRATCH0, r10; /* save one register */ \
mfspr r10, SPRN_SPRG_THREAD; \
stw r11, THREAD_NORMSAVE(0)(r10); \
@@ -82,6 +82,8 @@ END_BTB_FLUSH_SECTION
lis r10, STACK_FRAME_REGS_MARKER@ha;/* exception frame marker */ \
addi r10, r10, STACK_FRAME_REGS_MARKER@l; \
stw r10, 8(r11); \
+ li r10, trapno; \
+ stw r10,_TRAP(r11); \
SAVE_4GPRS(3, r11); \
SAVE_2GPRS(7, r11)
@@ -182,7 +184,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
* registers as the normal prolog above. Instead we use a portion of the
* critical/machine check exception stack at low physical addresses.
*/
-#define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, intno, exc_level_srr0, exc_level_srr1) \
+#define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, trapno, intno, exc_level_srr0, exc_level_srr1) \
mtspr SPRN_SPRG_WSCRATCH_##exc_level,r8; \
BOOKE_LOAD_EXC_LEVEL_STACK(exc_level);/* r8 points to the exc_level stack*/ \
stw r9,GPR9(r8); /* save various registers */\
@@ -225,6 +227,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r1,0(r11); \
mr r1,r11; \
rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\
+ li r10, trapno; \
+ stw r10,_TRAP(r11); \
stw r0,GPR0(r11); \
SAVE_4GPRS(3, r11); \
SAVE_2GPRS(7, r11)
@@ -259,12 +263,12 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#endif
.endm
-#define CRITICAL_EXCEPTION_PROLOG(intno) \
- EXC_LEVEL_EXCEPTION_PROLOG(CRIT, intno, SPRN_CSRR0, SPRN_CSRR1)
-#define DEBUG_EXCEPTION_PROLOG \
- EXC_LEVEL_EXCEPTION_PROLOG(DBG, DEBUG, SPRN_DSRR0, SPRN_DSRR1)
-#define MCHECK_EXCEPTION_PROLOG \
- EXC_LEVEL_EXCEPTION_PROLOG(MC, MACHINE_CHECK, \
+#define CRITICAL_EXCEPTION_PROLOG(trapno, intno) \
+ EXC_LEVEL_EXCEPTION_PROLOG(CRIT, trapno+2, intno, SPRN_CSRR0, SPRN_CSRR1)
+#define DEBUG_EXCEPTION_PROLOG(trapno) \
+ EXC_LEVEL_EXCEPTION_PROLOG(DBG, trapno+8, DEBUG, SPRN_DSRR0, SPRN_DSRR1)
+#define MCHECK_EXCEPTION_PROLOG(trapno) \
+ EXC_LEVEL_EXCEPTION_PROLOG(MC, trapno+4, MACHINE_CHECK, \
SPRN_MCSRR0, SPRN_MCSRR1)
/*
@@ -293,12 +297,12 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#define EXCEPTION(n, intno, label, hdlr, xfer) \
START_EXCEPTION(label); \
- NORMAL_EXCEPTION_PROLOG(intno); \
+ NORMAL_EXCEPTION_PROLOG(n, intno); \
xfer(n, hdlr)
#define CRITICAL_EXCEPTION(n, intno, label, hdlr) \
START_EXCEPTION(label); \
- CRITICAL_EXCEPTION_PROLOG(intno); \
+ CRITICAL_EXCEPTION_PROLOG(n, intno); \
SAVE_MMU_REGS; \
SAVE_xSRR(SRR); \
EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \
@@ -306,7 +310,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#define MCHECK_EXCEPTION(n, label, hdlr) \
START_EXCEPTION(label); \
- MCHECK_EXCEPTION_PROLOG; \
+ MCHECK_EXCEPTION_PROLOG(n); \
mfspr r5,SPRN_ESR; \
stw r5,_ESR(r11); \
SAVE_xSRR(DSRR); \
@@ -317,8 +321,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
mcheck_transfer_to_handler, ret_from_mcheck_exc)
#define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \
- li r10,trap; \
- stw r10,_TRAP(r11); \
bl tfer; \
bl hdlr; \
b ret; \
@@ -346,7 +348,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
*/
#define DEBUG_DEBUG_EXCEPTION \
START_EXCEPTION(DebugDebug); \
- DEBUG_EXCEPTION_PROLOG; \
+ DEBUG_EXCEPTION_PROLOG(2000); \
\
/* \
* If there is a single step or branch-taken exception in an \
@@ -402,7 +404,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#define DEBUG_CRIT_EXCEPTION \
START_EXCEPTION(DebugCrit); \
- CRITICAL_EXCEPTION_PROLOG(DEBUG); \
+ CRITICAL_EXCEPTION_PROLOG(2000,DEBUG); \
\
/* \
* If there is a single step or branch-taken exception in an \
@@ -457,7 +459,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#define DATA_STORAGE_EXCEPTION \
START_EXCEPTION(DataStorage) \
- NORMAL_EXCEPTION_PROLOG(DATA_STORAGE); \
+ NORMAL_EXCEPTION_PROLOG(0x300, DATA_STORAGE); \
mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
stw r5,_ESR(r11); \
mfspr r4,SPRN_DEAR; /* Grab the DEAR */ \
@@ -466,7 +468,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#define INSTRUCTION_STORAGE_EXCEPTION \
START_EXCEPTION(InstructionStorage) \
- NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \
+ NORMAL_EXCEPTION_PROLOG(0x400, INST_STORAGE); \
mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
stw r5,_ESR(r11); \
stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
@@ -474,28 +476,28 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#define ALIGNMENT_EXCEPTION \
START_EXCEPTION(Alignment) \
- NORMAL_EXCEPTION_PROLOG(ALIGNMENT); \
+ NORMAL_EXCEPTION_PROLOG(0x600, ALIGNMENT); \
mfspr r4,SPRN_DEAR; /* Grab the DEAR and save it */ \
stw r4,_DEAR(r11); \
EXC_XFER_STD(0x0600, alignment_exception)
#define PROGRAM_EXCEPTION \
START_EXCEPTION(Program) \
- NORMAL_EXCEPTION_PROLOG(PROGRAM); \
+ NORMAL_EXCEPTION_PROLOG(0x700, PROGRAM); \
mfspr r4,SPRN_ESR; /* Grab the ESR and save it */ \
stw r4,_ESR(r11); \
EXC_XFER_STD(0x0700, program_check_exception)
#define DECREMENTER_EXCEPTION \
START_EXCEPTION(Decrementer) \
- NORMAL_EXCEPTION_PROLOG(DECREMENTER); \
+ NORMAL_EXCEPTION_PROLOG(0x900, DECREMENTER); \
lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \
mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \
EXC_XFER_LITE(0x0900, timer_interrupt)
#define FP_UNAVAILABLE_EXCEPTION \
START_EXCEPTION(FloatingPointUnavailable) \
- NORMAL_EXCEPTION_PROLOG(FP_UNAVAIL); \
+ NORMAL_EXCEPTION_PROLOG(0x800, FP_UNAVAIL); \
beq 1f; \
bl load_up_fpu; /* if from user, just load it up */ \
b fast_exception_return; \
diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S
index 72e9aa45b99b..bf2730b4e43b 100644
--- a/arch/powerpc/kernel/head_fsl_booke.S
+++ b/arch/powerpc/kernel/head_fsl_booke.S
@@ -363,7 +363,7 @@ interrupt_base:
/* Data Storage Interrupt */
START_EXCEPTION(DataStorage)
- NORMAL_EXCEPTION_PROLOG(DATA_STORAGE)
+ NORMAL_EXCEPTION_PROLOG(0x300, DATA_STORAGE)
mfspr r5,SPRN_ESR /* Grab the ESR, save it */
stw r5,_ESR(r11)
mfspr r4,SPRN_DEAR /* Grab the DEAR, save it */
@@ -613,7 +613,7 @@ END_BTB_FLUSH_SECTION
#ifdef CONFIG_SPE
/* SPE Unavailable */
START_EXCEPTION(SPEUnavailable)
- NORMAL_EXCEPTION_PROLOG(SPE_UNAVAIL)
+ NORMAL_EXCEPTION_PROLOG(0x2010, SPE_UNAVAIL)
beq 1f
bl load_up_spe
b fast_exception_return
--
2.25.0
^ permalink raw reply related
* [PATCH v3 30/41] powerpc/32: Only restore non volatile registers when required
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
Until now, non volatile registers were restored everytime they
were saved, ie using EXC_XFER_STD meant saving and restoring
them while EXC_XFER_LITE meant neither saving not restoring them.
Now that they are always saved, EXC_XFER_STD means to restore
them and EXC_XFER_LITE means to not restore them.
Most of the users of EXC_XFER_STD only need to retrieve the
non volatile registers. For them there is no need to restore
the non volatile registers as they have not been modified.
Only very few exceptions require non volatile registers restore.
Opencode the few places which require saving of non volatile
registers.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 1 -
arch/powerpc/kernel/head_40x.S | 10 ++++++++--
arch/powerpc/kernel/head_8xx.S | 24 ++++++++++++++++++++----
arch/powerpc/kernel/head_book3s_32.S | 17 ++++++++++++++---
arch/powerpc/kernel/head_booke.h | 10 ++++++++--
arch/powerpc/kernel/head_fsl_booke.S | 16 ++++++++++++----
6 files changed, 62 insertions(+), 16 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 237b753720db..e6adb6882bde 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -358,7 +358,6 @@ _ASM_NOKPROBE_SYMBOL(fast_exception_return)
.globl ret_from_except_full
ret_from_except_full:
- REST_NVGPRS(r1)
/* fall through */
.globl ret_from_except
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index 7270caff665c..f3e5b462113f 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -228,12 +228,18 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
/* 0x0600 - Alignment Exception */
START_EXCEPTION(0x0600, Alignment)
EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
- EXC_XFER_STD(0x600, alignment_exception)
+ prepare_transfer_to_handler
+ bl alignment_exception
+ REST_NVGPRS(r1)
+ b interrupt_return
/* 0x0700 - Program Exception */
START_EXCEPTION(0x0700, ProgramCheck)
EXCEPTION_PROLOG 0x700 ProgramCheck handle_dar_dsisr=1
- EXC_XFER_STD(0x700, program_check_exception)
+ prepare_transfer_to_handler
+ bl program_check_exception
+ REST_NVGPRS(r1)
+ b interrupt_return
EXCEPTION(0x0800, Trap_08, unknown_exception, EXC_XFER_STD)
EXCEPTION(0x0900, Trap_09, unknown_exception, EXC_XFER_STD)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index c48de97f42fc..86f844eb0e5a 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -131,10 +131,18 @@ instruction_counter:
/* Alignment exception */
START_EXCEPTION(0x600, Alignment)
EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
- EXC_XFER_STD(0x600, alignment_exception)
+ prepare_transfer_to_handler
+ bl alignment_exception
+ REST_NVGPRS(r1)
+ b interrupt_return
/* Program check exception */
- EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
+ START_EXCEPTION(0x700, ProgramCheck)
+ EXCEPTION_PROLOG 0x700 ProgramCheck
+ prepare_transfer_to_handler
+ bl program_check_exception
+ REST_NVGPRS(r1)
+ b interrupt_return
/* Decrementer */
EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
@@ -149,7 +157,12 @@ instruction_counter:
/* On the MPC8xx, this is a software emulation interrupt. It occurs
* for all unimplemented and illegal instructions.
*/
- EXCEPTION(0x1000, SoftEmu, emulation_assist_interrupt, EXC_XFER_STD)
+ START_EXCEPTION(0x1000, SoftEmu)
+ EXCEPTION_PROLOG 0x1000 SoftEmu
+ prepare_transfer_to_handler
+ bl emulation_assist_interrupt
+ REST_NVGPRS(r1)
+ b interrupt_return
/*
* For the MPC8xx, this is a software tablewalk to load the instruction
@@ -348,7 +361,10 @@ DARFixed:/* Return from dcbx instruction bug workaround */
EXCEPTION_PROLOG_2 0x1c00 DataBreakpoint handle_dar_dsisr=1
mfspr r4,SPRN_BAR
stw r4,_DAR(r11)
- EXC_XFER_STD(0x1c00, do_break)
+ prepare_transfer_to_handler
+ bl do_break
+ REST_NVGPRS(r1)
+ b interrupt_return
#ifdef CONFIG_PERF_EVENTS
START_EXCEPTION(0x1d00, InstructionBreakpoint)
diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S
index 2b4875ceaea4..425a4f20ceac 100644
--- a/arch/powerpc/kernel/head_book3s_32.S
+++ b/arch/powerpc/kernel/head_book3s_32.S
@@ -300,7 +300,10 @@ ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
andis. r0, r5, DSISR_DABRMATCH@h
bne- 1f
EXC_XFER_LITE(0x300, do_page_fault)
-1: EXC_XFER_STD(0x300, do_break)
+1: prepare_transfer_to_handler
+ bl do_break
+ REST_NVGPRS(r1)
+ b interrupt_return
/* Instruction access exception. */
@@ -336,10 +339,18 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
/* Alignment exception */
START_EXCEPTION(0x600, Alignment)
EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
- EXC_XFER_STD(0x600, alignment_exception)
+ prepare_transfer_to_handler
+ bl alignment_exception
+ REST_NVGPRS(r1)
+ b interrupt_return
/* Program check exception */
- EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
+ START_EXCEPTION(0x700, ProgramCheck)
+ EXCEPTION_PROLOG 0x700 ProgramCheck
+ prepare_transfer_to_handler
+ bl program_check_exception
+ REST_NVGPRS(r1)
+ b interrupt_return
/* Floating-point unavailable */
START_EXCEPTION(0x800, FPUnavailable)
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 0f02b970e797..baf10556c587 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -483,14 +483,20 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
NORMAL_EXCEPTION_PROLOG(0x600, ALIGNMENT); \
mfspr r4,SPRN_DEAR; /* Grab the DEAR and save it */ \
stw r4,_DEAR(r11); \
- EXC_XFER_STD(0x0600, alignment_exception)
+ prepare_transfer_to_handler; \
+ bl alignment_exception; \
+ REST_NVGPRS(r1); \
+ b interrupt_return
#define PROGRAM_EXCEPTION \
START_EXCEPTION(Program) \
NORMAL_EXCEPTION_PROLOG(0x700, PROGRAM); \
mfspr r4,SPRN_ESR; /* Grab the ESR and save it */ \
stw r4,_ESR(r11); \
- EXC_XFER_STD(0x0700, program_check_exception)
+ prepare_transfer_to_handler; \
+ bl program_check_exception; \
+ REST_NVGPRS(r1); \
+ b interrupt_return
#define DECREMENTER_EXCEPTION \
START_EXCEPTION(Decrementer) \
diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S
index bf2730b4e43b..210871b2eb41 100644
--- a/arch/powerpc/kernel/head_fsl_booke.S
+++ b/arch/powerpc/kernel/head_fsl_booke.S
@@ -625,12 +625,20 @@ END_BTB_FLUSH_SECTION
/* SPE Floating Point Data */
#ifdef CONFIG_SPE
- EXCEPTION(0x2030, SPE_FP_DATA, SPEFloatingPointData,
- SPEFloatingPointException, EXC_XFER_STD)
+ START_EXCEPTION(SPEFloatingPointData)
+ NORMAL_EXCEPTION_PROLOG(0x2030, SPE_FP_DATA)
+ prepare_transfer_to_handler
+ bl SPEFloatingPointException
+ REST_NVGPRS(r1)
+ b interrupt_return
/* SPE Floating Point Round */
- EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, \
- SPEFloatingPointRoundException, EXC_XFER_STD)
+ START_EXCEPTION(SPEFloatingPointRound)
+ NORMAL_EXCEPTION_PROLOG(0x2050, SPE_FP_ROUND)
+ prepare_transfer_to_handler
+ bl SPEFloatingPointRoundException
+ REST_NVGPRS(r1)
+ b interrupt_return
#elif defined(CONFIG_SPE_POSSIBLE)
EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData,
unknown_exception, EXC_XFER_STD)
--
2.25.0
^ permalink raw reply related
* [PATCH v3 29/41] powerpc/32: Add a prepare_transfer_to_handler macro for exception prologs
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
In order to increase flexibility, add a macro that will for now
call transfer_to_handler.
As transfer_to_handler doesn't do the actual transfer anymore,
also name it prepare_transfer_to_handler. The following patches
will progressively remove the use of transfer_to_handler label.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 3 +++
arch/powerpc/kernel/head_32.h | 4 ++++
arch/powerpc/kernel/head_booke.h | 4 ++++
3 files changed, 11 insertions(+)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 76e1502b3e6f..237b753720db 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -86,6 +86,8 @@ _ASM_NOKPROBE_SYMBOL(transfer_to_handler_full)
.globl transfer_to_handler
transfer_to_handler:
+ .globl prepare_transfer_to_handler
+prepare_transfer_to_handler:
SAVE_NVGPRS(r11)
addi r3,r1,STACK_FRAME_OVERHEAD
stw r2,GPR2(r11)
@@ -133,6 +135,7 @@ transfer_to_handler_cont:
lwz r2, GPR2(r11)
b fast_exception_return
#endif
+_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
_ASM_NOKPROBE_SYMBOL(transfer_to_handler)
_ASM_NOKPROBE_SYMBOL(transfer_to_handler_cont)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index bf4c288173ad..3ab0f3ad9a6a 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -115,6 +115,10 @@
_ASM_NOKPROBE_SYMBOL(\name\()_virt)
.endm
+.macro prepare_transfer_to_handler
+ bl prepare_transfer_to_handler
+.endm
+
.macro SYSCALL_ENTRY trapno
mfspr r9, SPRN_SRR1
mfspr r10, SPRN_SRR0
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 27a7358c04bb..0f02b970e797 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -87,6 +87,10 @@ END_BTB_FLUSH_SECTION
SAVE_4GPRS(3, r11); \
SAVE_2GPRS(7, r11)
+.macro prepare_transfer_to_handler
+ bl prepare_transfer_to_handler
+.endm
+
.macro SYSCALL_ENTRY trapno intno srr1
mfspr r10, SPRN_SPRG_THREAD
#ifdef CONFIG_KVM_BOOKE_HV
--
2.25.0
^ permalink raw reply related
* [PATCH v3 31/41] powerpc/32: Dismantle EXC_XFER_STD/LITE/TEMPLATE
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
In order to get more control in exception prolog, dismantle
all non standard exception macros, finishing with EXC_XFER_STD
and EXC_XFER_LITE and EXC_XFER_TEMPLATE.
Also remove transfer_to_handler_full and ret_from_except and
ret_from_except_full as they are not used anymore.
Last parameter of EXCEPTION() is now ignored, will be removed
in a later patch to avoid too much churn.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 42 +-----------------------
arch/powerpc/kernel/head_32.h | 21 ++++--------
arch/powerpc/kernel/head_40x.S | 33 ++++++++++++-------
arch/powerpc/kernel/head_8xx.S | 12 +++++--
arch/powerpc/kernel/head_book3s_32.S | 27 ++++++++++-----
arch/powerpc/kernel/head_booke.h | 49 +++++++++++++++-------------
arch/powerpc/kernel/head_fsl_booke.S | 14 +++++---
7 files changed, 92 insertions(+), 106 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index e6adb6882bde..bcf8452ebb58 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -48,30 +48,6 @@
*/
.align 12
-#ifdef CONFIG_BOOKE
- .globl mcheck_transfer_to_handler
-mcheck_transfer_to_handler:
- /* fall through */
-_ASM_NOKPROBE_SYMBOL(mcheck_transfer_to_handler)
-
- .globl debug_transfer_to_handler
-debug_transfer_to_handler:
- /* fall through */
-_ASM_NOKPROBE_SYMBOL(debug_transfer_to_handler)
-
- .globl crit_transfer_to_handler
-crit_transfer_to_handler:
- /* fall through */
-_ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
-#endif
-
-#ifdef CONFIG_40x
- .globl crit_transfer_to_handler
-crit_transfer_to_handler:
- /* fall through */
-_ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
-#endif
-
/*
* This code finishes saving the registers to the exception frame
* and jumps to the appropriate handler for the exception, turning
@@ -79,13 +55,6 @@ _ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
* Note that we rely on the caller having set cr0.eq iff the exception
* occurred in kernel mode (i.e. MSR:PR = 0).
*/
- .globl transfer_to_handler_full
-transfer_to_handler_full:
-_ASM_NOKPROBE_SYMBOL(transfer_to_handler_full)
- /* fall through */
-
- .globl transfer_to_handler
-transfer_to_handler:
.globl prepare_transfer_to_handler
prepare_transfer_to_handler:
SAVE_NVGPRS(r11)
@@ -136,7 +105,6 @@ transfer_to_handler_cont:
b fast_exception_return
#endif
_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
-_ASM_NOKPROBE_SYMBOL(transfer_to_handler)
_ASM_NOKPROBE_SYMBOL(transfer_to_handler_cont)
.globl transfer_to_syscall
@@ -352,18 +320,10 @@ _ASM_NOKPROBE_SYMBOL(fast_exception_return)
3:
li r10,-1
stw r10,_TRAP(r11)
- bl transfer_to_handler_full
+ prepare_transfer_to_handler
bl unrecoverable_exception
trap /* should not get here */
- .globl ret_from_except_full
-ret_from_except_full:
- /* fall through */
-
- .globl ret_from_except
-ret_from_except:
-_ASM_NOKPROBE_SYMBOL(ret_from_except)
-
.globl interrupt_return
interrupt_return:
lwz r4,_MSR(r1)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index 3ab0f3ad9a6a..412ede8610f7 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -189,20 +189,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
#define EXCEPTION(n, label, hdlr, xfer) \
START_EXCEPTION(n, label) \
EXCEPTION_PROLOG n label; \
- xfer(n, hdlr)
-
-#define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \
- bl tfer; \
- bl hdlr; \
- b ret
-
-#define EXC_XFER_STD(n, hdlr) \
- EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \
- ret_from_except_full)
-
-#define EXC_XFER_LITE(n, hdlr) \
- EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler, \
- ret_from_except)
+ prepare_transfer_to_handler; \
+ bl hdlr; \
+ b interrupt_return
.macro vmap_stack_overflow_exception
__HEAD
@@ -218,7 +207,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
lwz r1, emergency_ctx@l(r1)
addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
EXCEPTION_PROLOG_2 0 vmap_stack_overflow
- EXC_XFER_STD(0, stack_overflow_exception)
+ prepare_transfer_to_handler
+ bl stack_overflow_exception
+ b interrupt_return
.endm
#endif /* __HEAD_32_H__ */
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index f3e5b462113f..7eb49ebd6000 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -187,8 +187,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
#define CRITICAL_EXCEPTION(n, label, hdlr) \
START_EXCEPTION(n, label); \
CRITICAL_EXCEPTION_PROLOG n label; \
- EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \
- crit_transfer_to_handler, ret_from_crit_exc)
+ prepare_transfer_to_handler; \
+ bl hdlr; \
+ b ret_from_crit_exc
/*
* 0x0100 - Critical Interrupt Exception
@@ -209,7 +210,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
*/
START_EXCEPTION(0x0300, DataStorage)
EXCEPTION_PROLOG 0x300 DataStorage handle_dar_dsisr=1
- EXC_XFER_LITE(0x300, do_page_fault)
+ prepare_transfer_to_handler
+ bl do_page_fault
+ b interrupt_return
/*
* 0x0400 - Instruction Storage Exception
@@ -220,7 +223,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
li r5,0
stw r5, _ESR(r11) /* Zero ESR */
stw r12, _DEAR(r11) /* SRR0 as DEAR */
- EXC_XFER_LITE(0x400, do_page_fault)
+ prepare_transfer_to_handler
+ bl do_page_fault
+ b interrupt_return
/* 0x0500 - External Interrupt Exception */
EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
@@ -499,9 +504,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
/* continue normal handling for a critical exception... */
2: mfspr r4,SPRN_DBSR
stw r4,_ESR(r11) /* DebugException takes DBSR in _ESR */
- EXC_XFER_TEMPLATE(DebugException, 0x2002, \
- (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \
- crit_transfer_to_handler, ret_from_crit_exc)
+ prepare_transfer_to_handler
+ bl DebugException
+ b ret_from_crit_exc
/* Programmable Interval Timer (PIT) Exception. (from 0x1000) */
__HEAD
@@ -509,21 +514,25 @@ Decrementer:
EXCEPTION_PROLOG 0x1000 Decrementer
lis r0,TSR_PIS@h
mtspr SPRN_TSR,r0 /* Clear the PIT exception */
- EXC_XFER_LITE(0x1000, timer_interrupt)
+ prepare_transfer_to_handler
+ bl timer_interrupt
+ b interrupt_return
/* Fixed Interval Timer (FIT) Exception. (from 0x1010) */
__HEAD
FITException:
EXCEPTION_PROLOG 0x1010 FITException
- EXC_XFER_STD(0x1010, unknown_exception)
+ prepare_transfer_to_handler
+ bl unknown_exception
+ b interrupt_return
/* Watchdog Timer (WDT) Exception. (from 0x1020) */
__HEAD
WDTException:
CRITICAL_EXCEPTION_PROLOG 0x1020 WDTException
- EXC_XFER_TEMPLATE(WatchdogException, 0x1020+2,
- (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)),
- crit_transfer_to_handler, ret_from_crit_exc)
+ prepare_transfer_to_handler
+ bl WatchdogException
+ b ret_from_crit_exc
/* Other PowerPC processors, namely those derived from the 6xx-series
* have vectors from 0x2100 through 0x2F00 defined, but marked as reserved.
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 86f844eb0e5a..4d73549722a1 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -123,7 +123,9 @@ instruction_counter:
/* Machine check */
START_EXCEPTION(0x200, MachineCheck)
EXCEPTION_PROLOG 0x200 MachineCheck handle_dar_dsisr=1
- EXC_XFER_STD(0x200, machine_check_exception)
+ prepare_transfer_to_handler
+ bl machine_check_exception
+ b interrupt_return
/* External interrupt */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
@@ -314,7 +316,9 @@ instruction_counter:
.Litlbie:
stw r12, _DAR(r11)
stw r5, _DSISR(r11)
- EXC_XFER_LITE(0x400, do_page_fault)
+ prepare_transfer_to_handler
+ bl do_page_fault
+ b interrupt_return
/* This is the data TLB error on the MPC8xx. This could be due to
* many reasons, including a dirty update to a pte. We bail out to
@@ -335,7 +339,9 @@ DARFixed:/* Return from dcbx instruction bug workaround */
beq+ .Ldtlbie
tlbie r4
.Ldtlbie:
- EXC_XFER_LITE(0x300, do_page_fault)
+ prepare_transfer_to_handler
+ bl do_page_fault
+ b interrupt_return
#ifdef CONFIG_VMAP_STACK
vmap_stack_overflow_exception
diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S
index 425a4f20ceac..0a3d7d4a9ec4 100644
--- a/arch/powerpc/kernel/head_book3s_32.S
+++ b/arch/powerpc/kernel/head_book3s_32.S
@@ -271,7 +271,9 @@ __secondary_hold_acknowledge:
beq cr1, 1f
twi 31, 0, 0
#endif
-1: EXC_XFER_STD(0x200, machine_check_exception)
+1: prepare_transfer_to_handler
+ bl machine_check_exception
+ b interrupt_return
/* Data access exception. */
START_EXCEPTION(0x300, DataAccess)
@@ -296,12 +298,13 @@ ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1
EXCEPTION_PROLOG_1
EXCEPTION_PROLOG_2 0x300 DataAccess handle_dar_dsisr=1
+ prepare_transfer_to_handler
lwz r5, _DSISR(r11)
andis. r0, r5, DSISR_DABRMATCH@h
bne- 1f
- EXC_XFER_LITE(0x300, do_page_fault)
-1: prepare_transfer_to_handler
- bl do_break
+ bl do_page_fault
+ b interrupt_return
+1: bl do_break
REST_NVGPRS(r1)
b interrupt_return
@@ -331,7 +334,9 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
stw r5, _DSISR(r11)
stw r12, _DAR(r11)
- EXC_XFER_LITE(0x400, do_page_fault)
+ prepare_transfer_to_handler
+ bl do_page_fault
+ b interrupt_return
/* External interrupt */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
@@ -366,7 +371,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
beq 1f
bl load_up_fpu /* if from user, just load it up */
b fast_exception_return
-1: EXC_XFER_LITE(0x800, kernel_fp_unavailable_exception)
+1: prepare_transfer_to_handler
+ bl kernel_fp_unavailable_exception
+ b interrupt_return
#else
b ProgramCheck
#endif
@@ -730,12 +737,16 @@ AltiVecUnavailable:
bl load_up_altivec /* if from user, just load it up */
b fast_exception_return
#endif /* CONFIG_ALTIVEC */
-1: EXC_XFER_LITE(0xf20, altivec_unavailable_exception)
+1: prepare_transfer_to_handler
+ bl altivec_unavailable_exception
+ b interrupt_return
__HEAD
PerformanceMonitor:
EXCEPTION_PROLOG 0xf00 PerformanceMonitor
- EXC_XFER_STD(0xf00, performance_monitor_exception)
+ prepare_transfer_to_handler
+ bl performance_monitor_exception
+ b interrupt_return
__HEAD
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index baf10556c587..bc69b9bf61a4 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -302,15 +302,18 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#define EXCEPTION(n, intno, label, hdlr, xfer) \
START_EXCEPTION(label); \
NORMAL_EXCEPTION_PROLOG(n, intno); \
- xfer(n, hdlr)
+ prepare_transfer_to_handler; \
+ bl hdlr; \
+ b interrupt_return
#define CRITICAL_EXCEPTION(n, intno, label, hdlr) \
START_EXCEPTION(label); \
CRITICAL_EXCEPTION_PROLOG(n, intno); \
SAVE_MMU_REGS; \
SAVE_xSRR(SRR); \
- EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \
- crit_transfer_to_handler, ret_from_crit_exc)
+ prepare_transfer_to_handler; \
+ bl hdlr; \
+ b ret_from_crit_exc
#define MCHECK_EXCEPTION(n, label, hdlr) \
START_EXCEPTION(label); \
@@ -321,21 +324,9 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
SAVE_xSRR(CSRR); \
SAVE_MMU_REGS; \
SAVE_xSRR(SRR); \
- EXC_XFER_TEMPLATE(hdlr, n+4, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \
- mcheck_transfer_to_handler, ret_from_mcheck_exc)
-
-#define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \
- bl tfer; \
+ prepare_transfer_to_handler; \
bl hdlr; \
- b ret; \
-
-#define EXC_XFER_STD(n, hdlr) \
- EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \
- ret_from_except_full)
-
-#define EXC_XFER_LITE(n, hdlr) \
- EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler, \
- ret_from_except)
+ b ret_from_mcheck_exc
/* Check for a single step debug exception while in an exception
* handler before state has been saved. This is to catch the case
@@ -404,7 +395,9 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
SAVE_xSRR(CSRR); \
SAVE_MMU_REGS; \
SAVE_xSRR(SRR); \
- EXC_XFER_TEMPLATE(DebugException, 0x2008, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), debug_transfer_to_handler, ret_from_debug_exc)
+ prepare_transfer_to_handler; \
+ bl DebugException; \
+ b ret_from_debug_exc
#define DEBUG_CRIT_EXCEPTION \
START_EXCEPTION(DebugCrit); \
@@ -459,7 +452,9 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\
SAVE_MMU_REGS; \
SAVE_xSRR(SRR); \
- EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), crit_transfer_to_handler, ret_from_crit_exc)
+ prepare_transfer_to_handler; \
+ bl DebugException; \
+ b ret_from_crit_exc
#define DATA_STORAGE_EXCEPTION \
START_EXCEPTION(DataStorage) \
@@ -468,7 +463,9 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r5,_ESR(r11); \
mfspr r4,SPRN_DEAR; /* Grab the DEAR */ \
stw r4, _DEAR(r11); \
- EXC_XFER_LITE(0x0300, do_page_fault)
+ prepare_transfer_to_handler; \
+ bl do_page_fault; \
+ b interrupt_return
#define INSTRUCTION_STORAGE_EXCEPTION \
START_EXCEPTION(InstructionStorage) \
@@ -476,7 +473,9 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
stw r5,_ESR(r11); \
stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
- EXC_XFER_LITE(0x0400, do_page_fault)
+ prepare_transfer_to_handler; \
+ bl do_page_fault; \
+ b interrupt_return
#define ALIGNMENT_EXCEPTION \
START_EXCEPTION(Alignment) \
@@ -503,7 +502,9 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
NORMAL_EXCEPTION_PROLOG(0x900, DECREMENTER); \
lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \
mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \
- EXC_XFER_LITE(0x0900, timer_interrupt)
+ prepare_transfer_to_handler; \
+ bl timer_interrupt; \
+ b interrupt_return
#define FP_UNAVAILABLE_EXCEPTION \
START_EXCEPTION(FloatingPointUnavailable) \
@@ -511,7 +512,9 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
beq 1f; \
bl load_up_fpu; /* if from user, just load it up */ \
b fast_exception_return; \
-1: EXC_XFER_STD(0x800, kernel_fp_unavailable_exception)
+1: prepare_transfer_to_handler; \
+ bl kernel_fp_unavailable_exception; \
+ b interrupt_return
#else /* __ASSEMBLY__ */
struct exception_regs {
diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S
index 210871b2eb41..48d022b1f508 100644
--- a/arch/powerpc/kernel/head_fsl_booke.S
+++ b/arch/powerpc/kernel/head_fsl_booke.S
@@ -370,9 +370,13 @@ interrupt_base:
stw r4, _DEAR(r11)
andis. r10,r5,(ESR_ILK|ESR_DLK)@h
bne 1f
- EXC_XFER_LITE(0x0300, do_page_fault)
+ prepare_transfer_to_handler
+ bl do_page_fault
+ b interrupt_return
1:
- EXC_XFER_LITE(0x0300, CacheLockingException)
+ prepare_transfer_to_handler
+ bl CacheLockingException
+ b interrupt_return
/* Instruction Storage Interrupt */
INSTRUCTION_STORAGE_EXCEPTION
@@ -617,7 +621,9 @@ END_BTB_FLUSH_SECTION
beq 1f
bl load_up_spe
b fast_exception_return
-1: EXC_XFER_LITE(0x2010, KernelSPE)
+1: prepare_transfer_to_handler
+ bl KernelSPE
+ b interrupt_return
#elif defined(CONFIG_SPE_POSSIBLE)
EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, \
unknown_exception, EXC_XFER_STD)
@@ -860,7 +866,7 @@ KernelSPE:
lwz r5,_NIP(r1)
bl printk
#endif
- b ret_from_except
+ b interrupt_return
#ifdef CONFIG_PRINTK
87: .string "SPE used in kernel (task=%p, pc=%x) \n"
#endif
--
2.25.0
^ permalink raw reply related
* [PATCH v3 32/41] powerpc/32: Remove the xfer parameter in EXCEPTION() macro
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
The xfer parameter is not used anymore, remove it.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/head_32.h | 2 +-
arch/powerpc/kernel/head_40x.S | 42 ++++++++--------
arch/powerpc/kernel/head_44x.S | 10 ++--
arch/powerpc/kernel/head_8xx.S | 14 +++---
arch/powerpc/kernel/head_book3s_32.S | 72 ++++++++++++++--------------
arch/powerpc/kernel/head_booke.h | 2 +-
arch/powerpc/kernel/head_fsl_booke.S | 28 +++++------
7 files changed, 81 insertions(+), 89 deletions(-)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index 412ede8610f7..84e6251622e8 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -186,7 +186,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
#endif
-#define EXCEPTION(n, label, hdlr, xfer) \
+#define EXCEPTION(n, label, hdlr) \
START_EXCEPTION(n, label) \
EXCEPTION_PROLOG n label; \
prepare_transfer_to_handler; \
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index 7eb49ebd6000..52b40bf529c6 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -228,7 +228,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
b interrupt_return
/* 0x0500 - External Interrupt Exception */
- EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+ EXCEPTION(0x0500, HardwareInterrupt, do_IRQ)
/* 0x0600 - Alignment Exception */
START_EXCEPTION(0x0600, Alignment)
@@ -246,19 +246,19 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
REST_NVGPRS(r1)
b interrupt_return
- EXCEPTION(0x0800, Trap_08, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x0900, Trap_09, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x0A00, Trap_0A, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x0B00, Trap_0B, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x0800, Trap_08, unknown_exception)
+ EXCEPTION(0x0900, Trap_09, unknown_exception)
+ EXCEPTION(0x0A00, Trap_0A, unknown_exception)
+ EXCEPTION(0x0B00, Trap_0B, unknown_exception)
/* 0x0C00 - System Call Exception */
START_EXCEPTION(0x0C00, SystemCall)
SYSCALL_ENTRY 0xc00
/* Trap_0D is commented out to get more space for system call exception */
-/* EXCEPTION(0x0D00, Trap_0D, unknown_exception, EXC_XFER_STD) */
- EXCEPTION(0x0E00, Trap_0E, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x0F00, Trap_0F, unknown_exception, EXC_XFER_STD)
+/* EXCEPTION(0x0D00, Trap_0D, unknown_exception) */
+ EXCEPTION(0x0E00, Trap_0E, unknown_exception)
+ EXCEPTION(0x0F00, Trap_0F, unknown_exception)
/* 0x1000 - Programmable Interval Timer (PIT) Exception */
START_EXCEPTION(0x1000, DecrementerTrap)
@@ -433,19 +433,19 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
mfspr r10, SPRN_SPRG_SCRATCH5
b InstructionAccess
- EXCEPTION(0x1300, Trap_13, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1400, Trap_14, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1A00, Trap_1A, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1B00, Trap_1B, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1C00, Trap_1C, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1D00, Trap_1D, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1E00, Trap_1E, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1F00, Trap_1F, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x1300, Trap_13, unknown_exception)
+ EXCEPTION(0x1400, Trap_14, unknown_exception)
+ EXCEPTION(0x1500, Trap_15, unknown_exception)
+ EXCEPTION(0x1600, Trap_16, unknown_exception)
+ EXCEPTION(0x1700, Trap_17, unknown_exception)
+ EXCEPTION(0x1800, Trap_18, unknown_exception)
+ EXCEPTION(0x1900, Trap_19, unknown_exception)
+ EXCEPTION(0x1A00, Trap_1A, unknown_exception)
+ EXCEPTION(0x1B00, Trap_1B, unknown_exception)
+ EXCEPTION(0x1C00, Trap_1C, unknown_exception)
+ EXCEPTION(0x1D00, Trap_1D, unknown_exception)
+ EXCEPTION(0x1E00, Trap_1E, unknown_exception)
+ EXCEPTION(0x1F00, Trap_1F, unknown_exception)
/* Check for a single step debug exception while in an exception
* handler before state has been saved. This is to catch the case
diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S
index 813fa305c33b..5c106ac36626 100644
--- a/arch/powerpc/kernel/head_44x.S
+++ b/arch/powerpc/kernel/head_44x.S
@@ -263,8 +263,7 @@ interrupt_base:
INSTRUCTION_STORAGE_EXCEPTION
/* External Input Interrupt */
- EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, \
- do_IRQ, EXC_XFER_LITE)
+ EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, do_IRQ)
/* Alignment Interrupt */
ALIGNMENT_EXCEPTION
@@ -277,7 +276,7 @@ interrupt_base:
FP_UNAVAILABLE_EXCEPTION
#else
EXCEPTION(0x2010, BOOKE_INTERRUPT_FP_UNAVAIL, \
- FloatingPointUnavailable, unknown_exception, EXC_XFER_STD)
+ FloatingPointUnavailable, unknown_exception)
#endif
/* System Call Interrupt */
START_EXCEPTION(SystemCall)
@@ -285,15 +284,14 @@ interrupt_base:
/* Auxiliary Processor Unavailable Interrupt */
EXCEPTION(0x2020, BOOKE_INTERRUPT_AP_UNAVAIL, \
- AuxillaryProcessorUnavailable, unknown_exception, EXC_XFER_STD)
+ AuxillaryProcessorUnavailable, unknown_exception)
/* Decrementer Interrupt */
DECREMENTER_EXCEPTION
/* Fixed Internal Timer Interrupt */
/* TODO: Add FIT support */
- EXCEPTION(0x1010, BOOKE_INTERRUPT_FIT, FixedIntervalTimer, \
- unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x1010, BOOKE_INTERRUPT_FIT, FixedIntervalTimer, unknown_exception)
/* Watchdog Timer Interrupt */
/* TODO: Add watchdog support */
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 4d73549722a1..34feb628c88d 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -118,7 +118,7 @@ instruction_counter:
#endif
/* System reset */
- EXCEPTION(0x100, Reset, system_reset_exception, EXC_XFER_STD)
+ EXCEPTION(0x100, Reset, system_reset_exception)
/* Machine check */
START_EXCEPTION(0x200, MachineCheck)
@@ -128,7 +128,7 @@ instruction_counter:
b interrupt_return
/* External interrupt */
- EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+ EXCEPTION(0x500, HardwareInterrupt, do_IRQ)
/* Alignment exception */
START_EXCEPTION(0x600, Alignment)
@@ -147,14 +147,14 @@ instruction_counter:
b interrupt_return
/* Decrementer */
- EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
+ EXCEPTION(0x900, Decrementer, timer_interrupt)
/* System call */
START_EXCEPTION(0xc00, SystemCall)
SYSCALL_ENTRY 0xc00
/* Single step - not used on 601 */
- EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
+ EXCEPTION(0xd00, SingleStep, single_step_exception)
/* On the MPC8xx, this is a software emulation interrupt. It occurs
* for all unimplemented and illegal instructions.
@@ -384,10 +384,10 @@ DARFixed:/* Return from dcbx instruction bug workaround */
mfspr r10, SPRN_SPRG_SCRATCH0
rfi
#else
- EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x1d00, Trap_1d, unknown_exception)
#endif
- EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x1e00, Trap_1e, unknown_exception)
+ EXCEPTION(0x1f00, Trap_1f, unknown_exception)
__HEAD
. = 0x2000
diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S
index 0a3d7d4a9ec4..18f4ae163f34 100644
--- a/arch/powerpc/kernel/head_book3s_32.S
+++ b/arch/powerpc/kernel/head_book3s_32.S
@@ -239,7 +239,7 @@ __secondary_hold_acknowledge:
/* System reset */
/* core99 pmac starts the seconary here by changing the vector, and
putting it back to what it was (unknown_async_exception) when done. */
- EXCEPTION(0x100, Reset, unknown_async_exception, EXC_XFER_STD)
+ EXCEPTION(0x100, Reset, unknown_async_exception)
/* Machine check */
/*
@@ -339,7 +339,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
b interrupt_return
/* External interrupt */
- EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+ EXCEPTION(0x500, HardwareInterrupt, do_IRQ)
/* Alignment exception */
START_EXCEPTION(0x600, Alignment)
@@ -379,17 +379,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
#endif
/* Decrementer */
- EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
+ EXCEPTION(0x900, Decrementer, timer_interrupt)
- EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0xa00, Trap_0a, unknown_exception)
+ EXCEPTION(0xb00, Trap_0b, unknown_exception)
/* System call */
START_EXCEPTION(0xc00, SystemCall)
SYSCALL_ENTRY 0xc00
- EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
- EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0xd00, SingleStep, single_step_exception)
+ EXCEPTION(0xe00, Trap_0e, unknown_exception)
/*
* The Altivec unavailable trap is at 0x0f20. Foo.
@@ -618,35 +618,35 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
#define TAUException unknown_async_exception
#endif
- EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD)
- EXCEPTION(0x1400, SMI, SMIException, EXC_XFER_STD)
- EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1600, Trap_16, altivec_assist_exception, EXC_XFER_STD)
- EXCEPTION(0x1700, Trap_17, TAUException, EXC_XFER_STD)
- EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1c00, Trap_1c, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2000, RunMode, RunModeException, EXC_XFER_STD)
- EXCEPTION(0x2100, Trap_21, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2200, Trap_22, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2300, Trap_23, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2400, Trap_24, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2500, Trap_25, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2600, Trap_26, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2700, Trap_27, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2800, Trap_28, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2900, Trap_29, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2a00, Trap_2a, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2b00, Trap_2b, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2c00, Trap_2c, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2d00, Trap_2d, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2e00, Trap_2e, unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2f00, Trap_2f, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception)
+ EXCEPTION(0x1400, SMI, SMIException)
+ EXCEPTION(0x1500, Trap_15, unknown_exception)
+ EXCEPTION(0x1600, Trap_16, altivec_assist_exception)
+ EXCEPTION(0x1700, Trap_17, TAUException)
+ EXCEPTION(0x1800, Trap_18, unknown_exception)
+ EXCEPTION(0x1900, Trap_19, unknown_exception)
+ EXCEPTION(0x1a00, Trap_1a, unknown_exception)
+ EXCEPTION(0x1b00, Trap_1b, unknown_exception)
+ EXCEPTION(0x1c00, Trap_1c, unknown_exception)
+ EXCEPTION(0x1d00, Trap_1d, unknown_exception)
+ EXCEPTION(0x1e00, Trap_1e, unknown_exception)
+ EXCEPTION(0x1f00, Trap_1f, unknown_exception)
+ EXCEPTION(0x2000, RunMode, RunModeException)
+ EXCEPTION(0x2100, Trap_21, unknown_exception)
+ EXCEPTION(0x2200, Trap_22, unknown_exception)
+ EXCEPTION(0x2300, Trap_23, unknown_exception)
+ EXCEPTION(0x2400, Trap_24, unknown_exception)
+ EXCEPTION(0x2500, Trap_25, unknown_exception)
+ EXCEPTION(0x2600, Trap_26, unknown_exception)
+ EXCEPTION(0x2700, Trap_27, unknown_exception)
+ EXCEPTION(0x2800, Trap_28, unknown_exception)
+ EXCEPTION(0x2900, Trap_29, unknown_exception)
+ EXCEPTION(0x2a00, Trap_2a, unknown_exception)
+ EXCEPTION(0x2b00, Trap_2b, unknown_exception)
+ EXCEPTION(0x2c00, Trap_2c, unknown_exception)
+ EXCEPTION(0x2d00, Trap_2d, unknown_exception)
+ EXCEPTION(0x2e00, Trap_2e, unknown_exception)
+ EXCEPTION(0x2f00, Trap_2f, unknown_exception)
__HEAD
. = 0x3000
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index bc69b9bf61a4..fa566e89f18b 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -299,7 +299,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
.align 5; \
label:
-#define EXCEPTION(n, intno, label, hdlr, xfer) \
+#define EXCEPTION(n, intno, label, hdlr) \
START_EXCEPTION(label); \
NORMAL_EXCEPTION_PROLOG(n, intno); \
prepare_transfer_to_handler; \
diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S
index 48d022b1f508..3efc5baa801a 100644
--- a/arch/powerpc/kernel/head_fsl_booke.S
+++ b/arch/powerpc/kernel/head_fsl_booke.S
@@ -382,7 +382,7 @@ interrupt_base:
INSTRUCTION_STORAGE_EXCEPTION
/* External Input Interrupt */
- EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ, EXC_XFER_LITE)
+ EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ)
/* Alignment Interrupt */
ALIGNMENT_EXCEPTION
@@ -394,8 +394,7 @@ interrupt_base:
#ifdef CONFIG_PPC_FPU
FP_UNAVAILABLE_EXCEPTION
#else
- EXCEPTION(0x0800, FP_UNAVAIL, FloatingPointUnavailable, \
- unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x0800, FP_UNAVAIL, FloatingPointUnavailable, unknown_exception)
#endif
/* System Call Interrupt */
@@ -403,16 +402,14 @@ interrupt_base:
SYSCALL_ENTRY 0xc00 BOOKE_INTERRUPT_SYSCALL SPRN_SRR1
/* Auxiliary Processor Unavailable Interrupt */
- EXCEPTION(0x2900, AP_UNAVAIL, AuxillaryProcessorUnavailable, \
- unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x2900, AP_UNAVAIL, AuxillaryProcessorUnavailable, unknown_exception)
/* Decrementer Interrupt */
DECREMENTER_EXCEPTION
/* Fixed Internal Timer Interrupt */
/* TODO: Add FIT support */
- EXCEPTION(0x3100, FIT, FixedIntervalTimer, \
- unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x3100, FIT, FixedIntervalTimer, unknown_exception)
/* Watchdog Timer Interrupt */
#ifdef CONFIG_BOOKE_WDT
@@ -625,8 +622,7 @@ END_BTB_FLUSH_SECTION
bl KernelSPE
b interrupt_return
#elif defined(CONFIG_SPE_POSSIBLE)
- EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, \
- unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, unknown_exception)
#endif /* CONFIG_SPE_POSSIBLE */
/* SPE Floating Point Data */
@@ -646,18 +642,16 @@ END_BTB_FLUSH_SECTION
REST_NVGPRS(r1)
b interrupt_return
#elif defined(CONFIG_SPE_POSSIBLE)
- EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData,
- unknown_exception, EXC_XFER_STD)
- EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, \
- unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData, unknown_exception)
+ EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, unknown_exception)
#endif /* CONFIG_SPE_POSSIBLE */
/* Performance Monitor */
EXCEPTION(0x2060, PERFORMANCE_MONITOR, PerformanceMonitor, \
- performance_monitor_exception, EXC_XFER_STD)
+ performance_monitor_exception)
- EXCEPTION(0x2070, DOORBELL, Doorbell, doorbell_exception, EXC_XFER_STD)
+ EXCEPTION(0x2070, DOORBELL, Doorbell, doorbell_exception)
CRITICAL_EXCEPTION(0x2080, DOORBELL_CRITICAL, \
CriticalDoorbell, unknown_exception)
@@ -672,10 +666,10 @@ END_BTB_FLUSH_SECTION
unknown_exception)
/* Hypercall */
- EXCEPTION(0, HV_SYSCALL, Hypercall, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0, HV_SYSCALL, Hypercall, unknown_exception)
/* Embedded Hypervisor Privilege */
- EXCEPTION(0, HV_PRIV, Ehvpriv, unknown_exception, EXC_XFER_STD)
+ EXCEPTION(0, HV_PRIV, Ehvpriv, unknown_exception)
interrupt_end:
--
2.25.0
^ permalink raw reply related
* [PATCH v3 33/41] powerpc/32: Refactor saving of volatile registers in exception prologs
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
Exception prologs all do the same at the end:
- Save trapno in stack
- Mark stack with exception marker
- Save r0
- Save r3 to r8
Refactor that into a COMMON_EXCEPTION_PROLOG_END macro.
At the same time use r1 instead of r11.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/head_32.h | 16 ++++++++++------
arch/powerpc/kernel/head_40x.S | 9 +--------
arch/powerpc/kernel/head_booke.h | 26 +++++++++++++-------------
3 files changed, 24 insertions(+), 27 deletions(-)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index 84e6251622e8..ba20bfabdf63 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -104,15 +104,19 @@
li r10, MSR_KERNEL /* can take exceptions */
mtmsr r10 /* (except for mach check in rtas) */
#endif
- stw r0,GPR0(r11)
+ COMMON_EXCEPTION_PROLOG_END \trapno
+_ASM_NOKPROBE_SYMBOL(\name\()_virt)
+.endm
+
+.macro COMMON_EXCEPTION_PROLOG_END trapno
+ stw r0,GPR0(r1)
lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
addi r10,r10,STACK_FRAME_REGS_MARKER@l
- stw r10,8(r11)
+ stw r10,8(r1)
li r10, \trapno
- stw r10,_TRAP(r11)
- SAVE_4GPRS(3, r11)
- SAVE_2GPRS(7, r11)
-_ASM_NOKPROBE_SYMBOL(\name\()_virt)
+ stw r10,_TRAP(r1)
+ SAVE_4GPRS(3, r1)
+ SAVE_2GPRS(7, r1)
.endm
.macro prepare_transfer_to_handler
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index 52b40bf529c6..e1360b88b6cb 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -157,14 +157,7 @@ _ENTRY(crit_esr)
mfspr r12,SPRN_SRR2
mfspr r9,SPRN_SRR3
rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */
- stw r0,GPR0(r11)
- lis r10, STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
- addi r10, r10, STACK_FRAME_REGS_MARKER@l
- stw r10, 8(r11)
- li r10, \trapno + 2
- stw r10,_TRAP(r11)
- SAVE_4GPRS(3, r11)
- SAVE_2GPRS(7, r11)
+ COMMON_EXCEPTION_PROLOG_END \trapno + 2
_ASM_NOKPROBE_SYMBOL(\name\()_virt)
.endm
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index fa566e89f18b..4d583fbef0b6 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -78,14 +78,18 @@ END_BTB_FLUSH_SECTION
stw r1, 0(r11); \
mr r1, r11; \
rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\
- stw r0,GPR0(r11); \
- lis r10, STACK_FRAME_REGS_MARKER@ha;/* exception frame marker */ \
- addi r10, r10, STACK_FRAME_REGS_MARKER@l; \
- stw r10, 8(r11); \
- li r10, trapno; \
- stw r10,_TRAP(r11); \
- SAVE_4GPRS(3, r11); \
- SAVE_2GPRS(7, r11)
+ COMMON_EXCEPTION_PROLOG_END trapno
+
+.macro COMMON_EXCEPTION_PROLOG_END trapno
+ stw r0,GPR0(r1)
+ lis r10, STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
+ addi r10, r10, STACK_FRAME_REGS_MARKER@l
+ stw r10, 8(r1)
+ li r10, \trapno
+ stw r10,_TRAP(r1)
+ SAVE_4GPRS(3, r1)
+ SAVE_2GPRS(7, r1)
+.endm
.macro prepare_transfer_to_handler
bl prepare_transfer_to_handler
@@ -231,11 +235,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r1,0(r11); \
mr r1,r11; \
rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\
- li r10, trapno; \
- stw r10,_TRAP(r11); \
- stw r0,GPR0(r11); \
- SAVE_4GPRS(3, r11); \
- SAVE_2GPRS(7, r11)
+ COMMON_EXCEPTION_PROLOG_END trapno
#define SAVE_xSRR(xSRR) \
mfspr r0,SPRN_##xSRR##0; \
--
2.25.0
^ permalink raw reply related
* [PATCH v3 34/41] powerpc/32: Save remaining registers in exception prolog
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
Save non volatile registers, XER, CTR, MSR and NIP in exception prolog.
Also assign proper value to r2 and r3 there.
For now, recalculate thread pointer in prepare_transfer_to_handler.
It will disappear once KUAP is ported to C.
And remove the comment which is now completely wrong.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 27 ++++-----------------------
arch/powerpc/kernel/head_32.h | 12 ++++++++++++
arch/powerpc/kernel/head_booke.h | 12 ++++++++++++
3 files changed, 28 insertions(+), 23 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index bcf8452ebb58..46b3854e7318 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -48,29 +48,11 @@
*/
.align 12
-/*
- * This code finishes saving the registers to the exception frame
- * and jumps to the appropriate handler for the exception, turning
- * on address translation.
- * Note that we rely on the caller having set cr0.eq iff the exception
- * occurred in kernel mode (i.e. MSR:PR = 0).
- */
.globl prepare_transfer_to_handler
prepare_transfer_to_handler:
- SAVE_NVGPRS(r11)
- addi r3,r1,STACK_FRAME_OVERHEAD
- stw r2,GPR2(r11)
- stw r12,_NIP(r11)
- stw r9,_MSR(r11)
- andi. r2,r9,MSR_PR
- mfctr r12
- mfspr r2,SPRN_XER
- stw r12,_CTR(r11)
- stw r2,_XER(r11)
- mfspr r12,SPRN_SPRG_THREAD
- tovirt(r12, r12)
+ andi. r12,r9,MSR_PR
+ addi r12,r2,THREAD
beq 2f
- addi r2, r12, -THREAD
#ifdef CONFIG_PPC_BOOK3S_32
kuep_lock r11, r12
#endif
@@ -78,8 +60,7 @@ prepare_transfer_to_handler:
/* if from kernel, check interrupted DOZE/NAP mode */
2:
- kuap_save_and_lock r11, r12, r9, r2, r6
- addi r2, r12, -THREAD
+ kuap_save_and_lock r11, r12, r9, r5, r6
#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
lwz r12,TI_LOCAL_FLAGS(r2)
mtcrf 0x01,r12
@@ -293,7 +274,7 @@ fast_exception_return:
2: REST_4GPRS(3, r11)
lwz r10,_CCR(r11)
- REST_GPR(1, r11)
+ REST_2GPRS(1, r11)
mtcr r10
lwz r10,_LINK(r11)
mtlr r10
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index ba20bfabdf63..80e45c800496 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -117,6 +117,18 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
stw r10,_TRAP(r1)
SAVE_4GPRS(3, r1)
SAVE_2GPRS(7, r1)
+ SAVE_NVGPRS(r1)
+ stw r2,GPR2(r1)
+ stw r12,_NIP(r1)
+ stw r9,_MSR(r1)
+ mfctr r10
+ mfspr r2,SPRN_SPRG_THREAD
+ stw r10,_CTR(r1)
+ tovirt(r2, r2)
+ mfspr r10,SPRN_XER
+ addi r2, r2, -THREAD
+ stw r10,_XER(r1)
+ addi r3,r1,STACK_FRAME_OVERHEAD
.endm
.macro prepare_transfer_to_handler
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 4d583fbef0b6..7f3dd5fae51d 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -89,6 +89,18 @@ END_BTB_FLUSH_SECTION
stw r10,_TRAP(r1)
SAVE_4GPRS(3, r1)
SAVE_2GPRS(7, r1)
+ SAVE_NVGPRS(r1)
+ stw r2,GPR2(r1)
+ stw r12,_NIP(r1)
+ stw r9,_MSR(r1)
+ mfctr r10
+ mfspr r2,SPRN_SPRG_THREAD
+ stw r10,_CTR(r1)
+ tovirt(r2, r2)
+ mfspr r10,SPRN_XER
+ addi r2, r2, -THREAD
+ stw r10,_XER(r1)
+ addi r3,r1,STACK_FRAME_OVERHEAD
.endm
.macro prepare_transfer_to_handler
--
2.25.0
^ permalink raw reply related
* [PATCH v3 35/41] powerpc/32: Return directly from power_save_ppc32_restore()
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
transfer_to_handler_cont: is now just a blr.
Directly perform blr in power_save_ppc32_restore().
Also remove useless setting of r11 in e500 version of
power_save_ppc32_restore().
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 3 ---
arch/powerpc/kernel/idle_6xx.S | 2 +-
arch/powerpc/kernel/idle_e500.S | 10 +---------
3 files changed, 2 insertions(+), 13 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 46b3854e7318..b9a2935efec1 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -67,8 +67,6 @@ prepare_transfer_to_handler:
bt- 31-TLF_NAPPING,4f
bt- 31-TLF_SLEEPING,7f
#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */
- .globl transfer_to_handler_cont
-transfer_to_handler_cont:
blr
#if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
@@ -86,7 +84,6 @@ transfer_to_handler_cont:
b fast_exception_return
#endif
_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
-_ASM_NOKPROBE_SYMBOL(transfer_to_handler_cont)
.globl transfer_to_syscall
transfer_to_syscall:
diff --git a/arch/powerpc/kernel/idle_6xx.S b/arch/powerpc/kernel/idle_6xx.S
index 153366e178c4..13cad9297d82 100644
--- a/arch/powerpc/kernel/idle_6xx.S
+++ b/arch/powerpc/kernel/idle_6xx.S
@@ -176,7 +176,7 @@ BEGIN_FTR_SECTION
lwz r9,nap_save_hid1@l(r9)
mtspr SPRN_HID1, r9
END_FTR_SECTION_IFSET(CPU_FTR_DUAL_PLL_750FX)
- b transfer_to_handler_cont
+ blr
_ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore)
.data
diff --git a/arch/powerpc/kernel/idle_e500.S b/arch/powerpc/kernel/idle_e500.S
index 7795727e7f08..9e1bc4502c50 100644
--- a/arch/powerpc/kernel/idle_e500.S
+++ b/arch/powerpc/kernel/idle_e500.S
@@ -81,13 +81,5 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
_GLOBAL(power_save_ppc32_restore)
lwz r9,_LINK(r11) /* interrupted in e500_idle */
stw r9,_NIP(r11) /* make it do a blr */
-
-#ifdef CONFIG_SMP
- lwz r11,TASK_CPU(r2) /* get cpu number * 4 */
- slwi r11,r11,2
-#else
- li r11,0
-#endif
-
- b transfer_to_handler_cont
+ blr
_ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore)
--
2.25.0
^ permalink raw reply related
* [PATCH v3 36/41] powerpc/32: Only use prepare_transfer_to_handler function on book3s/32 and e500
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
Only book3s/32 and e500 have significative work to do in
prepare_transfer_to_handler.
Other 32 bit have nothing to do at all.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 6 ++----
arch/powerpc/kernel/head_32.h | 2 ++
arch/powerpc/kernel/head_booke.h | 2 ++
3 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index b9a2935efec1..44d0eddf8738 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -48,6 +48,7 @@
*/
.align 12
+#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
.globl prepare_transfer_to_handler
prepare_transfer_to_handler:
andi. r12,r9,MSR_PR
@@ -61,15 +62,12 @@ prepare_transfer_to_handler:
/* if from kernel, check interrupted DOZE/NAP mode */
2:
kuap_save_and_lock r11, r12, r9, r5, r6
-#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
lwz r12,TI_LOCAL_FLAGS(r2)
mtcrf 0x01,r12
bt- 31-TLF_NAPPING,4f
bt- 31-TLF_SLEEPING,7f
-#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */
blr
-#if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
4: rlwinm r12,r12,0,~_TLF_NAPPING
stw r12,TI_LOCAL_FLAGS(r2)
b power_save_ppc32_restore
@@ -82,8 +80,8 @@ prepare_transfer_to_handler:
kuap_restore r11, r2, r3, r4, r5
lwz r2, GPR2(r11)
b fast_exception_return
-#endif
_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
+#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */
.globl transfer_to_syscall
transfer_to_syscall:
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index 80e45c800496..c018fcdf9157 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -132,7 +132,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
.endm
.macro prepare_transfer_to_handler
+#ifdef CONFIG_PPC_BOOK3S_32
bl prepare_transfer_to_handler
+#endif
.endm
.macro SYSCALL_ENTRY trapno
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 7f3dd5fae51d..cb96ae002af6 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -104,7 +104,9 @@ END_BTB_FLUSH_SECTION
.endm
.macro prepare_transfer_to_handler
+#ifdef CONFIG_E500
bl prepare_transfer_to_handler
+#endif
.endm
.macro SYSCALL_ENTRY trapno intno srr1
--
2.25.0
^ permalink raw reply related
* [PATCH v3 37/41] powerpc/32s: Move KUEP locking/unlocking in C
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
This can be done in C, do it.
Unrolling the loop gains approx. 15% performance.
From now on, prepare_transfer_to_handler() is only for
interrupts from kernel.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 31 -------------------
arch/powerpc/include/asm/interrupt.h | 6 +++-
arch/powerpc/include/asm/kup.h | 8 +++++
arch/powerpc/kernel/entry_32.S | 16 ----------
arch/powerpc/kernel/head_32.h | 3 ++
arch/powerpc/kernel/head_booke.h | 3 ++
arch/powerpc/kernel/interrupt.c | 4 +++
arch/powerpc/mm/book3s32/Makefile | 1 +
arch/powerpc/mm/book3s32/kuep.c | 38 ++++++++++++++++++++++++
9 files changed, 62 insertions(+), 48 deletions(-)
create mode 100644 arch/powerpc/mm/book3s32/kuep.c
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 73bc5d2c431d..b97ea60f6fa3 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -7,37 +7,6 @@
#ifdef __ASSEMBLY__
-.macro kuep_update_sr gpr1, gpr2 /* NEVER use r0 as gpr2 due to addis */
-101: mtsrin \gpr1, \gpr2
- addi \gpr1, \gpr1, 0x111 /* next VSID */
- rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */
- addis \gpr2, \gpr2, 0x1000 /* address of next segment */
- bdnz 101b
- isync
-.endm
-
-.macro kuep_lock gpr1, gpr2
-#ifdef CONFIG_PPC_KUEP
- li \gpr1, NUM_USER_SEGMENTS
- li \gpr2, 0
- mtctr \gpr1
- mfsrin \gpr1, \gpr2
- oris \gpr1, \gpr1, SR_NX@h /* set Nx */
- kuep_update_sr \gpr1, \gpr2
-#endif
-.endm
-
-.macro kuep_unlock gpr1, gpr2
-#ifdef CONFIG_PPC_KUEP
- li \gpr1, NUM_USER_SEGMENTS
- li \gpr2, 0
- mtctr \gpr1
- mfsrin \gpr1, \gpr2
- rlwinm \gpr1, \gpr1, 0, ~SR_NX /* Clear Nx */
- kuep_update_sr \gpr1, \gpr2
-#endif
-.endm
-
#ifdef CONFIG_PPC_KUAP
.macro kuap_update_sr gpr1, gpr2, gpr3 /* NEVER use r0 as gpr2 due to addis */
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 861e6eadc98c..857375309255 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -33,8 +33,10 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
if (!arch_irq_disabled_regs(regs))
trace_hardirqs_off();
- if (user_mode(regs))
+ if (user_mode(regs)) {
+ kuep_lock();
account_cpu_user_entry();
+ }
#endif
/*
* Book3E reconciles irq soft mask in asm
@@ -89,6 +91,8 @@ static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt
exception_exit(state->ctx_state);
#endif
+ if (user_mode(regs))
+ kuep_unlock();
/*
* Book3S exits to user via interrupt_exit_user_prepare(), which does
* context tracking, which is a cleaner way to handle PREEMPT=y
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index 7ec21af49a45..25671f711ec2 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -55,6 +55,14 @@ void setup_kuep(bool disabled);
static inline void setup_kuep(bool disabled) { }
#endif /* CONFIG_PPC_KUEP */
+#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
+void kuep_lock(void);
+void kuep_unlock(void);
+#else
+static inline void kuep_lock(void) { }
+static inline void kuep_unlock(void) { }
+#endif
+
#ifdef CONFIG_PPC_KUAP
void setup_kuap(bool disabled);
#else
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 44d0eddf8738..112d6247c391 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -51,16 +51,9 @@
#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
.globl prepare_transfer_to_handler
prepare_transfer_to_handler:
- andi. r12,r9,MSR_PR
addi r12,r2,THREAD
- beq 2f
-#ifdef CONFIG_PPC_BOOK3S_32
- kuep_lock r11, r12
-#endif
- blr
/* if from kernel, check interrupted DOZE/NAP mode */
-2:
kuap_save_and_lock r11, r12, r9, r5, r6
lwz r12,TI_LOCAL_FLAGS(r2)
mtcrf 0x01,r12
@@ -86,9 +79,6 @@ _ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
.globl transfer_to_syscall
transfer_to_syscall:
SAVE_NVGPRS(r1)
-#ifdef CONFIG_PPC_BOOK3S_32
- kuep_lock r11, r12
-#endif
/* Calling convention has r9 = orig r0, r10 = regs */
addi r10,r1,STACK_FRAME_OVERHEAD
@@ -105,9 +95,6 @@ ret_from_syscall:
cmplwi cr0,r5,0
bne- 2f
#endif /* CONFIG_PPC_47x */
-#ifdef CONFIG_PPC_BOOK3S_32
- kuep_unlock r5, r7
-#endif
kuap_check r2, r4
lwz r4,_LINK(r1)
lwz r5,_CCR(r1)
@@ -311,9 +298,6 @@ interrupt_return:
bne- .Lrestore_nvgprs
.Lfast_user_interrupt_return:
-#ifdef CONFIG_PPC_BOOK3S_32
- kuep_unlock r10, r11
-#endif
kuap_check r2, r4
lwz r11,_NIP(r1)
lwz r12,_MSR(r1)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index c018fcdf9157..a8221ddcbd66 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -133,7 +133,10 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
.macro prepare_transfer_to_handler
#ifdef CONFIG_PPC_BOOK3S_32
+ andi. r12,r9,MSR_PR
+ bne 777f
bl prepare_transfer_to_handler
+777:
#endif
.endm
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index cb96ae002af6..f82470091697 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -105,7 +105,10 @@ END_BTB_FLUSH_SECTION
.macro prepare_transfer_to_handler
#ifdef CONFIG_E500
+ andi. r12,r9,MSR_PR
+ bne 777f
bl prepare_transfer_to_handler
+777:
#endif
.endm
diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c
index 6875b82f613a..20ace874cd98 100644
--- a/arch/powerpc/kernel/interrupt.c
+++ b/arch/powerpc/kernel/interrupt.c
@@ -33,6 +33,8 @@ notrace long system_call_exception(long r3, long r4, long r5,
{
syscall_fn f;
+ kuep_lock();
+
regs->orig_gpr3 = r3;
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
@@ -354,6 +356,8 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
*/
kuap_user_restore(regs);
#endif
+ kuep_unlock();
+
return ret;
}
diff --git a/arch/powerpc/mm/book3s32/Makefile b/arch/powerpc/mm/book3s32/Makefile
index 446d9de88ce4..7f0c8a78ba0c 100644
--- a/arch/powerpc/mm/book3s32/Makefile
+++ b/arch/powerpc/mm/book3s32/Makefile
@@ -9,3 +9,4 @@ endif
obj-y += mmu.o mmu_context.o
obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o
obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o
+obj-$(CONFIG_PPC_KUEP) += kuep.o
diff --git a/arch/powerpc/mm/book3s32/kuep.c b/arch/powerpc/mm/book3s32/kuep.c
new file mode 100644
index 000000000000..c70532568a28
--- /dev/null
+++ b/arch/powerpc/mm/book3s32/kuep.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <asm/reg.h>
+#include <asm/task_size_32.h>
+#include <asm/mmu.h>
+
+#define KUEP_UPDATE_TWO_USER_SEGMENTS(n) do { \
+ if (TASK_SIZE > ((n) << 28)) \
+ mtsr(val1, (n) << 28); \
+ if (TASK_SIZE > (((n) + 1) << 28)) \
+ mtsr(val2, ((n) + 1) << 28); \
+ val1 = (val1 + 0x222) & 0xf0ffffff; \
+ val2 = (val2 + 0x222) & 0xf0ffffff; \
+} while (0)
+
+static __always_inline void kuep_update(u32 val)
+{
+ int val1 = val;
+ int val2 = (val + 0x111) & 0xf0ffffff;
+
+ KUEP_UPDATE_TWO_USER_SEGMENTS(0);
+ KUEP_UPDATE_TWO_USER_SEGMENTS(2);
+ KUEP_UPDATE_TWO_USER_SEGMENTS(4);
+ KUEP_UPDATE_TWO_USER_SEGMENTS(6);
+ KUEP_UPDATE_TWO_USER_SEGMENTS(8);
+ KUEP_UPDATE_TWO_USER_SEGMENTS(10);
+ KUEP_UPDATE_TWO_USER_SEGMENTS(12);
+ KUEP_UPDATE_TWO_USER_SEGMENTS(14);
+}
+
+void kuep_lock(void)
+{
+ kuep_update(mfsr(0) | SR_NX);
+}
+
+void kuep_unlock(void)
+{
+ kuep_update(mfsr(0) & ~SR_NX);
+}
--
2.25.0
^ permalink raw reply related
* [PATCH v3 38/41] powerpc/64s: Make kuap_check_amr() and kuap_get_and_check_amr() generic
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
In preparation of porting powerpc32 to C syscall entry/exit,
rename kuap_check_amr() and kuap_get_and_check_amr() as
kuap_assert_locked() and kuap_get_and_assert_locked(), and move in the
generic asm/kup.h the stub for when CONFIG_PPC_KUAP is not selected.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Reviewed-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/book3s/64/kup.h | 24 ++----------------------
arch/powerpc/include/asm/kup.h | 10 +++++++++-
arch/powerpc/kernel/interrupt.c | 12 ++++++------
arch/powerpc/kernel/irq.c | 2 +-
4 files changed, 18 insertions(+), 30 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h
index 8bd905050896..9700da3a4093 100644
--- a/arch/powerpc/include/asm/book3s/64/kup.h
+++ b/arch/powerpc/include/asm/book3s/64/kup.h
@@ -287,7 +287,7 @@ static inline void kuap_kernel_restore(struct pt_regs *regs,
*/
}
-static inline unsigned long kuap_get_and_check_amr(void)
+static inline unsigned long kuap_get_and_assert_locked(void)
{
if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
unsigned long amr = mfspr(SPRN_AMR);
@@ -298,27 +298,7 @@ static inline unsigned long kuap_get_and_check_amr(void)
return 0;
}
-#else /* CONFIG_PPC_PKEY */
-
-static inline void kuap_user_restore(struct pt_regs *regs)
-{
-}
-
-static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr)
-{
-}
-
-static inline unsigned long kuap_get_and_check_amr(void)
-{
- return 0;
-}
-
-#endif /* CONFIG_PPC_PKEY */
-
-
-#ifdef CONFIG_PPC_KUAP
-
-static inline void kuap_check_amr(void)
+static inline void kuap_assert_locked(void)
{
if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index 25671f711ec2..aa5f77459311 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -74,7 +74,15 @@ bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
return false;
}
-static inline void kuap_check_amr(void) { }
+static inline void kuap_assert_locked(void) { }
+static inline void kuap_save_and_lock(struct pt_regs *regs) { }
+static inline void kuap_user_restore(struct pt_regs *regs) { }
+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { }
+
+static inline unsigned long kuap_get_and_assert_locked(void)
+{
+ return 0;
+}
/*
* book3s/64/kup-radix.h defines these functions for the !KUAP case to flush
diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c
index 20ace874cd98..a7cb511bf945 100644
--- a/arch/powerpc/kernel/interrupt.c
+++ b/arch/powerpc/kernel/interrupt.c
@@ -76,7 +76,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
} else
#endif
#ifdef CONFIG_PPC64
- kuap_check_amr();
+ kuap_assert_locked();
#endif
booke_restore_dbcr0();
@@ -254,7 +254,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
CT_WARN_ON(ct_state() == CONTEXT_USER);
#ifdef CONFIG_PPC64
- kuap_check_amr();
+ kuap_assert_locked();
#endif
regs->result = r3;
@@ -380,7 +380,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
* AMR can only have been unlocked if we interrupted the kernel.
*/
#ifdef CONFIG_PPC64
- kuap_check_amr();
+ kuap_assert_locked();
#endif
local_irq_save(flags);
@@ -451,7 +451,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
unsigned long flags;
unsigned long ret = 0;
#ifdef CONFIG_PPC64
- unsigned long amr;
+ unsigned long kuap;
#endif
if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) &&
@@ -467,7 +467,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
CT_WARN_ON(ct_state() == CONTEXT_USER);
#ifdef CONFIG_PPC64
- amr = kuap_get_and_check_amr();
+ kuap = kuap_get_and_assert_locked();
#endif
if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {
@@ -511,7 +511,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
* value from the check above.
*/
#ifdef CONFIG_PPC64
- kuap_kernel_restore(regs, amr);
+ kuap_kernel_restore(regs, kuap);
#endif
return ret;
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index d71fd10a1dd4..5b72abbff96c 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -282,7 +282,7 @@ static inline void replay_soft_interrupts_irqrestore(void)
* and re-locking AMR but we shouldn't get here in the first place,
* hence the warning.
*/
- kuap_check_amr();
+ kuap_assert_locked();
if (kuap_state != AMR_KUAP_BLOCKED)
set_kuap(AMR_KUAP_BLOCKED);
--
2.25.0
^ permalink raw reply related
* [PATCH v3 39/41] powerpc/32s: Create C version of kuap save/restore/check helpers
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
In preparation of porting PPC32 to C syscall entry/exit,
create C version of kuap_save_and_lock() and kuap_user_restore() and
kuap_kernel_restore() and kuap_assert_locked() and
kuap_get_and_assert_locked() on book3s/32.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 45 ++++++++++++++++++++++++
1 file changed, 45 insertions(+)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index b97ea60f6fa3..46599bbc4525 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -72,6 +72,51 @@ static inline void kuap_update_sr(u32 sr, u32 addr, u32 end)
isync(); /* Context sync required after mtsr() */
}
+static inline void kuap_save_and_lock(struct pt_regs *regs)
+{
+ unsigned long kuap = current->thread.kuap;
+ u32 addr = kuap & 0xf0000000;
+ u32 end = kuap << 28;
+
+ regs->kuap = kuap;
+ if (unlikely(!kuap))
+ return;
+
+ current->thread.kuap = 0;
+ kuap_update_sr(mfsr(addr) | SR_KS, addr, end); /* Set Ks */
+}
+
+static inline void kuap_user_restore(struct pt_regs *regs)
+{
+}
+
+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
+{
+ u32 addr = regs->kuap & 0xf0000000;
+ u32 end = regs->kuap << 28;
+
+ current->thread.kuap = regs->kuap;
+
+ if (unlikely(regs->kuap == kuap))
+ return;
+
+ kuap_update_sr(mfsr(addr) & ~SR_KS, addr, end); /* Clear Ks */
+}
+
+static inline unsigned long kuap_get_and_assert_locked(void)
+{
+ unsigned long kuap = current->thread.kuap;
+
+ WARN_ON_ONCE(IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && kuap != 0);
+
+ return kuap;
+}
+
+static inline void kuap_assert_locked(void)
+{
+ kuap_get_and_assert_locked();
+}
+
static __always_inline void allow_user_access(void __user *to, const void __user *from,
u32 size, unsigned long dir)
{
--
2.25.0
^ permalink raw reply related
* [PATCH v3 40/41] powerpc/8xx: Create C version of kuap save/restore/check helpers
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
In preparation of porting PPC32 to C syscall entry/exit,
create C version of kuap_save_and_lock() and kuap_user_restore() and
kuap_kernel_restore() and kuap_assert_locked() and
kuap_get_and_assert_locked() on 8xx.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 31 ++++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
index 17a4a616436f..e6b5ebca47e5 100644
--- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
@@ -34,6 +34,37 @@
#include <asm/reg.h>
+static inline void kuap_save_and_lock(struct pt_regs *regs)
+{
+ regs->kuap = mfspr(SPRN_MD_AP);
+ mtspr(SPRN_MD_AP, MD_APG_KUAP);
+}
+
+static inline void kuap_user_restore(struct pt_regs *regs)
+{
+}
+
+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
+{
+ mtspr(SPRN_MD_AP, regs->kuap);
+}
+
+static inline unsigned long kuap_get_and_assert_locked(void)
+{
+ unsigned long kuap = mfspr(SPRN_MD_AP);
+
+ if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
+ WARN_ON_ONCE(kuap >> 16 != MD_APG_KUAP >> 16);
+
+ return kuap;
+}
+
+static inline void kuap_assert_locked(void)
+{
+ if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
+ kuap_get_and_assert_locked();
+}
+
static inline void allow_user_access(void __user *to, const void __user *from,
unsigned long size, unsigned long dir)
{
--
2.25.0
^ permalink raw reply related
* [PATCH v3 41/41] powerpc/32: Manage KUAP in C
From: Christophe Leroy @ 2021-03-12 12:50 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>
Move all KUAP management in C.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 50 +-------------------
arch/powerpc/include/asm/interrupt.h | 2 +
arch/powerpc/include/asm/kup.h | 9 ----
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 25 +---------
arch/powerpc/kernel/entry_32.S | 8 ----
arch/powerpc/kernel/interrupt.c | 19 ++------
arch/powerpc/kernel/process.c | 3 ++
7 files changed, 11 insertions(+), 105 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 46599bbc4525..1670dfe9d4f1 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -5,55 +5,7 @@
#include <asm/bug.h>
#include <asm/book3s/32/mmu-hash.h>
-#ifdef __ASSEMBLY__
-
-#ifdef CONFIG_PPC_KUAP
-
-.macro kuap_update_sr gpr1, gpr2, gpr3 /* NEVER use r0 as gpr2 due to addis */
-101: mtsrin \gpr1, \gpr2
- addi \gpr1, \gpr1, 0x111 /* next VSID */
- rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */
- addis \gpr2, \gpr2, 0x1000 /* address of next segment */
- cmplw \gpr2, \gpr3
- blt- 101b
- isync
-.endm
-
-.macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3
- lwz \gpr2, KUAP(\thread)
- rlwinm. \gpr3, \gpr2, 28, 0xf0000000
- stw \gpr2, STACK_REGS_KUAP(\sp)
- beq+ 102f
- li \gpr1, 0
- stw \gpr1, KUAP(\thread)
- mfsrin \gpr1, \gpr2
- oris \gpr1, \gpr1, SR_KS@h /* set Ks */
- kuap_update_sr \gpr1, \gpr2, \gpr3
-102:
-.endm
-
-.macro kuap_restore sp, current, gpr1, gpr2, gpr3
- lwz \gpr2, STACK_REGS_KUAP(\sp)
- rlwinm. \gpr3, \gpr2, 28, 0xf0000000
- stw \gpr2, THREAD + KUAP(\current)
- beq+ 102f
- mfsrin \gpr1, \gpr2
- rlwinm \gpr1, \gpr1, 0, ~SR_KS /* Clear Ks */
- kuap_update_sr \gpr1, \gpr2, \gpr3
-102:
-.endm
-
-.macro kuap_check current, gpr
-#ifdef CONFIG_PPC_KUAP_DEBUG
- lwz \gpr, THREAD + KUAP(\current)
-999: twnei \gpr, 0
- EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE)
-#endif
-.endm
-
-#endif /* CONFIG_PPC_KUAP */
-
-#else /* !__ASSEMBLY__ */
+#ifndef __ASSEMBLY__
#ifdef CONFIG_PPC_KUAP
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 857375309255..7c633896d758 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -36,6 +36,8 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
if (user_mode(regs)) {
kuep_lock();
account_cpu_user_entry();
+ } else {
+ kuap_save_and_lock(regs);
}
#endif
/*
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index aa5f77459311..ec96232529ac 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -28,15 +28,6 @@
#ifdef __ASSEMBLY__
#ifndef CONFIG_PPC_KUAP
-.macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3
-.endm
-
-.macro kuap_restore sp, current, gpr1, gpr2, gpr3
-.endm
-
-.macro kuap_check current, gpr
-.endm
-
.macro kuap_check_amr gpr1, gpr2
.endm
diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
index e6b5ebca47e5..295ef5639609 100644
--- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
@@ -7,30 +7,7 @@
#ifdef CONFIG_PPC_KUAP
-#ifdef __ASSEMBLY__
-
-.macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3
- lis \gpr2, MD_APG_KUAP@h /* only APG0 and APG1 are used */
- mfspr \gpr1, SPRN_MD_AP
- mtspr SPRN_MD_AP, \gpr2
- stw \gpr1, STACK_REGS_KUAP(\sp)
-.endm
-
-.macro kuap_restore sp, current, gpr1, gpr2, gpr3
- lwz \gpr1, STACK_REGS_KUAP(\sp)
- mtspr SPRN_MD_AP, \gpr1
-.endm
-
-.macro kuap_check current, gpr
-#ifdef CONFIG_PPC_KUAP_DEBUG
- mfspr \gpr, SPRN_MD_AP
- rlwinm \gpr, \gpr, 16, 0xffff
-999: twnei \gpr, MD_APG_KUAP@h
- EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE)
-#endif
-.endm
-
-#else /* !__ASSEMBLY__ */
+#ifndef __ASSEMBLY__
#include <asm/reg.h>
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 112d6247c391..9160285cb2f4 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -51,10 +51,7 @@
#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
.globl prepare_transfer_to_handler
prepare_transfer_to_handler:
- addi r12,r2,THREAD
-
/* if from kernel, check interrupted DOZE/NAP mode */
- kuap_save_and_lock r11, r12, r9, r5, r6
lwz r12,TI_LOCAL_FLAGS(r2)
mtcrf 0x01,r12
bt- 31-TLF_NAPPING,4f
@@ -70,7 +67,6 @@ prepare_transfer_to_handler:
lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
rlwinm r9,r9,0,~MSR_EE
lwz r12,_LINK(r11) /* and return to address in LR */
- kuap_restore r11, r2, r3, r4, r5
lwz r2, GPR2(r11)
b fast_exception_return
_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
@@ -95,7 +91,6 @@ ret_from_syscall:
cmplwi cr0,r5,0
bne- 2f
#endif /* CONFIG_PPC_47x */
- kuap_check r2, r4
lwz r4,_LINK(r1)
lwz r5,_CCR(r1)
mtlr r4
@@ -207,7 +202,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_SPE)
stw r10,_CCR(r1)
stw r1,KSP(r3) /* Set old stack pointer */
- kuap_check r2, r0
#ifdef CONFIG_SMP
/* We need a sync somewhere here to make sure that if the
* previous task gets rescheduled on another CPU, it sees all
@@ -298,7 +292,6 @@ interrupt_return:
bne- .Lrestore_nvgprs
.Lfast_user_interrupt_return:
- kuap_check r2, r4
lwz r11,_NIP(r1)
lwz r12,_MSR(r1)
mtspr SPRN_SRR0,r11
@@ -347,7 +340,6 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
.Lfast_kernel_interrupt_return:
cmpwi cr1,r3,0
- kuap_restore r1, r2, r3, r4, r5
lwz r11,_NIP(r1)
lwz r12,_MSR(r1)
mtspr SPRN_SRR0,r11
diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c
index a7cb511bf945..c4dd4b8f9cfa 100644
--- a/arch/powerpc/kernel/interrupt.c
+++ b/arch/powerpc/kernel/interrupt.c
@@ -34,6 +34,9 @@ notrace long system_call_exception(long r3, long r4, long r5,
syscall_fn f;
kuep_lock();
+#ifdef CONFIG_PPC32
+ kuap_save_and_lock(regs);
+#endif
regs->orig_gpr3 = r3;
@@ -75,9 +78,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
isync();
} else
#endif
-#ifdef CONFIG_PPC64
kuap_assert_locked();
-#endif
booke_restore_dbcr0();
@@ -253,9 +254,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
CT_WARN_ON(ct_state() == CONTEXT_USER);
-#ifdef CONFIG_PPC64
kuap_assert_locked();
-#endif
regs->result = r3;
@@ -350,7 +349,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
account_cpu_user_exit();
-#ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E and ppc32 not using this */
+#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not using this */
/*
* We do this at the end so that we do context switch with KERNEL AMR
*/
@@ -379,9 +378,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
* We don't need to restore AMR on the way back to userspace for KUAP.
* AMR can only have been unlocked if we interrupted the kernel.
*/
-#ifdef CONFIG_PPC64
kuap_assert_locked();
-#endif
local_irq_save(flags);
@@ -438,9 +435,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
/*
* We do this at the end so that we do context switch with KERNEL AMR
*/
-#ifdef CONFIG_PPC64
kuap_user_restore(regs);
-#endif
return ret;
}
@@ -450,9 +445,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
{
unsigned long flags;
unsigned long ret = 0;
-#ifdef CONFIG_PPC64
unsigned long kuap;
-#endif
if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) &&
unlikely(!(regs->msr & MSR_RI)))
@@ -466,9 +459,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
if (TRAP(regs) != 0x700)
CT_WARN_ON(ct_state() == CONTEXT_USER);
-#ifdef CONFIG_PPC64
kuap = kuap_get_and_assert_locked();
-#endif
if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {
clear_bits(_TIF_EMULATE_STACK_STORE, ¤t_thread_info()->flags);
@@ -510,9 +501,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
* which would cause Read-After-Write stalls. Hence, we take the AMR
* value from the check above.
*/
-#ifdef CONFIG_PPC64
kuap_kernel_restore(regs, kuap);
-#endif
return ret;
}
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 5d5d64be2679..7989d9ce468b 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1255,6 +1255,9 @@ struct task_struct *__switch_to(struct task_struct *prev,
*/
restore_sprs(old_thread, new_thread);
+#ifdef CONFIG_PPC32
+ kuap_assert_locked();
+#endif
last = _switch(old_thread, new_thread);
#ifdef CONFIG_PPC_BOOK3S_64
--
2.25.0
^ permalink raw reply related
* Re: [PATCH v3 23/41] KVM: PPC: Book3S HV P9: Reduce mftb per guest entry/exit
From: Fabiano Rosas @ 2021-03-12 12:55 UTC (permalink / raw)
To: Nicholas Piggin, kvm-ppc; +Cc: linuxppc-dev, Nicholas Piggin
In-Reply-To: <20210305150638.2675513-24-npiggin@gmail.com>
Nicholas Piggin <npiggin@gmail.com> writes:
> mftb is serialising (dispatch next-to-complete) so it is heavy weight
> for a mfspr. Avoid reading it multiple times in the entry or exit paths.
> A small number of cycles delay to timers is tolerable.
>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Reviewed-by: Fabiano Rosas <farosas@linux.ibm.com>
> ---
> arch/powerpc/kvm/book3s_hv.c | 9 +++++----
> 1 file changed, 5 insertions(+), 4 deletions(-)
>
> diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
> index c1965a9d8d00..6f3e3aed99aa 100644
> --- a/arch/powerpc/kvm/book3s_hv.c
> +++ b/arch/powerpc/kvm/book3s_hv.c
> @@ -3505,12 +3505,13 @@ static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit,
> host_dawrx1 = mfspr(SPRN_DAWRX1);
> }
>
> - hdec = time_limit - mftb();
> + tb = mftb();
> + hdec = time_limit - tb;
> if (hdec < 0)
> return BOOK3S_INTERRUPT_HV_DECREMENTER;
>
> if (vc->tb_offset) {
> - u64 new_tb = mftb() + vc->tb_offset;
> + u64 new_tb = tb + vc->tb_offset;
> mtspr(SPRN_TBU40, new_tb);
> tb = mftb();
> if ((tb & 0xffffff) < (new_tb & 0xffffff))
> @@ -3703,7 +3704,7 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,
> if (!(vcpu->arch.ctrl & 1))
> mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1);
>
> - mtspr(SPRN_DEC, vcpu->arch.dec_expires - mftb());
> + mtspr(SPRN_DEC, vcpu->arch.dec_expires - tb);
>
> if (kvmhv_on_pseries()) {
> /*
> @@ -3837,7 +3838,7 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,
> vc->entry_exit_map = 0x101;
> vc->in_guest = 0;
>
> - mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - mftb());
> + mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - tb);
> mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso);
>
> kvmhv_load_host_pmu();
^ permalink raw reply
* Re: [PATCH 4/4] tools/perf: Support pipeline stage cycles for powerpc
From: Jiri Olsa @ 2021-03-12 12:56 UTC (permalink / raw)
To: Athira Rajeev
Cc: ravi.bangoria, maddy, peterz, linux-kernel, acme,
linux-perf-users, jolsa, kjain, linuxppc-dev, kan.liang
In-Reply-To: <1615298640-1529-5-git-send-email-atrajeev@linux.vnet.ibm.com>
On Tue, Mar 09, 2021 at 09:04:00AM -0500, Athira Rajeev wrote:
> The pipeline stage cycles details can be recorded on powerpc from
> the contents of Performance Monitor Unit (PMU) registers. On
> ISA v3.1 platform, sampling registers exposes the cycles spent in
> different pipeline stages. Patch adds perf tools support to present
> two of the cycle counter information along with memory latency (weight).
>
> Re-use the field 'ins_lat' for storing the first pipeline stage cycle.
> This is stored in 'var2_w' field of 'perf_sample_weight'.
>
> Add a new field 'p_stage_cyc' to store the second pipeline stage cycle
> which is stored in 'var3_w' field of perf_sample_weight.
>
> Add new sort function 'Pipeline Stage Cycle' and include this in
> default_mem_sort_order[]. This new sort function may be used to denote
> some other pipeline stage in another architecture. So add this to
> list of sort entries that can have dynamic header string.
>
> Signed-off-by: Athira Rajeev <atrajeev@linux.vnet.ibm.com>
> ---
> tools/perf/Documentation/perf-report.txt | 1 +
> tools/perf/arch/powerpc/util/event.c | 18 ++++++++++++++++--
> tools/perf/util/event.h | 1 +
> tools/perf/util/hist.c | 11 ++++++++---
> tools/perf/util/hist.h | 1 +
> tools/perf/util/session.c | 4 +++-
> tools/perf/util/sort.c | 24 ++++++++++++++++++++++--
> tools/perf/util/sort.h | 2 ++
> 8 files changed, 54 insertions(+), 8 deletions(-)
>
> diff --git a/tools/perf/Documentation/perf-report.txt b/tools/perf/Documentation/perf-report.txt
> index f546b5e9db05..9691d9c227ba 100644
> --- a/tools/perf/Documentation/perf-report.txt
> +++ b/tools/perf/Documentation/perf-report.txt
> @@ -112,6 +112,7 @@ OPTIONS
> - ins_lat: Instruction latency in core cycles. This is the global instruction
> latency
> - local_ins_lat: Local instruction latency version
> + - p_stage_cyc: Number of cycles spent in a pipeline stage.
please specify in here that it's ppc only
SNIP
> +struct sort_entry sort_p_stage_cyc = {
> + .se_header = "Pipeline Stage Cycle",
> + .se_cmp = sort__global_p_stage_cyc_cmp,
> + .se_snprintf = hist_entry__p_stage_cyc_snprintf,
> + .se_width_idx = HISTC_P_STAGE_CYC,
> +};
> +
> struct sort_entry sort_mem_daddr_sym = {
> .se_header = "Data Symbol",
> .se_cmp = sort__daddr_cmp,
> @@ -1853,6 +1872,7 @@ static void sort_dimension_add_dynamic_header(struct sort_dimension *sd)
> DIM(SORT_CODE_PAGE_SIZE, "code_page_size", sort_code_page_size),
> DIM(SORT_LOCAL_INS_LAT, "local_ins_lat", sort_local_ins_lat),
> DIM(SORT_GLOBAL_INS_LAT, "ins_lat", sort_global_ins_lat),
> + DIM(SORT_P_STAGE_CYC, "p_stage_cyc", sort_p_stage_cyc),
this might be out of scope for this patch, but would it make sense
to add arch specific sort dimension? so the specific column is
not even visible on arch that it's not supported on
> };
>
> #undef DIM
> diff --git a/tools/perf/util/sort.h b/tools/perf/util/sort.h
> index 63f67a3f3630..23b20cbbc846 100644
> --- a/tools/perf/util/sort.h
> +++ b/tools/perf/util/sort.h
> @@ -51,6 +51,7 @@ struct he_stat {
> u64 period_guest_us;
> u64 weight;
> u64 ins_lat;
> + u64 p_stage_cyc;
> u32 nr_events;
> };
>
> @@ -234,6 +235,7 @@ enum sort_type {
> SORT_CODE_PAGE_SIZE,
> SORT_LOCAL_INS_LAT,
> SORT_GLOBAL_INS_LAT,
> + SORT_P_STAGE_CYC,
we could have the whole 'SORT_PEPELINE_STAGE_CYC',
so it's more obvious
thanks,
jirka
^ permalink raw reply
* Re: [PATCH 2/4] tools/perf: Add dynamic headers for perf report columns
From: Jiri Olsa @ 2021-03-12 12:57 UTC (permalink / raw)
To: Athira Rajeev
Cc: ravi.bangoria, maddy, peterz, linux-kernel, acme,
linux-perf-users, jolsa, kjain, linuxppc-dev, kan.liang
In-Reply-To: <1615298640-1529-3-git-send-email-atrajeev@linux.vnet.ibm.com>
On Tue, Mar 09, 2021 at 09:03:58AM -0500, Athira Rajeev wrote:
> Currently the header string for different columns in perf report
> is fixed. Some fields of perf sample could have different meaning
> for different architectures than the meaning conveyed by the header
> string. An example is the new field 'var2_w' of perf_sample_weight
> structure. This is presently captured as 'Local INSTR Latency' in
> perf mem report. But this could be used to denote a different latency
> cycle in another architecture.
>
> Introduce a weak function arch_perf_header_entry__add() to set
> the arch specific header string for the fields which can contain dynamic
> header. If the architecture do not have this function, fall back to the
> default header string value.
>
> Signed-off-by: Athira Rajeev <atrajeev@linux.vnet.ibm.com>
> ---
> tools/perf/util/event.h | 1 +
> tools/perf/util/sort.c | 19 ++++++++++++++++++-
> 2 files changed, 19 insertions(+), 1 deletion(-)
>
> diff --git a/tools/perf/util/event.h b/tools/perf/util/event.h
> index f603edbbbc6f..89b149e2e70a 100644
> --- a/tools/perf/util/event.h
> +++ b/tools/perf/util/event.h
> @@ -427,5 +427,6 @@ void cpu_map_data__synthesize(struct perf_record_cpu_map_data *data, struct per
>
> void arch_perf_parse_sample_weight(struct perf_sample *data, const __u64 *array, u64 type);
> void arch_perf_synthesize_sample_weight(const struct perf_sample *data, __u64 *array, u64 type);
> +const char *arch_perf_header_entry__add(const char *se_header);
>
> #endif /* __PERF_RECORD_H */
> diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c
> index 0d5ad42812b9..741a6df29fa0 100644
> --- a/tools/perf/util/sort.c
> +++ b/tools/perf/util/sort.c
> @@ -25,6 +25,7 @@
> #include <traceevent/event-parse.h>
> #include "mem-events.h"
> #include "annotate.h"
> +#include "event.h"
> #include "time-utils.h"
> #include "cgroup.h"
> #include "machine.h"
> @@ -45,6 +46,7 @@
> regex_t ignore_callees_regex;
> int have_ignore_callees = 0;
> enum sort_mode sort__mode = SORT_MODE__NORMAL;
> +const char *dynamic_headers[] = {"local_ins_lat"};
>
> /*
> * Replaces all occurrences of a char used with the:
> @@ -1816,6 +1818,16 @@ struct sort_dimension {
> int taken;
> };
>
> +const char * __weak arch_perf_header_entry__add(const char *se_header)
no need for the __add suffix in here
jirka
> +{
> + return se_header;
> +}
> +
> +static void sort_dimension_add_dynamic_header(struct sort_dimension *sd)
> +{
> + sd->entry->se_header = arch_perf_header_entry__add(sd->entry->se_header);
> +}
> +
> #define DIM(d, n, func) [d] = { .name = n, .entry = &(func) }
>
> static struct sort_dimension common_sort_dimensions[] = {
> @@ -2739,11 +2751,16 @@ int sort_dimension__add(struct perf_hpp_list *list, const char *tok,
> struct evlist *evlist,
> int level)
> {
> - unsigned int i;
> + unsigned int i, j;
>
> for (i = 0; i < ARRAY_SIZE(common_sort_dimensions); i++) {
> struct sort_dimension *sd = &common_sort_dimensions[i];
>
> + for (j = 0; j < ARRAY_SIZE(dynamic_headers); j++) {
> + if (!strcmp(dynamic_headers[j], sd->name))
> + sort_dimension_add_dynamic_header(sd);
> + }
> +
> if (strncasecmp(tok, sd->name, strlen(tok)))
> continue;
>
> --
> 1.8.3.1
>
^ permalink raw reply
* Re: [PATCH] powerpc/vdso32: Add missing _restgpr_31_x to fix build failure
From: Christophe Leroy @ 2021-03-12 13:09 UTC (permalink / raw)
To: Michael Ellerman; +Cc: linuxppc-dev, linux-kernel
In-Reply-To: <a7aa198a88bcd33c6e35e99f70f86c7b7f2f9440.1615270757.git.christophe.leroy@csgroup.eu>
Le 09/03/2021 à 07:19, Christophe Leroy a écrit :
> With some defconfig including CONFIG_CC_OPTIMIZE_FOR_SIZE,
> (for instance mvme5100_defconfig and ps3_defconfig), gcc 5
> generates a call to _restgpr_31_x.
>
> Until recently it went unnoticed, but
> commit 42ed6d56ade2 ("powerpc/vdso: Block R_PPC_REL24 relocations")
> made it rise to the surface.
>
> Provide that function (copied from lib/crtsavres.S) in
> gettimeofday.S
>
> Fixes: ab037dd87a2f ("powerpc/vdso: Switch VDSO to generic C implementation.")
> Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Fixes the following builds:
http://kisskb.ellerman.id.au/kisskb/buildresult/14492138/
http://kisskb.ellerman.id.au/kisskb/buildresult/14492041/
Christophe
> ---
> I don't know if there is a way to tell GCC not to emit that call, because at the end we get more instructions than needed.
> ---
> arch/powerpc/kernel/vdso32/gettimeofday.S | 11 +++++++++++
> 1 file changed, 11 insertions(+)
>
> diff --git a/arch/powerpc/kernel/vdso32/gettimeofday.S b/arch/powerpc/kernel/vdso32/gettimeofday.S
> index a6e29f880e0e..d21d08140a5e 100644
> --- a/arch/powerpc/kernel/vdso32/gettimeofday.S
> +++ b/arch/powerpc/kernel/vdso32/gettimeofday.S
> @@ -65,3 +65,14 @@ V_FUNCTION_END(__kernel_clock_getres)
> V_FUNCTION_BEGIN(__kernel_time)
> cvdso_call_time __c_kernel_time
> V_FUNCTION_END(__kernel_time)
> +
> +/* Routines for restoring integer registers, called by the compiler. */
> +/* Called with r11 pointing to the stack header word of the caller of the */
> +/* function, just beyond the end of the integer restore area. */
> +_GLOBAL(_restgpr_31_x)
> +_GLOBAL(_rest32gpr_31_x)
> + lwz r0,4(r11)
> + lwz r31,-4(r11)
> + mtlr r0
> + mr r1,r11
> + blr
>
^ permalink raw reply
* Re: [PATCH] powerpc: Force inlining of cpu_has_feature() to avoid build failure
From: Christophe Leroy @ 2021-03-12 13:12 UTC (permalink / raw)
To: Michael Ellerman; +Cc: linuxppc-dev, linux-kernel
In-Reply-To: <b231dfa040ce4cc37f702f5c3a595fdeabfe0462.1615378209.git.christophe.leroy@csgroup.eu>
Le 10/03/2021 à 13:10, Christophe Leroy a écrit :
> The code relies on constant folding of cpu_has_feature() based
> on possible and always true values as defined per
> CPU_FTRS_ALWAYS and CPU_FTRS_POSSIBLE.
>
> Build failure is encountered with for instance
> book3e_all_defconfig on kisskb in the AMDGPU driver which uses
> cpu_has_feature(CPU_FTR_VSX_COMP) to decide whether calling
> kernel_enable_vsx() or not.
>
> The failure is due to cpu_has_feature() not being inlined with
> that configuration with gcc 4.9.
>
> In the same way as commit acdad8fb4a15 ("powerpc: Force inlining of
> mmu_has_feature to fix build failure"), for inlining of
> cpu_has_feature().
>
> Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Fixes following build
http://kisskb.ellerman.id.au/kisskb/buildresult/14489391/
> ---
> arch/powerpc/include/asm/cpu_has_feature.h | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/cpu_has_feature.h b/arch/powerpc/include/asm/cpu_has_feature.h
> index 7897d16e0990..727d4b321937 100644
> --- a/arch/powerpc/include/asm/cpu_has_feature.h
> +++ b/arch/powerpc/include/asm/cpu_has_feature.h
> @@ -7,7 +7,7 @@
> #include <linux/bug.h>
> #include <asm/cputable.h>
>
> -static inline bool early_cpu_has_feature(unsigned long feature)
> +static __always_inline bool early_cpu_has_feature(unsigned long feature)
> {
> return !!((CPU_FTRS_ALWAYS & feature) ||
> (CPU_FTRS_POSSIBLE & cur_cpu_spec->cpu_features & feature));
> @@ -46,7 +46,7 @@ static __always_inline bool cpu_has_feature(unsigned long feature)
> return static_branch_likely(&cpu_feature_keys[i]);
> }
> #else
> -static inline bool cpu_has_feature(unsigned long feature)
> +static __always_inline bool cpu_has_feature(unsigned long feature)
> {
> return early_cpu_has_feature(feature);
> }
>
^ permalink raw reply
* [PATCH v3 03/15] powerpc/align: Convert emulate_spe() to user_access_begin
From: Christophe Leroy @ 2021-03-12 13:25 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Daniel Axtens
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <0ad4629c2d222019e82fcdfccc70d372beb4adf9.1615398265.git.christophe.leroy@csgroup.eu>
This patch converts emulate_spe() to using user_access_begin
logic.
Since commit 662bbcb2747c ("mm, sched: Allow uaccess in atomic with
pagefault_disable()"), might_fault() doesn't fire when called from
sections where pagefaults are disabled, which must be the case
when using _inatomic variants of __get_user and __put_user. So
the might_fault() in user_access_begin() is not a problem.
There was a verification of user_mode() together with the access_ok(),
but there is a second verification of user_mode() just after, that
leads to immediate return. The access_ok() is now part of the
user_access_begin which is called after that other user_mode()
verification, so no need to check user_mode() again.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Reviewed-by: Daniel Axtens <dja@axtens.net>
---
v3:
- Changed the second user_read_access_begin() to user_write_access_begin()
- Reworded explaination about the user_mode() with access_ok() in the commit message
---
arch/powerpc/kernel/align.c | 61 ++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 28 deletions(-)
diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c
index c7797eb958c7..f362c99213be 100644
--- a/arch/powerpc/kernel/align.c
+++ b/arch/powerpc/kernel/align.c
@@ -107,7 +107,6 @@ static struct aligninfo spe_aligninfo[32] = {
static int emulate_spe(struct pt_regs *regs, unsigned int reg,
struct ppc_inst ppc_instr)
{
- int ret;
union {
u64 ll;
u32 w[2];
@@ -127,11 +126,6 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg,
nb = spe_aligninfo[instr].len;
flags = spe_aligninfo[instr].flags;
- /* Verify the address of the operand */
- if (unlikely(user_mode(regs) &&
- !access_ok(addr, nb)))
- return -EFAULT;
-
/* userland only */
if (unlikely(!user_mode(regs)))
return 0;
@@ -169,26 +163,27 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg,
}
} else {
temp.ll = data.ll = 0;
- ret = 0;
p = addr;
+ if (!user_read_access_begin(addr, nb))
+ return -EFAULT;
+
switch (nb) {
case 8:
- ret |= __get_user_inatomic(temp.v[0], p++);
- ret |= __get_user_inatomic(temp.v[1], p++);
- ret |= __get_user_inatomic(temp.v[2], p++);
- ret |= __get_user_inatomic(temp.v[3], p++);
+ unsafe_get_user(temp.v[0], p++, Efault_read);
+ unsafe_get_user(temp.v[1], p++, Efault_read);
+ unsafe_get_user(temp.v[2], p++, Efault_read);
+ unsafe_get_user(temp.v[3], p++, Efault_read);
fallthrough;
case 4:
- ret |= __get_user_inatomic(temp.v[4], p++);
- ret |= __get_user_inatomic(temp.v[5], p++);
+ unsafe_get_user(temp.v[4], p++, Efault_read);
+ unsafe_get_user(temp.v[5], p++, Efault_read);
fallthrough;
case 2:
- ret |= __get_user_inatomic(temp.v[6], p++);
- ret |= __get_user_inatomic(temp.v[7], p++);
- if (unlikely(ret))
- return -EFAULT;
+ unsafe_get_user(temp.v[6], p++, Efault_read);
+ unsafe_get_user(temp.v[7], p++, Efault_read);
}
+ user_read_access_end();
switch (instr) {
case EVLDD:
@@ -255,31 +250,41 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg,
/* Store result to memory or update registers */
if (flags & ST) {
- ret = 0;
p = addr;
+
+ if (!user_write_access_begin(addr, nb))
+ return -EFAULT;
+
switch (nb) {
case 8:
- ret |= __put_user_inatomic(data.v[0], p++);
- ret |= __put_user_inatomic(data.v[1], p++);
- ret |= __put_user_inatomic(data.v[2], p++);
- ret |= __put_user_inatomic(data.v[3], p++);
+ unsafe_put_user(data.v[0], p++, Efault_write);
+ unsafe_put_user(data.v[1], p++, Efault_write);
+ unsafe_put_user(data.v[2], p++, Efault_write);
+ unsafe_put_user(data.v[3], p++, Efault_write);
fallthrough;
case 4:
- ret |= __put_user_inatomic(data.v[4], p++);
- ret |= __put_user_inatomic(data.v[5], p++);
+ unsafe_put_user(data.v[4], p++, Efault_write);
+ unsafe_put_user(data.v[5], p++, Efault_write);
fallthrough;
case 2:
- ret |= __put_user_inatomic(data.v[6], p++);
- ret |= __put_user_inatomic(data.v[7], p++);
+ unsafe_put_user(data.v[6], p++, Efault_write);
+ unsafe_put_user(data.v[7], p++, Efault_write);
}
- if (unlikely(ret))
- return -EFAULT;
+ user_write_access_end();
} else {
*evr = data.w[0];
regs->gpr[reg] = data.w[1];
}
return 1;
+
+Efault_read:
+ user_read_access_end();
+ return -EFAULT;
+
+Efault_write:
+ user_write_access_end();
+ return -EFAULT;
}
#endif /* CONFIG_SPE */
--
2.25.0
^ permalink raw reply related
* Re: [PATCH v2 1/8] powerpc/xive: Use cpu_to_node() instead of ibm,chip-id property
From: Cédric Le Goater @ 2021-03-12 13:28 UTC (permalink / raw)
To: Daniel Henrique Barboza, David Gibson
Cc: list@suse.de:PowerPC, linuxppc-dev, Greg Kurz, QEMU Developers
In-Reply-To: <0f27271d-cb4d-986c-95c6-3173b43f70e5@linux.ibm.com>
On 3/12/21 1:18 PM, Daniel Henrique Barboza wrote:
>
>
> On 3/12/21 6:53 AM, Cédric Le Goater wrote:
>> On 3/12/21 2:55 AM, David Gibson wrote:
>>> On Tue, 9 Mar 2021 18:26:35 +0100
>>> Cédric Le Goater <clg@kaod.org> wrote:
>>>
>>>> On 3/9/21 6:08 PM, Daniel Henrique Barboza wrote:
>>>>>
>>>>>
>>>>> On 3/9/21 12:33 PM, Cédric Le Goater wrote:
>>>>>> On 3/8/21 6:13 PM, Greg Kurz wrote:
>>>>>>> On Wed, 3 Mar 2021 18:48:50 +0100
>>>>>>> Cédric Le Goater <clg@kaod.org> wrote:
>>>>>>>
>>>>>>>> The 'chip_id' field of the XIVE CPU structure is used to choose a
>>>>>>>> target for a source located on the same chip when possible. This field
>>>>>>>> is assigned on the PowerNV platform using the "ibm,chip-id" property
>>>>>>>> on pSeries under KVM when NUMA nodes are defined but it is undefined
>>>>>>>
>>>>>>> This sentence seems to have a syntax problem... like it is missing an
>>>>>>> 'and' before 'on pSeries'.
>>>>>>
>>>>>> ah yes, or simply a comma.
>>>>>>
>>>>>>>> under PowerVM. The XIVE source structure has a similar field
>>>>>>>> 'src_chip' which is only assigned on the PowerNV platform.
>>>>>>>>
>>>>>>>> cpu_to_node() returns a compatible value on all platforms, 0 being the
>>>>>>>> default node. It will also give us the opportunity to set the affinity
>>>>>>>> of a source on pSeries when we can localize them.
>>>>>>>>
>>>>>>>
>>>>>>> IIUC this relies on the fact that the NUMA node id is == to chip id
>>>>>>> on PowerNV, i.e. xc->chip_id which is passed to OPAL remain stable
>>>>>>> with this change.
>>>>>>
>>>>>> Linux sets the NUMA node in numa_setup_cpu(). On pseries, the hcall
>>>>>> H_HOME_NODE_ASSOCIATIVITY returns the node id if I am correct (Daniel
>>>>>> in Cc:)
>>>> [...]
>>>>>>
>>>>>> On PowerNV, Linux uses "ibm,associativity" property of the CPU to find
>>>>>> the node id. This value is built from the chip id in OPAL, so the
>>>>>> value returned by cpu_to_node(cpu) and the value of the "ibm,chip-id"
>>>>>> property are unlikely to be different.
>>>>>>
>>>>>> cpu_to_node(cpu) is used in many places to allocate the structures
>>>>>> locally to the owning node. XIVE is not an exception (see below in the
>>>>>> same patch), it is better to be consistent and get the same information
>>>>>> (node id) using the same routine.
>>>>>>
>>>>>>
>>>>>> In Linux, "ibm,chip-id" is only used in low level PowerNV drivers :
>>>>>> LPC, XSCOM, RNG, VAS, NX. XIVE should be in that list also but skiboot
>>>>>> unifies the controllers of the system to only expose one the OS. This
>>>>>> is problematic and should be changed but it's another topic.
>>>>>>
>>>>>>
>>>>>>> On the other hand, you have the pSeries case under PowerVM that
>>>>>>> doesn't xc->chip_id, which isn't passed to any hcall AFAICT.
>>>>>>
>>>>>> yes "ibm,chip-id" is an OPAL concept unfortunately and it has no meaning
>>>>>> under PAPR. xc->chip_id on pseries (PowerVM) will contains an invalid
>>>>>> chip id.
>>>>>>
>>>>>> QEMU/KVM exposes "ibm,chip-id" but it's not used. (its value is not
>>>>>> always correct btw)
>>>>>
>>>>>
>>>>> If you have a way to reliably reproduce this, let me know and I'll fix it
>>>>> up in QEMU.
>>>>
>>>> with :
>>>>
>>>> -smp 4,cores=1,maxcpus=8 -object memory-backend-ram,id=ram-node0,size=2G -numa node,nodeid=0,cpus=0-1,cpus=4-5,memdev=ram-node0 -object memory-backend-ram,id=ram-node1,size=2G -numa node,nodeid=1,cpus=2-3,cpus=6-7,memdev=ram-node1
>>>>
>>>> # dmesg | grep numa
>>>> [ 0.013106] numa: Node 0 CPUs: 0-1
>>>> [ 0.013136] numa: Node 1 CPUs: 2-3
>>>>
>>>> # dtc -I fs /proc/device-tree/cpus/ -f | grep ibm,chip-id
>>>> ibm,chip-id = <0x01>;
>>>> ibm,chip-id = <0x02>;
>>>> ibm,chip-id = <0x00>;
>>>> ibm,chip-id = <0x03>;
>>>>
>>>> with :
>>>>
>>>> -smp 4,cores=4,maxcpus=8,threads=1 -object memory-backend-ram,id=ram-node0,size=2G -numa node,nodeid=0,cpus=0-1,cpus=4-5,memdev=ram-node0 -object memory-backend-ram,id=ram-node1,size=2G -numa node,nodeid=1,cpus=2-3,cpus=6-7,memdev=ram-node1
>>>>
>>>> # dmesg | grep numa
>>>> [ 0.013106] numa: Node 0 CPUs: 0-1
>>>> [ 0.013136] numa: Node 1 CPUs: 2-3
>>>>
>>>> # dtc -I fs /proc/device-tree/cpus/ -f | grep ibm,chip-id
>>>> ibm,chip-id = <0x00>;
>>>> ibm,chip-id = <0x00>;
>>>> ibm,chip-id = <0x00>;
>>>> ibm,chip-id = <0x00>;
>>>>
>>>> I think we should simply remove "ibm,chip-id" since it's not used and
>>>> not in the PAPR spec.
>>>
>>> As I mentioned to Daniel on our call this morning, oddly it *does*
>>> appear to be used in the RHEL kernel, even though that's 4.18 based.
>>> This patch seems to have caused a minor regression; not in the
>>> identification of NUMA nodes, but in the number of sockets shown be
>>> lscpu, etc. See https://bugzilla.redhat.com/show_bug.cgi?id=1934421
>>> for more information.
>>
>> Yes. The property "ibm,chip-id" is wrongly calculated in QEMU. If we
>> remove it, we get with 4.18.0-295.el8.ppc64le or 5.12.0-rc2 :
>>
>> [root@localhost ~]# lscpu
>> Architecture: ppc64le
>> Byte Order: Little Endian
>> CPU(s): 128
>> On-line CPU(s) list: 0-127
>> Thread(s) per core: 4
>> Core(s) per socket: 16
>> Socket(s): 2
>> NUMA node(s): 2
>> Model: 2.2 (pvr 004e 1202)
>> Model name: POWER9 (architected), altivec supported
>> Hypervisor vendor: KVM
>> Virtualization type: para
>> L1d cache: 32K
>> L1i cache: 32K
>> NUMA node0 CPU(s): 0-63
>> NUMA node1 CPU(s): 64-127
>>
>> [root@localhost ~]# grep . /sys/devices/system/cpu/*/topology/physical_package_id
>> /sys/devices/system/cpu/cpu0/topology/physical_package_id:-1
>> /sys/devices/system/cpu/cpu100/topology/physical_package_id:-1
>> /sys/devices/system/cpu/cpu101/topology/physical_package_id:-1
>> /sys/devices/system/cpu/cpu102/topology/physical_package_id:-1
>> /sys/devices/system/cpu/cpu103/topology/physical_package_id:-1
>> ....
>>
>> "ibm,chip-id" is still being used on some occasion on pSeries machines.
>> This is wrong :/ The problem is :
>>
>> #define topology_physical_package_id(cpu) (cpu_to_chip_id(cpu))
>>
>> We should be using cpu_to_node().
>
>
> IIUC the "real fix" then is this change you mentioned above, together with
> this xive patch as well,
These are independent.
The XIVE patch just raised the issue because it's another usage example of
cpu_to_chip_id() or directly "ibm,chip-id" in the XIVE case, on a pseries
machine.
The use of cpu_to_node(cpu) for topology_physical_package_id(cpu) is a fix
for the sysfs issue reported in the redhat BZ.
> to stop using ibm,chip-id for good in the pserie
> kernel. With these changes QEMU can remove 'ibm,chip-id' from the pseries
> machine without impact. Is this correct?
Linux is already "broken" on PowerVM today since we don't have the "ibm,chip-id"
property. QEMU is just hiding the problem on KVM.
But we have to be bug compatible :) if the QEMU fix is under the pseries-6.x
machine we should be fine.
> If that's the case, then I believe it's ok to go forward with the QEMU side
> change (just for 6.0.0 and newer machines). Or should I wait for the kernel
> changes to be merged upstream first?
Once Linux is fixed, we shouldn't care if QEMU exports 'ibm,chip-id' or not.
I don't think the order is very important. These are independent.
C.
^ permalink raw reply
* Re: [PATCH v2 1/8] powerpc/xive: Use cpu_to_node() instead of ibm,chip-id property
From: Greg Kurz @ 2021-03-12 13:03 UTC (permalink / raw)
To: Daniel Henrique Barboza
Cc: QEMU Developers, list@suse.de:PowerPC, Cédric Le Goater,
David Gibson, linuxppc-dev
In-Reply-To: <0f27271d-cb4d-986c-95c6-3173b43f70e5@linux.ibm.com>
On Fri, 12 Mar 2021 09:18:39 -0300
Daniel Henrique Barboza <danielhb@linux.ibm.com> wrote:
>
>
> On 3/12/21 6:53 AM, Cédric Le Goater wrote:
> > On 3/12/21 2:55 AM, David Gibson wrote:
> >> On Tue, 9 Mar 2021 18:26:35 +0100
> >> Cédric Le Goater <clg@kaod.org> wrote:
> >>
> >>> On 3/9/21 6:08 PM, Daniel Henrique Barboza wrote:
> >>>>
> >>>>
> >>>> On 3/9/21 12:33 PM, Cédric Le Goater wrote:
> >>>>> On 3/8/21 6:13 PM, Greg Kurz wrote:
> >>>>>> On Wed, 3 Mar 2021 18:48:50 +0100
> >>>>>> Cédric Le Goater <clg@kaod.org> wrote:
> >>>>>>
> >>>>>>> The 'chip_id' field of the XIVE CPU structure is used to choose a
> >>>>>>> target for a source located on the same chip when possible. This field
> >>>>>>> is assigned on the PowerNV platform using the "ibm,chip-id" property
> >>>>>>> on pSeries under KVM when NUMA nodes are defined but it is undefined
> >>>>>>
> >>>>>> This sentence seems to have a syntax problem... like it is missing an
> >>>>>> 'and' before 'on pSeries'.
> >>>>>
> >>>>> ah yes, or simply a comma.
> >>>>>
> >>>>>>> under PowerVM. The XIVE source structure has a similar field
> >>>>>>> 'src_chip' which is only assigned on the PowerNV platform.
> >>>>>>>
> >>>>>>> cpu_to_node() returns a compatible value on all platforms, 0 being the
> >>>>>>> default node. It will also give us the opportunity to set the affinity
> >>>>>>> of a source on pSeries when we can localize them.
> >>>>>>>
> >>>>>>
> >>>>>> IIUC this relies on the fact that the NUMA node id is == to chip id
> >>>>>> on PowerNV, i.e. xc->chip_id which is passed to OPAL remain stable
> >>>>>> with this change.
> >>>>>
> >>>>> Linux sets the NUMA node in numa_setup_cpu(). On pseries, the hcall
> >>>>> H_HOME_NODE_ASSOCIATIVITY returns the node id if I am correct (Daniel
> >>>>> in Cc:)
> >>> [...]
> >>>>>
> >>>>> On PowerNV, Linux uses "ibm,associativity" property of the CPU to find
> >>>>> the node id. This value is built from the chip id in OPAL, so the
> >>>>> value returned by cpu_to_node(cpu) and the value of the "ibm,chip-id"
> >>>>> property are unlikely to be different.
> >>>>>
> >>>>> cpu_to_node(cpu) is used in many places to allocate the structures
> >>>>> locally to the owning node. XIVE is not an exception (see below in the
> >>>>> same patch), it is better to be consistent and get the same information
> >>>>> (node id) using the same routine.
> >>>>>
> >>>>>
> >>>>> In Linux, "ibm,chip-id" is only used in low level PowerNV drivers :
> >>>>> LPC, XSCOM, RNG, VAS, NX. XIVE should be in that list also but skiboot
> >>>>> unifies the controllers of the system to only expose one the OS. This
> >>>>> is problematic and should be changed but it's another topic.
> >>>>>
> >>>>>
> >>>>>> On the other hand, you have the pSeries case under PowerVM that
> >>>>>> doesn't xc->chip_id, which isn't passed to any hcall AFAICT.
> >>>>>
> >>>>> yes "ibm,chip-id" is an OPAL concept unfortunately and it has no meaning
> >>>>> under PAPR. xc->chip_id on pseries (PowerVM) will contains an invalid
> >>>>> chip id.
> >>>>>
> >>>>> QEMU/KVM exposes "ibm,chip-id" but it's not used. (its value is not
> >>>>> always correct btw)
> >>>>
> >>>>
> >>>> If you have a way to reliably reproduce this, let me know and I'll fix it
> >>>> up in QEMU.
> >>>
> >>> with :
> >>>
> >>> -smp 4,cores=1,maxcpus=8 -object memory-backend-ram,id=ram-node0,size=2G -numa node,nodeid=0,cpus=0-1,cpus=4-5,memdev=ram-node0 -object memory-backend-ram,id=ram-node1,size=2G -numa node,nodeid=1,cpus=2-3,cpus=6-7,memdev=ram-node1
> >>>
> >>> # dmesg | grep numa
> >>> [ 0.013106] numa: Node 0 CPUs: 0-1
> >>> [ 0.013136] numa: Node 1 CPUs: 2-3
> >>>
> >>> # dtc -I fs /proc/device-tree/cpus/ -f | grep ibm,chip-id
> >>> ibm,chip-id = <0x01>;
> >>> ibm,chip-id = <0x02>;
> >>> ibm,chip-id = <0x00>;
> >>> ibm,chip-id = <0x03>;
> >>>
> >>> with :
> >>>
> >>> -smp 4,cores=4,maxcpus=8,threads=1 -object memory-backend-ram,id=ram-node0,size=2G -numa node,nodeid=0,cpus=0-1,cpus=4-5,memdev=ram-node0 -object memory-backend-ram,id=ram-node1,size=2G -numa node,nodeid=1,cpus=2-3,cpus=6-7,memdev=ram-node1
> >>>
> >>> # dmesg | grep numa
> >>> [ 0.013106] numa: Node 0 CPUs: 0-1
> >>> [ 0.013136] numa: Node 1 CPUs: 2-3
> >>>
> >>> # dtc -I fs /proc/device-tree/cpus/ -f | grep ibm,chip-id
> >>> ibm,chip-id = <0x00>;
> >>> ibm,chip-id = <0x00>;
> >>> ibm,chip-id = <0x00>;
> >>> ibm,chip-id = <0x00>;
> >>>
> >>> I think we should simply remove "ibm,chip-id" since it's not used and
> >>> not in the PAPR spec.
> >>
> >> As I mentioned to Daniel on our call this morning, oddly it *does*
> >> appear to be used in the RHEL kernel, even though that's 4.18 based.
> >> This patch seems to have caused a minor regression; not in the
> >> identification of NUMA nodes, but in the number of sockets shown be
> >> lscpu, etc. See https://bugzilla.redhat.com/show_bug.cgi?id=1934421
> >> for more information.
> >
> > Yes. The property "ibm,chip-id" is wrongly calculated in QEMU. If we
> > remove it, we get with 4.18.0-295.el8.ppc64le or 5.12.0-rc2 :
> >
> > [root@localhost ~]# lscpu
> > Architecture: ppc64le
> > Byte Order: Little Endian
> > CPU(s): 128
> > On-line CPU(s) list: 0-127
> > Thread(s) per core: 4
> > Core(s) per socket: 16
> > Socket(s): 2
> > NUMA node(s): 2
> > Model: 2.2 (pvr 004e 1202)
> > Model name: POWER9 (architected), altivec supported
> > Hypervisor vendor: KVM
> > Virtualization type: para
> > L1d cache: 32K
> > L1i cache: 32K
> > NUMA node0 CPU(s): 0-63
> > NUMA node1 CPU(s): 64-127
> >
> > [root@localhost ~]# grep . /sys/devices/system/cpu/*/topology/physical_package_id
> > /sys/devices/system/cpu/cpu0/topology/physical_package_id:-1
> > /sys/devices/system/cpu/cpu100/topology/physical_package_id:-1
> > /sys/devices/system/cpu/cpu101/topology/physical_package_id:-1
> > /sys/devices/system/cpu/cpu102/topology/physical_package_id:-1
> > /sys/devices/system/cpu/cpu103/topology/physical_package_id:-1
> > ....
> >
> > "ibm,chip-id" is still being used on some occasion on pSeries machines.
> > This is wrong :/ The problem is :
> >
> > #define topology_physical_package_id(cpu) (cpu_to_chip_id(cpu))
> >
> > We should be using cpu_to_node().
>
>
> IIUC the "real fix" then is this change you mentioned above, together with
> this xive patch as well, to stop using ibm,chip-id for good in the pserie
> kernel. With these changes QEMU can remove 'ibm,chip-id' from the pseries
> machine without impact. Is this correct?
>
> If that's the case, then I believe it's ok to go forward with the QEMU side
> change (just for 6.0.0 and newer machines). Or should I wait for the kernel
> changes to be merged upstream first?
>
I'd say the latter since this is a breaking change and people will want
to identify the upstream commits they have to backport to their kernel
in order to support the disappearance of "ibm,chip-id".
Cheers,
--
Greg
>
> Thanks,
>
>
> DHB
>
>
> >
> > C.
> >
> >>
> >> Since the value was used by some PAPR kernels - even if they shouldn't
> >> have - I think we should only remove this for newer machine types. We
> >> also need to check what we're not supplying that the guest kernel is
> >> showing a different number of sockets than specified on the qemu
> >> command line.
> >>
> >>>
> >>> Thanks,
> >>>
> >>> C.
> >>>
> >>>
> >>>
> >>> [...]
> >>> [...]
> >>> [...]
> >>> [...]
> >>> [...]
> >>> [...]
> >>> [...]
> >>> [...]
> >>> [...]
> >>>
> >>
> >>
> >
^ permalink raw reply
* Re: [PATCH] powerpc: define the variable 'uaccess_flush' as static
From: Cédric Le Goater @ 2021-03-12 13:33 UTC (permalink / raw)
To: He Ying, mpe, benh, paulus, npiggin, dja, akpm, rppt,
aneesh.kumar
Cc: johnny.chenyi, linuxppc-dev, linux-kernel
In-Reply-To: <20210312110638.178974-1-heying24@huawei.com>
On 3/12/21 12:06 PM, He Ying wrote:
> The variable 'uaccess_fulsh' is not referenced outside the file. Perhaps we
> should define it as static to avoid the warning as follows:
>
> arch/powerpc/kernel/setup_64.c:953:6: warning: symbol 'uaccess_flush'
> was not declared. Should it be static?
>
> Reported-by: Hulk Robot <hulkci@huawei.com>
> Signed-off-by: He Ying <heying24@huawei.com>
> ---
> arch/powerpc/kernel/setup_64.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
> index 560ed8b975e7..22aca271496b 100644
> --- a/arch/powerpc/kernel/setup_64.c
> +++ b/arch/powerpc/kernel/setup_64.c
> @@ -950,7 +950,7 @@ static bool no_entry_flush;
> static bool no_uaccess_flush;
> bool rfi_flush;
> bool entry_flush;
I think this is the case also for entry_flush. compiling with W=1 will tell you more.
Thanks,
C.
> -bool uaccess_flush;
> +static bool uaccess_flush;
> DEFINE_STATIC_KEY_FALSE(uaccess_flush_key);
> EXPORT_SYMBOL(uaccess_flush_key);
>
>
^ permalink raw reply
* [PATCH] powerpc/numa: Fix topology_physical_package_id() on pSeries
From: Cédric Le Goater @ 2021-03-12 14:31 UTC (permalink / raw)
To: linuxppc-dev
Cc: Nathan Lynch, Srikar Dronamraju, Daniel Henrique Barboza,
Greg Kurz, Vasant Hegde, Cédric Le Goater, David Gibson
Initial commit 15863ff3b8da ("powerpc: Make chip-id information
available to userspace") introduce a cpu_to_chip_id() routine for the
PowerNV platform using the "ibm,chip-id" property to query the chip id
of a CPU. But PAPR does not specify such a property and the node id
query is broken.
Use cpu_to_node() instead which guarantees to have a correct value on
all platforms, PowerNV an pSeries.
Cc: Nathan Lynch <nathanl@linux.ibm.com>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Vasant Hegde <hegdevasant@linux.vnet.ibm.com>
Signed-off-by: Cédric Le Goater <clg@kaod.org>
---
arch/powerpc/include/asm/topology.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h
index 3beeb030cd78..887c42a4e43d 100644
--- a/arch/powerpc/include/asm/topology.h
+++ b/arch/powerpc/include/asm/topology.h
@@ -123,7 +123,7 @@ static inline int cpu_to_coregroup_id(int cpu)
#ifdef CONFIG_PPC64
#include <asm/smp.h>
-#define topology_physical_package_id(cpu) (cpu_to_chip_id(cpu))
+#define topology_physical_package_id(cpu) (cpu_to_node(cpu))
#define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu))
#define topology_core_cpumask(cpu) (cpu_cpu_mask(cpu))
--
2.26.2
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox