* [PATCH v4 18/23] powerpc/32: Remove verification of MSR_PR on syscall in the ASM entry
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
system_call_exception() checks MSR_PR and BUGs if a syscall
is issued from kernel mode.
No need to handle it anymore from the ASM entry code.
null_syscall reduction 2 cycles (348 => 346 cycles)
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 30 ------------------------------
arch/powerpc/kernel/head_32.h | 3 ---
arch/powerpc/kernel/head_booke.h | 3 ---
3 files changed, 36 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index ce5fdb23ed7c..9922a04650f7 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -416,36 +416,6 @@ ret_from_kernel_thread:
li r3,0
b ret_from_syscall
- /*
- * System call was called from kernel. We get here with SRR1 in r9.
- * Mark the exception as recoverable once we have retrieved SRR0,
- * trap a warning and return ENOSYS with CR[SO] set.
- */
- .globl ret_from_kernel_syscall
-ret_from_kernel_syscall:
- mfspr r9, SPRN_SRR0
- mfspr r10, SPRN_SRR1
-#if !defined(CONFIG_4xx) && !defined(CONFIG_BOOKE)
- LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_IR|MSR_DR))
- mtmsr r11
-#endif
-
-0: trap
- EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
-
- li r3, ENOSYS
- crset so
-#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
- mtspr SPRN_NRI, r0
-#endif
- mtspr SPRN_SRR0, r9
- mtspr SPRN_SRR1, r10
- rfi
-#ifdef CONFIG_40x
- b . /* Prevent prefetch past rfi */
-#endif
-_ASM_NOKPROBE_SYMBOL(ret_from_kernel_syscall)
-
/*
* Top-level page fault handling.
* This is in assembler because if do_page_fault tells us that
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index c2aa0d8f1f63..c0de4acbe3f8 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -118,8 +118,6 @@
.macro SYSCALL_ENTRY trapno
mfspr r9, SPRN_SRR1
mfspr r10, SPRN_SRR0
- andi. r11, r9, MSR_PR
- beq- 99f
LOAD_REG_IMMEDIATE(r11, MSR_KERNEL) /* can take exceptions */
lis r12, 1f@h
ori r12, r12, 1f@l
@@ -176,7 +174,6 @@
3:
#endif
b transfer_to_syscall /* jump to handler */
-99: b ret_from_kernel_syscall
.endm
.macro save_dar_dsisr_on_stack reg1, reg2, sp
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index faff094b650e..7af84e1e717b 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -106,10 +106,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
#endif
mfspr r9, SPRN_SRR1
BOOKE_CLEAR_BTB(r11)
- andi. r11, r9, MSR_PR
lwz r11, TASK_STACK - THREAD(r10)
rlwinm r12,r12,0,4,2 /* Clear SO bit in CR */
- beq- 99f
ALLOC_STACK_FRAME(r11, THREAD_SIZE - INT_FRAME_SIZE)
stw r12, _CCR(r11) /* save various registers */
mflr r12
@@ -157,7 +155,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
3:
b transfer_to_syscall /* jump to handler */
-99: b ret_from_kernel_syscall
.endm
/* To handle the additional exception priority levels on 40x and Book-E
--
2.25.0
^ permalink raw reply related
* [PATCH v4 17/23] powerpc/syscall: implement system call entry/exit logic in C for PPC32
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
That's port of PPC64 syscall entry/exit logic in C to PPC32.
Performancewise on 8xx:
Before : 304 cycles on null_syscall
After : 348 cycles on null_syscall
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 227 ++++---------------------------
arch/powerpc/kernel/head_32.h | 16 ---
arch/powerpc/kernel/head_booke.h | 15 --
3 files changed, 29 insertions(+), 229 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 97dc28a68465..ce5fdb23ed7c 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -329,117 +329,22 @@ stack_ovf:
_ASM_NOKPROBE_SYMBOL(stack_ovf)
#endif
-#ifdef CONFIG_TRACE_IRQFLAGS
-trace_syscall_entry_irq_off:
- /*
- * Syscall shouldn't happen while interrupts are disabled,
- * so let's do a warning here.
- */
-0: trap
- EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
- bl trace_hardirqs_on
-
- /* Now enable for real */
- LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE)
- mtmsr r10
-
- REST_GPR(0, r1)
- REST_4GPRS(3, r1)
- REST_2GPRS(7, r1)
- b DoSyscall
-#endif /* CONFIG_TRACE_IRQFLAGS */
-
.globl transfer_to_syscall
transfer_to_syscall:
SAVE_NVGPRS(r1)
#ifdef CONFIG_PPC_BOOK3S_32
kuep_lock r11, r12
#endif
-#ifdef CONFIG_TRACE_IRQFLAGS
- andi. r12,r9,MSR_EE
- beq- trace_syscall_entry_irq_off
-#endif /* CONFIG_TRACE_IRQFLAGS */
-/*
- * Handle a system call.
- */
- .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
- .stabs "entry_32.S",N_SO,0,0,0f
-0:
-
-_GLOBAL(DoSyscall)
- stw r3,ORIG_GPR3(r1)
- li r12,0
- stw r12,RESULT(r1)
-#ifdef CONFIG_TRACE_IRQFLAGS
- /* Make sure interrupts are enabled */
- mfmsr r11
- andi. r12,r11,MSR_EE
- /* We came in with interrupts disabled, we WARN and mark them enabled
- * for lockdep now */
-0: tweqi r12, 0
- EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
-#endif /* CONFIG_TRACE_IRQFLAGS */
- lwz r11,TI_FLAGS(r2)
- andi. r11,r11,_TIF_SYSCALL_DOTRACE
- bne- syscall_dotrace
-syscall_dotrace_cont:
- cmplwi 0,r0,NR_syscalls
- lis r10,sys_call_table@h
- ori r10,r10,sys_call_table@l
- slwi r0,r0,2
- bge- 66f
-
- barrier_nospec_asm
- /*
- * Prevent the load of the handler below (based on the user-passed
- * system call number) being speculatively executed until the test
- * against NR_syscalls and branch to .66f above has
- * committed.
- */
+ /* Calling convention has r9 = orig r0, r10 = regs */
+ mr r9,r0
+ addi r10,r1,STACK_FRAME_OVERHEAD
+ bl system_call_exception
- lwzx r10,r10,r0 /* Fetch system call handler [ptr] */
- mtlr r10
- addi r9,r1,STACK_FRAME_OVERHEAD
- PPC440EP_ERR42
- blrl /* Call handler */
- .globl ret_from_syscall
ret_from_syscall:
-#ifdef CONFIG_DEBUG_RSEQ
- /* Check whether the syscall is issued inside a restartable sequence */
- stw r3,GPR3(r1)
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl rseq_syscall
- lwz r3,GPR3(r1)
-#endif
- mr r6,r3
- /* disable interrupts so current_thread_info()->flags can't change */
- LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) /* doesn't include MSR_EE */
- /* Note: We don't bother telling lockdep about it */
- mtmsr r10
- lwz r9,TI_FLAGS(r2)
- li r8,-MAX_ERRNO
- andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
- bne- syscall_exit_work
- cmplw 0,r3,r8
- blt+ syscall_exit_cont
- lwz r11,_CCR(r1) /* Load CR */
- neg r3,r3
- oris r11,r11,0x1000 /* Set SO bit in CR */
- stw r11,_CCR(r1)
-syscall_exit_cont:
- lwz r8,_MSR(r1)
-#ifdef CONFIG_TRACE_IRQFLAGS
- /* If we are going to return from the syscall with interrupts
- * off, we trace that here. It shouldn't normally happen.
- */
- andi. r10,r8,MSR_EE
- bne+ 1f
- stw r3,GPR3(r1)
- bl trace_hardirqs_off
- lwz r3,GPR3(r1)
-1:
-#endif /* CONFIG_TRACE_IRQFLAGS */
+ addi r4,r1,STACK_FRAME_OVERHEAD
+ li r5,0
+ bl syscall_exit_prepare
#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
/* If the process has its own DBCR0 value, load it up. The internal
debug mode bit tells us that dbcr0 should be loaded. */
@@ -453,34 +358,39 @@ syscall_exit_cont:
cmplwi cr0,r5,0
bne- 2f
#endif /* CONFIG_PPC_47x */
-1:
-BEGIN_FTR_SECTION
- lwarx r7,0,r1
-END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
- stwcx. r0,0,r1 /* to clear the reservation */
- ACCOUNT_CPU_USER_EXIT(r2, r5, r7)
#ifdef CONFIG_PPC_BOOK3S_32
kuep_unlock r5, r7
#endif
- kuap_check r2, r4
lwz r4,_LINK(r1)
lwz r5,_CCR(r1)
mtlr r4
- mtcr r5
lwz r7,_NIP(r1)
- lwz r2,GPR2(r1)
- lwz r1,GPR1(r1)
+ lwz r8,_MSR(r1)
+ cmpwi r3,0
+ lwz r3,GPR3(r1)
syscall_exit_finish:
-#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
- mtspr SPRN_NRI, r0
-#endif
mtspr SPRN_SRR0,r7
mtspr SPRN_SRR1,r8
+
+ bne 3f
+ mtcr r5
+
+1: lwz r2,GPR2(r1)
+ lwz r1,GPR1(r1)
rfi
-#ifdef CONFIG_40x
- b . /* Prevent prefetch past rfi */
-#endif
-_ASM_NOKPROBE_SYMBOL(syscall_exit_finish)
+
+3: mtcr r5
+ lwz r4,_CTR(r1)
+ lwz r5,_XER(r1)
+ REST_NVGPRS(r1)
+ mtctr r4
+ mtxer r5
+ lwz r0,GPR0(r1)
+ lwz r3,GPR3(r1)
+ REST_8GPRS(4,r1)
+ lwz r12,GPR12(r1)
+ b 1b
+
#ifdef CONFIG_44x
2: li r7,0
iccci r0,r0
@@ -488,9 +398,6 @@ _ASM_NOKPROBE_SYMBOL(syscall_exit_finish)
b 1b
#endif /* CONFIG_44x */
-66: li r3,-ENOSYS
- b ret_from_syscall
-
.globl ret_from_fork
ret_from_fork:
REST_NVGPRS(r1)
@@ -509,82 +416,6 @@ ret_from_kernel_thread:
li r3,0
b ret_from_syscall
-/* Traced system call support */
-syscall_dotrace:
- SAVE_NVGPRS(r1)
- li r0,0xc00
- stw r0,_TRAP(r1)
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl do_syscall_trace_enter
- /*
- * Restore argument registers possibly just changed.
- * We use the return value of do_syscall_trace_enter
- * for call number to look up in the table (r0).
- */
- mr r0,r3
- lwz r3,GPR3(r1)
- lwz r4,GPR4(r1)
- lwz r5,GPR5(r1)
- lwz r6,GPR6(r1)
- lwz r7,GPR7(r1)
- lwz r8,GPR8(r1)
- REST_NVGPRS(r1)
-
- cmplwi r0,NR_syscalls
- /* Return code is already in r3 thanks to do_syscall_trace_enter() */
- bge- ret_from_syscall
- b syscall_dotrace_cont
-
-syscall_exit_work:
- andi. r0,r9,_TIF_RESTOREALL
- beq+ 0f
- REST_NVGPRS(r1)
- b 2f
-0: cmplw 0,r3,r8
- blt+ 1f
- andi. r0,r9,_TIF_NOERROR
- bne- 1f
- lwz r11,_CCR(r1) /* Load CR */
- neg r3,r3
- oris r11,r11,0x1000 /* Set SO bit in CR */
- stw r11,_CCR(r1)
-
-1: stw r6,RESULT(r1) /* Save result */
- stw r3,GPR3(r1) /* Update return value */
-2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
- beq 4f
-
- /* Clear per-syscall TIF flags if any are set. */
-
- li r11,_TIF_PERSYSCALL_MASK
- addi r12,r2,TI_FLAGS
-3: lwarx r8,0,r12
- andc r8,r8,r11
- stwcx. r8,0,r12
- bne- 3b
-
-4: /* Anything which requires enabling interrupts? */
- andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
- beq ret_from_except
-
- /* Re-enable interrupts. There is no need to trace that with
- * lockdep as we are supposed to have IRQs on at this point
- */
- ori r10,r10,MSR_EE
- mtmsr r10
-
- /* Save NVGPRS if they're not saved already */
- lwz r4,_TRAP(r1)
- andi. r4,r4,1
- beq 5f
- SAVE_NVGPRS(r1)
- li r4,0xc00
- stw r4,_TRAP(r1)
-5:
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl do_syscall_trace_leave
- b ret_from_except_full
-
/*
* System call was called from kernel. We get here with SRR1 in r9.
* Mark the exception as recoverable once we have retrieved SRR0,
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index 7b12736ec546..c2aa0d8f1f63 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -162,9 +162,6 @@
internal debug mode bit to do this. */
lwz r12,THREAD_DBCR0(r12)
andis. r12,r12,DBCR0_IDM@h
-#endif
- ACCOUNT_CPU_USER_ENTRY(r2, r11, r12)
-#if defined(CONFIG_40x)
beq+ 3f
/* From user and task is ptraced - load up global dbcr0 */
li r12,-1 /* clear all pending debug events */
@@ -176,21 +173,8 @@
lwz r12,4(r11)
addi r12,r12,-1
stw r12,4(r11)
-#endif
-
3:
-#ifdef CONFIG_TRACE_IRQFLAGS
- /*
- * If MSR is changing we need to keep interrupts disabled at this point
- * otherwise we might risk taking an interrupt before we tell lockdep
- * they are enabled.
- */
- LOAD_REG_IMMEDIATE(r10, MSR_KERNEL)
- rlwimi r10, r9, 0, MSR_EE
-#else
- LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE)
#endif
- mtmsr r10
b transfer_to_syscall /* jump to handler */
99: b ret_from_kernel_syscall
.endm
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 626e716576ce..faff094b650e 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -138,7 +138,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
internal debug mode bit to do this. */
lwz r12,THREAD_DBCR0(r10)
andis. r12,r12,DBCR0_IDM@h
- ACCOUNT_CPU_USER_ENTRY(r2, r11, r12)
beq+ 3f
/* From user and task is ptraced - load up global dbcr0 */
li r12,-1 /* clear all pending debug events */
@@ -157,20 +156,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r12,4(r11)
3:
-#ifdef CONFIG_TRACE_IRQFLAGS
- /*
- * If MSR is changing we need to keep interrupts disabled at this point
- * otherwise we might risk taking an interrupt before we tell lockdep
- * they are enabled.
- */
- lis r10, MSR_KERNEL@h
- ori r10, r10, MSR_KERNEL@l
- rlwimi r10, r9, 0, MSR_EE
-#else
- lis r10, (MSR_KERNEL | MSR_EE)@h
- ori r10, r10, (MSR_KERNEL | MSR_EE)@l
-#endif
- mtmsr r10
b transfer_to_syscall /* jump to handler */
99: b ret_from_kernel_syscall
.endm
--
2.25.0
^ permalink raw reply related
* [PATCH v4 16/23] powerpc/32: Always save non volatile GPRs at syscall entry
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
In preparation for porting syscall entry/exit to C, inconditionally
save non volatile general purpose registers.
Commit 965dd3ad3076 ("powerpc/64/syscall: Remove non-volatile GPR save
optimisation") provides detailed explanation.
This increases the number of cycles by 24 cycles on 8xx with
null_syscall benchmark (280 => 304 cycles)
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 46 +-----------------------
arch/powerpc/kernel/head_32.h | 2 +-
arch/powerpc/kernel/head_booke.h | 2 +-
arch/powerpc/kernel/syscalls/syscall.tbl | 20 +++--------
4 files changed, 8 insertions(+), 62 deletions(-)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index b1e36602c013..97dc28a68465 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -351,6 +351,7 @@ trace_syscall_entry_irq_off:
.globl transfer_to_syscall
transfer_to_syscall:
+ SAVE_NVGPRS(r1)
#ifdef CONFIG_PPC_BOOK3S_32
kuep_lock r11, r12
#endif
@@ -614,51 +615,6 @@ ret_from_kernel_syscall:
#endif
_ASM_NOKPROBE_SYMBOL(ret_from_kernel_syscall)
-/*
- * The fork/clone functions need to copy the full register set into
- * the child process. Therefore we need to save all the nonvolatile
- * registers (r13 - r31) before calling the C code.
- */
- .globl ppc_fork
-ppc_fork:
- SAVE_NVGPRS(r1)
- lwz r0,_TRAP(r1)
- rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
- stw r0,_TRAP(r1) /* register set saved */
- b sys_fork
-
- .globl ppc_vfork
-ppc_vfork:
- SAVE_NVGPRS(r1)
- lwz r0,_TRAP(r1)
- rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
- stw r0,_TRAP(r1) /* register set saved */
- b sys_vfork
-
- .globl ppc_clone
-ppc_clone:
- SAVE_NVGPRS(r1)
- lwz r0,_TRAP(r1)
- rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
- stw r0,_TRAP(r1) /* register set saved */
- b sys_clone
-
- .globl ppc_clone3
-ppc_clone3:
- SAVE_NVGPRS(r1)
- lwz r0,_TRAP(r1)
- rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
- stw r0,_TRAP(r1) /* register set saved */
- b sys_clone3
-
- .globl ppc_swapcontext
-ppc_swapcontext:
- SAVE_NVGPRS(r1)
- lwz r0,_TRAP(r1)
- rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
- stw r0,_TRAP(r1) /* register set saved */
- b sys_swapcontext
-
/*
* Top-level page fault handling.
* This is in assembler because if do_page_fault tells us that
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index 24dc326e0d56..7b12736ec546 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -148,7 +148,7 @@
stw r2,GPR2(r11)
addi r10,r10,STACK_FRAME_REGS_MARKER@l
stw r9,_MSR(r11)
- li r2, \trapno + 1
+ li r2, \trapno
stw r10,8(r11)
stw r2,_TRAP(r11)
SAVE_GPR(0, r11)
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index b3c502c503a0..626e716576ce 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -124,7 +124,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r2,GPR2(r11)
addi r12, r12, STACK_FRAME_REGS_MARKER@l
stw r9,_MSR(r11)
- li r2, \trapno + 1
+ li r2, \trapno
stw r12, 8(r11)
stw r2,_TRAP(r11)
SAVE_GPR(0, r11)
diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl
index f744eb5cba88..96b2157f0371 100644
--- a/arch/powerpc/kernel/syscalls/syscall.tbl
+++ b/arch/powerpc/kernel/syscalls/syscall.tbl
@@ -9,9 +9,7 @@
#
0 nospu restart_syscall sys_restart_syscall
1 nospu exit sys_exit
-2 32 fork ppc_fork sys_fork
-2 64 fork sys_fork
-2 spu fork sys_ni_syscall
+2 nospu fork sys_fork
3 common read sys_read
4 common write sys_write
5 common open sys_open compat_sys_open
@@ -160,9 +158,7 @@
119 32 sigreturn sys_sigreturn compat_sys_sigreturn
119 64 sigreturn sys_ni_syscall
119 spu sigreturn sys_ni_syscall
-120 32 clone ppc_clone sys_clone
-120 64 clone sys_clone
-120 spu clone sys_ni_syscall
+120 nospu clone sys_clone
121 common setdomainname sys_setdomainname
122 common uname sys_newuname
123 common modify_ldt sys_ni_syscall
@@ -244,9 +240,7 @@
186 spu sendfile sys_sendfile64
187 common getpmsg sys_ni_syscall
188 common putpmsg sys_ni_syscall
-189 32 vfork ppc_vfork sys_vfork
-189 64 vfork sys_vfork
-189 spu vfork sys_ni_syscall
+189 nospu vfork sys_vfork
190 common ugetrlimit sys_getrlimit compat_sys_getrlimit
191 common readahead sys_readahead compat_sys_readahead
192 32 mmap2 sys_mmap2 compat_sys_mmap2
@@ -322,9 +316,7 @@
248 32 clock_nanosleep sys_clock_nanosleep_time32
248 64 clock_nanosleep sys_clock_nanosleep
248 spu clock_nanosleep sys_clock_nanosleep
-249 32 swapcontext ppc_swapcontext compat_sys_swapcontext
-249 64 swapcontext sys_swapcontext
-249 spu swapcontext sys_ni_syscall
+249 nospu swapcontext sys_swapcontext compat_sys_swapcontext
250 common tgkill sys_tgkill
251 32 utimes sys_utimes_time32
251 64 utimes sys_utimes
@@ -522,9 +514,7 @@
432 common fsmount sys_fsmount
433 common fspick sys_fspick
434 common pidfd_open sys_pidfd_open
-435 32 clone3 ppc_clone3 sys_clone3
-435 64 clone3 sys_clone3
-435 spu clone3 sys_ni_syscall
+435 nospu clone3 sys_clone3
436 common close_range sys_close_range
437 common openat2 sys_openat2
438 common pidfd_getfd sys_pidfd_getfd
--
2.25.0
^ permalink raw reply related
* [PATCH v4 15/23] powerpc/syscall: Change condition to check MSR_RI
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
In system_call_exception(), MSR_RI also needs to be checked on 8xx.
Only booke and 40x doesn't have MSR_RI.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/syscall.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
index b66cfcbcb755..a3510fa4e641 100644
--- a/arch/powerpc/kernel/syscall.c
+++ b/arch/powerpc/kernel/syscall.c
@@ -39,7 +39,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
trace_hardirqs_off(); /* finish reconciling */
- if (IS_ENABLED(CONFIG_PPC_BOOK3S))
+ if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
BUG_ON(!(regs->msr & MSR_RI));
BUG_ON(!(regs->msr & MSR_PR));
BUG_ON(!FULL_REGS(regs));
--
2.25.0
^ permalink raw reply related
* [PATCH v4 14/23] powerpc/syscall: Save r3 in regs->orig_r3
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
Save r3 in regs->orig_r3 in system_call_exception()
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_64.S | 1 -
arch/powerpc/kernel/syscall.c | 2 ++
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index aa1af139d947..a562a4240aa6 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -278,7 +278,6 @@ END_BTB_FLUSH_SECTION
std r10,_LINK(r1)
std r11,_TRAP(r1)
std r12,_CCR(r1)
- std r3,ORIG_GPR3(r1)
addi r10,r1,STACK_FRAME_OVERHEAD
ld r11,exception_marker@toc(r2)
std r11,-16(r10) /* "regshere" marker */
diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
index cb415170b8f2..b66cfcbcb755 100644
--- a/arch/powerpc/kernel/syscall.c
+++ b/arch/powerpc/kernel/syscall.c
@@ -29,6 +29,8 @@ notrace long system_call_exception(long r3, long r4, long r5,
{
syscall_fn f;
+ regs->orig_gpr3 = r3;
+
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
--
2.25.0
^ permalink raw reply related
* [PATCH v4 13/23] powerpc/syscall: Use is_compat_task()
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
Instead of hard comparing task flags with _TIF_32BIT, use
is_compat_task(). The advantage is that it returns 0 on PPC32
allthough _TIF_32BIT is always set.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/syscall.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
index bf9bf4b5bc41..cb415170b8f2 100644
--- a/arch/powerpc/kernel/syscall.c
+++ b/arch/powerpc/kernel/syscall.c
@@ -2,6 +2,8 @@
#include <linux/context_tracking.h>
#include <linux/err.h>
+#include <linux/compat.h>
+
#include <asm/asm-prototypes.h>
#include <asm/kup.h>
#include <asm/cputime.h>
@@ -116,7 +118,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
/* May be faster to do array_index_nospec? */
barrier_nospec();
- if (unlikely(is_32bit_task())) {
+ if (unlikely(is_compat_task())) {
f = (void *)compat_sys_call_table[r0];
r3 &= 0x00000000ffffffffULL;
--
2.25.0
^ permalink raw reply related
* [PATCH v4 12/23] powerpc/syscall: Make syscall.c buildable on PPC32
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
ifdef out specific PPC64 stuff to allow building
syscall.c on PPC32.
Modify Makefile to always build syscall.o
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/Makefile | 4 ++--
arch/powerpc/kernel/syscall.c | 9 +++++----
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 1cbc51fc82fd..23c127db0d0c 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -46,10 +46,10 @@ obj-y := cputable.o syscalls.o \
prom.o traps.o setup-common.o \
udbg.o misc.o io.o misc_$(BITS).o \
of_platform.o prom_parse.o firmware.o \
- hw_breakpoint_constraints.o
+ hw_breakpoint_constraints.o syscall.o
obj-y += ptrace/
obj-$(CONFIG_PPC64) += setup_64.o \
- paca.o nvram_64.o note.o syscall.o
+ paca.o nvram_64.o note.o
obj-$(CONFIG_COMPAT) += sys_ppc32.o signal_32.o
obj-$(CONFIG_VDSO32) += vdso32/
obj-$(CONFIG_PPC_WATCHDOG) += watchdog.o
diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
index b627a6384029..bf9bf4b5bc41 100644
--- a/arch/powerpc/kernel/syscall.c
+++ b/arch/powerpc/kernel/syscall.c
@@ -39,7 +39,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
BUG_ON(!(regs->msr & MSR_RI));
BUG_ON(!(regs->msr & MSR_PR));
BUG_ON(!FULL_REGS(regs));
- BUG_ON(regs->softe != IRQS_ENABLED);
+ BUG_ON(arch_irq_disabled_regs(regs));
#ifdef CONFIG_PPC_PKEY
if (mmu_has_feature(MMU_FTR_PKEY)) {
@@ -77,7 +77,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
* frame, or if the unwinder was taught the first stack frame always
* returns to user with IRQS_ENABLED, this store could be avoided!
*/
- regs->softe = IRQS_ENABLED;
+ irq_soft_mask_regs_set_state(regs, IRQS_ENABLED);
local_irq_enable();
@@ -147,6 +147,7 @@ static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri)
__hard_EE_RI_disable();
else
__hard_irq_disable();
+#ifdef CONFIG_PPC64
if (unlikely(lazy_irq_pending_nocheck())) {
/* Took an interrupt, may have more exit work to do. */
if (clear_ri)
@@ -158,7 +159,7 @@ static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri)
}
local_paca->irq_happened = 0;
irq_soft_mask_set(IRQS_ENABLED);
-
+#endif
return true;
}
@@ -281,7 +282,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
return ret;
}
-#ifdef CONFIG_PPC_BOOK3S /* BOOK3E not yet using this */
+#ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E not yet using this */
notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)
{
#ifdef CONFIG_PPC_BOOK3E
--
2.25.0
^ permalink raw reply related
* [PATCH v4 11/23] powerpc/syscall: Rename syscall_64.c into syscall.c
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
syscall_64.c will be reused almost as is for PPC32.
Rename it syscall.c
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/Makefile | 2 +-
arch/powerpc/kernel/{syscall_64.c => syscall.c} | 0
2 files changed, 1 insertion(+), 1 deletion(-)
rename arch/powerpc/kernel/{syscall_64.c => syscall.c} (100%)
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index fe2ef598e2ea..1cbc51fc82fd 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -49,7 +49,7 @@ obj-y := cputable.o syscalls.o \
hw_breakpoint_constraints.o
obj-y += ptrace/
obj-$(CONFIG_PPC64) += setup_64.o \
- paca.o nvram_64.o note.o syscall_64.o
+ paca.o nvram_64.o note.o syscall.o
obj-$(CONFIG_COMPAT) += sys_ppc32.o signal_32.o
obj-$(CONFIG_VDSO32) += vdso32/
obj-$(CONFIG_PPC_WATCHDOG) += watchdog.o
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall.c
similarity index 100%
rename from arch/powerpc/kernel/syscall_64.c
rename to arch/powerpc/kernel/syscall.c
--
2.25.0
^ permalink raw reply related
* [PATCH v4 10/23] powerpc/irq: Add stub irq_soft_mask_return() for PPC32
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
To allow building syscall_64.c smoothly on PPC32, add stub version
of irq_soft_mask_return().
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/hw_irq.h | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index 4739f61e632c..56a98936a6a9 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -330,6 +330,11 @@ static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned l
}
#else /* CONFIG_PPC64 */
+static inline notrace unsigned long irq_soft_mask_return(void)
+{
+ return 0;
+}
+
static inline unsigned long arch_local_save_flags(void)
{
return mfmsr();
--
2.25.0
^ permalink raw reply related
* [PATCH v4 09/23] powerpc/irq: Rework helpers that manipulate MSR[EE/RI]
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
In preparation of porting PPC32 to C syscall entry/exit,
rewrite the following helpers as static inline functions and
add support for PPC32 in them:
__hard_irq_enable()
__hard_irq_disable()
__hard_EE_RI_disable()
__hard_RI_enable()
Then use them in PPC32 version of arch_local_irq_disable()
and arch_local_irq_enable() to avoid code duplication.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/hw_irq.h | 75 +++++++++++++++++++++----------
arch/powerpc/include/asm/reg.h | 1 +
2 files changed, 52 insertions(+), 24 deletions(-)
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index ed0c3b049dfd..4739f61e632c 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -50,6 +50,55 @@
#ifndef __ASSEMBLY__
+static inline void __hard_irq_enable(void)
+{
+ if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x))
+ wrtee(MSR_EE);
+ else if (IS_ENABLED(CONFIG_PPC_8xx))
+ wrtspr(SPRN_EIE);
+ else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
+ __mtmsrd(MSR_EE | MSR_RI, 1);
+ else
+ mtmsr(mfmsr() | MSR_EE);
+}
+
+static inline void __hard_irq_disable(void)
+{
+ if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x))
+ wrtee(0);
+ else if (IS_ENABLED(CONFIG_PPC_8xx))
+ wrtspr(SPRN_EID);
+ else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
+ __mtmsrd(MSR_RI, 1);
+ else
+ mtmsr(mfmsr() & ~MSR_EE);
+}
+
+static inline void __hard_EE_RI_disable(void)
+{
+ if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x))
+ wrtee(0);
+ else if (IS_ENABLED(CONFIG_PPC_8xx))
+ wrtspr(SPRN_NRI);
+ else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
+ __mtmsrd(0, 1);
+ else
+ mtmsr(mfmsr() & ~(MSR_EE | MSR_RI));
+}
+
+static inline void __hard_RI_enable(void)
+{
+ if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x))
+ return;
+
+ if (IS_ENABLED(CONFIG_PPC_8xx))
+ wrtspr(SPRN_EID);
+ else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
+ __mtmsrd(MSR_RI, 1);
+ else
+ mtmsr(mfmsr() | MSR_RI);
+}
+
#ifdef CONFIG_PPC64
#include <asm/paca.h>
@@ -212,18 +261,6 @@ static inline bool arch_irqs_disabled(void)
#endif /* CONFIG_PPC_BOOK3S */
-#ifdef CONFIG_PPC_BOOK3E
-#define __hard_irq_enable() wrtee(MSR_EE)
-#define __hard_irq_disable() wrtee(0)
-#define __hard_EE_RI_disable() wrtee(0)
-#define __hard_RI_enable() do { } while (0)
-#else
-#define __hard_irq_enable() __mtmsrd(MSR_EE|MSR_RI, 1)
-#define __hard_irq_disable() __mtmsrd(MSR_RI, 1)
-#define __hard_EE_RI_disable() __mtmsrd(0, 1)
-#define __hard_RI_enable() __mtmsrd(MSR_RI, 1)
-#endif
-
#define hard_irq_disable() do { \
unsigned long flags; \
__hard_irq_disable(); \
@@ -322,22 +359,12 @@ static inline unsigned long arch_local_irq_save(void)
static inline void arch_local_irq_disable(void)
{
- if (IS_ENABLED(CONFIG_BOOKE))
- wrtee(0);
- else if (IS_ENABLED(CONFIG_PPC_8xx))
- wrtspr(SPRN_EID);
- else
- mtmsr(mfmsr() & ~MSR_EE);
+ __hard_irq_disable();
}
static inline void arch_local_irq_enable(void)
{
- if (IS_ENABLED(CONFIG_BOOKE))
- wrtee(MSR_EE);
- else if (IS_ENABLED(CONFIG_PPC_8xx))
- wrtspr(SPRN_EIE);
- else
- mtmsr(mfmsr() | MSR_EE);
+ __hard_irq_enable();
}
static inline bool arch_irqs_disabled_flags(unsigned long flags)
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index e40a921d78f9..d05dca30604d 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -1375,6 +1375,7 @@
#define mtmsr(v) asm volatile("mtmsr %0" : \
: "r" ((unsigned long)(v)) \
: "memory")
+#define __mtmsrd(v, l) BUILD_BUG()
#define __MTMSR "mtmsr"
#endif
--
2.25.0
^ permalink raw reply related
* [PATCH v4 05/23] powerpc/64s: Make kuap_check_amr() and kuap_get_and_check_amr() generic
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
In preparation of porting powerpc32 to C syscall entry/exit,
rename kuap_check_amr() and kuap_get_and_check_amr() as kuap_check()
and kuap_get_and_check(), and move in the generic asm/kup.h the stub
for when CONFIG_PPC_KUAP is not selected.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/64/kup.h | 24 ++----------------------
arch/powerpc/include/asm/kup.h | 9 ++++++++-
arch/powerpc/kernel/syscall_64.c | 12 ++++++------
3 files changed, 16 insertions(+), 29 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h
index f50f72e535aa..1507681ad4ef 100644
--- a/arch/powerpc/include/asm/book3s/64/kup.h
+++ b/arch/powerpc/include/asm/book3s/64/kup.h
@@ -281,7 +281,7 @@ static inline void kuap_kernel_restore(struct pt_regs *regs,
*/
}
-static inline unsigned long kuap_get_and_check_amr(void)
+static inline unsigned long kuap_get_and_check(void)
{
if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
unsigned long amr = mfspr(SPRN_AMR);
@@ -292,27 +292,7 @@ static inline unsigned long kuap_get_and_check_amr(void)
return 0;
}
-#else /* CONFIG_PPC_PKEY */
-
-static inline void kuap_user_restore(struct pt_regs *regs)
-{
-}
-
-static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr)
-{
-}
-
-static inline unsigned long kuap_get_and_check_amr(void)
-{
- return 0;
-}
-
-#endif /* CONFIG_PPC_PKEY */
-
-
-#ifdef CONFIG_PPC_KUAP
-
-static inline void kuap_check_amr(void)
+static inline void kuap_check(void)
{
if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index bf221a2a523e..6ef9f9cfbed0 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -66,7 +66,14 @@ bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
return false;
}
-static inline void kuap_check_amr(void) { }
+static inline void kuap_check(void) { }
+static inline void kuap_user_restore(struct pt_regs *regs) { }
+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { }
+
+static inline unsigned long kuap_get_and_check(void)
+{
+ return 0;
+}
/*
* book3s/64/kup-radix.h defines these functions for the !KUAP case to flush
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index 32f72965da26..b627a6384029 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -65,7 +65,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
isync();
} else
#endif
- kuap_check_amr();
+ kuap_check();
account_cpu_user_entry();
@@ -181,7 +181,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
CT_WARN_ON(ct_state() == CONTEXT_USER);
- kuap_check_amr();
+ kuap_check();
regs->result = r3;
@@ -303,7 +303,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
* We don't need to restore AMR on the way back to userspace for KUAP.
* AMR can only have been unlocked if we interrupted the kernel.
*/
- kuap_check_amr();
+ kuap_check();
local_irq_save(flags);
@@ -381,7 +381,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
unsigned long *ti_flagsp = ¤t_thread_info()->flags;
unsigned long flags;
unsigned long ret = 0;
- unsigned long amr;
+ unsigned long kuap;
if (IS_ENABLED(CONFIG_PPC_BOOK3S) && unlikely(!(regs->msr & MSR_RI)))
unrecoverable_exception(regs);
@@ -394,7 +394,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
if (TRAP(regs) != 0x700)
CT_WARN_ON(ct_state() == CONTEXT_USER);
- amr = kuap_get_and_check_amr();
+ kuap = kuap_get_and_check();
if (unlikely(*ti_flagsp & _TIF_EMULATE_STACK_STORE)) {
clear_bits(_TIF_EMULATE_STACK_STORE, ti_flagsp);
@@ -446,7 +446,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
* which would cause Read-After-Write stalls. Hence, we take the AMR
* value from the check above.
*/
- kuap_kernel_restore(regs, amr);
+ kuap_kernel_restore(regs, kuap);
return ret;
}
--
2.25.0
^ permalink raw reply related
* [PATCH v4 06/23] powerpc/32s: Create C version of kuap_user/kernel_restore() and friends
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
In preparation of porting PPC32 to C syscall entry/exit,
create C version of kuap_user_restore() and kuap_kernel_restore()
and kuap_check() and kuap_get_and_check() on book3s/32.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 33 ++++++++++++++++++++++++
1 file changed, 33 insertions(+)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index a0117a9d5b06..a3e72e1141c5 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -103,6 +103,39 @@ static inline void kuap_update_sr(u32 sr, u32 addr, u32 end)
isync(); /* Context sync required after mtsrin() */
}
+static inline void kuap_user_restore(struct pt_regs *regs)
+{
+}
+
+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
+{
+ u32 addr = kuap & 0xf0000000;
+ u32 end = kuap << 28;
+
+ if (unlikely(!kuap))
+ return;
+
+ current->thread.kuap = 0;
+ kuap_update_sr(mfsrin(addr) & ~SR_KS, addr, end); /* Clear Ks */
+}
+
+static inline void kuap_check(void)
+{
+ if (!IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
+ return;
+
+ WARN_ON_ONCE(current->thread.kuap != 0);
+}
+
+static inline unsigned long kuap_get_and_check(void)
+{
+ unsigned long kuap = current->thread.kuap;
+
+ WARN_ON_ONCE(IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && kuap != 0);
+
+ return kuap;
+}
+
static __always_inline void allow_user_access(void __user *to, const void __user *from,
u32 size, unsigned long dir)
{
--
2.25.0
^ permalink raw reply related
* [PATCH v4 08/23] powerpc/irq: Add helper to set regs->softe
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
regs->softe doesn't exist on PPC32.
Add irq_soft_mask_regs_set_state() helper to set regs->softe.
This helper will void on PPC32.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/hw_irq.h | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index 614957f74cee..ed0c3b049dfd 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -38,6 +38,8 @@
#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE)
#endif
+#endif /* CONFIG_PPC64 */
+
/*
* flags for paca->irq_soft_mask
*/
@@ -46,8 +48,6 @@
#define IRQS_PMI_DISABLED 2
#define IRQS_ALL_DISABLED (IRQS_DISABLED | IRQS_PMI_DISABLED)
-#endif /* CONFIG_PPC64 */
-
#ifndef __ASSEMBLY__
#ifdef CONFIG_PPC64
@@ -287,6 +287,10 @@ extern void irq_set_pending_from_srr1(unsigned long srr1);
extern void force_external_irq_replay(void);
+static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
+{
+ regs->softe = val;
+}
#else /* CONFIG_PPC64 */
static inline unsigned long arch_local_save_flags(void)
@@ -355,6 +359,9 @@ static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
static inline void may_hard_irq_enable(void) { }
+static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
+{
+}
#endif /* CONFIG_PPC64 */
#define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST
--
2.25.0
^ permalink raw reply related
* [PATCH v4 07/23] powerpc/8xx: Create C version of kuap_user/kernel_restore() and friends
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
In preparation of porting PPC32 to C syscall entry/exit,
create C version of kuap_user_restore() and kuap_kernel_restore()
and kuap_check() and kuap_get_and_check() on 8xx
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 27 ++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
index 17a4a616436f..5ca6c375f767 100644
--- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
@@ -34,6 +34,33 @@
#include <asm/reg.h>
+static inline void kuap_user_restore(struct pt_regs *regs)
+{
+}
+
+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
+{
+ mtspr(SPRN_MD_AP, kuap);
+}
+
+static inline void kuap_check(void)
+{
+ if (!IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
+ return;
+
+ WARN_ON_ONCE(mfspr(SPRN_MD_AP) >> 16 != MD_APG_KUAP >> 16);
+}
+
+static inline unsigned long kuap_get_and_check(void)
+{
+ unsigned long kuap = mfspr(SPRN_MD_AP);
+
+ if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
+ WARN_ON_ONCE(mfspr(SPRN_MD_AP) >> 16 != MD_APG_KUAP >> 16);
+
+ return kuap;
+}
+
static inline void allow_user_access(void __user *to, const void __user *from,
unsigned long size, unsigned long dir)
{
--
2.25.0
^ permalink raw reply related
* [PATCH v4 02/23] powerpc/32: Always enable data translation on syscall entry
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
If the code can use a stack in vm area, it can also use a
stack in linear space.
Simplify code by removing old non VMAP stack code on PPC32 in syscall.
That means the data translation is now re-enabled early in
syscall entry in all cases, not only when using VMAP stacks.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/head_32.h | 23 +----------------------
arch/powerpc/kernel/head_booke.h | 2 --
2 files changed, 1 insertion(+), 24 deletions(-)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index a2f72c966baf..fdc07beab844 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -118,7 +118,6 @@
.macro SYSCALL_ENTRY trapno
mfspr r12,SPRN_SPRG_THREAD
mfspr r9, SPRN_SRR1
-#ifdef CONFIG_VMAP_STACK
mfspr r11, SPRN_SRR0
mtctr r11
andi. r11, r9, MSR_PR
@@ -126,30 +125,16 @@
lwz r1,TASK_STACK-THREAD(r12)
beq- 99f
addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
- li r10, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */
+ LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~(MSR_IR | MSR_RI)) /* can take DTLB miss */
mtmsr r10
isync
tovirt(r12, r12)
stw r11,GPR1(r1)
stw r11,0(r1)
mr r11, r1
-#else
- andi. r11, r9, MSR_PR
- lwz r11,TASK_STACK-THREAD(r12)
- beq- 99f
- addi r11, r11, THREAD_SIZE - INT_FRAME_SIZE
- tophys(r11, r11)
- stw r1,GPR1(r11)
- stw r1,0(r11)
- tovirt(r1, r11) /* set new kernel sp */
-#endif
mflr r10
stw r10, _LINK(r11)
-#ifdef CONFIG_VMAP_STACK
mfctr r10
-#else
- mfspr r10,SPRN_SRR0
-#endif
stw r10,_NIP(r11)
mfcr r10
rlwinm r10,r10,0,4,2 /* Clear SO bit in CR */
@@ -157,11 +142,7 @@
#ifdef CONFIG_40x
rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */
#else
-#ifdef CONFIG_VMAP_STACK
LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~MSR_IR) /* can take exceptions */
-#else
- LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~(MSR_IR|MSR_DR)) /* can take exceptions */
-#endif
mtmsr r10 /* (except for mach check in rtas) */
#endif
lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
@@ -190,7 +171,6 @@
li r12,-1 /* clear all pending debug events */
mtspr SPRN_DBSR,r12
lis r11,global_dbcr0@ha
- tophys(r11,r11)
addi r11,r11,global_dbcr0@l
lwz r12,0(r11)
mtspr SPRN_DBCR0,r12
@@ -200,7 +180,6 @@
#endif
3:
- tovirt_novmstack r2, r2 /* set r2 to current */
lis r11, transfer_to_syscall@h
ori r11, r11, transfer_to_syscall@l
#ifdef CONFIG_TRACE_IRQFLAGS
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index bf33af714d11..706cd9368992 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -144,7 +144,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
li r12,-1 /* clear all pending debug events */
mtspr SPRN_DBSR,r12
lis r11,global_dbcr0@ha
- tophys(r11,r11)
addi r11,r11,global_dbcr0@l
#ifdef CONFIG_SMP
lwz r10, TASK_CPU(r2)
@@ -158,7 +157,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r12,4(r11)
3:
- tovirt(r2, r2) /* set r2 to current */
lis r11, transfer_to_syscall@h
ori r11, r11, transfer_to_syscall@l
#ifdef CONFIG_TRACE_IRQFLAGS
--
2.25.0
^ permalink raw reply related
* [PATCH v4 00/23] powerpc/32: Implement C syscall entry/exit
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
This series implements C syscall entry/exit for PPC32. It reuses
the work already done for PPC64.
This series is based on Nick's v6 series "powerpc: interrupt wrappers".
Patch 1 is a bug fix submitted separately but this series depends on it.
Patches 2-4 are an extract from the series "powerpc/32: Reduce head
complexity and re-activate MMU earlier". The changes here are limited
to system calls. That series will be respined to only contain exception
related changes and the syscall changes will remain in this series.
Patches 5-16 are preparatory changes.
Patch 17 is THE patch that changes to C syscall entry/exit
Patches 18-23 are optimisations.
In terms on performance we have the following number of cycles on an
8xx running null_syscall benchmark:
- mainline: 296 cycles
- after patch 4: 283 cycles
- after patch 16: 304 cycles
- after patch 17: 348 cycles
- at the end of the series: 320 cycles
So in summary, we have a degradation of performance of 8% on null_syscall.
I think it is not a big degradation, it is worth it.
v4 is the first mature version.
Christophe Leroy (23):
powerpc/32s: Add missing call to kuep_lock on syscall entry
powerpc/32: Always enable data translation on syscall entry
powerpc/32: On syscall entry, enable instruction translation at the
same time as data
powerpc/32: Reorder instructions to avoid using CTR in syscall entry
powerpc/64s: Make kuap_check_amr() and kuap_get_and_check_amr()
generic
powerpc/32s: Create C version of kuap_user/kernel_restore() and
friends
powerpc/8xx: Create C version of kuap_user/kernel_restore() and
friends
powerpc/irq: Add helper to set regs->softe
powerpc/irq: Rework helpers that manipulate MSR[EE/RI]
powerpc/irq: Add stub irq_soft_mask_return() for PPC32
powerpc/syscall: Rename syscall_64.c into syscall.c
powerpc/syscall: Make syscall.c buildable on PPC32
powerpc/syscall: Use is_compat_task()
powerpc/syscall: Save r3 in regs->orig_r3
powerpc/syscall: Change condition to check MSR_RI
powerpc/32: Always save non volatile GPRs at syscall entry
powerpc/syscall: implement system call entry/exit logic in C for PPC32
powerpc/32: Remove verification of MSR_PR on syscall in the ASM entry
powerpc/syscall: Avoid stack frame in likely part of
system_call_exception()
powerpc/syscall: Do not check unsupported scv vector on PPC32
powerpc/syscall: Remove FULL_REGS verification in
system_call_exception
powerpc/syscall: Optimise checks in beginning of
system_call_exception()
powerpc/syscall: Avoid storing 'current' in another pointer
arch/powerpc/include/asm/book3s/32/kup.h | 33 ++
arch/powerpc/include/asm/book3s/64/kup.h | 24 +-
arch/powerpc/include/asm/hw_irq.h | 91 ++++--
arch/powerpc/include/asm/kup.h | 9 +-
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 27 ++
arch/powerpc/include/asm/reg.h | 1 +
arch/powerpc/kernel/Makefile | 4 +-
arch/powerpc/kernel/entry_32.S | 305 ++----------------
arch/powerpc/kernel/entry_64.S | 1 -
arch/powerpc/kernel/head_32.h | 76 +----
arch/powerpc/kernel/head_booke.h | 27 +-
.../kernel/{syscall_64.c => syscall.c} | 57 ++--
arch/powerpc/kernel/syscalls/syscall.tbl | 20 +-
13 files changed, 225 insertions(+), 450 deletions(-)
rename arch/powerpc/kernel/{syscall_64.c => syscall.c} (90%)
--
2.25.0
^ permalink raw reply
* [PATCH v4 03/23] powerpc/32: On syscall entry, enable instruction translation at the same time as data
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
On 40x and 8xx, kernel text is pinned.
On book3s/32, kernel text is mapped by BATs.
Enable instruction translation at the same time as data translation, it
makes things simpler.
MSR_RI can also be set at the same time because srr0/srr1 are already
saved and r1 is set properly.
On booke, translation is always on, so at the end all PPC32
have translation on early.
This reduces null_syscall benchmark by 13 cycles on 8xx
(296 ==> 283 cycles).
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/head_32.h | 26 +++++++++-----------------
arch/powerpc/kernel/head_booke.h | 7 ++-----
2 files changed, 11 insertions(+), 22 deletions(-)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index fdc07beab844..4029c51dce5d 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -125,9 +125,13 @@
lwz r1,TASK_STACK-THREAD(r12)
beq- 99f
addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
- LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~(MSR_IR | MSR_RI)) /* can take DTLB miss */
- mtmsr r10
- isync
+ LOAD_REG_IMMEDIATE(r10, MSR_KERNEL) /* can take exceptions */
+ mtspr SPRN_SRR1, r10
+ lis r10, 1f@h
+ ori r10, r10, 1f@l
+ mtspr SPRN_SRR0, r10
+ rfi
+1:
tovirt(r12, r12)
stw r11,GPR1(r1)
stw r11,0(r1)
@@ -141,9 +145,6 @@
stw r10,_CCR(r11) /* save registers */
#ifdef CONFIG_40x
rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */
-#else
- LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~MSR_IR) /* can take exceptions */
- mtmsr r10 /* (except for mach check in rtas) */
#endif
lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
stw r2,GPR2(r11)
@@ -180,8 +181,6 @@
#endif
3:
- lis r11, transfer_to_syscall@h
- ori r11, r11, transfer_to_syscall@l
#ifdef CONFIG_TRACE_IRQFLAGS
/*
* If MSR is changing we need to keep interrupts disabled at this point
@@ -193,15 +192,8 @@
#else
LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE)
#endif
-#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
- mtspr SPRN_NRI, r0
-#endif
- mtspr SPRN_SRR1,r10
- mtspr SPRN_SRR0,r11
- rfi /* jump to handler, enable MMU */
-#ifdef CONFIG_40x
- b . /* Prevent prefetch past rfi */
-#endif
+ mtmsr r10
+ b transfer_to_syscall /* jump to handler */
99: b ret_from_kernel_syscall
.endm
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index 706cd9368992..b3c502c503a0 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -157,8 +157,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
stw r12,4(r11)
3:
- lis r11, transfer_to_syscall@h
- ori r11, r11, transfer_to_syscall@l
#ifdef CONFIG_TRACE_IRQFLAGS
/*
* If MSR is changing we need to keep interrupts disabled at this point
@@ -172,9 +170,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)
lis r10, (MSR_KERNEL | MSR_EE)@h
ori r10, r10, (MSR_KERNEL | MSR_EE)@l
#endif
- mtspr SPRN_SRR1,r10
- mtspr SPRN_SRR0,r11
- rfi /* jump to handler, enable MMU */
+ mtmsr r10
+ b transfer_to_syscall /* jump to handler */
99: b ret_from_kernel_syscall
.endm
--
2.25.0
^ permalink raw reply related
* [PATCH v4 01/23] powerpc/32s: Add missing call to kuep_lock on syscall entry
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
Userspace Execution protection and fast syscall entry were implemented
independently from each other and were both merged in kernel 5.2,
leading to syscall entry missing userspace execution protection.
On syscall entry, execution of user space memory must be
locked in the same way as on exception entry.
Fixes: b86fb88855ea ("powerpc/32: implement fast entry for syscalls on non BOOKE")
Cc: stable@vger.kernel.org
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/entry_32.S | 3 +++
1 file changed, 3 insertions(+)
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index b102b40c4988..b1e36602c013 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -351,6 +351,9 @@ trace_syscall_entry_irq_off:
.globl transfer_to_syscall
transfer_to_syscall:
+#ifdef CONFIG_PPC_BOOK3S_32
+ kuep_lock r11, r12
+#endif
#ifdef CONFIG_TRACE_IRQFLAGS
andi. r12,r9,MSR_EE
beq- trace_syscall_entry_irq_off
--
2.25.0
^ permalink raw reply related
* [PATCH v4 04/23] powerpc/32: Reorder instructions to avoid using CTR in syscall entry
From: Christophe Leroy @ 2021-01-25 14:48 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin,
msuchanek
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <cover.1611585031.git.christophe.leroy@csgroup.eu>
Now that we are using rfi instead of mtmsr to reactivate MMU, it is
possible to reorder instructions and avoid the need to use CTR for
stashing SRR0.
null_syscall on 8xx is reduced by 3 cycles (283 => 280 cycles).
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/kernel/head_32.h | 22 ++++++++++------------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index 4029c51dce5d..24dc326e0d56 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -116,30 +116,28 @@
.endm
.macro SYSCALL_ENTRY trapno
- mfspr r12,SPRN_SPRG_THREAD
mfspr r9, SPRN_SRR1
- mfspr r11, SPRN_SRR0
- mtctr r11
+ mfspr r10, SPRN_SRR0
andi. r11, r9, MSR_PR
+ beq- 99f
+ LOAD_REG_IMMEDIATE(r11, MSR_KERNEL) /* can take exceptions */
+ lis r12, 1f@h
+ ori r12, r12, 1f@l
+ mtspr SPRN_SRR1, r11
+ mtspr SPRN_SRR0, r12
+ mfspr r12,SPRN_SPRG_THREAD
mr r11, r1
lwz r1,TASK_STACK-THREAD(r12)
- beq- 99f
+ tovirt(r12, r12)
addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
- LOAD_REG_IMMEDIATE(r10, MSR_KERNEL) /* can take exceptions */
- mtspr SPRN_SRR1, r10
- lis r10, 1f@h
- ori r10, r10, 1f@l
- mtspr SPRN_SRR0, r10
rfi
1:
- tovirt(r12, r12)
stw r11,GPR1(r1)
stw r11,0(r1)
mr r11, r1
+ stw r10,_NIP(r11)
mflr r10
stw r10, _LINK(r11)
- mfctr r10
- stw r10,_NIP(r11)
mfcr r10
rlwinm r10,r10,0,4,2 /* Clear SO bit in CR */
stw r10,_CCR(r11) /* save registers */
--
2.25.0
^ permalink raw reply related
* [PATCH v2 2/2] powerpc/sstep: Fix incorrect return from analyze_instr()
From: Ananth N Mavinakayanahalli @ 2021-01-25 13:06 UTC (permalink / raw)
To: linuxppc-dev; +Cc: ravi.bangoria, paulus, naveen.n.rao, msuchanek, dja
In-Reply-To: <161157995977.64773.13794501093457185080.stgit@thinktux.local>
We currently just percolate the return value from analyze_instr()
to the caller of emulate_step(), especially if it is a -1.
For one particular case (opcode = 4) for instructions that aren't
currently emulated, we are returning 'should not be single-stepped'
while we should have returned 0 which says 'did not emulate, may
have to single-step'.
Fixes: 930d6288a26787 ("powerpc: sstep: Add support for maddhd, maddhdu, maddld instructions")
Signed-off-by: Ananth N Mavinakayanahalli <ananth@linux.ibm.com>
Suggested-by: Michael Ellerman <mpe@ellerman.id.au>
Tested-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
Reviewed-by: Sandipan Das <sandipan@linux.ibm.com>
---
arch/powerpc/lib/sstep.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index f859cbbb6375..e96cff845ef7 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -1445,6 +1445,11 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
#ifdef __powerpc64__
case 4:
+ /*
+ * There are very many instructions with this primary opcode
+ * introduced in the ISA as early as v2.03. However, the ones
+ * we currently emulate were all introduced with ISA 3.0
+ */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
goto unknown_opcode;
@@ -1472,7 +1477,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
* There are other instructions from ISA 3.0 with the same
* primary opcode which do not have emulation support yet.
*/
- return -1;
+ goto unknown_opcode;
#endif
case 7: /* mulli */
^ permalink raw reply related
* [PATCH v4 1/2] [PATCH] powerpc/sstep: Check instruction validity against ISA version before emulation
From: Ananth N Mavinakayanahalli @ 2021-01-25 13:06 UTC (permalink / raw)
To: linuxppc-dev; +Cc: ravi.bangoria, paulus, naveen.n.rao, msuchanek, dja
We currently unconditionally try to emulate newer instructions on older
Power versions that could cause issues. Gate it.
Fixes: 350779a29f11 ("powerpc: Handle most loads and stores in instruction emulation code")
Signed-off-by: Ananth N Mavinakayanahalli <ananth@linux.ibm.com>
---
[v4] Based on feedback from Paul Mackerras, Naveen Rao and Michael Ellerman,
changed return code to 0, after setting opcode type to UNKNOWN
[v3] Addressed Naveen's comments on scv and addpcis
[v2] Fixed description
---
arch/powerpc/lib/sstep.c | 78 +++++++++++++++++++++++++++++++++++++---------
1 file changed, 62 insertions(+), 16 deletions(-)
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index bf7a7d62ae8b..f859cbbb6375 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -1304,9 +1304,11 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
if ((word & 0xfe2) == 2)
op->type = SYSCALL;
else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
- (word & 0xfe3) == 1)
+ (word & 0xfe3) == 1) { /* scv */
op->type = SYSCALL_VECTORED_0;
- else
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
+ } else
op->type = UNKNOWN;
return 0;
#endif
@@ -1410,7 +1412,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
#ifdef __powerpc64__
case 1:
if (!cpu_has_feature(CPU_FTR_ARCH_31))
- return -1;
+ goto unknown_opcode;
prefix_r = GET_PREFIX_R(word);
ra = GET_PREFIX_RA(suffix);
@@ -1444,7 +1446,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
#ifdef __powerpc64__
case 4:
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
switch (word & 0x3f) {
case 48: /* maddhd */
@@ -1530,6 +1532,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
case 19:
if (((word >> 1) & 0x1f) == 2) {
/* addpcis */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
imm = (short) (word & 0xffc1); /* d0 + d2 fields */
imm |= (word >> 15) & 0x3e; /* d1 field */
op->val = regs->nip + (imm << 16) + 4;
@@ -1842,7 +1846,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
#ifdef __powerpc64__
case 265: /* modud */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
op->val = regs->gpr[ra] % regs->gpr[rb];
goto compute_done;
#endif
@@ -1852,7 +1856,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
case 267: /* moduw */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
op->val = (unsigned int) regs->gpr[ra] %
(unsigned int) regs->gpr[rb];
goto compute_done;
@@ -1889,7 +1893,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
#endif
case 755: /* darn */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
switch (ra & 0x3) {
case 0:
/* 32-bit conditioned */
@@ -1911,14 +1915,14 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
#ifdef __powerpc64__
case 777: /* modsd */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
op->val = (long int) regs->gpr[ra] %
(long int) regs->gpr[rb];
goto compute_done;
#endif
case 779: /* modsw */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
op->val = (int) regs->gpr[ra] %
(int) regs->gpr[rb];
goto compute_done;
@@ -1995,14 +1999,14 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
#endif
case 538: /* cnttzw */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
val = (unsigned int) regs->gpr[rd];
op->val = (val ? __builtin_ctz(val) : 32);
goto logical_done;
#ifdef __powerpc64__
case 570: /* cnttzd */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
val = regs->gpr[rd];
op->val = (val ? __builtin_ctzl(val) : 64);
goto logical_done;
@@ -2112,7 +2116,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
case 890: /* extswsli with sh_5 = 0 */
case 891: /* extswsli with sh_5 = 1 */
if (!cpu_has_feature(CPU_FTR_ARCH_300))
- return -1;
+ goto unknown_opcode;
op->type = COMPUTE + SETREG;
sh = rb | ((word & 2) << 4);
val = (signed int) regs->gpr[rd];
@@ -2439,6 +2443,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 268: /* lxvx */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(LOAD_VSX, 0, 16);
op->element_size = 16;
@@ -2448,6 +2454,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
case 269: /* lxvl */
case 301: { /* lxvll */
int nb;
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->ea = ra ? regs->gpr[ra] : 0;
nb = regs->gpr[rb] & 0xff;
@@ -2468,13 +2476,15 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
case 333: /* lxvpx */
if (!cpu_has_feature(CPU_FTR_ARCH_31))
- return -1;
+ goto unknown_opcode;
op->reg = VSX_REGISTER_XTP(rd);
op->type = MKOP(LOAD_VSX, 0, 32);
op->element_size = 32;
break;
case 364: /* lxvwsx */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(LOAD_VSX, 0, 4);
op->element_size = 4;
@@ -2482,6 +2492,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 396: /* stxvx */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(STORE_VSX, 0, 16);
op->element_size = 16;
@@ -2491,6 +2503,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
case 397: /* stxvl */
case 429: { /* stxvll */
int nb;
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->ea = ra ? regs->gpr[ra] : 0;
nb = regs->gpr[rb] & 0xff;
@@ -2504,7 +2518,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
}
case 461: /* stxvpx */
if (!cpu_has_feature(CPU_FTR_ARCH_31))
- return -1;
+ goto unknown_opcode;
op->reg = VSX_REGISTER_XTP(rd);
op->type = MKOP(STORE_VSX, 0, 32);
op->element_size = 32;
@@ -2542,6 +2556,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 781: /* lxsibzx */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(LOAD_VSX, 0, 1);
op->element_size = 8;
@@ -2549,6 +2565,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 812: /* lxvh8x */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(LOAD_VSX, 0, 16);
op->element_size = 2;
@@ -2556,6 +2574,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 813: /* lxsihzx */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(LOAD_VSX, 0, 2);
op->element_size = 8;
@@ -2569,6 +2589,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 876: /* lxvb16x */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(LOAD_VSX, 0, 16);
op->element_size = 1;
@@ -2582,6 +2604,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 909: /* stxsibx */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(STORE_VSX, 0, 1);
op->element_size = 8;
@@ -2589,6 +2613,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 940: /* stxvh8x */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(STORE_VSX, 0, 16);
op->element_size = 2;
@@ -2596,6 +2622,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 941: /* stxsihx */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(STORE_VSX, 0, 2);
op->element_size = 8;
@@ -2609,6 +2637,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 1004: /* stxvb16x */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd | ((word & 1) << 5);
op->type = MKOP(STORE_VSX, 0, 16);
op->element_size = 1;
@@ -2717,12 +2747,16 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
op->type = MKOP(LOAD_FP, 0, 16);
break;
case 2: /* lxsd */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd + 32;
op->type = MKOP(LOAD_VSX, 0, 8);
op->element_size = 8;
op->vsx_flags = VSX_CHECK_VEC;
break;
case 3: /* lxssp */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->reg = rd + 32;
op->type = MKOP(LOAD_VSX, 0, 4);
op->element_size = 8;
@@ -2752,7 +2786,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
#ifdef CONFIG_VSX
case 6:
if (!cpu_has_feature(CPU_FTR_ARCH_31))
- return -1;
+ goto unknown_opcode;
op->ea = dqform_ea(word, regs);
op->reg = VSX_REGISTER_XTP(rd);
op->element_size = 32;
@@ -2775,6 +2809,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 1: /* lxv */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->ea = dqform_ea(word, regs);
if (word & 8)
op->reg = rd + 32;
@@ -2785,6 +2821,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
case 2: /* stxsd with LSB of DS field = 0 */
case 6: /* stxsd with LSB of DS field = 1 */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->ea = dsform_ea(word, regs);
op->reg = rd + 32;
op->type = MKOP(STORE_VSX, 0, 8);
@@ -2794,6 +2832,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
case 3: /* stxssp with LSB of DS field = 0 */
case 7: /* stxssp with LSB of DS field = 1 */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->ea = dsform_ea(word, regs);
op->reg = rd + 32;
op->type = MKOP(STORE_VSX, 0, 4);
@@ -2802,6 +2842,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 5: /* stxv */
+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
+ goto unknown_opcode;
op->ea = dqform_ea(word, regs);
if (word & 8)
op->reg = rd + 32;
@@ -2831,7 +2873,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
break;
case 1: /* Prefixed instructions */
if (!cpu_has_feature(CPU_FTR_ARCH_31))
- return -1;
+ goto unknown_opcode;
prefix_r = GET_PREFIX_R(word);
ra = GET_PREFIX_RA(suffix);
@@ -2980,6 +3022,10 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
return 0;
+ unknown_opcode:
+ op->type = UNKNOWN;
+ return 0;
+
logical_done:
if (word & 1)
set_cr0(regs, op);
^ permalink raw reply related
* RE: [PATCH v10 11/12] mm/vmalloc: Hugepage vmalloc mappings
From: David Laight @ 2021-01-25 12:24 UTC (permalink / raw)
To: 'Christophe Leroy', Nicholas Piggin, linux-mm@kvack.org,
Andrew Morton
Cc: linux-arch@vger.kernel.org, Ding Tianhong,
linux-kernel@vger.kernel.org, Christoph Hellwig, Zefan Li,
Jonathan Cameron, Rick Edgecombe, linuxppc-dev@lists.ozlabs.org
In-Reply-To: <933352bd-dcf3-c483-4d7a-07afe1116cf1@csgroup.eu>
From: Christophe Leroy
> Sent: 25 January 2021 09:15
>
> Le 24/01/2021 à 09:22, Nicholas Piggin a écrit :
> > Support huge page vmalloc mappings. Config option HAVE_ARCH_HUGE_VMALLOC
> > enables support on architectures that define HAVE_ARCH_HUGE_VMAP and
> > supports PMD sized vmap mappings.
> >
> > vmalloc will attempt to allocate PMD-sized pages if allocating PMD size
> > or larger, and fall back to small pages if that was unsuccessful.
> >
> > Architectures must ensure that any arch specific vmalloc allocations
> > that require PAGE_SIZE mappings (e.g., module allocations vs strict
> > module rwx) use the VM_NOHUGE flag to inhibit larger mappings.
> >
> > When hugepage vmalloc mappings are enabled in the next patch, this
> > reduces TLB misses by nearly 30x on a `git diff` workload on a 2-node
> > POWER9 (59,800 -> 2,100) and reduces CPU cycles by 0.54%.
> >
> > This can result in more internal fragmentation and memory overhead for a
> > given allocation, an option nohugevmalloc is added to disable at boot.
> >
> > Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> > ---
> > arch/Kconfig | 10 +++
> > include/linux/vmalloc.h | 18 ++++
> > mm/page_alloc.c | 5 +-
> > mm/vmalloc.c | 192 ++++++++++++++++++++++++++++++----------
> > 4 files changed, 177 insertions(+), 48 deletions(-)
> >
>
> > diff --git a/mm/vmalloc.c b/mm/vmalloc.c
> > index 0377e1d059e5..eef61e0f5170 100644
> > --- a/mm/vmalloc.c
> > +++ b/mm/vmalloc.c
>
> > @@ -2691,15 +2746,18 @@ EXPORT_SYMBOL_GPL(vmap_pfn);
> > #endif /* CONFIG_VMAP_PFN */
> >
> > static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> > - pgprot_t prot, int node)
> > + pgprot_t prot, unsigned int page_shift,
> > + int node)
> > {
> > const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
> > - unsigned int nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
> > - unsigned long array_size;
> > - unsigned int i;
> > + unsigned int page_order = page_shift - PAGE_SHIFT;
> > + unsigned long addr = (unsigned long)area->addr;
> > + unsigned long size = get_vm_area_size(area);
> > + unsigned int nr_small_pages = size >> PAGE_SHIFT;
> > struct page **pages;
> > + unsigned int i;
> >
> > - array_size = (unsigned long)nr_pages * sizeof(struct page *);
> > + array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
>
> array_size() is a function in include/linux/overflow.h
>
> For some reason, it breaks the build with your series.
I can't see the replacement definition for array_size.
The old local variable is deleted.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK
Registration No: 1397386 (Wales)
^ permalink raw reply
* Re: [PATCH v10 11/12] mm/vmalloc: Hugepage vmalloc mappings
From: Christophe Leroy @ 2021-01-25 12:13 UTC (permalink / raw)
To: Nicholas Piggin, Andrew Morton, linux-mm
Cc: linux-arch, Ding Tianhong, linux-kernel, Christoph Hellwig,
Zefan Li, Jonathan Cameron, Rick Edgecombe, linuxppc-dev
In-Reply-To: <1611574637.k9njsi2um5.astroid@bobo.none>
Le 25/01/2021 à 12:37, Nicholas Piggin a écrit :
> Excerpts from Christophe Leroy's message of January 25, 2021 7:14 pm:
>>
>>
>> Le 24/01/2021 à 09:22, Nicholas Piggin a écrit :
>>> Support huge page vmalloc mappings. Config option HAVE_ARCH_HUGE_VMALLOC
>>> enables support on architectures that define HAVE_ARCH_HUGE_VMAP and
>>> supports PMD sized vmap mappings.
>>>
>>> vmalloc will attempt to allocate PMD-sized pages if allocating PMD size
>>> or larger, and fall back to small pages if that was unsuccessful.
>>>
>>> Architectures must ensure that any arch specific vmalloc allocations
>>> that require PAGE_SIZE mappings (e.g., module allocations vs strict
>>> module rwx) use the VM_NOHUGE flag to inhibit larger mappings.
>>>
>>> When hugepage vmalloc mappings are enabled in the next patch, this
>>> reduces TLB misses by nearly 30x on a `git diff` workload on a 2-node
>>> POWER9 (59,800 -> 2,100) and reduces CPU cycles by 0.54%.
>>>
>>> This can result in more internal fragmentation and memory overhead for a
>>> given allocation, an option nohugevmalloc is added to disable at boot.
>>>
>>> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
>>> ---
>>> arch/Kconfig | 10 +++
>>> include/linux/vmalloc.h | 18 ++++
>>> mm/page_alloc.c | 5 +-
>>> mm/vmalloc.c | 192 ++++++++++++++++++++++++++++++----------
>>> 4 files changed, 177 insertions(+), 48 deletions(-)
>>>
>>
>>> diff --git a/mm/vmalloc.c b/mm/vmalloc.c
>>> index 0377e1d059e5..eef61e0f5170 100644
>>> --- a/mm/vmalloc.c
>>> +++ b/mm/vmalloc.c
>>
>>> @@ -2691,15 +2746,18 @@ EXPORT_SYMBOL_GPL(vmap_pfn);
>>> #endif /* CONFIG_VMAP_PFN */
>>>
>>> static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
>>> - pgprot_t prot, int node)
>>> + pgprot_t prot, unsigned int page_shift,
>>> + int node)
>>> {
>>> const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
>>> - unsigned int nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
>>> - unsigned long array_size;
>>> - unsigned int i;
>>> + unsigned int page_order = page_shift - PAGE_SHIFT;
>>> + unsigned long addr = (unsigned long)area->addr;
>>> + unsigned long size = get_vm_area_size(area);
>>> + unsigned int nr_small_pages = size >> PAGE_SHIFT;
>>> struct page **pages;
>>> + unsigned int i;
>>>
>>> - array_size = (unsigned long)nr_pages * sizeof(struct page *);
>>> + array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
>>
>> array_size() is a function in include/linux/overflow.h
>>
>> For some reason, it breaks the build with your series.
>
> What config? I haven't seen it.
>
Several configs I believe. I saw it this morning in
https://patchwork.ozlabs.org/project/linuxppc-dev/patch/20210124082230.2118861-13-npiggin@gmail.com/
Though the reports have all disappeared now.
^ permalink raw reply
* Re: [PATCH v10 11/12] mm/vmalloc: Hugepage vmalloc mappings
From: Nicholas Piggin @ 2021-01-25 11:37 UTC (permalink / raw)
To: Andrew Morton, Christophe Leroy, linux-mm
Cc: linux-arch, Ding Tianhong, linux-kernel, Christoph Hellwig,
Zefan Li, Jonathan Cameron, Rick Edgecombe, linuxppc-dev
In-Reply-To: <933352bd-dcf3-c483-4d7a-07afe1116cf1@csgroup.eu>
Excerpts from Christophe Leroy's message of January 25, 2021 7:14 pm:
>
>
> Le 24/01/2021 à 09:22, Nicholas Piggin a écrit :
>> Support huge page vmalloc mappings. Config option HAVE_ARCH_HUGE_VMALLOC
>> enables support on architectures that define HAVE_ARCH_HUGE_VMAP and
>> supports PMD sized vmap mappings.
>>
>> vmalloc will attempt to allocate PMD-sized pages if allocating PMD size
>> or larger, and fall back to small pages if that was unsuccessful.
>>
>> Architectures must ensure that any arch specific vmalloc allocations
>> that require PAGE_SIZE mappings (e.g., module allocations vs strict
>> module rwx) use the VM_NOHUGE flag to inhibit larger mappings.
>>
>> When hugepage vmalloc mappings are enabled in the next patch, this
>> reduces TLB misses by nearly 30x on a `git diff` workload on a 2-node
>> POWER9 (59,800 -> 2,100) and reduces CPU cycles by 0.54%.
>>
>> This can result in more internal fragmentation and memory overhead for a
>> given allocation, an option nohugevmalloc is added to disable at boot.
>>
>> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
>> ---
>> arch/Kconfig | 10 +++
>> include/linux/vmalloc.h | 18 ++++
>> mm/page_alloc.c | 5 +-
>> mm/vmalloc.c | 192 ++++++++++++++++++++++++++++++----------
>> 4 files changed, 177 insertions(+), 48 deletions(-)
>>
>
>> diff --git a/mm/vmalloc.c b/mm/vmalloc.c
>> index 0377e1d059e5..eef61e0f5170 100644
>> --- a/mm/vmalloc.c
>> +++ b/mm/vmalloc.c
>
>> @@ -2691,15 +2746,18 @@ EXPORT_SYMBOL_GPL(vmap_pfn);
>> #endif /* CONFIG_VMAP_PFN */
>>
>> static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
>> - pgprot_t prot, int node)
>> + pgprot_t prot, unsigned int page_shift,
>> + int node)
>> {
>> const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
>> - unsigned int nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
>> - unsigned long array_size;
>> - unsigned int i;
>> + unsigned int page_order = page_shift - PAGE_SHIFT;
>> + unsigned long addr = (unsigned long)area->addr;
>> + unsigned long size = get_vm_area_size(area);
>> + unsigned int nr_small_pages = size >> PAGE_SHIFT;
>> struct page **pages;
>> + unsigned int i;
>>
>> - array_size = (unsigned long)nr_pages * sizeof(struct page *);
>> + array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
>
> array_size() is a function in include/linux/overflow.h
>
> For some reason, it breaks the build with your series.
What config? I haven't seen it.
Thanks,
Nick
^ permalink raw reply
* Re: [PATCH v10 06/12] powerpc: inline huge vmap supported functions
From: Nicholas Piggin @ 2021-01-25 11:37 UTC (permalink / raw)
To: Andrew Morton, Christophe Leroy, linux-mm
Cc: linux-arch, Ding Tianhong, linux-kernel, Christoph Hellwig,
Zefan Li, Jonathan Cameron, Rick Edgecombe, linuxppc-dev
In-Reply-To: <c03010a7-a358-0321-d5d4-80a770c2213f@csgroup.eu>
Excerpts from Christophe Leroy's message of January 25, 2021 6:42 pm:
>
>
> Le 24/01/2021 à 09:22, Nicholas Piggin a écrit :
>> This allows unsupported levels to be constant folded away, and so
>> p4d_free_pud_page can be removed because it's no longer linked to.
>
> Ah, ok, you did it here. Why not squashing this patch into patch 5 directly ?
To reduce arch code movement in the first patch and split up these arch
patches to get separate acks for them.
Maybe overkill for these changes but doesn't hurt I think.
Thanks,
Nick
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox