From: Nicholas Piggin <npiggin@gmail.com>
To: linuxppc-dev@lists.ozlabs.org
Cc: Nicholas Piggin <npiggin@gmail.com>
Subject: [PATCH 14/14] powerpc/64s: use the same default PPR for user and kernel
Date: Tue, 16 Mar 2021 08:04:02 +1000 [thread overview]
Message-ID: <20210315220402.260594-15-npiggin@gmail.com> (raw)
In-Reply-To: <20210315220402.260594-1-npiggin@gmail.com>
Change the default PPR to userspace to 4 (medium), matching the
normal kernel PPR.
This allows system calls and user interrupts to avoid setting PPR on
entry and exit, providing a significant speedup.
This is a change to the user environment. The problem with changing
the kernel to match userspace at 3 (medium-low), is that userspace
can then boost priority above the kernel which is also undesirable.
glibc does not seem to change PPR anywhere, so the decision is to
go with this.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 2 ++
arch/powerpc/include/asm/processor.h | 4 ++--
arch/powerpc/kernel/exceptions-64s.S | 3 ---
arch/powerpc/kernel/interrupt.c | 33 ++++++++++++++++++++++++++++
arch/powerpc/kernel/interrupt_64.S | 17 --------------
5 files changed, 37 insertions(+), 22 deletions(-)
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index d6d54bbcba2f..293e6be9fd71 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -57,6 +57,8 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
#ifdef CONFIG_PPC64
bool trace_enable = false;
+ if (unlikely(regs->ppr != DEFAULT_PPR))
+ mtspr(SPRN_PPR, DEFAULT_PPR);
if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS)) {
if (irq_soft_mask_set_return(IRQS_DISABLED) == IRQS_ENABLED)
trace_enable = true;
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index cb1edf21a82e..5ff589042103 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -27,8 +27,8 @@
#endif
#ifdef CONFIG_PPC64
-/* Default SMT priority is set to 3. Use 11- 13bits to save priority. */
-#define PPR_PRIORITY 3
+/* Default SMT priority is set to 4. Use 11- 13bits to save priority. */
+#define PPR_PRIORITY 4
#ifdef __ASSEMBLY__
#define DEFAULT_PPR (PPR_PRIORITY << 50)
#else
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 75cee7cdf887..0d40614d13e0 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -367,7 +367,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
BEGIN_FTR_SECTION
mfspr r9,SPRN_PPR
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
- HMT_MEDIUM
std r10,IAREA+EX_R10(r13) /* save r10 - r12 */
BEGIN_FTR_SECTION
mfspr r10,SPRN_CFAR
@@ -1962,8 +1961,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
mfspr r11,SPRN_SRR0
mfspr r12,SPRN_SRR1
- HMT_MEDIUM
-
.if ! \virt
__LOAD_HANDLER(r10, system_call_common_real)
mtctr r10
diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c
index 09cf699d0e2e..a6e0595da0dd 100644
--- a/arch/powerpc/kernel/interrupt.c
+++ b/arch/powerpc/kernel/interrupt.c
@@ -40,6 +40,11 @@ notrace long system_call_exception(long r3, long r4, long r5,
regs->orig_gpr3 = r3;
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (unlikely(regs->ppr != DEFAULT_PPR))
+ mtspr(SPRN_PPR, DEFAULT_PPR);
+#endif
+
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
@@ -237,6 +242,11 @@ notrace unsigned long syscall_exit_prepare_main(unsigned long r3,
account_cpu_user_exit();
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (unlikely(regs->ppr != DEFAULT_PPR))
+ mtspr(SPRN_PPR, regs->ppr);
+#endif
+
#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not using this */
/*
* We do this at the end so that we do context switch with KERNEL AMR
@@ -315,6 +325,11 @@ notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *reg
*/
hard_irq_disable();
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (unlikely(regs->ppr != DEFAULT_PPR))
+ mtspr(SPRN_PPR, DEFAULT_PPR);
+#endif
+
trace_hardirqs_off();
user_exit_irqoff();
account_cpu_user_entry();
@@ -398,6 +413,11 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs)
account_cpu_user_exit();
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (unlikely(regs->ppr != DEFAULT_PPR))
+ mtspr(SPRN_PPR, regs->ppr);
+#endif
+
/*
* We do this at the end so that we do context switch with KERNEL AMR
*/
@@ -489,6 +509,11 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs)
local_paca->tm_scratch = regs->msr;
#endif
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (unlikely(regs->ppr != DEFAULT_PPR))
+ mtspr(SPRN_PPR, regs->ppr);
+#endif
+
/*
* Don't want to mfspr(SPRN_AMR) here, because this comes after mtmsr,
* which would cause Read-After-Write stalls. Hence, we take the AMR
@@ -505,6 +530,10 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs)
notrace unsigned long interrupt_exit_user_restart(struct pt_regs *regs)
{
hard_irq_disable();
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (unlikely(regs->ppr != DEFAULT_PPR))
+ mtspr(SPRN_PPR, DEFAULT_PPR);
+#endif
trace_hardirqs_off();
user_exit_irqoff();
@@ -523,6 +552,10 @@ notrace unsigned long interrupt_exit_user_restart(struct pt_regs *regs)
notrace unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs)
{
hard_irq_disable();
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (unlikely(regs->ppr != DEFAULT_PPR))
+ mtspr(SPRN_PPR, DEFAULT_PPR);
+#endif
#ifndef CONFIG_PPC_BOOK3E_64
set_kuap(AMR_KUAP_BLOCKED);
diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
index eef61800f734..53fc446dcbeb 100644
--- a/arch/powerpc/kernel/interrupt_64.S
+++ b/arch/powerpc/kernel/interrupt_64.S
@@ -99,10 +99,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM)
ld r11,exception_marker@toc(r2)
std r11,-16(r10) /* "regshere" marker */
-BEGIN_FTR_SECTION
- HMT_MEDIUM
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
ENTER_KERNEL_SECURITY_FALLBACK
/*
@@ -142,10 +138,6 @@ BEGIN_FTR_SECTION
stdcx. r0,0,r1 /* to clear the reservation */
END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
-BEGIN_FTR_SECTION
- HMT_MEDIUM_LOW
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
cmpdi r3,0
bne .Lsyscall_vectored_\name\()_restore_regs
@@ -377,10 +369,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
mtspr SPRN_XER,r0
.Lsyscall_restore_regs_cont:
-BEGIN_FTR_SECTION
- HMT_MEDIUM_LOW
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
/*
* We don't need to restore AMR on the way back to userspace for KUAP.
* The value of AMR only matters while we're in the kernel.
@@ -533,11 +521,6 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
tdnei r4,IRQS_ENABLED
#ifdef CONFIG_PPC_BOOK3S
-BEGIN_FTR_SECTION
- ld r10,_PPR(r1)
- mtspr SPRN_PPR,r10
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
.ifc \srr,srr
lbz r4,PACASRR_VALID(r13)
.else
--
2.23.0
next prev parent reply other threads:[~2021-03-15 22:11 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-03-15 22:03 [PATCH 00/14] powerpc/64: fast interrupt exits Nicholas Piggin
2021-03-15 22:03 ` [PATCH 01/14] powerpc: remove interrupt exit helpers unused argument Nicholas Piggin
2021-05-17 13:49 ` Christophe Leroy
2021-03-15 22:03 ` [PATCH 02/14] powerpc/64s: security fallback improvement Nicholas Piggin
2021-03-15 22:03 ` [PATCH 03/14] powerpc/64s: introduce different functions to return from SRR vs HSRR interrupts Nicholas Piggin
2021-03-15 22:03 ` [PATCH 04/14] powerpc/64s: avoid reloading (H)SRR registers if they are still valid Nicholas Piggin
2021-04-02 22:39 ` Michael Ellerman
2021-04-04 0:49 ` Nicholas Piggin
2021-04-03 2:28 ` Michael Ellerman
2021-04-04 0:51 ` Nicholas Piggin
2021-03-15 22:03 ` [PATCH 05/14] powerpc/64: move interrupt return asm to interrupt_64.S Nicholas Piggin
2021-03-15 22:03 ` [PATCH 06/14] powerpc/64s: save one more register in the masked interrupt handler Nicholas Piggin
2021-03-15 22:03 ` [PATCH 07/14] powerpc/64: allow alternate return locations for soft-masked interrupts Nicholas Piggin
2021-03-15 22:03 ` [PATCH 08/14] powerpc/64: interrupt soft-enable race fix Nicholas Piggin
2021-03-15 22:03 ` [PATCH 09/14] powerpc/64: treat low kernel text as irqs soft-masked Nicholas Piggin
2021-03-15 22:03 ` [PATCH 10/14] powerpc/64: use interrupt restart table to speed up return from interrupt Nicholas Piggin
2021-03-16 19:34 ` Christophe Leroy
2021-03-16 23:46 ` Nicholas Piggin
2021-03-15 22:03 ` [PATCH 11/14] powerpc/64e: Remove PPR from pt_regs Nicholas Piggin
2021-03-15 22:04 ` [PATCH 12/14] powerpc/64s: system call avoid setting MSR[RI] until we set MSR[EE] Nicholas Piggin
2021-03-16 7:21 ` Christophe Leroy
2021-03-16 8:13 ` Nicholas Piggin
2021-03-19 11:29 ` Michael Ellerman
2021-03-15 22:04 ` [PATCH 13/14] powerpc/64: handle MSR EE and RI in interrupt entry wrapper Nicholas Piggin
2021-03-15 22:04 ` Nicholas Piggin [this message]
2021-05-17 14:09 ` [PATCH 14/14] powerpc/64s: use the same default PPR for user and kernel Christophe Leroy
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210315220402.260594-15-npiggin@gmail.com \
--to=npiggin@gmail.com \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).