From: Rohan McLure <rmclure@linux.ibm.com>
To: linuxppc-dev@lists.ozlabs.org
Cc: Rohan McLure <rmclure@linux.ibm.com>
Subject: [PATCH v6 24/25] powerpc/64s: Clear gprs on interrupt routine entry in Book3S
Date: Wed, 21 Sep 2022 16:56:04 +1000 [thread overview]
Message-ID: <20220921065605.1051927-25-rmclure@linux.ibm.com> (raw)
In-Reply-To: <20220921065605.1051927-1-rmclure@linux.ibm.com>
Zero GPRS r0, r2-r11, r14-r31, on entry into the kernel for all
other interrupt sources to limit influence of user-space values
in potential speculation gadgets. The remaining gprs are overwritten by
entry macros to interrupt handlers, irrespective of whether or not a
given handler consumes these register values.
Prior to this commit, r14-r31 are restored on a per-interrupt basis at
exit, but now they are always restored. Remove explicit REST_NVGPRS
invocations as non-volatiles must now always be restored. 32-bit systems
do not clear user registers on interrupt, and continue to depend on the
return value of interrupt_exit_user_prepare to determine whether or not
to restore non-volatiles.
The mmap_bench benchmark in selftests should rapidly invoke pagefaults.
See ~0.8% performance regression with this mitigation, but this
indicates the worst-case performance due to heavier-weight interrupt
handlers. This mitigation is disabled by default, but enabled with
CONFIG_INTERRUPT_SANITIZE_REGISTERS.
Signed-off-by: Rohan McLure <rmclure@linux.ibm.com>
---
V2: Add benchmark data
V3: Use ZEROIZE_GPR{,S} macro renames, clarify
upt_exit_user_prepare changes in summary.
V5: Configurable now with INTERRUPT_SANITIZE_REGISTERS. Zero r12
(containing MSR) from common macro on per-interrupt basis with IOPTION.
V6: Replace ifdefs with invocations of conditionally defined macros.
---
arch/powerpc/kernel/exceptions-64s.S | 47 +++++++++++++++++++++-----
arch/powerpc/kernel/interrupt_64.S | 15 ++++++++
2 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index a3b51441b039..b3f5ef1c712f 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -21,6 +21,19 @@
#include <asm/feature-fixups.h>
#include <asm/kup.h>
+/*
+ * macros for handling user register sanitisation
+ */
+#ifdef CONFIG_INTERRUPT_SANITIZE_REGISTERS
+#define SANITIZE_ZEROIZE_NVGPRS() ZEROIZE_NVGPRS()
+#define SANITIZE_RESTORE_NVGPRS() REST_NVGPRS(r1)
+#define HANDLER_RESTORE_NVGPRS()
+#else
+#define SANITIZE_ZEROIZE_NVGPRS()
+#define SANITIZE_RESTORE_NVGPRS()
+#define HANDLER_RESTORE_NVGPRS() REST_NVGPRS(r1)
+#endif /* CONFIG_INTERRUPT_SANITIZE_REGISTERS */
+
/*
* Following are fixed section helper macros.
*
@@ -111,6 +124,7 @@ name:
#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */
#define __ISTACK(name) .L_ISTACK_ ## name
#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */
+#define IMSR_R12 .L_IMSR_R12_\name\() /* Assumes MSR saved to r12 */
#define INT_DEFINE_BEGIN(n) \
.macro int_define_ ## n name
@@ -176,6 +190,9 @@ do_define_int n
.ifndef IKUAP
IKUAP=1
.endif
+ .ifndef IMSR_R12
+ IMSR_R12=0
+ .endif
.endm
/*
@@ -502,6 +519,7 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real, text)
std r10,0(r1) /* make stack chain pointer */
std r0,GPR0(r1) /* save r0 in stackframe */
std r10,GPR1(r1) /* save r1 in stackframe */
+ ZEROIZE_GPR(0)
/* Mark our [H]SRRs valid for return */
li r10,1
@@ -544,8 +562,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
std r9,GPR11(r1)
std r10,GPR12(r1)
std r11,GPR13(r1)
+ .if !IMSR_R12
+ ZEROIZE_GPRS(9, 12)
+ .else
+ ZEROIZE_GPRS(9, 11)
+ .endif
SAVE_NVGPRS(r1)
+ SANITIZE_ZEROIZE_NVGPRS()
.if IDAR
.if IISIDE
@@ -577,8 +601,8 @@ BEGIN_FTR_SECTION
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
ld r10,IAREA+EX_CTR(r13)
std r10,_CTR(r1)
- std r2,GPR2(r1) /* save r2 in stackframe */
- SAVE_GPRS(3, 8, r1) /* save r3 - r8 in stackframe */
+ SAVE_GPRS(2, 8, r1) /* save r2 - r8 in stackframe */
+ ZEROIZE_GPRS(2, 8)
mflr r9 /* Get LR, later save to stack */
ld r2,PACATOC(r13) /* get kernel TOC into r2 */
std r9,_LINK(r1)
@@ -696,6 +720,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
mtlr r9
ld r9,_CCR(r1)
mtcr r9
+ SANITIZE_RESTORE_NVGPRS()
REST_GPRS(2, 13, r1)
REST_GPR(0, r1)
/* restore original r1. */
@@ -1372,7 +1397,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
* do_break() may have changed the NV GPRS while handling a breakpoint.
* If so, we need to restore them with their updated values.
*/
- REST_NVGPRS(r1)
+ HANDLER_RESTORE_NVGPRS()
b interrupt_return_srr
@@ -1598,7 +1623,7 @@ EXC_COMMON_BEGIN(alignment_common)
GEN_COMMON alignment
addi r3,r1,STACK_FRAME_OVERHEAD
bl alignment_exception
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
b interrupt_return_srr
@@ -1708,7 +1733,7 @@ EXC_COMMON_BEGIN(program_check_common)
.Ldo_program_check:
addi r3,r1,STACK_FRAME_OVERHEAD
bl program_check_exception
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
b interrupt_return_srr
@@ -1726,6 +1751,7 @@ INT_DEFINE_BEGIN(fp_unavailable)
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
+ IMSR_R12=1
INT_DEFINE_END(fp_unavailable)
EXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100)
@@ -2139,7 +2165,7 @@ EXC_COMMON_BEGIN(emulation_assist_common)
GEN_COMMON emulation_assist
addi r3,r1,STACK_FRAME_OVERHEAD
bl emulation_assist_interrupt
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
b interrupt_return_hsrr
@@ -2347,6 +2373,7 @@ INT_DEFINE_BEGIN(altivec_unavailable)
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
+ IMSR_R12=1
INT_DEFINE_END(altivec_unavailable)
EXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20)
@@ -2396,6 +2423,7 @@ INT_DEFINE_BEGIN(vsx_unavailable)
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
+ IMSR_R12=1
INT_DEFINE_END(vsx_unavailable)
EXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20)
@@ -2457,7 +2485,7 @@ EXC_COMMON_BEGIN(facility_unavailable_common)
GEN_COMMON facility_unavailable
addi r3,r1,STACK_FRAME_OVERHEAD
bl facility_unavailable_exception
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
b interrupt_return_srr
@@ -2485,7 +2513,8 @@ EXC_COMMON_BEGIN(h_facility_unavailable_common)
GEN_COMMON h_facility_unavailable
addi r3,r1,STACK_FRAME_OVERHEAD
bl facility_unavailable_exception
- REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */
+ /* XXX Shouldn't be necessary in practice */
+ HANDLER_RESTORE_NVGPRS()
b interrupt_return_hsrr
@@ -2711,7 +2740,7 @@ EXC_COMMON_BEGIN(altivec_assist_common)
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_ALTIVEC
bl altivec_assist_exception
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
#else
bl unknown_exception
#endif
diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
index 40147558e1a6..6b60d39f113b 100644
--- a/arch/powerpc/kernel/interrupt_64.S
+++ b/arch/powerpc/kernel/interrupt_64.S
@@ -13,6 +13,15 @@
#include <asm/ppc_asm.h>
#include <asm/ptrace.h>
+/*
+ * macros for handling user register sanitisation
+ */
+#ifdef CONFIG_INTERRUPT_SANITIZE_REGISTERS
+#define SANITIZE_RESTORE_NVGPRS() REST_NVGPRS(r1)
+#else
+#define SANITIZE_RESTORE_NVGPRS()
+#endif /* CONFIG_INTERRUPT_SANITIZE_REGISTERS */
+
.section ".toc","aw"
SYS_CALL_TABLE:
.tc sys_call_table[TC],sys_call_table
@@ -433,9 +442,11 @@ interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
addi r3,r1,STACK_FRAME_OVERHEAD
bl interrupt_exit_user_prepare
+#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
cmpdi r3,0
bne- .Lrestore_nvgprs_\srr
.Lrestore_nvgprs_\srr\()_cont:
+#endif
std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
#ifdef CONFIG_PPC_BOOK3S
.Linterrupt_return_\srr\()_user_rst_start:
@@ -449,6 +460,7 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
.Lfast_user_interrupt_return_\srr\():
+ SANITIZE_RESTORE_NVGPRS()
#ifdef CONFIG_PPC_BOOK3S
.ifc \srr,srr
lbz r4,PACASRR_VALID(r13)
@@ -518,9 +530,11 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
b . /* prevent speculative execution */
.Linterrupt_return_\srr\()_user_rst_end:
+#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
.Lrestore_nvgprs_\srr\():
REST_NVGPRS(r1)
b .Lrestore_nvgprs_\srr\()_cont
+#endif
#ifdef CONFIG_PPC_BOOK3S
interrupt_return_\srr\()_user_restart:
@@ -562,6 +576,7 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel)
1:
.Lfast_kernel_interrupt_return_\srr\():
+ SANITIZE_RESTORE_NVGPRS()
cmpdi cr1,r3,0
#ifdef CONFIG_PPC_BOOK3S
.ifc \srr,srr
--
2.34.1
next prev parent reply other threads:[~2022-09-21 7:13 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-09-21 6:55 [PATCH v6 00/25] powerpc: Syscall wrapper and register clearing Rohan McLure
2022-09-21 6:55 ` [PATCH v6 01/25] powerpc: Remove asmlinkage from syscall handler definitions Rohan McLure
2022-09-21 6:55 ` [PATCH v6 02/25] powerpc: Save caller r3 prior to system_call_exception Rohan McLure
2022-09-21 6:55 ` [PATCH v6 03/25] powerpc: Add ZEROIZE_GPRS macros for register clears Rohan McLure
2022-09-21 6:55 ` [PATCH v6 04/25] powerpc/64s: Use {ZEROIZE,SAVE,REST}_GPRS macros in sc, scv 0 handlers Rohan McLure
2022-09-21 6:55 ` [PATCH v6 05/25] powerpc/32: Clarify interrupt restores with REST_GPR macro in entry_32.S Rohan McLure
2022-09-21 6:55 ` [PATCH v6 06/25] powerpc/64e: Clarify register saves and clears with {SAVE,ZEROIZE}_GPRS Rohan McLure
2022-09-21 6:55 ` [PATCH v6 07/25] powerpc/64s: Fix comment on interrupt handler prologue Rohan McLure
2022-09-21 6:55 ` [PATCH v6 08/25] powerpc: Fix fallocate and fadvise64_64 compat parameter combination Rohan McLure
2022-09-21 6:55 ` [PATCH v6 09/25] asm-generic: compat: Support BE for long long args in 32-bit ABIs Rohan McLure
2022-10-31 13:23 ` [PATCH] asm-generic: compat: fix compat_arg_u64 and compat_arg_u64_dual Andreas Schwab
2022-11-01 12:25 ` Michael Ellerman
2022-11-03 8:20 ` Christophe Leroy
2022-11-03 8:48 ` Arnd Bergmann
2022-09-21 6:55 ` [PATCH v6 10/25] powerpc: Use generic fallocate compatibility syscall Rohan McLure
2022-09-21 6:55 ` [PATCH v6 11/25] powerpc/32: Remove powerpc select specialisation Rohan McLure
2022-09-21 6:55 ` [PATCH v6 12/25] powerpc: Remove direct call to personality syscall handler Rohan McLure
2022-09-21 6:55 ` [PATCH v6 13/25] powerpc: Remove direct call to mmap2 syscall handlers Rohan McLure
2022-09-28 12:15 ` Michael Ellerman
2022-09-28 13:00 ` Arnd Bergmann
2022-09-30 13:19 ` Michael Ellerman
2022-09-30 14:09 ` Arnd Bergmann
2022-09-21 6:55 ` [PATCH v6 14/25] powerpc: Provide do_ppc64_personality helper Rohan McLure
2022-09-21 6:55 ` [PATCH v6 15/25] powerpc: Adopt SYSCALL_DEFINE for arch-specific syscall handlers Rohan McLure
2022-09-21 6:55 ` [PATCH v6 16/25] powerpc: Include all arch-specific syscall prototypes Rohan McLure
2022-09-21 6:55 ` [PATCH v6 17/25] powerpc: Enable compile-time check for syscall handlers Rohan McLure
2022-09-21 6:55 ` [PATCH v6 18/25] powerpc: Use common syscall handler type Rohan McLure
2022-09-21 6:55 ` [PATCH v6 19/25] powerpc: Remove high-order word clearing on compat syscall entry Rohan McLure
2022-09-23 7:40 ` Nicholas Piggin
2022-09-28 11:56 ` Michael Ellerman
2022-09-21 6:56 ` [PATCH v6 20/25] powerpc: Change system_call_exception calling convention Rohan McLure
2022-09-23 7:43 ` Nicholas Piggin
2022-09-21 6:56 ` [PATCH v6 21/25] powerpc: Provide syscall wrapper Rohan McLure
2022-09-23 7:50 ` Nicholas Piggin
2022-10-30 15:34 ` Andreas Schwab
2022-10-30 15:50 ` Andreas Schwab
2022-10-30 19:43 ` Arnd Bergmann
2022-10-30 20:05 ` Andreas Schwab
2022-10-31 3:09 ` Michael Ellerman
2022-10-31 14:47 ` [PATCH] powerpc/32: fix syscall wrappers with 64-bit arguments Andreas Schwab
2022-10-31 19:37 ` Arnd Bergmann
2022-11-01 12:25 ` Michael Ellerman
2022-09-21 6:56 ` [PATCH v6 22/25] powerpc/64s: Clear user GPRs in syscall interrupt entry Rohan McLure
2022-09-23 8:02 ` Nicholas Piggin
2022-10-31 23:22 ` Rohan McLure
2022-09-21 6:56 ` [PATCH v6 23/25] powerpc/64: Add INTERRUPT_SANITIZE_REGISTERS Kconfig Rohan McLure
2022-09-21 6:56 ` Rohan McLure [this message]
2022-09-21 6:56 ` [PATCH v6 25/25] powerpc/64e: Clear gprs on interrupt routine entry on Book3E Rohan McLure
2022-10-04 13:24 ` [PATCH v6 00/25] powerpc: Syscall wrapper and register clearing Michael Ellerman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220921065605.1051927-25-rmclure@linux.ibm.com \
--to=rmclure@linux.ibm.com \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).