From: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
To: linuxppc-dev@lists.ozlabs.org, mpe@ellerman.id.au
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>,
linuxram@us.ibm.com, bauerman@linux.ibm.com, npiggin@gmail.com
Subject: [PATCH v2 12/28] powerpc/book3s64/pkeys: Store/restore userspace AMR correctly on entry and exit from kernel
Date: Sat, 2 May 2020 16:52:13 +0530 [thread overview]
Message-ID: <20200502112229.545331-13-aneesh.kumar@linux.ibm.com> (raw)
In-Reply-To: <20200502112229.545331-1-aneesh.kumar@linux.ibm.com>
This prepare kernel to operate with a different value than userspace AMR.
For this, AMR needs to be saved and restored on entry and return from the
kernel.
With KUAP we modify kernel AMR when accessing user address from the kernel
via copy_to/from_user interfaces.
If MMU_FTR_KEY is enabled we always use the key mechanism to implement KUAP
feature. If MMU_FTR_KEY is not supported and if we support MMU_FTR_KUAP
(radix translation on POWER9), we can skip restoring AMR on return
to userspace. Userspace won't be using AMR in that specific config.
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
---
arch/powerpc/include/asm/book3s/64/kup.h | 129 +++++++++++++++++++----
arch/powerpc/kernel/entry_64.S | 6 +-
arch/powerpc/kernel/exceptions-64s.S | 4 +-
arch/powerpc/kernel/syscall_64.c | 25 ++++-
4 files changed, 136 insertions(+), 28 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h
index 1b350bf781ec..82bef3901672 100644
--- a/arch/powerpc/include/asm/book3s/64/kup.h
+++ b/arch/powerpc/include/asm/book3s/64/kup.h
@@ -13,18 +13,47 @@
#ifdef __ASSEMBLY__
-.macro kuap_restore_amr gpr1, gpr2
-#ifdef CONFIG_PPC_KUAP
+.macro kuap_restore_user_amr gpr1
+#if defined(CONFIG_PPC_MEM_KEYS)
BEGIN_MMU_FTR_SECTION_NESTED(67)
- mfspr \gpr1, SPRN_AMR
+ /*
+ * AMR is going to be different when
+ * returning to userspace.
+ */
+ ld \gpr1, STACK_REGS_KUAP(r1)
+ isync
+ mtspr SPRN_AMR, \gpr1
+
+ /* No isync required, see kuap_restore_user_amr() */
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_PKEY , 67)
+#endif
+.endm
+
+.macro kuap_restore_kernel_amr gpr1, gpr2
+#if defined(CONFIG_PPC_MEM_KEYS)
+ BEGIN_MMU_FTR_SECTION_NESTED(67)
+ b 99f // handle_pkey_restore_amr
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_PKEY , 67)
+
+ BEGIN_MMU_FTR_SECTION_NESTED(68)
+ b 99f // handle_kuap_restore_amr
+ MMU_FTR_SECTION_ELSE_NESTED(68)
+ b 100f // skip_restore_amr
+ ALT_MMU_FTR_SECTION_END_NESTED_IFSET(MMU_FTR_KUAP, 68)
+
+99:
+ /*
+ * AMR is going to be mostly the same since we are
+ * returning to the kernel. Compare and do a mtspr.
+ */
ld \gpr2, STACK_REGS_KUAP(r1)
+ mfspr \gpr1, SPRN_AMR
cmpd \gpr1, \gpr2
- beq 998f
+ beq 100f
isync
mtspr SPRN_AMR, \gpr2
/* No isync required, see kuap_restore_amr() */
-998:
- END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67)
+100: // skip_restore_amr
#endif
.endm
@@ -40,38 +69,89 @@
#endif
.endm
+/*
+ * MMU_FTR_PKEY and MMU_FTR_KUAP can both be enabled on a platform. We prefer
+ * PKEY over KUAP if both can be enabled on the platform.
+ *
+ * With KUAP only enabled on exception if we are coming from userspace we don't
+ * save the AMR at all, because the expectation is that userspace can't change
+ * the AMR if KUAP feature is enabled.
+ */
.macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr
-#ifdef CONFIG_PPC_KUAP
+#if defined(CONFIG_PPC_MEM_KEYS)
+
BEGIN_MMU_FTR_SECTION_NESTED(67)
+ b 101f // handle_pkey_save_amr
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_PKEY , 67)
+
+ BEGIN_MMU_FTR_SECTION_NESTED(68)
+ b 99f // handle_kuap_save_amr
+ MMU_FTR_SECTION_ELSE_NESTED(68)
+ b 100f // skip_save_amr
+ ALT_MMU_FTR_SECTION_END_NESTED_IFSET(MMU_FTR_KUAP, 68)
+
+
+99: // handle_kuap_save_amr
.ifnb \msr_pr_cr
- bne \msr_pr_cr, 99f
+ /*
+ * We avoid changing AMR outside the kernel
+ * hence skip this completely.
+ */
+ bne \msr_pr_cr, 100f // from userspace
.endif
+
+101: // handle_pkey_save_amr
mfspr \gpr1, SPRN_AMR
std \gpr1, STACK_REGS_KUAP(r1)
- li \gpr2, (AMR_KUAP_BLOCKED >> AMR_KUAP_SHIFT)
- sldi \gpr2, \gpr2, AMR_KUAP_SHIFT
+
+ /*
+ * update kernel AMR with AMR_KUAP_BLOCKED only
+ * if KUAP feature is enabled
+ */
+ BEGIN_MMU_FTR_SECTION_NESTED(69)
+ LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED)
cmpd \use_cr, \gpr1, \gpr2
- beq \use_cr, 99f
- // We don't isync here because we very recently entered via rfid
+ beq \use_cr, 102f
+ /*
+ * We don't isync here because we very recently entered via an interrupt
+ */
mtspr SPRN_AMR, \gpr2
isync
-99:
- END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67)
+102:
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 69)
+
+100: // skip_save_amr
#endif
.endm
#else /* !__ASSEMBLY__ */
-#ifdef CONFIG_PPC_KUAP
+#ifdef CONFIG_PPC_MEM_KEYS
#include <asm/mmu.h>
#include <asm/ptrace.h>
extern u64 default_uamor;
-static inline void kuap_restore_amr(struct pt_regs *regs, unsigned long amr)
+static inline void kuap_restore_user_amr(struct pt_regs *regs)
{
- if (mmu_has_feature(MMU_FTR_KUAP)) {
+ if (!mmu_has_feature(MMU_FTR_PKEY))
+ return;
+
+ isync();
+ mtspr(SPRN_AMR, regs->kuap);
+ /*
+ * No isync required here because we are about to rfi
+ * back to previous context before any user accesses
+ * would be made, which is a CSI.
+ */
+}
+
+static inline void kuap_restore_kernel_amr(struct pt_regs *regs,
+ unsigned long amr)
+{
+ if (mmu_has_feature(MMU_FTR_KUAP) || mmu_has_feature(MMU_FTR_PKEY)) {
+
if (unlikely(regs->kuap != amr)) {
isync();
mtspr(SPRN_AMR, regs->kuap);
@@ -86,7 +166,7 @@ static inline void kuap_restore_amr(struct pt_regs *regs, unsigned long amr)
static inline unsigned long kuap_get_and_check_amr(void)
{
- if (mmu_has_feature(MMU_FTR_KUAP)) {
+ if (mmu_has_feature(MMU_FTR_KUAP) || mmu_has_feature(MMU_FTR_PKEY)) {
unsigned long amr = mfspr(SPRN_AMR);
if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) /* kuap_check_amr() */
WARN_ON_ONCE(amr != AMR_KUAP_BLOCKED);
@@ -97,7 +177,8 @@ static inline unsigned long kuap_get_and_check_amr(void)
static inline void kuap_check_amr(void)
{
- if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_KUAP))
+ if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) &&
+ (mmu_has_feature(MMU_FTR_KUAP) || mmu_has_feature(MMU_FTR_PKEY)))
WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);
}
@@ -170,8 +251,12 @@ bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
(regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)),
"Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read");
}
-#else /* CONFIG_PPC_KUAP */
-static inline void kuap_restore_amr(struct pt_regs *regs, unsigned long amr)
+#else /* CONFIG_PPC_MEM_KEYS */
+static inline void kuap_restore_user_amr(struct pt_regs *regs)
+{
+}
+
+static inline void kuap_restore_kernel_amr(struct pt_regs *regs, unsigned long amr)
{
}
@@ -183,7 +268,7 @@ static inline unsigned long kuap_get_and_check_amr(void)
{
return 0;
}
-#endif /* CONFIG_PPC_KUAP */
+#endif /* CONFIG_PPC_MEM_KEYS */
#endif /* __ASSEMBLY__ */
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 9d49338e0c85..a087cbe0b17d 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -481,8 +481,8 @@ _ASM_NOKPROBE_SYMBOL(fast_interrupt_return)
kuap_check_amr r3, r4
ld r5,_MSR(r1)
andi. r0,r5,MSR_PR
- bne .Lfast_user_interrupt_return
- kuap_restore_amr r3, r4
+ bne .Lfast_user_interrupt_return_amr
+ kuap_restore_kernel_amr r3, r4
andi. r0,r5,MSR_RI
li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
bne+ .Lfast_kernel_interrupt_return
@@ -502,6 +502,8 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return)
cmpdi r3,0
bne- .Lrestore_nvgprs
+.Lfast_user_interrupt_return_amr:
+ kuap_restore_user_amr r3
.Lfast_user_interrupt_return:
ld r11,_NIP(r1)
ld r12,_MSR(r1)
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index ef4a90212664..11abe8fbf0ba 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -971,7 +971,7 @@ EXC_COMMON_BEGIN(system_reset_common)
ld r10,SOFTE(r1)
stb r10,PACAIRQSOFTMASK(r13)
- kuap_restore_amr r9, r10
+ kuap_restore_kernel_amr r9, r10
EXCEPTION_RESTORE_REGS
RFI_TO_USER_OR_KERNEL
@@ -2757,7 +2757,7 @@ EXC_COMMON_BEGIN(soft_nmi_common)
ld r10,SOFTE(r1)
stb r10,PACAIRQSOFTMASK(r13)
- kuap_restore_amr r9, r10
+ kuap_restore_kernel_amr r9, r10
EXCEPTION_RESTORE_REGS hsrr=0
RFI_TO_KERNEL
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index f704f657e1f7..4d417fdadb2a 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -35,7 +35,20 @@ notrace long system_call_exception(long r3, long r4, long r5,
BUG_ON(!FULL_REGS(regs));
BUG_ON(regs->softe != IRQS_ENABLED);
- kuap_check_amr();
+#ifdef CONFIG_PPC_MEM_KEYS
+ if (mmu_has_feature(MMU_FTR_PKEY)) {
+ unsigned long amr;
+ /*
+ * When entering from userspace we mostly have the AMR
+ * different from kernel default values. Hence don't compare.
+ */
+ amr = mfspr(SPRN_AMR);
+ regs->kuap = amr;
+ mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
+ isync();
+ } else
+#endif
+ kuap_check_amr();
account_cpu_user_entry();
@@ -208,6 +221,10 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
account_cpu_user_exit();
+ /*
+ * We do this at the end so that we do context switch with KERNEL AMR
+ */
+ kuap_restore_user_amr(regs);
return ret;
}
@@ -300,6 +317,10 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
account_cpu_user_exit();
+ /*
+ * We do this at the end so that we do context switch with KERNEL AMR
+ */
+ kuap_restore_user_amr(regs);
return ret;
}
@@ -378,7 +399,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
* mtmsr, which would cause RAW stalls. Hence, we take the AMR value
* from the check above.
*/
- kuap_restore_amr(regs, amr);
+ kuap_restore_kernel_amr(regs, amr);
return ret;
}
--
2.26.2
next prev parent reply other threads:[~2020-05-02 12:24 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-05-02 11:22 [PATCH v2 00/28] Kernel userspace access/execution prevention with hash translation Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 01/28] powerpc/book3s64/pkeys: Enable MMU_FTR_PKEY Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 02/28] powerpc/book3s64/kuep: Add MMU_FTR_KUEP Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 03/28] powerpc/book3s64/pkeys: Use execute_pkey_disable static key Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 04/28] powerpc/book3s64/pkeys: Use MMU_FTR_PKEY instead of pkey_disabled " Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 05/28] powerpc/book3s64/kuap: Move KUAP related function outside radix Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 06/28] powerpc/book3s64/kuep: Move KUEP " Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 07/28] powerpc/book3s64/kuap: Rename MMU_FTR_RADIX_KUAP to MMU_FTR_KUAP Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 08/28] powerpc/book3s64/kuap/kuep: Make KUAP and KUEP a subfeature of PPC_MEM_KEYS Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 09/28] powerpc/book3s64/kuap: Move UAMOR setup to key init function Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 10/28] powerpc/book3s64/kuap: Use Key 3 for kernel mapping with hash translation Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 11/28] powerpc/exec: Set thread.regs early during exec Aneesh Kumar K.V
2020-05-02 11:22 ` Aneesh Kumar K.V [this message]
2020-05-02 11:22 ` [PATCH v2 13/28] powerpc/book3s64/kuep: Store/restore userspace IAMR correctly on entry and exit from kernel Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 14/28] powerpc/book3s64/pkeys: Inherit correctly on fork Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 15/28] powerpc/book3s64/pkeys: Reset userspace AMR correctly on exec Aneesh Kumar K.V
2020-05-07 14:25 ` Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 16/28] powerpc/ptrace-view: Use pt_regs values instead of thread_struct based one Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 17/28] powerpc/book3s64/pkeys: Don't update SPRN_AMR when in kernel mode Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 18/28] powerpc/book3s64/kuap: Restrict access to userspace based on userspace AMR Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 19/28] powerpc/book3s64/kuap: Improve error reporting with KUAP Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 20/28] powerpc/book3s64/kuap: Use Key 3 to implement KUAP with hash translation Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 21/28] powerpc/book3s64/kuep: Use Key 3 to implement KUEP " Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 22/28] powerpc/book3s64/hash/kuap: Enable kuap on hash Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 23/28] powerpc/book3s64/hash/kuep: Enable KUEP " Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 24/28] powerpc/book3s64/keys: Print information during boot Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 25/28] powerpc/selftest/ptrave-pkey: Rename variables to make it easier to follow code Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 26/28] powerpc/selftest/ptrace-pkey: Update the test to mark an invalid pkey correctly Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 27/28] powerpc/selftest/ptrace-pkey: IAMR and uamor cannot be updated by ptrace Aneesh Kumar K.V
2020-05-02 11:22 ` [PATCH v2 28/28] powerpc/book3s64/keys/kuap: Reset AMR/IAMR values on kexec Aneesh Kumar K.V
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200502112229.545331-13-aneesh.kumar@linux.ibm.com \
--to=aneesh.kumar@linux.ibm.com \
--cc=bauerman@linux.ibm.com \
--cc=linuxppc-dev@lists.ozlabs.org \
--cc=linuxram@us.ibm.com \
--cc=mpe@ellerman.id.au \
--cc=npiggin@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).