* [PATCH v2 1/9] powerpc/kuap: Avoid unnecessary reads of MD_AP
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 2/9] powerpc/kuap: Avoid useless jump_label on empty function Christophe Leroy
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
A disassembly of interrupt_exit_kernel_prepare() shows a useless read
of MD_AP register. This is shown by r9 being re-used immediately without
doing anything with the value read.
c000e0e0: 60 00 00 00 nop
c000e0e4: ===> 7d 3a c2 a6 mfmd_ap r9 <====
c000e0e8: 7d 20 00 a6 mfmsr r9
c000e0ec: 7c 51 13 a6 mtspr 81,r2
c000e0f0: 81 3f 00 84 lwz r9,132(r31)
c000e0f4: 71 29 80 00 andi. r9,r9,32768
kuap_get_and_assert_locked() is paired with kuap_kernel_restore()
and are only used in interrupt_exit_kernel_prepare(). The value
returned by kuap_get_and_assert_locked() is only used by
kuap_kernel_restore().
On 8xx, kuap_kernel_restore() doesn't use the value read by
kuap_get_and_assert_locked() so modify kuap_get_and_assert_locked()
to not perform the read of MD_AP and return 0 instead.
The same applies on BOOKE.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 8 ++------
arch/powerpc/include/asm/nohash/kup-booke.h | 6 ++----
2 files changed, 4 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
index c44d97751723..8579210f2a6a 100644
--- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
@@ -41,14 +41,10 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kua
static inline unsigned long __kuap_get_and_assert_locked(void)
{
- unsigned long kuap;
-
- kuap = mfspr(SPRN_MD_AP);
-
if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
- WARN_ON_ONCE(kuap >> 16 != MD_APG_KUAP >> 16);
+ WARN_ON_ONCE(mfspr(SPRN_MD_AP) >> 16 != MD_APG_KUAP >> 16);
- return kuap;
+ return 0;
}
static inline void __allow_user_access(void __user *to, const void __user *from,
diff --git a/arch/powerpc/include/asm/nohash/kup-booke.h b/arch/powerpc/include/asm/nohash/kup-booke.h
index 49bb41ed0816..823c5a3a96d8 100644
--- a/arch/powerpc/include/asm/nohash/kup-booke.h
+++ b/arch/powerpc/include/asm/nohash/kup-booke.h
@@ -58,12 +58,10 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kua
static inline unsigned long __kuap_get_and_assert_locked(void)
{
- unsigned long kuap = mfspr(SPRN_PID);
-
if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
- WARN_ON_ONCE(kuap);
+ WARN_ON_ONCE(mfspr(SPRN_PID));
- return kuap;
+ return 0;
}
static inline void __allow_user_access(void __user *to, const void __user *from,
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 2/9] powerpc/kuap: Avoid useless jump_label on empty function
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 1/9] powerpc/kuap: Avoid unnecessary reads of MD_AP Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 3/9] powerpc/kuap: Fold kuep_is_disabled() into its only user Christophe Leroy
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
Disassembly of interrupt_enter_prepare() shows a pointless nop
before the mftb
c000abf0 <interrupt_enter_prepare>:
c000abf0: 81 23 00 84 lwz r9,132(r3)
c000abf4: 71 29 40 00 andi. r9,r9,16384
c000abf8: 41 82 00 28 beq- c000ac20 <interrupt_enter_prepare+0x30>
c000abfc: ===> 60 00 00 00 nop <====
c000ac00: 7d 0c 42 e6 mftb r8
c000ac04: 80 e2 00 08 lwz r7,8(r2)
c000ac08: 81 22 00 28 lwz r9,40(r2)
c000ac0c: 91 02 00 24 stw r8,36(r2)
c000ac10: 7d 29 38 50 subf r9,r9,r7
c000ac14: 7d 29 42 14 add r9,r9,r8
c000ac18: 91 22 00 08 stw r9,8(r2)
c000ac1c: 4e 80 00 20 blr
c000ac20: 60 00 00 00 nop
c000ac24: 7d 5a c2 a6 mfmd_ap r10
c000ac28: 3d 20 de 00 lis r9,-8704
c000ac2c: 91 43 00 b0 stw r10,176(r3)
c000ac30: 7d 3a c3 a6 mtspr 794,r9
c000ac34: 4e 80 00 20 blr
That comes from the call to kuap_loc(), allthough __kuap_lock() is an
empty function on the 8xx.
To avoid that, only perform kuap_is_disabled() check when there is
something to do with __kuap_lock().
Do the same with __kuap_save_and_lock() and
__kuap_get_and_assert_locked().
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Reviewed-by: Nicholas Piggin <npiggin@gmail.com>
---
v2: Add back comment about __kupa_lock() not needed on 64s
---
arch/powerpc/include/asm/book3s/32/kup.h | 6 ++--
arch/powerpc/include/asm/book3s/64/kup.h | 10 ++----
arch/powerpc/include/asm/kup.h | 33 +++++++++-----------
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 11 +++----
arch/powerpc/include/asm/nohash/kup-booke.h | 8 +++--
5 files changed, 29 insertions(+), 39 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 678f9c9d89b6..466a19cfb4df 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -77,10 +77,6 @@ static inline void kuap_unlock(unsigned long addr, bool ool)
kuap_unlock_all_ool();
}
-static inline void __kuap_lock(void)
-{
-}
-
static inline void __kuap_save_and_lock(struct pt_regs *regs)
{
unsigned long kuap = current->thread.kuap;
@@ -92,6 +88,7 @@ static inline void __kuap_save_and_lock(struct pt_regs *regs)
current->thread.kuap = KUAP_NONE;
kuap_lock_addr(kuap, false);
}
+#define __kuap_save_and_lock __kuap_save_and_lock
static inline void kuap_user_restore(struct pt_regs *regs)
{
@@ -120,6 +117,7 @@ static inline unsigned long __kuap_get_and_assert_locked(void)
return kuap;
}
+#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
static __always_inline void __allow_user_access(void __user *to, const void __user *from,
u32 size, unsigned long dir)
diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h
index 84c09e546115..2a7bd3ecc556 100644
--- a/arch/powerpc/include/asm/book3s/64/kup.h
+++ b/arch/powerpc/include/asm/book3s/64/kup.h
@@ -298,15 +298,9 @@ static inline unsigned long __kuap_get_and_assert_locked(void)
WARN_ON_ONCE(amr != AMR_KUAP_BLOCKED);
return amr;
}
+#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
-/* Do nothing, book3s/64 does that in ASM */
-static inline void __kuap_lock(void)
-{
-}
-
-static inline void __kuap_save_and_lock(struct pt_regs *regs)
-{
-}
+/* __kuap_lock() not required, book3s/64 does that in ASM */
/*
* We support individually allowing read or write, but we don't support nesting
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index d751ddd08110..24cde16c4fbe 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -52,16 +52,9 @@ __bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
return false;
}
-static inline void __kuap_lock(void) { }
-static inline void __kuap_save_and_lock(struct pt_regs *regs) { }
static inline void kuap_user_restore(struct pt_regs *regs) { }
static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { }
-static inline unsigned long __kuap_get_and_assert_locked(void)
-{
- return 0;
-}
-
/*
* book3s/64/kup-radix.h defines these functions for the !KUAP case to flush
* the L1D cache after user accesses. Only include the empty stubs for other
@@ -85,29 +78,24 @@ bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
return __bad_kuap_fault(regs, address, is_write);
}
-static __always_inline void kuap_assert_locked(void)
-{
- if (kuap_is_disabled())
- return;
-
- if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
- __kuap_get_and_assert_locked();
-}
-
static __always_inline void kuap_lock(void)
{
+#ifdef __kuap_lock
if (kuap_is_disabled())
return;
__kuap_lock();
+#endif
}
static __always_inline void kuap_save_and_lock(struct pt_regs *regs)
{
+#ifdef __kuap_save_and_lock
if (kuap_is_disabled())
return;
__kuap_save_and_lock(regs);
+#endif
}
static __always_inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr)
@@ -120,10 +108,17 @@ static __always_inline void kuap_kernel_restore(struct pt_regs *regs, unsigned l
static __always_inline unsigned long kuap_get_and_assert_locked(void)
{
- if (kuap_is_disabled())
- return 0;
+#ifdef __kuap_get_and_assert_locked
+ if (!kuap_is_disabled())
+ return __kuap_get_and_assert_locked();
+#endif
+ return 0;
+}
- return __kuap_get_and_assert_locked();
+static __always_inline void kuap_assert_locked(void)
+{
+ if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
+ kuap_get_and_assert_locked();
}
#ifndef CONFIG_PPC_BOOK3S_64
diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
index 8579210f2a6a..a372cd822887 100644
--- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
@@ -20,15 +20,12 @@ static __always_inline bool kuap_is_disabled(void)
return static_branch_unlikely(&disable_kuap_key);
}
-static inline void __kuap_lock(void)
-{
-}
-
static inline void __kuap_save_and_lock(struct pt_regs *regs)
{
regs->kuap = mfspr(SPRN_MD_AP);
mtspr(SPRN_MD_AP, MD_APG_KUAP);
}
+#define __kuap_save_and_lock __kuap_save_and_lock
static inline void kuap_user_restore(struct pt_regs *regs)
{
@@ -39,13 +36,15 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kua
mtspr(SPRN_MD_AP, regs->kuap);
}
+#ifdef CONFIG_PPC_KUAP_DEBUG
static inline unsigned long __kuap_get_and_assert_locked(void)
{
- if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
- WARN_ON_ONCE(mfspr(SPRN_MD_AP) >> 16 != MD_APG_KUAP >> 16);
+ WARN_ON_ONCE(mfspr(SPRN_MD_AP) >> 16 != MD_APG_KUAP >> 16);
return 0;
}
+#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
+#endif
static inline void __allow_user_access(void __user *to, const void __user *from,
unsigned long size, unsigned long dir)
diff --git a/arch/powerpc/include/asm/nohash/kup-booke.h b/arch/powerpc/include/asm/nohash/kup-booke.h
index 823c5a3a96d8..71182cbe20c3 100644
--- a/arch/powerpc/include/asm/nohash/kup-booke.h
+++ b/arch/powerpc/include/asm/nohash/kup-booke.h
@@ -30,6 +30,7 @@ static inline void __kuap_lock(void)
mtspr(SPRN_PID, 0);
isync();
}
+#define __kuap_lock __kuap_lock
static inline void __kuap_save_and_lock(struct pt_regs *regs)
{
@@ -37,6 +38,7 @@ static inline void __kuap_save_and_lock(struct pt_regs *regs)
mtspr(SPRN_PID, 0);
isync();
}
+#define __kuap_save_and_lock __kuap_save_and_lock
static inline void kuap_user_restore(struct pt_regs *regs)
{
@@ -56,13 +58,15 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kua
/* Context synchronisation is performed by rfi */
}
+#ifdef CONFIG_PPC_KUAP_DEBUG
static inline unsigned long __kuap_get_and_assert_locked(void)
{
- if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
- WARN_ON_ONCE(mfspr(SPRN_PID));
+ WARN_ON_ONCE(mfspr(SPRN_PID));
return 0;
}
+#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
+#endif
static inline void __allow_user_access(void __user *to, const void __user *from,
unsigned long size, unsigned long dir)
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 3/9] powerpc/kuap: Fold kuep_is_disabled() into its only user
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 1/9] powerpc/kuap: Avoid unnecessary reads of MD_AP Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 2/9] powerpc/kuap: Avoid useless jump_label on empty function Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 4/9] powerpc/features: Add capability to update mmu features later Christophe Leroy
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
kuep_is_disabled() was introduced by commit 91bb30822a2e ("powerpc/32s:
Refactor update of user segment registers") but then all users but one
were removed by commit 526d4a4c77ae ("powerpc/32s: Do kuep_lock() and
kuep_unlock() in assembly").
Fold kuep_is_disabled() into init_new_context() which is its only user.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 5 -----
arch/powerpc/mm/book3s32/mmu_context.c | 2 +-
2 files changed, 1 insertion(+), 6 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 466a19cfb4df..0da0dea76c47 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -13,11 +13,6 @@
extern struct static_key_false disable_kuap_key;
-static __always_inline bool kuep_is_disabled(void)
-{
- return !IS_ENABLED(CONFIG_PPC_KUEP);
-}
-
#ifdef CONFIG_PPC_KUAP
#include <linux/sched.h>
diff --git a/arch/powerpc/mm/book3s32/mmu_context.c b/arch/powerpc/mm/book3s32/mmu_context.c
index 269a3eb25a73..1922f9a6b058 100644
--- a/arch/powerpc/mm/book3s32/mmu_context.c
+++ b/arch/powerpc/mm/book3s32/mmu_context.c
@@ -71,7 +71,7 @@ int init_new_context(struct task_struct *t, struct mm_struct *mm)
mm->context.id = __init_new_context();
mm->context.sr0 = CTX_TO_VSID(mm->context.id, 0);
- if (!kuep_is_disabled())
+ if (IS_ENABLED(CONFIG_PPC_KUEP))
mm->context.sr0 |= SR_NX;
if (!kuap_is_disabled())
mm->context.sr0 |= SR_KS;
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 4/9] powerpc/features: Add capability to update mmu features later
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
` (2 preceding siblings ...)
2023-07-03 8:48 ` [PATCH v2 3/9] powerpc/kuap: Fold kuep_is_disabled() into its only user Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 5/9] powerpc/kuap: MMU_FTR_BOOK3S_KUAP becomes MMU_FTR_KUAP Christophe Leroy
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
On powerpc32, features fixup is performed very early and that's too
early to read the cmdline and take into account 'nosmap' parameter.
On the other hand, no userspace access is performed that early and
KUAP feature fixup can be performed later.
Add a function to update mmu features. The function is passed a
mask with the features that can be updated.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/feature-fixups.h | 1 +
arch/powerpc/lib/feature-fixups.c | 31 ++++++++++++++++++++---
2 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/include/asm/feature-fixups.h b/arch/powerpc/include/asm/feature-fixups.h
index ac605fc369c4..77824bd289a3 100644
--- a/arch/powerpc/include/asm/feature-fixups.h
+++ b/arch/powerpc/include/asm/feature-fixups.h
@@ -292,6 +292,7 @@ extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;
extern long __start__btb_flush_fixup, __stop__btb_flush_fixup;
void apply_feature_fixups(void);
+void update_mmu_feature_fixups(unsigned long mask);
void setup_feature_keys(void);
#endif
diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c
index 80def1c2afcb..4f82581ca203 100644
--- a/arch/powerpc/lib/feature-fixups.c
+++ b/arch/powerpc/lib/feature-fixups.c
@@ -67,7 +67,8 @@ static int patch_alt_instruction(u32 *src, u32 *dest, u32 *alt_start, u32 *alt_e
return 0;
}
-static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
+static int patch_feature_section_mask(unsigned long value, unsigned long mask,
+ struct fixup_entry *fcur)
{
u32 *start, *end, *alt_start, *alt_end, *src, *dest;
@@ -79,7 +80,7 @@ static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
if ((alt_end - alt_start) > (end - start))
return 1;
- if ((value & fcur->mask) == fcur->value)
+ if ((value & fcur->mask & mask) == (fcur->value & mask))
return 0;
src = alt_start;
@@ -97,7 +98,8 @@ static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
return 0;
}
-void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
+static void do_feature_fixups_mask(unsigned long value, unsigned long mask,
+ void *fixup_start, void *fixup_end)
{
struct fixup_entry *fcur, *fend;
@@ -105,7 +107,7 @@ void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
fend = fixup_end;
for (; fcur < fend; fcur++) {
- if (patch_feature_section(value, fcur)) {
+ if (patch_feature_section_mask(value, mask, fcur)) {
WARN_ON(1);
printk("Unable to patch feature section at %p - %p" \
" with %p - %p\n",
@@ -117,6 +119,11 @@ void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
}
}
+void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
+{
+ do_feature_fixups_mask(value, ~0, fixup_start, fixup_end);
+}
+
#ifdef CONFIG_PPC_BARRIER_NOSPEC
static bool is_fixup_addr_valid(void *dest, size_t size)
{
@@ -651,6 +658,17 @@ void __init apply_feature_fixups(void)
do_final_fixups();
}
+void __init update_mmu_feature_fixups(unsigned long mask)
+{
+ saved_mmu_features &= ~mask;
+ saved_mmu_features |= cur_cpu_spec->mmu_features & mask;
+
+ do_feature_fixups_mask(cur_cpu_spec->mmu_features, mask,
+ PTRRELOC(&__start___mmu_ftr_fixup),
+ PTRRELOC(&__stop___mmu_ftr_fixup));
+ mmu_feature_keys_init();
+}
+
void __init setup_feature_keys(void)
{
/*
@@ -683,6 +701,11 @@ late_initcall(check_features);
#define check(x) \
if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__);
+static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
+{
+ return patch_feature_section_mask(value, ~0, fcur);
+}
+
/* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */
static struct fixup_entry fixup;
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 5/9] powerpc/kuap: MMU_FTR_BOOK3S_KUAP becomes MMU_FTR_KUAP
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
` (3 preceding siblings ...)
2023-07-03 8:48 ` [PATCH v2 4/9] powerpc/features: Add capability to update mmu features later Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 6/9] powerpc/kuap: Use MMU_FTR_KUAP on all and refactor disabling kuap Christophe Leroy
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
In order to reuse MMU_FTR_BOOK3S_KUAP for other targets than BOOK3S,
rename it MMU_FTR_KUAP.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/64/hash-pkey.h | 2 +-
arch/powerpc/include/asm/book3s/64/kup.h | 18 +++++++++---------
arch/powerpc/include/asm/mmu.h | 4 ++--
arch/powerpc/kernel/syscall.c | 2 +-
arch/powerpc/mm/book3s64/pkeys.c | 2 +-
5 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/hash-pkey.h b/arch/powerpc/include/asm/book3s/64/hash-pkey.h
index f1e60d579f6c..6c5564c4fae4 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-pkey.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-pkey.h
@@ -24,7 +24,7 @@ static inline u64 pte_to_hpte_pkey_bits(u64 pteflags, unsigned long flags)
((pteflags & H_PTE_PKEY_BIT1) ? HPTE_R_KEY_BIT1 : 0x0UL) |
((pteflags & H_PTE_PKEY_BIT0) ? HPTE_R_KEY_BIT0 : 0x0UL));
- if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP) ||
+ if (mmu_has_feature(MMU_FTR_KUAP) ||
mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
if ((pte_pkey == 0) && (flags & HPTE_USE_KERNEL_KEY))
return HASH_DEFAULT_KERNEL_KEY;
diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h
index 2a7bd3ecc556..72fc4263ed26 100644
--- a/arch/powerpc/include/asm/book3s/64/kup.h
+++ b/arch/powerpc/include/asm/book3s/64/kup.h
@@ -31,7 +31,7 @@
mfspr \gpr2, SPRN_AMR
cmpd \gpr1, \gpr2
beq 99f
- END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUAP, 68)
+ END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_KUAP, 68)
isync
mtspr SPRN_AMR, \gpr1
@@ -78,7 +78,7 @@
* No need to restore IAMR when returning to kernel space.
*/
100:
- END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 67)
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67)
#endif
.endm
@@ -91,7 +91,7 @@
LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED)
999: tdne \gpr1, \gpr2
EMIT_WARN_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE)
- END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 67)
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67)
#endif
.endm
#endif
@@ -130,7 +130,7 @@
*/
BEGIN_MMU_FTR_SECTION_NESTED(68)
b 100f // skip_save_amr
- END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY | MMU_FTR_BOOK3S_KUAP, 68)
+ END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY | MMU_FTR_KUAP, 68)
/*
* if pkey is disabled and we are entering from userspace
@@ -166,7 +166,7 @@
mtspr SPRN_AMR, \gpr2
isync
102:
- END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 69)
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 69)
/*
* if entering from kernel we don't need save IAMR
@@ -232,7 +232,7 @@ static inline u64 current_thread_iamr(void)
static __always_inline bool kuap_is_disabled(void)
{
- return !mmu_has_feature(MMU_FTR_BOOK3S_KUAP);
+ return !mmu_has_feature(MMU_FTR_KUAP);
}
static inline void kuap_user_restore(struct pt_regs *regs)
@@ -243,7 +243,7 @@ static inline void kuap_user_restore(struct pt_regs *regs)
if (!mmu_has_feature(MMU_FTR_PKEY))
return;
- if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
+ if (!mmu_has_feature(MMU_FTR_KUAP)) {
amr = mfspr(SPRN_AMR);
if (amr != regs->amr)
restore_amr = true;
@@ -317,7 +317,7 @@ static inline unsigned long get_kuap(void)
* This has no effect in terms of actually blocking things on hash,
* so it doesn't break anything.
*/
- if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
+ if (!mmu_has_feature(MMU_FTR_KUAP))
return AMR_KUAP_BLOCKED;
return mfspr(SPRN_AMR);
@@ -325,7 +325,7 @@ static inline unsigned long get_kuap(void)
static __always_inline void set_kuap(unsigned long value)
{
- if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
+ if (!mmu_has_feature(MMU_FTR_KUAP))
return;
/*
diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h
index 94b981152667..82af2e2c5eca 100644
--- a/arch/powerpc/include/asm/mmu.h
+++ b/arch/powerpc/include/asm/mmu.h
@@ -33,7 +33,7 @@
* key 0 controlling userspace addresses on radix
* Key 3 on hash
*/
-#define MMU_FTR_BOOK3S_KUAP ASM_CONST(0x00000200)
+#define MMU_FTR_KUAP ASM_CONST(0x00000200)
/*
* Supports KUEP feature
@@ -188,7 +188,7 @@ enum {
#endif /* CONFIG_PPC_RADIX_MMU */
#endif
#ifdef CONFIG_PPC_KUAP
- MMU_FTR_BOOK3S_KUAP |
+ MMU_FTR_KUAP |
#endif /* CONFIG_PPC_KUAP */
#ifdef CONFIG_PPC_MEM_KEYS
MMU_FTR_PKEY |
diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
index 18b9d325395f..77fedb190c93 100644
--- a/arch/powerpc/kernel/syscall.c
+++ b/arch/powerpc/kernel/syscall.c
@@ -46,7 +46,7 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
iamr = mfspr(SPRN_IAMR);
regs->amr = amr;
regs->iamr = iamr;
- if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
+ if (mmu_has_feature(MMU_FTR_KUAP)) {
mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
flush_needed = true;
}
diff --git a/arch/powerpc/mm/book3s64/pkeys.c b/arch/powerpc/mm/book3s64/pkeys.c
index 1d2675ab6711..125733962033 100644
--- a/arch/powerpc/mm/book3s64/pkeys.c
+++ b/arch/powerpc/mm/book3s64/pkeys.c
@@ -291,7 +291,7 @@ void setup_kuap(bool disabled)
if (smp_processor_id() == boot_cpuid) {
pr_info("Activating Kernel Userspace Access Prevention\n");
- cur_cpu_spec->mmu_features |= MMU_FTR_BOOK3S_KUAP;
+ cur_cpu_spec->mmu_features |= MMU_FTR_KUAP;
}
/*
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 6/9] powerpc/kuap: Use MMU_FTR_KUAP on all and refactor disabling kuap
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
` (4 preceding siblings ...)
2023-07-03 8:48 ` [PATCH v2 5/9] powerpc/kuap: MMU_FTR_BOOK3S_KUAP becomes MMU_FTR_KUAP Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 7/9] powerpc/kuap: Simplify KUAP lock/unlock on BOOK3S/32 Christophe Leroy
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
All but book3s/64 use a static branch key for disabling kuap.
book3s/64 uses an mmu feature.
Refactor all targets to use MMU_FTR_KUAP like book3s/64.
For PPC32 that implies updating mmu features fixups once KUAP
has been initialised.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 9 ---------
arch/powerpc/include/asm/book3s/64/kup.h | 5 -----
arch/powerpc/include/asm/kup.h | 11 +++++++++++
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 9 ---------
arch/powerpc/include/asm/nohash/kup-booke.h | 8 --------
arch/powerpc/kernel/cputable.c | 4 ++++
arch/powerpc/mm/book3s32/kuap.c | 5 +----
arch/powerpc/mm/init_32.c | 2 ++
arch/powerpc/mm/nohash/kup.c | 6 +-----
9 files changed, 19 insertions(+), 40 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 0da0dea76c47..4ca6122ef0e1 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -9,10 +9,6 @@
#ifndef __ASSEMBLY__
-#include <linux/jump_label.h>
-
-extern struct static_key_false disable_kuap_key;
-
#ifdef CONFIG_PPC_KUAP
#include <linux/sched.h>
@@ -20,11 +16,6 @@ extern struct static_key_false disable_kuap_key;
#define KUAP_NONE (~0UL)
#define KUAP_ALL (~1UL)
-static __always_inline bool kuap_is_disabled(void)
-{
- return static_branch_unlikely(&disable_kuap_key);
-}
-
static inline void kuap_lock_one(unsigned long addr)
{
mtsr(mfsr(addr) | SR_KS, addr);
diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h
index 72fc4263ed26..a014f4d9a2aa 100644
--- a/arch/powerpc/include/asm/book3s/64/kup.h
+++ b/arch/powerpc/include/asm/book3s/64/kup.h
@@ -230,11 +230,6 @@ static inline u64 current_thread_iamr(void)
#ifdef CONFIG_PPC_KUAP
-static __always_inline bool kuap_is_disabled(void)
-{
- return !mmu_has_feature(MMU_FTR_KUAP);
-}
-
static inline void kuap_user_restore(struct pt_regs *regs)
{
bool restore_amr = false, restore_iamr = false;
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index 24cde16c4fbe..bab161b609c1 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -6,6 +6,12 @@
#define KUAP_WRITE 2
#define KUAP_READ_WRITE (KUAP_READ | KUAP_WRITE)
+#ifndef __ASSEMBLY__
+#include <linux/types.h>
+
+static __always_inline bool kuap_is_disabled(void);
+#endif
+
#ifdef CONFIG_PPC_BOOK3S_64
#include <asm/book3s/64/kup.h>
#endif
@@ -41,6 +47,11 @@ void setup_kuep(bool disabled);
#ifdef CONFIG_PPC_KUAP
void setup_kuap(bool disabled);
+
+static __always_inline bool kuap_is_disabled(void)
+{
+ return !mmu_has_feature(MMU_FTR_KUAP);
+}
#else
static inline void setup_kuap(bool disabled) { }
diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
index a372cd822887..d0601859c45a 100644
--- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
@@ -9,17 +9,8 @@
#ifndef __ASSEMBLY__
-#include <linux/jump_label.h>
-
#include <asm/reg.h>
-extern struct static_key_false disable_kuap_key;
-
-static __always_inline bool kuap_is_disabled(void)
-{
- return static_branch_unlikely(&disable_kuap_key);
-}
-
static inline void __kuap_save_and_lock(struct pt_regs *regs)
{
regs->kuap = mfspr(SPRN_MD_AP);
diff --git a/arch/powerpc/include/asm/nohash/kup-booke.h b/arch/powerpc/include/asm/nohash/kup-booke.h
index 71182cbe20c3..8e4734c8fef1 100644
--- a/arch/powerpc/include/asm/nohash/kup-booke.h
+++ b/arch/powerpc/include/asm/nohash/kup-booke.h
@@ -13,18 +13,10 @@
#else
-#include <linux/jump_label.h>
#include <linux/sched.h>
#include <asm/reg.h>
-extern struct static_key_false disable_kuap_key;
-
-static __always_inline bool kuap_is_disabled(void)
-{
- return static_branch_unlikely(&disable_kuap_key);
-}
-
static inline void __kuap_lock(void)
{
mtspr(SPRN_PID, 0);
diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
index 8a32bffefa5b..e97a0fd0ae90 100644
--- a/arch/powerpc/kernel/cputable.c
+++ b/arch/powerpc/kernel/cputable.c
@@ -75,6 +75,10 @@ static struct cpu_spec * __init setup_cpu_spec(unsigned long offset,
t->cpu_features |= old.cpu_features & CPU_FTR_PMAO_BUG;
}
+ /* Set kuap ON at startup, will be disabled later if cmdline has 'nosmap' */
+ if (IS_ENABLED(CONFIG_PPC_KUAP) && IS_ENABLED(CONFIG_PPC32))
+ t->mmu_features |= MMU_FTR_KUAP;
+
*PTRRELOC(&cur_cpu_spec) = &the_cpu_spec;
/*
diff --git a/arch/powerpc/mm/book3s32/kuap.c b/arch/powerpc/mm/book3s32/kuap.c
index 28676cabb005..24c1c686e6b9 100644
--- a/arch/powerpc/mm/book3s32/kuap.c
+++ b/arch/powerpc/mm/book3s32/kuap.c
@@ -3,9 +3,6 @@
#include <asm/kup.h>
#include <asm/smp.h>
-struct static_key_false disable_kuap_key;
-EXPORT_SYMBOL(disable_kuap_key);
-
void kuap_lock_all_ool(void)
{
kuap_lock_all();
@@ -30,7 +27,7 @@ void setup_kuap(bool disabled)
return;
if (disabled)
- static_branch_enable(&disable_kuap_key);
+ cur_cpu_spec->mmu_features &= ~MMU_FTR_KUAP;
else
pr_info("Activating Kernel Userspace Access Protection\n");
}
diff --git a/arch/powerpc/mm/init_32.c b/arch/powerpc/mm/init_32.c
index d4cc3749e621..d8adc452f431 100644
--- a/arch/powerpc/mm/init_32.c
+++ b/arch/powerpc/mm/init_32.c
@@ -126,6 +126,8 @@ void __init MMU_init(void)
setup_kup();
+ update_mmu_feature_fixups(MMU_FTR_KUAP);
+
/* Shortly after that, the entire linear mapping will be available */
memblock_set_current_limit(lowmem_end_addr);
}
diff --git a/arch/powerpc/mm/nohash/kup.c b/arch/powerpc/mm/nohash/kup.c
index 552becf90e97..94ff82b9ae60 100644
--- a/arch/powerpc/mm/nohash/kup.c
+++ b/arch/powerpc/mm/nohash/kup.c
@@ -5,7 +5,6 @@
#include <linux/export.h>
#include <linux/init.h>
-#include <linux/jump_label.h>
#include <linux/printk.h>
#include <linux/smp.h>
@@ -13,16 +12,13 @@
#include <asm/smp.h>
#ifdef CONFIG_PPC_KUAP
-struct static_key_false disable_kuap_key;
-EXPORT_SYMBOL(disable_kuap_key);
-
void setup_kuap(bool disabled)
{
if (disabled) {
if (IS_ENABLED(CONFIG_40x))
disable_kuep = true;
if (smp_processor_id() == boot_cpuid)
- static_branch_enable(&disable_kuap_key);
+ cur_cpu_spec->mmu_features &= ~MMU_FTR_KUAP;
return;
}
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 7/9] powerpc/kuap: Simplify KUAP lock/unlock on BOOK3S/32
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
` (5 preceding siblings ...)
2023-07-03 8:48 ` [PATCH v2 6/9] powerpc/kuap: Use MMU_FTR_KUAP on all and refactor disabling kuap Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 8/9] powerpc/kuap: KUAP enabling/disabling functions must be __always_inline Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 9/9] powerpc/kuap: Use ASM feature fixups instead of static branches Christophe Leroy
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
On book3s/32 KUAP is performed at segment level. At the moment,
when enabling userspace access, only current segment is modified.
Then if a write is performed on another user segment, a fault is
taken and all other user segments get enabled for userspace
access. This then require special attention when disabling
userspace access.
Having a userspace write access crossing a segment boundary is
unlikely. Having a userspace write access crossing a segment boundary
back and forth is even more unlikely. So, instead of enabling
userspace access on all segments when a write fault occurs, just
change which segment has userspace access enabled in order to
eliminate the case when more than one segment has userspace access
enabled. That simplifies userspace access deactivation.
There is however a corner case which is even more unlikely but has
to be handled anyway: an unaligned access which is crossing a
segment boundary. That would definitely require at least having
userspace access enabled on the two segments. To avoid complicating
the likely case for a so unlikely happening, handle such situation
like an alignment exception and emulate the store.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 71 ++++++++----------------
arch/powerpc/include/asm/bug.h | 1 +
arch/powerpc/kernel/traps.c | 2 +-
arch/powerpc/mm/book3s32/kuap.c | 15 +----
4 files changed, 26 insertions(+), 63 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 4ca6122ef0e1..5d4f787244d5 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -14,55 +14,19 @@
#include <linux/sched.h>
#define KUAP_NONE (~0UL)
-#define KUAP_ALL (~1UL)
-static inline void kuap_lock_one(unsigned long addr)
+static inline void uaccess_end_32s(unsigned long addr)
{
mtsr(mfsr(addr) | SR_KS, addr);
isync(); /* Context sync required after mtsr() */
}
-static inline void kuap_unlock_one(unsigned long addr)
+static inline void uaccess_begin_32s(unsigned long addr)
{
mtsr(mfsr(addr) & ~SR_KS, addr);
isync(); /* Context sync required after mtsr() */
}
-static inline void kuap_lock_all(void)
-{
- update_user_segments(mfsr(0) | SR_KS);
- isync(); /* Context sync required after mtsr() */
-}
-
-static inline void kuap_unlock_all(void)
-{
- update_user_segments(mfsr(0) & ~SR_KS);
- isync(); /* Context sync required after mtsr() */
-}
-
-void kuap_lock_all_ool(void);
-void kuap_unlock_all_ool(void);
-
-static inline void kuap_lock_addr(unsigned long addr, bool ool)
-{
- if (likely(addr != KUAP_ALL))
- kuap_lock_one(addr);
- else if (!ool)
- kuap_lock_all();
- else
- kuap_lock_all_ool();
-}
-
-static inline void kuap_unlock(unsigned long addr, bool ool)
-{
- if (likely(addr != KUAP_ALL))
- kuap_unlock_one(addr);
- else if (!ool)
- kuap_unlock_all();
- else
- kuap_unlock_all_ool();
-}
-
static inline void __kuap_save_and_lock(struct pt_regs *regs)
{
unsigned long kuap = current->thread.kuap;
@@ -72,7 +36,7 @@ static inline void __kuap_save_and_lock(struct pt_regs *regs)
return;
current->thread.kuap = KUAP_NONE;
- kuap_lock_addr(kuap, false);
+ uaccess_end_32s(kuap);
}
#define __kuap_save_and_lock __kuap_save_and_lock
@@ -84,7 +48,7 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kua
{
if (unlikely(kuap != KUAP_NONE)) {
current->thread.kuap = KUAP_NONE;
- kuap_lock_addr(kuap, false);
+ uaccess_end_32s(kuap);
}
if (likely(regs->kuap == KUAP_NONE))
@@ -92,7 +56,7 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kua
current->thread.kuap = regs->kuap;
- kuap_unlock(regs->kuap, false);
+ uaccess_begin_32s(regs->kuap);
}
static inline unsigned long __kuap_get_and_assert_locked(void)
@@ -114,7 +78,7 @@ static __always_inline void __allow_user_access(void __user *to, const void __us
return;
current->thread.kuap = (__force u32)to;
- kuap_unlock_one((__force u32)to);
+ uaccess_begin_32s((__force u32)to);
}
static __always_inline void __prevent_user_access(unsigned long dir)
@@ -127,7 +91,7 @@ static __always_inline void __prevent_user_access(unsigned long dir)
return;
current->thread.kuap = KUAP_NONE;
- kuap_lock_addr(kuap, true);
+ uaccess_end_32s(kuap);
}
static inline unsigned long __prevent_user_access_return(void)
@@ -136,7 +100,7 @@ static inline unsigned long __prevent_user_access_return(void)
if (flags != KUAP_NONE) {
current->thread.kuap = KUAP_NONE;
- kuap_lock_addr(flags, true);
+ uaccess_end_32s(flags);
}
return flags;
@@ -146,7 +110,7 @@ static inline void __restore_user_access(unsigned long flags)
{
if (flags != KUAP_NONE) {
current->thread.kuap = flags;
- kuap_unlock(flags, true);
+ uaccess_begin_32s(flags);
}
}
@@ -155,14 +119,23 @@ __bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
{
unsigned long kuap = regs->kuap;
- if (!is_write || kuap == KUAP_ALL)
+ if (!is_write)
return false;
if (kuap == KUAP_NONE)
return true;
- /* If faulting address doesn't match unlocked segment, unlock all */
- if ((kuap ^ address) & 0xf0000000)
- regs->kuap = KUAP_ALL;
+ /*
+ * If faulting address doesn't match unlocked segment, change segment.
+ * In case of unaligned store crossing two segments, emulate store.
+ */
+ if ((kuap ^ address) & 0xf0000000) {
+ if (!(kuap & 0x0fffffff) && address > kuap - 4 && fix_alignment(regs)) {
+ regs_add_return_ip(regs, 4);
+ emulate_single_step(regs);
+ } else {
+ regs->kuap = address;
+ }
+ }
return false;
}
diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
index ef42adb44aa3..492530adecc2 100644
--- a/arch/powerpc/include/asm/bug.h
+++ b/arch/powerpc/include/asm/bug.h
@@ -163,6 +163,7 @@ __label_warn_on: \
struct pt_regs;
void hash__do_page_fault(struct pt_regs *);
void bad_page_fault(struct pt_regs *, int);
+void emulate_single_step(struct pt_regs *regs);
extern void _exception(int, struct pt_regs *, int, unsigned long);
extern void _exception_pkey(struct pt_regs *, unsigned long, int);
extern void die(const char *, struct pt_regs *, long);
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index e59ec6d32d37..ab95105c69ca 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -1158,7 +1158,7 @@ DEFINE_INTERRUPT_HANDLER(single_step_exception)
* pretend we got a single-step exception. This was pointed out
* by Kumar Gala. -- paulus
*/
-static void emulate_single_step(struct pt_regs *regs)
+void emulate_single_step(struct pt_regs *regs)
{
if (single_stepping(regs))
__single_step_exception(regs);
diff --git a/arch/powerpc/mm/book3s32/kuap.c b/arch/powerpc/mm/book3s32/kuap.c
index 24c1c686e6b9..3a8815555a48 100644
--- a/arch/powerpc/mm/book3s32/kuap.c
+++ b/arch/powerpc/mm/book3s32/kuap.c
@@ -3,22 +3,11 @@
#include <asm/kup.h>
#include <asm/smp.h>
-void kuap_lock_all_ool(void)
-{
- kuap_lock_all();
-}
-EXPORT_SYMBOL(kuap_lock_all_ool);
-
-void kuap_unlock_all_ool(void)
-{
- kuap_unlock_all();
-}
-EXPORT_SYMBOL(kuap_unlock_all_ool);
-
void setup_kuap(bool disabled)
{
if (!disabled) {
- kuap_lock_all_ool();
+ update_user_segments(mfsr(0) | SR_KS);
+ isync(); /* Context sync required after mtsr() */
init_mm.context.sr0 |= SR_KS;
current->thread.sr0 |= SR_KS;
}
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 8/9] powerpc/kuap: KUAP enabling/disabling functions must be __always_inline
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
` (6 preceding siblings ...)
2023-07-03 8:48 ` [PATCH v2 7/9] powerpc/kuap: Simplify KUAP lock/unlock on BOOK3S/32 Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
2023-07-03 8:48 ` [PATCH v2 9/9] powerpc/kuap: Use ASM feature fixups instead of static branches Christophe Leroy
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
Objtool reports following warnings:
arch/powerpc/kernel/signal_32.o: warning: objtool:
__prevent_user_access.constprop.0+0x4 (.text+0x4):
redundant UACCESS disable
arch/powerpc/kernel/signal_32.o: warning: objtool: user_access_begin+0x2c
(.text+0x4c): return with UACCESS enabled
arch/powerpc/kernel/signal_32.o: warning: objtool: handle_rt_signal32+0x188
(.text+0x360): call to __prevent_user_access.constprop.0() with UACCESS enabled
arch/powerpc/kernel/signal_32.o: warning: objtool: handle_signal32+0x150
(.text+0x4d4): call to __prevent_user_access.constprop.0() with UACCESS enabled
This is due to some KUAP enabling/disabling functions being outline
allthough they are marked inline. Use __always_inline instead.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 18 +++++++--------
arch/powerpc/include/asm/book3s/64/kup.h | 23 ++++++++++----------
arch/powerpc/include/asm/kup.h | 16 +++++++-------
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 20 ++++++++---------
arch/powerpc/include/asm/nohash/kup-booke.h | 22 +++++++++----------
arch/powerpc/include/asm/uaccess.h | 6 ++---
6 files changed, 53 insertions(+), 52 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 5d4f787244d5..f40a210bf1bb 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -15,19 +15,19 @@
#define KUAP_NONE (~0UL)
-static inline void uaccess_end_32s(unsigned long addr)
+static __always_inline void uaccess_end_32s(unsigned long addr)
{
mtsr(mfsr(addr) | SR_KS, addr);
isync(); /* Context sync required after mtsr() */
}
-static inline void uaccess_begin_32s(unsigned long addr)
+static __always_inline void uaccess_begin_32s(unsigned long addr)
{
mtsr(mfsr(addr) & ~SR_KS, addr);
isync(); /* Context sync required after mtsr() */
}
-static inline void __kuap_save_and_lock(struct pt_regs *regs)
+static __always_inline void __kuap_save_and_lock(struct pt_regs *regs)
{
unsigned long kuap = current->thread.kuap;
@@ -40,11 +40,11 @@ static inline void __kuap_save_and_lock(struct pt_regs *regs)
}
#define __kuap_save_and_lock __kuap_save_and_lock
-static inline void kuap_user_restore(struct pt_regs *regs)
+static __always_inline void kuap_user_restore(struct pt_regs *regs)
{
}
-static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
+static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
{
if (unlikely(kuap != KUAP_NONE)) {
current->thread.kuap = KUAP_NONE;
@@ -59,7 +59,7 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kua
uaccess_begin_32s(regs->kuap);
}
-static inline unsigned long __kuap_get_and_assert_locked(void)
+static __always_inline unsigned long __kuap_get_and_assert_locked(void)
{
unsigned long kuap = current->thread.kuap;
@@ -94,7 +94,7 @@ static __always_inline void __prevent_user_access(unsigned long dir)
uaccess_end_32s(kuap);
}
-static inline unsigned long __prevent_user_access_return(void)
+static __always_inline unsigned long __prevent_user_access_return(void)
{
unsigned long flags = current->thread.kuap;
@@ -106,7 +106,7 @@ static inline unsigned long __prevent_user_access_return(void)
return flags;
}
-static inline void __restore_user_access(unsigned long flags)
+static __always_inline void __restore_user_access(unsigned long flags)
{
if (flags != KUAP_NONE) {
current->thread.kuap = flags;
@@ -114,7 +114,7 @@ static inline void __restore_user_access(unsigned long flags)
}
}
-static inline bool
+static __always_inline bool
__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
{
unsigned long kuap = regs->kuap;
diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h
index a014f4d9a2aa..497a7bd31ecc 100644
--- a/arch/powerpc/include/asm/book3s/64/kup.h
+++ b/arch/powerpc/include/asm/book3s/64/kup.h
@@ -213,14 +213,14 @@ extern u64 __ro_after_init default_iamr;
* access restrictions. Because of this ignore AMR value when accessing
* userspace via kernel thread.
*/
-static inline u64 current_thread_amr(void)
+static __always_inline u64 current_thread_amr(void)
{
if (current->thread.regs)
return current->thread.regs->amr;
return default_amr;
}
-static inline u64 current_thread_iamr(void)
+static __always_inline u64 current_thread_iamr(void)
{
if (current->thread.regs)
return current->thread.regs->iamr;
@@ -230,7 +230,7 @@ static inline u64 current_thread_iamr(void)
#ifdef CONFIG_PPC_KUAP
-static inline void kuap_user_restore(struct pt_regs *regs)
+static __always_inline void kuap_user_restore(struct pt_regs *regs)
{
bool restore_amr = false, restore_iamr = false;
unsigned long amr, iamr;
@@ -269,7 +269,7 @@ static inline void kuap_user_restore(struct pt_regs *regs)
*/
}
-static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr)
+static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr)
{
if (likely(regs->amr == amr))
return;
@@ -285,7 +285,7 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr
*/
}
-static inline unsigned long __kuap_get_and_assert_locked(void)
+static __always_inline unsigned long __kuap_get_and_assert_locked(void)
{
unsigned long amr = mfspr(SPRN_AMR);
@@ -302,7 +302,7 @@ static inline unsigned long __kuap_get_and_assert_locked(void)
* because that would require an expensive read/modify write of the AMR.
*/
-static inline unsigned long get_kuap(void)
+static __always_inline unsigned long get_kuap(void)
{
/*
* We return AMR_KUAP_BLOCKED when we don't support KUAP because
@@ -332,7 +332,8 @@ static __always_inline void set_kuap(unsigned long value)
isync();
}
-static inline bool __bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
+static __always_inline bool
+__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
{
/*
* For radix this will be a storage protection fault (DSISR_PROTFAULT).
@@ -375,12 +376,12 @@ static __always_inline void allow_user_access(void __user *to, const void __user
#else /* CONFIG_PPC_KUAP */
-static inline unsigned long get_kuap(void)
+static __always_inline unsigned long get_kuap(void)
{
return AMR_KUAP_BLOCKED;
}
-static inline void set_kuap(unsigned long value) { }
+static __always_inline void set_kuap(unsigned long value) { }
static __always_inline void allow_user_access(void __user *to, const void __user *from,
unsigned long size, unsigned long dir)
@@ -395,7 +396,7 @@ static __always_inline void prevent_user_access(unsigned long dir)
do_uaccess_flush();
}
-static inline unsigned long prevent_user_access_return(void)
+static __always_inline unsigned long prevent_user_access_return(void)
{
unsigned long flags = get_kuap();
@@ -406,7 +407,7 @@ static inline unsigned long prevent_user_access_return(void)
return flags;
}
-static inline void restore_user_access(unsigned long flags)
+static __always_inline void restore_user_access(unsigned long flags)
{
set_kuap(flags);
if (static_branch_unlikely(&uaccess_flush_key) && flags == AMR_KUAP_BLOCKED)
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index bab161b609c1..77adb9cd2da5 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -57,14 +57,14 @@ static inline void setup_kuap(bool disabled) { }
static __always_inline bool kuap_is_disabled(void) { return true; }
-static inline bool
+static __always_inline bool
__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
{
return false;
}
-static inline void kuap_user_restore(struct pt_regs *regs) { }
-static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { }
+static __always_inline void kuap_user_restore(struct pt_regs *regs) { }
+static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { }
/*
* book3s/64/kup-radix.h defines these functions for the !KUAP case to flush
@@ -72,11 +72,11 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr
* platforms.
*/
#ifndef CONFIG_PPC_BOOK3S_64
-static inline void __allow_user_access(void __user *to, const void __user *from,
- unsigned long size, unsigned long dir) { }
-static inline void __prevent_user_access(unsigned long dir) { }
-static inline unsigned long __prevent_user_access_return(void) { return 0UL; }
-static inline void __restore_user_access(unsigned long flags) { }
+static __always_inline void __allow_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir) { }
+static __always_inline void __prevent_user_access(unsigned long dir) { }
+static __always_inline unsigned long __prevent_user_access_return(void) { return 0UL; }
+static __always_inline void __restore_user_access(unsigned long flags) { }
#endif /* CONFIG_PPC_BOOK3S_64 */
#endif /* CONFIG_PPC_KUAP */
diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
index d0601859c45a..e231b3afed98 100644
--- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
@@ -11,24 +11,24 @@
#include <asm/reg.h>
-static inline void __kuap_save_and_lock(struct pt_regs *regs)
+static __always_inline void __kuap_save_and_lock(struct pt_regs *regs)
{
regs->kuap = mfspr(SPRN_MD_AP);
mtspr(SPRN_MD_AP, MD_APG_KUAP);
}
#define __kuap_save_and_lock __kuap_save_and_lock
-static inline void kuap_user_restore(struct pt_regs *regs)
+static __always_inline void kuap_user_restore(struct pt_regs *regs)
{
}
-static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
+static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
{
mtspr(SPRN_MD_AP, regs->kuap);
}
#ifdef CONFIG_PPC_KUAP_DEBUG
-static inline unsigned long __kuap_get_and_assert_locked(void)
+static __always_inline unsigned long __kuap_get_and_assert_locked(void)
{
WARN_ON_ONCE(mfspr(SPRN_MD_AP) >> 16 != MD_APG_KUAP >> 16);
@@ -37,18 +37,18 @@ static inline unsigned long __kuap_get_and_assert_locked(void)
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
#endif
-static inline void __allow_user_access(void __user *to, const void __user *from,
- unsigned long size, unsigned long dir)
+static __always_inline void __allow_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir)
{
mtspr(SPRN_MD_AP, MD_APG_INIT);
}
-static inline void __prevent_user_access(unsigned long dir)
+static __always_inline void __prevent_user_access(unsigned long dir)
{
mtspr(SPRN_MD_AP, MD_APG_KUAP);
}
-static inline unsigned long __prevent_user_access_return(void)
+static __always_inline unsigned long __prevent_user_access_return(void)
{
unsigned long flags;
@@ -59,12 +59,12 @@ static inline unsigned long __prevent_user_access_return(void)
return flags;
}
-static inline void __restore_user_access(unsigned long flags)
+static __always_inline void __restore_user_access(unsigned long flags)
{
mtspr(SPRN_MD_AP, flags);
}
-static inline bool
+static __always_inline bool
__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
{
return !((regs->kuap ^ MD_APG_KUAP) & 0xff000000);
diff --git a/arch/powerpc/include/asm/nohash/kup-booke.h b/arch/powerpc/include/asm/nohash/kup-booke.h
index 8e4734c8fef1..98780a2d3dcd 100644
--- a/arch/powerpc/include/asm/nohash/kup-booke.h
+++ b/arch/powerpc/include/asm/nohash/kup-booke.h
@@ -17,14 +17,14 @@
#include <asm/reg.h>
-static inline void __kuap_lock(void)
+static __always_inline void __kuap_lock(void)
{
mtspr(SPRN_PID, 0);
isync();
}
#define __kuap_lock __kuap_lock
-static inline void __kuap_save_and_lock(struct pt_regs *regs)
+static __always_inline void __kuap_save_and_lock(struct pt_regs *regs)
{
regs->kuap = mfspr(SPRN_PID);
mtspr(SPRN_PID, 0);
@@ -32,7 +32,7 @@ static inline void __kuap_save_and_lock(struct pt_regs *regs)
}
#define __kuap_save_and_lock __kuap_save_and_lock
-static inline void kuap_user_restore(struct pt_regs *regs)
+static __always_inline void kuap_user_restore(struct pt_regs *regs)
{
if (kuap_is_disabled())
return;
@@ -42,7 +42,7 @@ static inline void kuap_user_restore(struct pt_regs *regs)
/* Context synchronisation is performed by rfi */
}
-static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
+static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
{
if (regs->kuap)
mtspr(SPRN_PID, current->thread.pid);
@@ -51,7 +51,7 @@ static inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long kua
}
#ifdef CONFIG_PPC_KUAP_DEBUG
-static inline unsigned long __kuap_get_and_assert_locked(void)
+static __always_inline unsigned long __kuap_get_and_assert_locked(void)
{
WARN_ON_ONCE(mfspr(SPRN_PID));
@@ -60,20 +60,20 @@ static inline unsigned long __kuap_get_and_assert_locked(void)
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
#endif
-static inline void __allow_user_access(void __user *to, const void __user *from,
- unsigned long size, unsigned long dir)
+static __always_inline void __allow_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir)
{
mtspr(SPRN_PID, current->thread.pid);
isync();
}
-static inline void __prevent_user_access(unsigned long dir)
+static __always_inline void __prevent_user_access(unsigned long dir)
{
mtspr(SPRN_PID, 0);
isync();
}
-static inline unsigned long __prevent_user_access_return(void)
+static __always_inline unsigned long __prevent_user_access_return(void)
{
unsigned long flags = mfspr(SPRN_PID);
@@ -83,7 +83,7 @@ static inline unsigned long __prevent_user_access_return(void)
return flags;
}
-static inline void __restore_user_access(unsigned long flags)
+static __always_inline void __restore_user_access(unsigned long flags)
{
if (flags) {
mtspr(SPRN_PID, current->thread.pid);
@@ -91,7 +91,7 @@ static inline void __restore_user_access(unsigned long flags)
}
}
-static inline bool
+static __always_inline bool
__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
{
return !regs->kuap;
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index a2d255aa9627..fb725ec77926 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -386,7 +386,7 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n)
extern long __copy_from_user_flushcache(void *dst, const void __user *src,
unsigned size);
-static __must_check inline bool user_access_begin(const void __user *ptr, size_t len)
+static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
{
if (unlikely(!access_ok(ptr, len)))
return false;
@@ -401,7 +401,7 @@ static __must_check inline bool user_access_begin(const void __user *ptr, size_t
#define user_access_save prevent_user_access_return
#define user_access_restore restore_user_access
-static __must_check inline bool
+static __must_check __always_inline bool
user_read_access_begin(const void __user *ptr, size_t len)
{
if (unlikely(!access_ok(ptr, len)))
@@ -415,7 +415,7 @@ user_read_access_begin(const void __user *ptr, size_t len)
#define user_read_access_begin user_read_access_begin
#define user_read_access_end prevent_current_read_from_user
-static __must_check inline bool
+static __must_check __always_inline bool
user_write_access_begin(const void __user *ptr, size_t len)
{
if (unlikely(!access_ok(ptr, len)))
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 9/9] powerpc/kuap: Use ASM feature fixups instead of static branches
2023-07-03 8:48 [PATCH v2 0/9] Cleanup/Optimise KUAP (v2) Christophe Leroy
` (7 preceding siblings ...)
2023-07-03 8:48 ` [PATCH v2 8/9] powerpc/kuap: KUAP enabling/disabling functions must be __always_inline Christophe Leroy
@ 2023-07-03 8:48 ` Christophe Leroy
8 siblings, 0 replies; 10+ messages in thread
From: Christophe Leroy @ 2023-07-03 8:48 UTC (permalink / raw)
To: Michael Ellerman, Nicholas Piggin
Cc: Christophe Leroy, linux-kernel, linuxppc-dev
To avoid a useless nop on top of every uaccess enable/disable and
make life easier for objtool, replace static branches by ASM feature
fixups that will nop KUAP enabling instructions out in the unlikely
case KUAP is disabled at boottime.
Leave it as is on book3s/64 for now, it will be handled later when
objtool is activated on PPC64.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
arch/powerpc/include/asm/book3s/32/kup.h | 34 +++++++++++----
arch/powerpc/include/asm/kup.h | 45 +++-----------------
arch/powerpc/include/asm/nohash/32/kup-8xx.h | 30 +++++++++----
arch/powerpc/include/asm/nohash/kup-booke.h | 38 ++++++++++-------
arch/powerpc/mm/nohash/kup.c | 2 +-
5 files changed, 75 insertions(+), 74 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index f40a210bf1bb..73d02815f52d 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -17,14 +17,30 @@
static __always_inline void uaccess_end_32s(unsigned long addr)
{
- mtsr(mfsr(addr) | SR_KS, addr);
- isync(); /* Context sync required after mtsr() */
+ unsigned long tmp;
+
+ asm volatile(ASM_MMU_FTR_IFSET(
+ "mfsrin %0, %1;"
+ "oris %0, %0, %2;"
+ "mtsrin %0, %1;"
+ "isync", "", %3)
+ : "=&r"(tmp)
+ : "r"(addr), "i"(SR_KS >> 16), "i"(MMU_FTR_KUAP)
+ : "memory");
}
static __always_inline void uaccess_begin_32s(unsigned long addr)
{
- mtsr(mfsr(addr) & ~SR_KS, addr);
- isync(); /* Context sync required after mtsr() */
+ unsigned long tmp;
+
+ asm volatile(ASM_MMU_FTR_IFSET(
+ "mfsrin %0, %1;"
+ "rlwinm %0, %0, 0, %2;"
+ "mtsrin %0, %1;"
+ "isync", "", %3)
+ : "=&r"(tmp)
+ : "r"(addr), "i"(~SR_KS), "i"(MMU_FTR_KUAP)
+ : "memory");
}
static __always_inline void __kuap_save_and_lock(struct pt_regs *regs)
@@ -69,8 +85,8 @@ static __always_inline unsigned long __kuap_get_and_assert_locked(void)
}
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
-static __always_inline void __allow_user_access(void __user *to, const void __user *from,
- u32 size, unsigned long dir)
+static __always_inline void allow_user_access(void __user *to, const void __user *from,
+ u32 size, unsigned long dir)
{
BUILD_BUG_ON(!__builtin_constant_p(dir));
@@ -81,7 +97,7 @@ static __always_inline void __allow_user_access(void __user *to, const void __us
uaccess_begin_32s((__force u32)to);
}
-static __always_inline void __prevent_user_access(unsigned long dir)
+static __always_inline void prevent_user_access(unsigned long dir)
{
u32 kuap = current->thread.kuap;
@@ -94,7 +110,7 @@ static __always_inline void __prevent_user_access(unsigned long dir)
uaccess_end_32s(kuap);
}
-static __always_inline unsigned long __prevent_user_access_return(void)
+static __always_inline unsigned long prevent_user_access_return(void)
{
unsigned long flags = current->thread.kuap;
@@ -106,7 +122,7 @@ static __always_inline unsigned long __prevent_user_access_return(void)
return flags;
}
-static __always_inline void __restore_user_access(unsigned long flags)
+static __always_inline void restore_user_access(unsigned long flags)
{
if (flags != KUAP_NONE) {
current->thread.kuap = flags;
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index 77adb9cd2da5..ad7e8c5aec3f 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -72,11 +72,11 @@ static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned
* platforms.
*/
#ifndef CONFIG_PPC_BOOK3S_64
-static __always_inline void __allow_user_access(void __user *to, const void __user *from,
- unsigned long size, unsigned long dir) { }
-static __always_inline void __prevent_user_access(unsigned long dir) { }
-static __always_inline unsigned long __prevent_user_access_return(void) { return 0UL; }
-static __always_inline void __restore_user_access(unsigned long flags) { }
+static __always_inline void allow_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir) { }
+static __always_inline void prevent_user_access(unsigned long dir) { }
+static __always_inline unsigned long prevent_user_access_return(void) { return 0UL; }
+static __always_inline void restore_user_access(unsigned long flags) { }
#endif /* CONFIG_PPC_BOOK3S_64 */
#endif /* CONFIG_PPC_KUAP */
@@ -132,41 +132,6 @@ static __always_inline void kuap_assert_locked(void)
kuap_get_and_assert_locked();
}
-#ifndef CONFIG_PPC_BOOK3S_64
-static __always_inline void allow_user_access(void __user *to, const void __user *from,
- unsigned long size, unsigned long dir)
-{
- if (kuap_is_disabled())
- return;
-
- __allow_user_access(to, from, size, dir);
-}
-
-static __always_inline void prevent_user_access(unsigned long dir)
-{
- if (kuap_is_disabled())
- return;
-
- __prevent_user_access(dir);
-}
-
-static __always_inline unsigned long prevent_user_access_return(void)
-{
- if (kuap_is_disabled())
- return 0;
-
- return __prevent_user_access_return();
-}
-
-static __always_inline void restore_user_access(unsigned long flags)
-{
- if (kuap_is_disabled())
- return;
-
- __restore_user_access(flags);
-}
-#endif /* CONFIG_PPC_BOOK3S_64 */
-
static __always_inline void allow_read_from_user(const void __user *from, unsigned long size)
{
barrier_nospec();
diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
index e231b3afed98..46bc5925e5fd 100644
--- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h
@@ -37,31 +37,43 @@ static __always_inline unsigned long __kuap_get_and_assert_locked(void)
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
#endif
-static __always_inline void __allow_user_access(void __user *to, const void __user *from,
- unsigned long size, unsigned long dir)
+static __always_inline void uaccess_begin_8xx(unsigned long val)
{
- mtspr(SPRN_MD_AP, MD_APG_INIT);
+ asm(ASM_MMU_FTR_IFSET("mtspr %0, %1", "", %2) : :
+ "i"(SPRN_MD_AP), "r"(val), "i"(MMU_FTR_KUAP) : "memory");
}
-static __always_inline void __prevent_user_access(unsigned long dir)
+static __always_inline void uaccess_end_8xx(void)
{
- mtspr(SPRN_MD_AP, MD_APG_KUAP);
+ asm(ASM_MMU_FTR_IFSET("mtspr %0, %1", "", %2) : :
+ "i"(SPRN_MD_AP), "r"(MD_APG_KUAP), "i"(MMU_FTR_KUAP) : "memory");
+}
+
+static __always_inline void allow_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir)
+{
+ uaccess_begin_8xx(MD_APG_INIT);
}
-static __always_inline unsigned long __prevent_user_access_return(void)
+static __always_inline void prevent_user_access(unsigned long dir)
+{
+ uaccess_end_8xx();
+}
+
+static __always_inline unsigned long prevent_user_access_return(void)
{
unsigned long flags;
flags = mfspr(SPRN_MD_AP);
- mtspr(SPRN_MD_AP, MD_APG_KUAP);
+ uaccess_end_8xx();
return flags;
}
-static __always_inline void __restore_user_access(unsigned long flags)
+static __always_inline void restore_user_access(unsigned long flags)
{
- mtspr(SPRN_MD_AP, flags);
+ uaccess_begin_8xx(flags);
}
static __always_inline bool
diff --git a/arch/powerpc/include/asm/nohash/kup-booke.h b/arch/powerpc/include/asm/nohash/kup-booke.h
index 98780a2d3dcd..0c7c3258134c 100644
--- a/arch/powerpc/include/asm/nohash/kup-booke.h
+++ b/arch/powerpc/include/asm/nohash/kup-booke.h
@@ -3,6 +3,7 @@
#define _ASM_POWERPC_KUP_BOOKE_H_
#include <asm/bug.h>
+#include <asm/mmu.h>
#ifdef CONFIG_PPC_KUAP
@@ -60,35 +61,42 @@ static __always_inline unsigned long __kuap_get_and_assert_locked(void)
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
#endif
-static __always_inline void __allow_user_access(void __user *to, const void __user *from,
- unsigned long size, unsigned long dir)
+static __always_inline void uaccess_begin_booke(unsigned long val)
{
- mtspr(SPRN_PID, current->thread.pid);
- isync();
+ asm(ASM_MMU_FTR_IFSET("mtspr %0, %1; isync", "", %2) : :
+ "i"(SPRN_PID), "r"(val), "i"(MMU_FTR_KUAP) : "memory");
}
-static __always_inline void __prevent_user_access(unsigned long dir)
+static __always_inline void uaccess_end_booke(void)
{
- mtspr(SPRN_PID, 0);
- isync();
+ asm(ASM_MMU_FTR_IFSET("mtspr %0, %1; isync", "", %2) : :
+ "i"(SPRN_PID), "r"(0), "i"(MMU_FTR_KUAP) : "memory");
+}
+
+static __always_inline void allow_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir)
+{
+ uaccess_begin_booke(current->thread.pid);
}
-static __always_inline unsigned long __prevent_user_access_return(void)
+static __always_inline void prevent_user_access(unsigned long dir)
+{
+ uaccess_end_booke();
+}
+
+static __always_inline unsigned long prevent_user_access_return(void)
{
unsigned long flags = mfspr(SPRN_PID);
- mtspr(SPRN_PID, 0);
- isync();
+ uaccess_end_booke();
return flags;
}
-static __always_inline void __restore_user_access(unsigned long flags)
+static __always_inline void restore_user_access(unsigned long flags)
{
- if (flags) {
- mtspr(SPRN_PID, current->thread.pid);
- isync();
- }
+ if (flags)
+ uaccess_begin_booke(current->thread.pid);
}
static __always_inline bool
diff --git a/arch/powerpc/mm/nohash/kup.c b/arch/powerpc/mm/nohash/kup.c
index 94ff82b9ae60..e1f7de2e54ec 100644
--- a/arch/powerpc/mm/nohash/kup.c
+++ b/arch/powerpc/mm/nohash/kup.c
@@ -24,6 +24,6 @@ void setup_kuap(bool disabled)
pr_info("Activating Kernel Userspace Access Protection\n");
- __prevent_user_access(KUAP_READ_WRITE);
+ prevent_user_access(KUAP_READ_WRITE);
}
#endif
--
2.41.0
^ permalink raw reply related [flat|nested] 10+ messages in thread