From: Will Deacon <will@kernel.org>
To: linux-arm-kernel@lists.infradead.org
Cc: Mark Rutland <mark.rutland@arm.com>,
Vladimir Murzin <vladimir.murzin@arm.com>,
Arnd Bergmann <arnd@arndb.de>,
Catalin Marinas <catalin.marinas@arm.com>,
Linus Walleij <linusw@kernel.org>,
linux-kernel@vger.kernel.org, Mostafa Saleh <smostafa@google.com>,
Marc Zyngier <maz@kernel.org>,
David Hildenbrand <david@kernel.org>,
Lorenzo Stoakes <ljs@kernel.org>,
Oliver Upton <oupton@kernel.org>, Will Deacon <will@kernel.org>,
Ard Biesheuvel <ardb@kernel.org>
Subject: [PATCH v2 19/21] arm64: entry: The great stack switcheroo
Date: Fri, 18 Sep 2026 17:14:03 +0100 [thread overview]
Message-ID: <20260918161407.2300-20-will@kernel.org> (raw)
In-Reply-To: <20260918161407.2300-1-will@kernel.org>
With the kernel stack pointer in SP_EL1 and the overflow stack pointer
in SP_EL0, it is now straightforward to switch between the two on
exception entry from EL1 by writing to SPSel. However, since exception
entry sets PSTATE.SP to 1 (selecting SP_EL1 as the stack pointer),
repurposing the overflow stack as a more general kernel exception stack
would require writing to SPSel on every exception entry from the kernel.
Switch things around so that the overflow stack resides in SP_EL1, with
the kernel stack residing in SP_EL0.
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/kvm_asm.h | 16 +++++++++++++++
arch/arm64/kernel/entry-common.c | 34 ++++++++++++++++----------------
arch/arm64/kernel/entry.S | 26 +++++++++++++-----------
arch/arm64/kernel/head.S | 5 ++++-
arch/arm64/kernel/sleep.S | 3 ++-
arch/arm64/kernel/smp.c | 8 +++++---
arch/arm64/kvm/hyp/entry.S | 3 +++
arch/arm64/mm/proc.S | 10 ++++++++--
8 files changed, 70 insertions(+), 35 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 1cfe9216a2ff..bde046c3ca79 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -398,6 +398,22 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, u64 elr_virt,
ldp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
.endm
+#ifdef __KVM_VHE_HYPERVISOR__
+.macro activate_exception_stack
+msr spsel, #1
+.endm
+
+.macro deactivate_exception_stack
+msr spsel, #0
+.endm
+#else
+.macro activate_exception_stack
+.endm
+
+.macro deactivate_exception_stack
+.endm
+#endif
+
.macro save_sp_el0 ctxt, tmp
mrs \tmp, sp_el0
str \tmp, [\ctxt, #CPU_SP_EL0_OFFSET]
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 9738142780df..9d734cd09f62 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -327,7 +327,7 @@ static void debug_exception_exit(struct pt_regs *regs)
}
NOKPROBE_SYMBOL(debug_exception_exit);
-static void noinstr el1t_64_check_overflow_stack(struct pt_regs *regs)
+static void noinstr el1h_64_check_overflow_stack(struct pt_regs *regs)
{
unsigned long sp = kernel_stack_pointer(regs) - sizeof(*regs);
unsigned long ovf_stack = (unsigned long)this_cpu_ptr(overflow_stack);
@@ -343,28 +343,28 @@ static void noinstr el1t_64_check_overflow_stack(struct pt_regs *regs)
cpu_park_loop();
}
-asmlinkage void noinstr el1t_64_sync_handler(struct pt_regs *regs)
+asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
{
- el1t_64_check_overflow_stack(regs);
- el1h_64_sync_handler(regs);
+ el1h_64_check_overflow_stack(regs);
+ el1t_64_sync_handler(regs);
}
-asmlinkage void noinstr el1t_64_irq_handler(struct pt_regs *regs)
+asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
{
- el1t_64_check_overflow_stack(regs);
- el1h_64_irq_handler(regs);
+ el1h_64_check_overflow_stack(regs);
+ el1t_64_irq_handler(regs);
}
-asmlinkage void noinstr el1t_64_fiq_handler(struct pt_regs *regs)
+asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
{
- el1t_64_check_overflow_stack(regs);
- el1h_64_fiq_handler(regs);
+ el1h_64_check_overflow_stack(regs);
+ el1t_64_fiq_handler(regs);
}
-asmlinkage void noinstr el1t_64_error_handler(struct pt_regs *regs)
+asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
{
- el1t_64_check_overflow_stack(regs);
- el1h_64_error_handler(regs);
+ el1h_64_check_overflow_stack(regs);
+ el1t_64_error_handler(regs);
}
static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
@@ -494,7 +494,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
arm64_exit_to_kernel_mode(regs, state);
}
-asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
+asmlinkage void noinstr el1t_64_sync_handler(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
@@ -578,17 +578,17 @@ static void noinstr el1_interrupt(struct pt_regs *regs,
__el1_irq(regs, handler);
}
-asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
+asmlinkage void noinstr el1t_64_irq_handler(struct pt_regs *regs)
{
el1_interrupt(regs, handle_arch_irq);
}
-asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
+asmlinkage void noinstr el1t_64_fiq_handler(struct pt_regs *regs)
{
el1_interrupt(regs, handle_arch_fiq);
}
-asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
+asmlinkage void noinstr el1t_64_error_handler(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
irqentry_state_t state;
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 38f9327e6a0a..afcd84510daf 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -54,6 +54,12 @@
.macro kernel_ventry_el1h, regsize:req, label:req
sub sp, sp, #PT_REGS_SIZE
+ b el1h_\regsize\()_\label
+ .endm
+
+ .macro kernel_ventry_el1t, regsize:req, label:req
+ msr spsel, #0 // Stay on the kernel stack
+ sub sp, sp, #PT_REGS_SIZE
/*
* Test whether the SP has overflowed, without corrupting a GPR.
@@ -65,12 +71,6 @@
tbnz x0, #THREAD_SHIFT, __bad_stack
sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
- b el1h_\regsize\()_\label
- .endm
-
- .macro kernel_ventry_el1t, regsize:req, label:req
- msr spsel, #0 // Stay on the overflow stack
- sub sp, sp, #PT_REGS_SIZE
b el1t_\regsize\()_\label
.endm
@@ -208,7 +208,10 @@ alternative_cb_end
ldr_this_cpu tsk, __entry_task, x20
msr tpidrro_el0, tsk
adr_this_cpu x19, overflow_stack + OVERFLOW_STACK_SIZE, x20
- msr sp_el0, x19
+ mov x20, sp
+ msr sp_el0, x20
+ mov sp, x19
+ msr spsel, #0
/*
* Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions
@@ -346,6 +349,9 @@ alternative_else_nop_endif
.if \el == 0
ldr x23, [sp, #S_SP] // load return stack pointer
+ mov x0, sp
+ msr spsel, #1
+ mov sp, x0
msr sp_el0, x23
msr tpidrro_el0, xzr
tst x22, #PSR_MODE32_BIT // native task?
@@ -532,16 +538,14 @@ SYM_CODE_START_LOCAL(__bad_stack)
add sp, sp, #PT_REGS_SIZE
/* Switch to the overflow stack */
- msr spsel, #0
+ msr spsel, #1
/* Stash the exception regs */
sub sp, sp, #PT_REGS_SIZE
kernel_entry 1
/* Fix-up the saved SP */
- msr spsel, #1
- mov x0, sp
- msr spsel, #0
+ mrs x0, sp_el0
str x0, [sp, #S_SP]
/* Stash the regs for handle_bad_stack */
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 27a19e1a0ee6..7419e1d8b7ec 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -211,8 +211,11 @@ SYM_CODE_END(preserve_boot_args)
ldr \tmp1, [\tmp1, \tmp2, lsl #3]
set_this_cpu_offset \tmp1
- adr_this_cpu \tmp1, overflow_stack + OVERFLOW_STACK_SIZE, \tmp2
+ mov \tmp1, sp
msr sp_el0, \tmp1
+ adr_this_cpu \tmp1, overflow_stack + OVERFLOW_STACK_SIZE, \tmp2
+ mov sp, \tmp1
+ msr spsel, #0
.endm
/*
diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index f093cdf71be1..facf3f1cc3b1 100644
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -133,7 +133,8 @@ SYM_FUNC_START(_cpu_resume)
add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
/* load sp from context */
ldr x2, [x0, #CPU_CTX_SP]
- mov sp, x2
+ msr sp_el0, x2
+
/*
* cpu_do_resume expects x0 to contain context address pointer
*/
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 4c122d6598de..b2e778aae881 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -447,14 +447,16 @@ void __init smp_cpus_done(unsigned int max_cpus)
mark_linear_text_alias_ro();
}
-static void __init set_boot_cpu_offset(void)
+static void __init update_boot_cpu_offset_and_overflow_stack(void)
{
u64 ovf_sp = (u64)raw_cpu_ptr(overflow_stack) + OVERFLOW_STACK_SIZE;
asm volatile(
" msr tpidr_el1, %1\n"
" add %0, %0, %1\n"
- " msr sp_el0, %0" /* Update the overflow stack pointer */
+ " msr spsel, #1\n"
+ " mov sp, %0\n" /* Update the overflow stack pointer */
+ " msr spsel, #0"
: "+r" (ovf_sp)
: "r" (per_cpu_offset(0))
: "memory");
@@ -467,7 +469,7 @@ void __init smp_prepare_boot_cpu(void)
* setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be
* freed shortly, so we must move over to the runtime per-cpu area.
*/
- set_boot_cpu_offset();
+ update_boot_cpu_offset_and_overflow_stack();
cpuinfo_store_boot_cpu();
setup_boot_cpu_features();
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index 4c89931a6a92..4d1205d04383 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -31,6 +31,7 @@ SYM_FUNC_START(__guest_enter)
save_callee_saved_regs x1
// Save hyp's sp_el0 and tpidrro_el0
+ activate_exception_stack
save_sp_el0 x1, x2
save_tpidrro_el0 x1, x2
@@ -50,6 +51,7 @@ alternative_else_nop_endif
// that would usually be synchonized by the ERET.
isb
mov x0, #ARM_EXCEPTION_IRQ
+ deactivate_exception_stack
ret
1:
@@ -167,6 +169,7 @@ SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL)
// Restore hyp's sp_el0 and tpidrro_el0
restore_sp_el0 x2, x3
restore_tpidrro_el0 x2, x3
+ deactivate_exception_stack
// Now restore the hyp regs
restore_callee_saved_regs x2
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index ad4965d34978..bec7f858fae9 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -87,6 +87,7 @@
* This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
*/
SYM_FUNC_START(cpu_do_suspend)
+ msr spsel, #1
mrs x2, tpidr_el0
mrs x3, tpidrro_el0
mrs x4, contextidr_el1
@@ -98,7 +99,7 @@ SYM_FUNC_START(cpu_do_suspend)
mrs x10, oslsr_el1
mrs x11, sctlr_el1
get_this_cpu_offset x12
- mrs x13, sp_el0
+ mov x13, sp // SP_EL1
stp x2, x3, [x0]
stp x4, x5, [x0, #16]
stp x6, x7, [x0, #32]
@@ -114,6 +115,7 @@ alternative_if ARM64_HAS_TCR2
mrs x2, REG_TCR2_EL1
str x2, [x0, #104]
alternative_else_nop_endif
+ msr spsel, #0
ret
SYM_FUNC_END(cpu_do_suspend)
@@ -121,6 +123,8 @@ SYM_FUNC_END(cpu_do_suspend)
* cpu_do_resume - restore CPU register context
*
* x0: Address of context pointer
+ *
+ * Entered with SPSel == 1, returns with SPSel == 0.
*/
SYM_FUNC_START(cpu_do_resume)
ldp x2, x3, [x0]
@@ -159,7 +163,8 @@ alternative_else_nop_endif
msr sctlr_el1, x12
set_this_cpu_offset x13
- msr sp_el0, x14
+ mov sp, x14 // SP_EL1
+
/*
* Restore oslsr_el1 by writing oslar_el1
*/
@@ -182,6 +187,7 @@ alternative_else_nop_endif
ptrauth_keys_install_kernel_nosync x15, x1, x2, x3
isb
+ msr spsel, #0
ret
SYM_FUNC_END(cpu_do_resume)
#endif
--
2.55.0.1082.g2b9226bbc0-goog
next prev parent reply other threads:[~2026-09-18 16:15 UTC|newest]
Thread overview: 65+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-18 16:13 [PATCH v2 00/21] arm64: Move overflow sp into SP_EL1 and kernel sp into SP_EL0 Will Deacon
2026-09-18 16:13 ` [PATCH v2 01/21] arm64: entry: Defer setting of TPIDRRO_EL0 until exit to userspace Will Deacon
2026-09-30 20:47 ` Linus Walleij
2026-09-18 16:13 ` [PATCH v2 02/21] arm64: entry: Only check for stack overflow on exceptions from EL1 Will Deacon
2026-09-30 20:50 ` Linus Walleij
2026-10-08 6:54 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 03/21] arm64: stackprotector: Temporarily disable per-task stackprotector Will Deacon
2026-09-30 21:01 ` Linus Walleij
2026-09-18 16:13 ` [PATCH v2 04/21] arm64: bpf: Add support for generating reads of TPIDRRO_EL0 Will Deacon
2026-09-30 20:53 ` Linus Walleij
2026-10-08 7:07 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 05/21] KVM: arm64: Protect TPIDRRO_EL0 across guest entry/exit Will Deacon
2026-09-30 20:55 ` Linus Walleij
2026-10-08 7:45 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 06/21] arm64: Store 'current' in TPIDRRO_EL0 instead of SP_EL0 Will Deacon
2026-09-30 20:58 ` Linus Walleij
2026-10-08 7:29 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 07/21] selftests/bpf: arm64: Use TPIDRRO_EL0 instead of SP_EL0 for 'current' Will Deacon
2026-09-30 20:58 ` Linus Walleij
2026-10-08 7:55 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 08/21] scripts/gdb: " Will Deacon
2026-09-30 20:59 ` Linus Walleij
2026-10-08 7:56 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 09/21] arm64: stackprotector: Re-enable per-task stackprotector Will Deacon
2026-09-30 21:00 ` Linus Walleij
2026-10-08 7:58 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 10/21] arm64: percpu: Specialise set_my_cpu_offset() for the primary CPU Will Deacon
2026-09-30 21:02 ` Linus Walleij
2026-10-08 8:12 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 11/21] arm64: percpu: Annotate __kern_my_cpu_offset() as '__always_inline' Will Deacon
2026-09-30 21:05 ` Linus Walleij
2026-09-30 21:05 ` Linus Walleij
2026-10-08 8:14 ` Jinjie Ruan
2026-09-18 16:13 ` [PATCH v2 12/21] KVM: arm64: Preserve handler/thread bit of EL1 mode in __finalise_el2() Will Deacon
2026-09-18 16:13 ` [PATCH v2 13/21] arm64: sdei: Guard most of asm/sdei.h with CONFIG_ARM_SDE_INTERFACE Will Deacon
2026-09-18 16:13 ` [PATCH v2 14/21] arm64: sdei: Support SDEI events from kernel handler and thread modes Will Deacon
2026-09-18 16:13 ` [PATCH v2 15/21] arm64: entry: Point SP_EL0 at the overflow stack Will Deacon
2026-09-21 15:56 ` Catalin Marinas
2026-10-08 11:22 ` Jinjie Ruan
2026-09-18 16:14 ` [PATCH v2 16/21] arm64: entry: Implement EL1t exception handlers for " Will Deacon
2026-09-30 21:17 ` Linus Walleij
2026-10-08 11:47 ` Jinjie Ruan
2026-09-18 16:14 ` [PATCH v2 17/21] arm64: entry: Use SPSel to switch to " Will Deacon
2026-09-30 21:22 ` Linus Walleij
2026-10-08 11:59 ` Jinjie Ruan
2026-09-18 16:14 ` [PATCH v2 18/21] arm64: entry: Split up kernel_ventry macro into separate helper macros Will Deacon
2026-09-30 21:26 ` Linus Walleij
2026-10-08 8:53 ` Jinjie Ruan
2026-09-18 16:14 ` Will Deacon [this message]
2026-09-19 11:41 ` [PATCH v2 19/21] arm64: entry: The great stack switcheroo Bradley Morgan
2026-09-22 13:51 ` Catalin Marinas
2026-09-30 21:36 ` Linus Walleij
2026-10-01 7:27 ` Will Deacon
2026-10-01 8:38 ` Linus Walleij
2026-10-01 11:32 ` Will Deacon
2026-10-01 11:35 ` Linus Walleij
2026-09-18 16:14 ` [PATCH v2 20/21] arm64: tracing: Advertise a mode of EL1t in synthetic kernel regs Will Deacon
2026-09-22 14:00 ` Catalin Marinas
2026-09-18 16:14 ` [PATCH v2 21/21] arm64: Rename 'overflow_stack' and OVERFLOW_STACK_SIZE Will Deacon
2026-09-22 14:01 ` Catalin Marinas
2026-09-24 12:30 ` [PATCH v2 00/21] arm64: Move overflow sp into SP_EL1 and kernel sp into SP_EL0 Usama Anjum
2026-09-30 21:42 ` Linus Walleij
2026-10-01 7:21 ` Will Deacon
2026-10-01 9:00 ` Muhammad Usama Anjum
2026-10-01 21:19 ` Linus Walleij
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260918161407.2300-20-will@kernel.org \
--to=will@kernel.org \
--cc=ardb@kernel.org \
--cc=arnd@arndb.de \
--cc=catalin.marinas@arm.com \
--cc=david@kernel.org \
--cc=linusw@kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=ljs@kernel.org \
--cc=mark.rutland@arm.com \
--cc=maz@kernel.org \
--cc=oupton@kernel.org \
--cc=smostafa@google.com \
--cc=vladimir.murzin@arm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.