From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
To: linux-kernel@vger.kernel.org
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
stable@vger.kernel.org, Catalin Marinas <catalin.marinas@arm.com>,
James Morse <james.morse@arm.com>
Subject: [PATCH 4.14 19/27] arm64: entry: Add vectors that have the bhb mitigation sequences
Date: Fri, 1 Apr 2022 08:36:29 +0200 [thread overview]
Message-ID: <20220401063624.777642682@linuxfoundation.org> (raw)
In-Reply-To: <20220401063624.232282121@linuxfoundation.org>
From: James Morse <james.morse@arm.com>
commit ba2689234be92024e5635d30fe744f4853ad97db upstream.
Some CPUs affected by Spectre-BHB need a sequence of branches, or a
firmware call to be run before any indirect branch. This needs to go
in the vectors. No CPU needs both.
While this can be patched in, it would run on all CPUs as there is a
single set of vectors. If only one part of a big/little combination is
affected, the unaffected CPUs have to run the mitigation too.
Create extra vectors that include the sequence. Subsequent patches will
allow affected CPUs to select this set of vectors. Later patches will
modify the loop count to match what the CPU requires.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
arch/arm64/include/asm/assembler.h | 25 +++++++++++++++++
arch/arm64/include/asm/vectors.h | 34 +++++++++++++++++++++++
arch/arm64/kernel/entry.S | 53 ++++++++++++++++++++++++++++++-------
include/linux/arm-smccc.h | 7 ++++
4 files changed, 110 insertions(+), 9 deletions(-)
create mode 100644 arch/arm64/include/asm/vectors.h
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -549,4 +549,29 @@ alternative_endif
.Ldone\@:
.endm
+ .macro __mitigate_spectre_bhb_loop tmp
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+ mov \tmp, #32
+.Lspectre_bhb_loop\@:
+ b . + 4
+ subs \tmp, \tmp, #1
+ b.ne .Lspectre_bhb_loop\@
+ dsb nsh
+ isb
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+ .endm
+
+ /* Save/restores x0-x3 to the stack */
+ .macro __mitigate_spectre_bhb_fw
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+ stp x0, x1, [sp, #-16]!
+ stp x2, x3, [sp, #-16]!
+ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3
+alternative_cb arm64_update_smccc_conduit
+ nop // Patched to SMC/HVC #0
+alternative_cb_end
+ ldp x2, x3, [sp], #16
+ ldp x0, x1, [sp], #16
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+ .endm
#endif /* __ASM_ASSEMBLER_H */
--- /dev/null
+++ b/arch/arm64/include/asm/vectors.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2022 ARM Ltd.
+ */
+#ifndef __ASM_VECTORS_H
+#define __ASM_VECTORS_H
+
+/*
+ * Note: the order of this enum corresponds to two arrays in entry.S:
+ * tramp_vecs and __bp_harden_el1_vectors. By default the canonical
+ * 'full fat' vectors are used directly.
+ */
+enum arm64_bp_harden_el1_vectors {
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+ /*
+ * Perform the BHB loop mitigation, before branching to the canonical
+ * vectors.
+ */
+ EL1_VECTOR_BHB_LOOP,
+
+ /*
+ * Make the SMC call for firmware mitigation, before branching to the
+ * canonical vectors.
+ */
+ EL1_VECTOR_BHB_FW,
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+
+ /*
+ * Remap the kernel before branching to the canonical vectors.
+ */
+ EL1_VECTOR_KPTI,
+};
+
+#endif /* __ASM_VECTORS_H */
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -1022,13 +1022,26 @@ alternative_else_nop_endif
sub \dst, \dst, PAGE_SIZE
.endm
- .macro tramp_ventry, vector_start, regsize, kpti
+
+#define BHB_MITIGATION_NONE 0
+#define BHB_MITIGATION_LOOP 1
+#define BHB_MITIGATION_FW 2
+
+ .macro tramp_ventry, vector_start, regsize, kpti, bhb
.align 7
1:
.if \regsize == 64
msr tpidrro_el0, x30 // Restored in kernel_ventry
.endif
+ .if \bhb == BHB_MITIGATION_LOOP
+ /*
+ * This sequence must appear before the first indirect branch. i.e. the
+ * ret out of tramp_ventry. It appears here because x30 is free.
+ */
+ __mitigate_spectre_bhb_loop x30
+ .endif // \bhb == BHB_MITIGATION_LOOP
+
.if \kpti == 1
/*
* Defend against branch aliasing attacks by pushing a dummy
@@ -1053,6 +1066,15 @@ alternative_insn isb, nop, ARM64_WORKARO
ldr x30, =vectors
.endif // \kpti == 1
+ .if \bhb == BHB_MITIGATION_FW
+ /*
+ * The firmware sequence must appear before the first indirect branch.
+ * i.e. the ret out of tramp_ventry. But it also needs the stack to be
+ * mapped to save/restore the registers the SMC clobbers.
+ */
+ __mitigate_spectre_bhb_fw
+ .endif // \bhb == BHB_MITIGATION_FW
+
add x30, x30, #(1b - \vector_start + 4)
ret
.org 1b + 128 // Did we overflow the ventry slot?
@@ -1060,6 +1082,9 @@ alternative_insn isb, nop, ARM64_WORKARO
.macro tramp_exit, regsize = 64
adr x30, tramp_vectors
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+ add x30, x30, SZ_4K
+#endif
msr vbar_el1, x30
ldr lr, [sp, #S_LR]
tramp_unmap_kernel x29
@@ -1070,26 +1095,32 @@ alternative_insn isb, nop, ARM64_WORKARO
eret
.endm
- .macro generate_tramp_vector, kpti
+ .macro generate_tramp_vector, kpti, bhb
.Lvector_start\@:
.space 0x400
.rept 4
- tramp_ventry .Lvector_start\@, 64, \kpti
+ tramp_ventry .Lvector_start\@, 64, \kpti, \bhb
.endr
.rept 4
- tramp_ventry .Lvector_start\@, 32, \kpti
+ tramp_ventry .Lvector_start\@, 32, \kpti, \bhb
.endr
.endm
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
/*
* Exception vectors trampoline.
+ * The order must match __bp_harden_el1_vectors and the
+ * arm64_bp_harden_el1_vectors enum.
*/
.pushsection ".entry.tramp.text", "ax"
.align 11
ENTRY(tramp_vectors)
- generate_tramp_vector kpti=1
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
END(tramp_vectors)
ENTRY(tramp_exit_native)
@@ -1116,7 +1147,7 @@ __entry_tramp_data_start:
* Exception vectors for spectre mitigations on entry from EL1 when
* kpti is not in use.
*/
- .macro generate_el1_vector
+ .macro generate_el1_vector, bhb
.Lvector_start\@:
kernel_ventry 1, sync_invalid // Synchronous EL1t
kernel_ventry 1, irq_invalid // IRQ EL1t
@@ -1129,17 +1160,21 @@ __entry_tramp_data_start:
kernel_ventry 1, error_invalid // Error EL1h
.rept 4
- tramp_ventry .Lvector_start\@, 64, kpti=0
+ tramp_ventry .Lvector_start\@, 64, 0, \bhb
.endr
.rept 4
- tramp_ventry .Lvector_start\@, 32, kpti=0
+ tramp_ventry .Lvector_start\@, 32, 0, \bhb
.endr
.endm
+/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */
.pushsection ".entry.text", "ax"
.align 11
ENTRY(__bp_harden_el1_vectors)
- generate_el1_vector
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+ generate_el1_vector bhb=BHB_MITIGATION_LOOP
+ generate_el1_vector bhb=BHB_MITIGATION_FW
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
END(__bp_harden_el1_vectors)
.popsection
--- a/include/linux/arm-smccc.h
+++ b/include/linux/arm-smccc.h
@@ -85,6 +85,13 @@
ARM_SMCCC_SMC_32, \
0, 0x7fff)
+#define ARM_SMCCC_ARCH_WORKAROUND_3 \
+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
+ ARM_SMCCC_SMC_32, \
+ 0, 0x3fff)
+
+#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED 1
+
#ifndef __ASSEMBLY__
#include <linux/linkage.h>
next prev parent reply other threads:[~2022-04-01 6:37 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-04-01 6:36 [PATCH 4.14 00/27] 4.14.275-rc1 review Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 01/27] arm64: arch_timer: Add workaround for ARM erratum 1188873 Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 02/27] arm64: arch_timer: avoid unused function warning Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 03/27] arm64: Add silicon-errata.txt entry for ARM erratum 1188873 Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 04/27] arm64: Make ARM64_ERRATUM_1188873 depend on COMPAT Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 05/27] arm64: Add part number for Neoverse N1 Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 06/27] arm64: Add part number for Arm Cortex-A77 Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 07/27] arm64: Add Neoverse-N2, Cortex-A710 CPU part definition Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 08/27] arm64: Add Cortex-X2 " Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 09/27] arm64: entry.S: Add ventry overflow sanity checks Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 10/27] arm64: entry: Make the trampoline cleanup optional Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 11/27] arm64: entry: Free up another register on kptis tramp_exit path Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 12/27] arm64: entry: Move the trampoline data page before the text page Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 13/27] arm64: entry: Allow tramp_alias to access symbols after the 4K boundary Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 14/27] arm64: entry: Dont assume tramp_vectors is the start of the vectors Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 15/27] arm64: entry: Move trampoline macros out of ifdefd section Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 16/27] arm64: entry: Make the kpti trampolines kpti sequence optional Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 17/27] arm64: entry: Allow the trampoline text to occupy multiple pages Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 18/27] arm64: entry: Add non-kpti __bp_harden_el1_vectors for mitigations Greg Kroah-Hartman
2022-04-01 6:36 ` Greg Kroah-Hartman [this message]
2022-04-01 6:36 ` [PATCH 4.14 20/27] arm64: entry: Add macro for reading symbol addresses from the trampoline Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 21/27] arm64: Add percpu vectors for EL1 Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 22/27] arm64: proton-pack: Report Spectre-BHB vulnerabilities as part of Spectre-v2 Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 23/27] KVM: arm64: Add templates for BHB mitigation sequences Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 24/27] arm64: Mitigate spectre style branch history side channels Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 25/27] KVM: arm64: Allow SMCCC_ARCH_WORKAROUND_3 to be discovered and migrated Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 26/27] arm64: add ID_AA64ISAR2_EL1 sys register Greg Kroah-Hartman
2022-04-01 6:36 ` [PATCH 4.14 27/27] arm64: Use the clearbhb instruction in mitigations Greg Kroah-Hartman
2022-04-01 10:43 ` [PATCH 4.14 00/27] 4.14.275-rc1 review Guenter Roeck
2022-04-01 18:26 ` Naresh Kamboju
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220401063624.777642682@linuxfoundation.org \
--to=gregkh@linuxfoundation.org \
--cc=catalin.marinas@arm.com \
--cc=james.morse@arm.com \
--cc=linux-kernel@vger.kernel.org \
--cc=stable@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).