From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
To: linux-kernel@vger.kernel.org
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
stable@vger.kernel.org,
"Russell King (Oracle)" <rmk+kernel@armlinux.org.uk>,
Catalin Marinas <catalin.marinas@arm.com>,
James Morse <james.morse@arm.com>
Subject: [PATCH 5.16 35/37] arm64: Use the clearbhb instruction in mitigations
Date: Wed, 9 Mar 2022 17:00:36 +0100 [thread overview]
Message-ID: <20220309155900.103924301@linuxfoundation.org> (raw)
In-Reply-To: <20220309155859.086952723@linuxfoundation.org>
From: James Morse <james.morse@arm.com>
commit 228a26b912287934789023b4132ba76065d9491c upstream.
Future CPUs may implement a clearbhb instruction that is sufficient
to mitigate SpectreBHB. CPUs that implement this instruction, but
not CSV2.3 must be affected by Spectre-BHB.
Add support to use this instruction as the BHB mitigation on CPUs
that support it. The instruction is in the hint space, so it will
be treated by a NOP as older CPUs.
Reviewed-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
arch/arm64/include/asm/assembler.h | 17 +++++++++++++++++
arch/arm64/include/asm/cpufeature.h | 13 +++++++++++++
arch/arm64/include/asm/insn.h | 1 +
arch/arm64/include/asm/sysreg.h | 1 +
arch/arm64/include/asm/vectors.h | 7 +++++++
arch/arm64/kernel/cpufeature.c | 1 +
arch/arm64/kernel/entry.S | 8 ++++++++
arch/arm64/kernel/image-vars.h | 1 +
arch/arm64/kernel/proton-pack.c | 29 +++++++++++++++++++++++++++++
arch/arm64/kvm/hyp/hyp-entry.S | 1 +
10 files changed, 79 insertions(+)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -109,6 +109,13 @@
.endm
/*
+ * Clear Branch History instruction
+ */
+ .macro clearbhb
+ hint #22
+ .endm
+
+/*
* Speculation barrier
*/
.macro sb
@@ -876,4 +883,14 @@ alternative_cb_end
ldp x0, x1, [sp], #16
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
.endm
+
+ .macro mitigate_spectre_bhb_clear_insn
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
+alternative_cb spectre_bhb_patch_clearbhb
+ /* Patched to NOP when not supported */
+ clearbhb
+ isb
+alternative_cb_end
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
+ .endm
#endif /* __ASM_ASSEMBLER_H */
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -653,6 +653,19 @@ static inline bool supports_csv2p3(int s
return csv2_val == 3;
}
+static inline bool supports_clearbhb(int scope)
+{
+ u64 isar2;
+
+ if (scope == SCOPE_LOCAL_CPU)
+ isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
+ else
+ isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
+
+ return cpuid_feature_extract_unsigned_field(isar2,
+ ID_AA64ISAR2_CLEARBHB_SHIFT);
+}
+
const struct cpumask *system_32bit_el0_cpumask(void);
DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -65,6 +65,7 @@ enum aarch64_insn_hint_cr_op {
AARCH64_INSN_HINT_PSB = 0x11 << 5,
AARCH64_INSN_HINT_TSB = 0x12 << 5,
AARCH64_INSN_HINT_CSDB = 0x14 << 5,
+ AARCH64_INSN_HINT_CLEARBHB = 0x16 << 5,
AARCH64_INSN_HINT_BTI = 0x20 << 5,
AARCH64_INSN_HINT_BTIC = 0x22 << 5,
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -773,6 +773,7 @@
#define ID_AA64ISAR1_GPI_IMP_DEF 0x1
/* id_aa64isar2 */
+#define ID_AA64ISAR2_CLEARBHB_SHIFT 28
#define ID_AA64ISAR2_RPRES_SHIFT 4
#define ID_AA64ISAR2_WFXT_SHIFT 0
--- a/arch/arm64/include/asm/vectors.h
+++ b/arch/arm64/include/asm/vectors.h
@@ -32,6 +32,12 @@ enum arm64_bp_harden_el1_vectors {
* canonical vectors.
*/
EL1_VECTOR_BHB_FW,
+
+ /*
+ * Use the ClearBHB instruction, before branching to the canonical
+ * vectors.
+ */
+ EL1_VECTOR_BHB_CLEAR_INSN,
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
/*
@@ -43,6 +49,7 @@ enum arm64_bp_harden_el1_vectors {
#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
#define EL1_VECTOR_BHB_LOOP -1
#define EL1_VECTOR_BHB_FW -1
+#define EL1_VECTOR_BHB_CLEAR_INSN -1
#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
/* The vectors to use on return from EL0. e.g. to remap the kernel */
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -231,6 +231,7 @@ static const struct arm64_ftr_bits ftr_i
};
static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0),
ARM64_FTR_END,
};
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -657,6 +657,7 @@ alternative_else_nop_endif
#define BHB_MITIGATION_NONE 0
#define BHB_MITIGATION_LOOP 1
#define BHB_MITIGATION_FW 2
+#define BHB_MITIGATION_INSN 3
.macro tramp_ventry, vector_start, regsize, kpti, bhb
.align 7
@@ -673,6 +674,11 @@ alternative_else_nop_endif
__mitigate_spectre_bhb_loop x30
.endif // \bhb == BHB_MITIGATION_LOOP
+ .if \bhb == BHB_MITIGATION_INSN
+ clearbhb
+ isb
+ .endif // \bhb == BHB_MITIGATION_INSN
+
.if \kpti == 1
/*
* Defend against branch aliasing attacks by pushing a dummy
@@ -749,6 +755,7 @@ SYM_CODE_START_NOALIGN(tramp_vectors)
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
SYM_CODE_END(tramp_vectors)
@@ -811,6 +818,7 @@ SYM_CODE_START(__bp_harden_el1_vectors)
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
generate_el1_vector bhb=BHB_MITIGATION_LOOP
generate_el1_vector bhb=BHB_MITIGATION_FW
+ generate_el1_vector bhb=BHB_MITIGATION_INSN
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
SYM_CODE_END(__bp_harden_el1_vectors)
.popsection
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -69,6 +69,7 @@ KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0
KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter);
KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable);
KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
+KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
/* Global kernel state accessed by nVHE hyp code. */
KVM_NVHE_ALIAS(kvm_vgic_global_state);
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -805,6 +805,7 @@ int arch_prctl_spec_ctrl_get(struct task
* - Mitigated by a branchy loop a CPU specific number of times, and listed
* in our "loop mitigated list".
* - Mitigated in software by the firmware Spectre v2 call.
+ * - Has the ClearBHB instruction to perform the mitigation.
* - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
* software mitigation in the vectors is needed.
* - Has CSV2.3, so is unaffected.
@@ -820,6 +821,7 @@ enum bhb_mitigation_bits {
BHB_LOOP,
BHB_FW,
BHB_HW,
+ BHB_INSN,
};
static unsigned long system_bhb_mitigations;
@@ -937,6 +939,9 @@ bool is_spectre_bhb_affected(const struc
if (supports_csv2p3(scope))
return false;
+ if (supports_clearbhb(scope))
+ return true;
+
if (spectre_bhb_loop_affected(scope))
return true;
@@ -984,6 +989,17 @@ void spectre_bhb_enable_mitigation(const
} else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
state = SPECTRE_MITIGATED;
set_bit(BHB_HW, &system_bhb_mitigations);
+ } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
+ /*
+ * Ensure KVM uses the indirect vector which will have ClearBHB
+ * added.
+ */
+ if (!data->slot)
+ data->slot = HYP_VECTOR_INDIRECT;
+
+ this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
+ state = SPECTRE_MITIGATED;
+ set_bit(BHB_INSN, &system_bhb_mitigations);
} else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) {
/*
* Ensure KVM uses the indirect vector which will have the
@@ -1096,3 +1112,16 @@ void noinstr spectre_bhb_patch_wa3(struc
*updptr++ = cpu_to_le32(insn);
}
+
+/* Patched to NOP when not supported */
+void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt,
+ __le32 *origptr, __le32 *updptr, int nr_inst)
+{
+ BUG_ON(nr_inst != 2);
+
+ if (test_bit(BHB_INSN, &system_bhb_mitigations))
+ return;
+
+ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
+ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
+}
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -213,6 +213,7 @@ SYM_CODE_END(__kvm_hyp_vector)
.else
stp x0, x1, [sp, #-16]!
mitigate_spectre_bhb_loop x0
+ mitigate_spectre_bhb_clear_insn
.endif
.if \indirect != 0
alternative_cb kvm_patch_vector_branch
next prev parent reply other threads:[~2022-03-09 16:22 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-03-09 16:00 [PATCH 5.16 00/37] 5.16.14-rc1 review Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 01/37] x86/speculation: Rename RETPOLINE_AMD to RETPOLINE_LFENCE Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 02/37] x86/speculation: Add eIBRS + Retpoline options Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 03/37] Documentation/hw-vuln: Update spectre doc Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 04/37] x86/speculation: Include unprivileged eBPF status in Spectre v2 mitigation reporting Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 05/37] x86/speculation: Use generic retpoline by default on AMD Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 06/37] x86/speculation: Update link to AMD speculation whitepaper Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 07/37] x86/speculation: Warn about Spectre v2 LFENCE mitigation Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 08/37] x86/speculation: Warn about eIBRS + LFENCE + Unprivileged eBPF + SMT Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 09/37] ARM: report Spectre v2 status through sysfs Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 10/37] ARM: early traps initialisation Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 11/37] ARM: use LOADADDR() to get load address of sections Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 12/37] ARM: Spectre-BHB workaround Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 13/37] ARM: include unprivileged BPF status in Spectre V2 reporting Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 14/37] arm64: add ID_AA64ISAR2_EL1 sys register Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 15/37] arm64: cpufeature: add HWCAP for FEAT_AFP Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 16/37] arm64: cpufeature: add HWCAP for FEAT_RPRES Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 17/37] arm64: entry.S: Add ventry overflow sanity checks Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 18/37] arm64: spectre: Rename spectre_v4_patch_fw_mitigation_conduit Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 19/37] KVM: arm64: Allow indirect vectors to be used without SPECTRE_V3A Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 20/37] arm64: entry: Make the trampoline cleanup optional Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 21/37] arm64: entry: Free up another register on kptis tramp_exit path Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 22/37] arm64: entry: Move the trampoline data page before the text page Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 23/37] arm64: entry: Allow tramp_alias to access symbols after the 4K boundary Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 24/37] arm64: entry: Dont assume tramp_vectors is the start of the vectors Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 25/37] arm64: entry: Move trampoline macros out of ifdefd section Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 26/37] arm64: entry: Make the kpti trampolines kpti sequence optional Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 27/37] arm64: entry: Allow the trampoline text to occupy multiple pages Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 28/37] arm64: entry: Add non-kpti __bp_harden_el1_vectors for mitigations Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 29/37] arm64: entry: Add vectors that have the bhb mitigation sequences Greg Kroah-Hartman
2022-03-10 23:27 ` Pavel Machek
2022-03-11 8:15 ` Greg Kroah-Hartman
2022-03-11 8:39 ` Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 30/37] arm64: entry: Add macro for reading symbol addresses from the trampoline Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 31/37] arm64: Add percpu vectors for EL1 Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 32/37] arm64: proton-pack: Report Spectre-BHB vulnerabilities as part of Spectre-v2 Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 33/37] arm64: Mitigate spectre style branch history side channels Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 34/37] KVM: arm64: Allow SMCCC_ARCH_WORKAROUND_3 to be discovered and migrated Greg Kroah-Hartman
2022-03-09 16:00 ` Greg Kroah-Hartman [this message]
2022-03-09 16:00 ` [PATCH 5.16 36/37] arm64: proton-pack: Include unprivileged eBPF status in Spectre v2 mitigation reporting Greg Kroah-Hartman
2022-03-09 16:00 ` [PATCH 5.16 37/37] ARM: fix build error when BPF_SYSCALL is disabled Greg Kroah-Hartman
2022-03-09 19:05 ` [PATCH 5.16 00/37] 5.16.14-rc1 review Guenter Roeck
2022-03-09 20:01 ` Greg Kroah-Hartman
2022-03-09 21:17 ` Greg Kroah-Hartman
2022-03-09 20:23 ` Shuah Khan
2022-03-09 21:17 ` Daniel Díaz
2022-03-09 21:19 ` Daniel Díaz
2022-03-09 21:49 ` Fox Chen
2022-03-10 10:50 ` Bagas Sanjaya
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220309155900.103924301@linuxfoundation.org \
--to=gregkh@linuxfoundation.org \
--cc=catalin.marinas@arm.com \
--cc=james.morse@arm.com \
--cc=linux-kernel@vger.kernel.org \
--cc=rmk+kernel@armlinux.org.uk \
--cc=stable@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox