[PATCH 2/2] KVM: arm64: Avoid blocking irqs when tlb flushing/ATing with HCR.TGE=0

D Scott Phillips
2025-04-15T15:47:08Z

pNMIs are intended to be deliverable during operations like guest
tlb flushing or nested AT, however the setting of HCR_EL2 controls
are accidentally blocking async exceptions.

You can see this by doing:

# perf record -e cycles:Hk -g ./dirty_log_perf_test -m 3 \
-i 4 -s anonymous -b 4G -v 32

Where no samples will be collected in __kvm_tlb_flush_vmid_ipa_nsh()
between enter_vmid_context() and exit_vmid_context() then many
samples are collected right after the write to HCR_EL2 in
exit_vmid_context(), where pNMIs actually get unmasked.

Set HCR_EL2.IMO so that pNMIs are not blocked during guest tlb
flushing or nested AT.

Signed-off-by: D Scott Phillips <scott@os.amperecomputing.com>
---
arch/arm64/include/asm/hardirq.h | 3 ++-
arch/arm64/kvm/at.c | 4 +++-
arch/arm64/kvm/hyp/vhe/tlb.c | 10 ++++++++++
3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h
index cbfa7b6f2e098..6eb3f93851023 100644
--- a/arch/arm64/include/asm/hardirq.h
+++ b/arch/arm64/include/asm/hardirq.h
@@ -41,7 +41,8 @@ do { \
\
___hcr = read_sysreg(hcr_el2); \
if (!(___hcr & HCR_TGE)) { \
- write_sysreg(___hcr | HCR_TGE, hcr_el2); \
+ write_sysreg((___hcr & ~(HCR_AMO | HCR_IMO | HCR_FMO)) |\
+ HCR_TGE, hcr_el2); \
isb(); \
} \
/* \
diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c
index ff4b06ce661af..f31f0d78c5813 100644
--- a/arch/arm64/kvm/at.c
+++ b/arch/arm64/kvm/at.c
@@ -1269,7 +1269,9 @@ static u64 __kvm_at_s1e01_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)

skip_mmu_switch:
/* Clear TGE, enable S2 translation, we're rolling */
- write_sysreg((config.hcr & ~HCR_TGE) | HCR_VM, hcr_el2);
+ write_sysreg((config.hcr & ~HCR_TGE) |
+ HCR_AMO | HCR_IMO | HCR_FMO | HCR_VM,
+ hcr_el2);
isb();

switch (op) {
diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c
index 3d50a1bd2bdbc..ecb700bab3b8f 100644
--- a/arch/arm64/kvm/hyp/vhe/tlb.c
+++ b/arch/arm64/kvm/hyp/vhe/tlb.c
@@ -55,6 +55,15 @@ static void enter_vmid_context(struct kvm_s2_mmu *mmu,
* bits. Changing E2H is impossible (goodbye TTBR1_EL2), so
* let's flip TGE before executing the TLB operation.
*
+ * One other fun complication to consider is the target EL for
+ * asynchronous exceptions. We want to allow NMIs during tlb flushing,
+ * so we need to ensure that the target EL for IRQs remains as EL2.
+ * HCR_EL2.{E2H,TGE,IMO} = {1,0,0} would set the target EL for IRQs as
+ * EL1, and IRQs at EL2 would be "C" (Interrupts not taken regardless
+ * of the value of interrupt masks). So we need to set
+ * HCR_EL2.{E2H,TGE,IMO} = {1,0,1} so that NMIs will still be
+ * delivered.
+ *
* ARM erratum 1165522 requires some special handling (again),
* as we need to make sure both stages of translation are in
* place before clearing TGE. __load_stage2() already
@@ -63,6 +72,7 @@ static void enter_vmid_context(struct kvm_s2_mmu *mmu,
__load_stage2(mmu, mmu->arch);
val = read_sysreg(hcr_el2);
val &= ~HCR_TGE;
+ val |= HCR_AMO | HCR_IMO | HCR_FMO;
write_sysreg(val, hcr_el2);
isb();
}
--
2.48.1