[PATCH v2 03/13] KVM: arm64: Support host MMIO trap handlers for unmapped devices

[Sebastian Ene <sebastianene@google.com>]

Hook a handler to the host mem abort so that the hypervisor can
intercept host accesses to unmapped memory regions.
When a Stage-2 fault occurs on a registered device region, the
hypervisor will look if there is any registered function that
can handle the access. On the back of this, mediate host accesses
to devices and emulate them in pKVM.

Signed-off-by: Sebastian Ene <sebastianene@google.com>
Signed-off-by: Bartłomiej Grzesik <bgrzesik@google.com>
---
 arch/arm64/include/asm/kvm_arm.h      |  2 ++
 arch/arm64/include/asm/kvm_pkvm.h     |  4 +++
 arch/arm64/kvm/hyp/nvhe/mem_protect.c | 49 +++++++++++++++++++++++++++
 arch/arm64/kvm/hyp/nvhe/setup.c       |  3 ++
 4 files changed, 58 insertions(+)

diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 3f9233b5a130..6360c90f9855 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -304,6 +304,8 @@
 
 /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
 #define HPFAR_MASK	(~UL(0xf))
+#define FAR_MASK	GENMASK_ULL(11, 0)
+
 /*
  * We have
  *	PAR	[PA_Shift - 1	: 12] = PA	[PA_Shift - 1 : 12]
diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h
index ab26bec079d6..0a471564be00 100644
--- a/arch/arm64/include/asm/kvm_pkvm.h
+++ b/arch/arm64/include/asm/kvm_pkvm.h
@@ -20,9 +20,13 @@
 /* The maximum number of hypervisor protected regions from the host */
 #define PKVM_PROTECTED_REGS_NUM	8
 
+struct pkvm_protected_reg;
+typedef void(pkvm_emulate_handler)(struct pkvm_protected_reg *region, u64 offset,
+				   bool write, u64 *reg, u8 reg_size);
 struct pkvm_protected_reg {
 	u64 pfn;
 	u64 nr_pages;
+	pkvm_emulate_handler *cb;
 };
 
 extern struct pkvm_protected_reg kvm_nvhe_sym(pkvm_protected_regs)[];
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 500c18c2fd48..7e978e0c44b9 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -14,6 +14,7 @@
 #include <asm/stage2_pgtable.h>
 
 #include <hyp/fault.h>
+#include <hyp/adjust_pc.h>
 
 #include <nvhe/arm-smccc.h>
 #include <nvhe/gfp.h>
@@ -752,6 +753,50 @@ static void host_inject_mem_abort(struct kvm_cpu_context *host_ctxt)
 	inject_host_exception(esr);
 }
 
+static bool handle_host_mmio_trap(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr)
+{
+	u64 offset, reg_value = 0, start, end;
+	u8 reg_size, reg_index;
+	bool write;
+	int i;
+
+	for (i = 0; i < num_protected_reg; i++) {
+		if (!pkvm_protected_regs[i].pfn || !pkvm_protected_regs[i].nr_pages ||
+		    !pkvm_protected_regs[i].cb)
+			continue;
+
+		start = PFN_PHYS(pkvm_protected_regs[i].pfn);
+		end = start + PFN_PHYS(pkvm_protected_regs[i].nr_pages);
+		reg_size = BIT((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
+
+		if (start > addr || addr + reg_size > end)
+			continue;
+
+		reg_index = (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
+		write = (esr & ESR_ELx_WNR) == ESR_ELx_WNR;
+		offset = addr - start;
+
+		if (write && reg_index != 31)
+			reg_value = host_ctxt->regs.regs[reg_index];
+
+		pkvm_protected_regs[i].cb(&pkvm_protected_regs[i], offset, write,
+					  &reg_value, reg_size);
+
+		if (!write && reg_index != 31)
+			host_ctxt->regs.regs[reg_index] = reg_value;
+
+		kvm_skip_host_instr();
+		return true;
+	}
+
+	return false;
+}
+
+static bool is_dabt(u64 esr)
+{
+	return (ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_LOW) && (esr & ESR_ELx_ISV);
+}
+
 void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
 {
 	struct kvm_vcpu_fault_info fault;
@@ -774,6 +819,10 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
 	BUG_ON(!(fault.hpfar_el2 & HPFAR_EL2_NS));
 	addr = FIELD_GET(HPFAR_EL2_FIPA, fault.hpfar_el2) << 12;
 
+	if (is_dabt(esr) && !addr_is_memory(addr) &&
+	    handle_host_mmio_trap(host_ctxt, esr, addr | (fault.far_el2 & FAR_MASK)))
+		return;
+
 	switch (host_stage2_idmap(addr)) {
 	case -EPERM:
 		host_inject_mem_abort(host_ctxt);
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index 64c0290da888..4395595b7f7e 100644
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -294,6 +294,9 @@ static int donate_protected_mmio_regions(void)
 						  pkvm_protected_regs[i].nr_pages << PAGE_SHIFT);
 		if (ret)
 			goto err_setup;
+
+		if (pkvm_protected_regs[i].cb)
+			pkvm_protected_regs[i].cb = kern_hyp_va(pkvm_protected_regs[i].cb);
 	}
 
 	return 0;
-- 
2.55.0.654.g21b8a5bc05-goog