[PATCH v9 09/25] KVM: arm64: iommu: Support DABT for IOMMU

[Mostafa Saleh <smostafa@google.com>]

The pKVM SMMUv3 driver emulates the SMMU registers for the host: the
MMIO is donated to the hypervisor so that host accesses trap, and the
hypervisor performs or rejects each access on the host's behalf.

Add a dabt_handler op and call it from the host data abort path for
aborts with a valid instruction syndrome on non-memory addresses. If
the driver emulates the access, skip the faulting instruction,
otherwise take the normal path, where hyp-owned MMIO is denied.

Only aborts with a valid instruction syndrome can be emulated, so the
host must access emulated registers with single-register loads and
stores (readl/writel and friends).

Signed-off-by: Mostafa Saleh <smostafa@google.com>
---
 arch/arm64/kvm/hyp/include/nvhe/iommu.h |  9 +++++++++
 arch/arm64/kvm/hyp/nvhe/iommu.c         |  6 ++++++
 arch/arm64/kvm/hyp/nvhe/mem_protect.c   | 17 +++++++++++++++++
 3 files changed, 32 insertions(+)

diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h
index acd5058ef3ed..bd6604f194b5 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h
@@ -13,6 +13,13 @@ struct pkvm_iommu_ops {
 	 * with the same @prot, which must succeed.
 	 */
 	int (*host_stage2_idmap)(phys_addr_t start, phys_addr_t end, int prot);
+	/*
+	 * Emulate a host data abort on MMIO @addr
+	 * @esr has a valid instruction syndrome and @regs holds the host x0-x30,
+	 * SRT 31 is the zero register. Return true if the access was emulated,
+	 * which would skip the instruction.
+	 */
+	bool (*dabt_handler)(struct user_pt_regs *regs, u64 esr, u64 addr);
 };
 
 int pkvm_iommu_init(void *pool_base, unsigned int nr_pages);
@@ -25,4 +32,6 @@ void *pkvm_iommu_alloc_pages(u8 order);
 /* Free pages from pkvm_iommu_alloc_pages(). */
 void pkvm_iommu_free_pages(void *ptr);
 
+bool pkvm_iommu_host_dabt_handler(struct user_pt_regs *regs, u64 esr, u64 addr);
+
 #endif /* __ARM64_KVM_NVHE_IOMMU_H__ */
diff --git a/arch/arm64/kvm/hyp/nvhe/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu.c
index 36432bd46c78..cdf6e87bb68e 100644
--- a/arch/arm64/kvm/hyp/nvhe/iommu.c
+++ b/arch/arm64/kvm/hyp/nvhe/iommu.c
@@ -149,3 +149,9 @@ void pkvm_iommu_free_pages(void *ptr)
 {
 	hyp_put_page(&iommu_pages_pool, ptr);
 }
+
+bool pkvm_iommu_host_dabt_handler(struct user_pt_regs *regs, u64 esr, u64 addr)
+{
+	return pkvm_iommu_ops && pkvm_iommu_ops->dabt_handler &&
+	       pkvm_iommu_ops->dabt_handler(regs, esr, addr);
+}
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index c14af6a4877c..523bb6bd793b 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -13,6 +13,7 @@
 #include <asm/kvm_pkvm.h>
 #include <asm/stage2_pgtable.h>
 
+#include <hyp/adjust_pc.h>
 #include <hyp/fault.h>
 
 #include <nvhe/arm-smccc.h>
@@ -778,6 +779,12 @@ static void host_inject_mem_abort(struct kvm_cpu_context *host_ctxt)
 	inject_host_exception(esr);
 }
 
+static bool is_dabt_with_isv(u64 esr)
+{
+	return (ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_LOW) &&
+		(esr & ESR_ELx_ISV);
+}
+
 void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
 {
 	struct kvm_vcpu_fault_info fault;
@@ -800,6 +807,16 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
 	BUG_ON(!(fault.hpfar_el2 & HPFAR_EL2_NS));
 	addr = FIELD_GET(HPFAR_EL2_FIPA, fault.hpfar_el2) << 12;
 
+	/*
+	 * Let the IOMMU driver emulate accesses to its MMIO.
+	 */
+	if (is_dabt_with_isv(esr) && !addr_is_memory(addr) &&
+	    pkvm_iommu_host_dabt_handler(&host_ctxt->regs, esr,
+					 addr | FAR_TO_FIPA_OFFSET(fault.far_el2))) {
+		kvm_skip_host_instr();
+		return;
+	}
+
 	switch (host_stage2_idmap(addr)) {
 	case -EPERM:
 		host_inject_mem_abort(host_ctxt);
-- 
2.56.0.176.g3575e65682-goog