[PATCH v6 22/25] iommu/io-pgtable-arm: Support io-pgtable-arm in the hypervisor

[Mostafa Saleh <smostafa@google.com>]

To support DMA isolation in pKVM through SMMUv3 nested trap and
emulate, io-pgtable-arm needs to be compiled for the hypervisor
to create the SMMUs page tables.

Instead of factoring out the kernel-specific code and providing
parallel implementations for the hypervisor, we directly utilize
and abstract the differences within the iommu-pages API.

This introduces a set of hypervisor-specific wrappers in iommu-pages.h
when compiled under __KVM_NVHE_HYPERVISOR__, routing allocations,
frees, virt/phys conversions, and DMA API mapping to the appropriate
pKVM hypervisor functions (kvm_iommu_donate_pages, etc). The generic
kernel definitions are now appropriately excluded in this case.

It also introduces kvm_alloc_io_pgtable_ops at the end of
io-pgtable-arm.c to instantiate the page table for pKVM and adds
the io-pgtable-arm.o object to the hypervisor Makefile.

Signed-off-by: Mostafa Saleh <smostafa@google.com>
---
 arch/arm64/kvm/hyp/nvhe/Makefile |  3 +-
 drivers/iommu/io-pgtable-arm.c   | 31 +++++++++++++++-
 drivers/iommu/io-pgtable-arm.h   |  6 +++
 drivers/iommu/iommu-pages.h      | 63 ++++++++++++++++++++++++++++++++
 4 files changed, 100 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index 8a75739db947..4e9e0f1ed2b5 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -36,7 +36,8 @@ hyp-obj-y += $(lib-objs)
 HYP_SMMU_V3_DRV_PATH = ../../../../../drivers/iommu/arm/arm-smmu-v3
 
 hyp-obj-$(CONFIG_ARM_SMMU_V3_PKVM) += $(HYP_SMMU_V3_DRV_PATH)/pkvm/arm-smmu-v3.o \
-	$(HYP_SMMU_V3_DRV_PATH)/arm-smmu-v3-common-lib.o
+	$(HYP_SMMU_V3_DRV_PATH)/arm-smmu-v3-common-lib.o \
+	$(HYP_SMMU_V3_DRV_PATH)/../../io-pgtable-arm.o
 
 # Path to simple_ring_buffer.c
 CFLAGS_trace.nvhe.o += -I$(srctree)/kernel/trace/
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index e765021308f9..8a0ffea3ae2c 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -252,10 +252,14 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
 				    void *cookie)
 {
 	struct device *dev = cfg->iommu_dev;
+	int nid = NUMA_NO_NODE;
 	size_t alloc_size;
 	dma_addr_t dma;
 	void *pages;
 
+	if (dev)
+		nid = dev_to_node(dev);
+
 	/*
 	 * For very small starting-level translation tables the HW requires a
 	 * minimum alignment of at least 64 to cover all cases.
@@ -264,8 +268,7 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
 	if (cfg->alloc)
 		pages = cfg->alloc(cookie, alloc_size, gfp);
 	else
-		pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp,
-						  alloc_size);
+		pages = iommu_alloc_pages_node_sz(nid, gfp, alloc_size);
 
 	if (!pages)
 		return NULL;
@@ -1262,3 +1265,27 @@ struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
 	.alloc	= arm_mali_lpae_alloc_pgtable,
 	.free	= arm_lpae_free_pgtable,
 };
+
+#ifdef __KVM_NVHE_HYPERVISOR__
+#include <nvhe/iommu.h>
+
+struct io_pgtable_ops *kvm_alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
+						struct io_pgtable_cfg *cfg,
+						void *cookie)
+{
+	struct io_pgtable *iop;
+
+	if (fmt != ARM_64_LPAE_S2)
+		return NULL;
+
+	iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
+	if (!iop)
+		return NULL;
+
+	iop->fmt        = fmt;
+	iop->cookie     = cookie;
+	iop->cfg        = *cfg;
+
+	return &iop->ops;
+}
+#endif
diff --git a/drivers/iommu/io-pgtable-arm.h b/drivers/iommu/io-pgtable-arm.h
index ba7cfdf7afa0..af3a3f1e765e 100644
--- a/drivers/iommu/io-pgtable-arm.h
+++ b/drivers/iommu/io-pgtable-arm.h
@@ -27,4 +27,10 @@
 #define ARM_LPAE_TCR_PS_48_BIT		0x5ULL
 #define ARM_LPAE_TCR_PS_52_BIT		0x6ULL
 
+#ifdef __KVM_NVHE_HYPERVISOR__
+struct io_pgtable_ops *kvm_alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
+						struct io_pgtable_cfg *cfg,
+						void *cookie);
+#endif
+
 #endif /* IO_PGTABLE_ARM_H_ */
diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h
index e1945193ad7f..749f0f4f870c 100644
--- a/drivers/iommu/iommu-pages.h
+++ b/drivers/iommu/iommu-pages.h
@@ -10,6 +10,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/iommu.h>
 
+#ifndef __KVM_NVHE_HYPERVISOR__
 /**
  * struct ioptdesc - Memory descriptor for IOMMU page tables
  * @iopt_freelist_elm: List element for a struct iommu_pages_list
@@ -181,4 +182,66 @@ static inline void iommu_pages_dma_unmap(struct device *dev, dma_addr_t dma, siz
 	dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
 }
 
+#else /* __KVM_NVHE_HYPERVISOR__ */
+
+#include <nvhe/memory.h>
+#include <nvhe/iommu.h>
+
+static inline void *iommu_alloc_pages_node_sz(int nid, gfp_t gfp, size_t size)
+{
+	return kvm_iommu_donate_pages(get_order(size));
+}
+
+static inline void iommu_free_pages(void *virt)
+{
+	kvm_iommu_reclaim_pages(virt);
+}
+
+static inline void *iommu_alloc_data(size_t size, gfp_t gfp)
+{
+	return kvm_iommu_donate_pages(get_order(size));
+}
+
+static inline void iommu_free_data(void *p)
+{
+	kvm_iommu_reclaim_pages(p);
+}
+
+#undef WARN_ONCE
+#define WARN_ONCE(condition, format...) WARN_ON(condition)
+
+static inline phys_addr_t iommu_virt_to_phys(void *virt)
+{
+	return hyp_virt_to_phys(virt);
+}
+
+static inline void *iommu_phys_to_virt(phys_addr_t phys)
+{
+	return hyp_phys_to_virt(phys);
+}
+
+static inline void iommu_pages_flush_incoherent(struct device *dma_dev,
+						void *virt, size_t offset,
+						size_t len)
+{
+	kvm_flush_dcache_to_poc(virt + offset, len);
+}
+
+static inline dma_addr_t iommu_pages_dma_map(struct device *dev, void *virt, size_t size)
+{
+	kvm_flush_dcache_to_poc(virt, size);
+	return (dma_addr_t)hyp_virt_to_phys(virt);
+}
+
+static inline int iommu_pages_dma_mapping_error(struct device *dev, dma_addr_t dma)
+{
+	return 0;
+}
+
+static inline void iommu_pages_dma_unmap(struct device *dev, dma_addr_t dma, size_t size)
+{
+}
+
+#endif /* __KVM_NVHE_HYPERVISOR__ */
+
 #endif /* __IOMMU_PAGES_H */
-- 
2.54.0.545.g6539524ca2-goog