[PATCH v4 27/28] iommu/arm-smmu-v3-kvm: Shadow the CPU stage-2 page table

Mostafa Saleh
2025-08-19T21:52:51Z

Based on the callbacks from the hypervisor, update the SMMUv3
Identity mapped page table.

Signed-off-by: Mostafa Saleh <smostafa@google.com>
---
.../iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c | 171 +++++++++++++++++-
1 file changed, 169 insertions(+), 2 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c
index db9d9caaca2c..2d4ff21f83f9 100644
--- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c
@@ -11,6 +11,7 @@
#include <nvhe/trap_handler.h>

#include "arm_smmu_v3.h"
+#include "../../../io-pgtable-arm.h"

size_t __ro_after_init kvm_hyp_arm_smmu_v3_count;
struct hyp_arm_smmu_v3_device *kvm_hyp_arm_smmu_v3_smmus;
@@ -58,6 +59,9 @@ struct hyp_arm_smmu_v3_device *kvm_hyp_arm_smmu_v3_smmus;
smmu_wait(_cond); \
})

+/* Protected by host_mmu.lock from core code. */
+static struct io_pgtable *idmap_pgtable;
+
/* Transfer ownership of memory */
static int smmu_take_pages(u64 phys, size_t size)
{
@@ -166,7 +170,6 @@ static int smmu_sync_cmd(struct hyp_arm_smmu_v3_device *smmu)
return smmu_wait_event(smmu, smmu_cmdq_empty(&smmu->cmdq));
}

-__maybe_unused
static int smmu_send_cmd(struct hyp_arm_smmu_v3_device *smmu,
struct arm_smmu_cmdq_ent *cmd)
{
@@ -178,6 +181,66 @@ static int smmu_send_cmd(struct hyp_arm_smmu_v3_device *smmu,
return smmu_sync_cmd(smmu);
}

+static void __smmu_add_cmd(struct hyp_arm_smmu_v3_device *smmu, void *unused,
+ struct arm_smmu_cmdq_ent *cmd)
+{
+ WARN_ON(smmu_add_cmd(smmu, cmd));
+}
+
+static int smmu_tlb_inv_range_smmu(struct hyp_arm_smmu_v3_device *smmu,
+ struct arm_smmu_cmdq_ent *cmd,
+ unsigned long iova, size_t size, size_t granule)
+{
+ arm_smmu_tlb_inv_build(cmd, iova, size, granule,
+ idmap_pgtable->cfg.pgsize_bitmap, smmu,
+ __smmu_add_cmd, NULL);
+ return smmu_sync_cmd(smmu);
+}
+
+static void smmu_tlb_inv_range(unsigned long iova, size_t size, size_t granule,
+ bool leaf)
+{
+ struct arm_smmu_cmdq_ent cmd = {
+ .opcode = CMDQ_OP_TLBI_S2_IPA,
+ .tlbi = {
+ .leaf = leaf,
+ .vmid = 0,
+ },
+ };
+ struct arm_smmu_cmdq_ent cmd_s1 = {
+ .opcode = CMDQ_OP_TLBI_NH_ALL,
+ .tlbi = {
+ .vmid = 0,
+ },
+ };
+ struct hyp_arm_smmu_v3_device *smmu;
+
+ for_each_smmu(smmu) {
+ hyp_spin_lock(&smmu->lock);
+ WARN_ON(smmu_tlb_inv_range_smmu(smmu, &cmd, iova, size, granule));
+ WARN_ON(smmu_send_cmd(smmu, &cmd_s1));
+ hyp_spin_unlock(&smmu->lock);
+ }
+}
+
+static void smmu_tlb_flush_walk(unsigned long iova, size_t size,
+ size_t granule, void *cookie)
+{
+ smmu_tlb_inv_range(iova, size, granule, false);
+}
+
+static void smmu_tlb_add_page(struct iommu_iotlb_gather *gather,
+ unsigned long iova, size_t granule,
+ void *cookie)
+{
+ smmu_tlb_inv_range(iova, granule, granule, true);
+}
+
+static const struct iommu_flush_ops smmu_tlb_ops = {
+ .tlb_flush_walk = smmu_tlb_flush_walk,
+ .tlb_add_page = smmu_tlb_add_page,
+};
+
/* Put the device in a state that can be probed by the host driver. */
static void smmu_deinit_device(struct hyp_arm_smmu_v3_device *smmu)
{
@@ -434,6 +497,37 @@ static int smmu_init_device(struct hyp_arm_smmu_v3_device *smmu)
return ret;
}

+static int smmu_init_pgt(void)
+{
+ /* Default values overridden based on SMMUs common features. */
+ struct io_pgtable_cfg cfg = (struct io_pgtable_cfg) {
+ .tlb = &smmu_tlb_ops,
+ .pgsize_bitmap = -1,
+ .ias = 48,
+ .oas = 48,
+ .coherent_walk = true,
+ };
+ struct hyp_arm_smmu_v3_device *smmu;
+ struct io_pgtable_ops *ops;
+
+ for_each_smmu(smmu) {
+ cfg.ias = min(cfg.ias, smmu->ias);
+ cfg.oas = min(cfg.oas, smmu->oas);
+ cfg.pgsize_bitmap &= smmu->pgsize_bitmap;
+ cfg.coherent_walk &= !!(smmu->features & ARM_SMMU_FEAT_COHERENCY);
+ }
+
+ /* At least PAGE_SIZE must be supported by all SMMUs*/
+ if ((cfg.pgsize_bitmap & PAGE_SIZE) == 0)
+ return -EINVAL;
+
+ ops = kvm_alloc_io_pgtable_ops(ARM_64_LPAE_S2, &cfg, NULL);
+ if (!ops)
+ return -ENOMEM;
+ idmap_pgtable = io_pgtable_ops_to_pgtable(ops);
+ return 0;
+}
+
static int smmu_init(void)
{
int ret;
@@ -455,7 +549,7 @@ static int smmu_init(void)

BUILD_BUG_ON(sizeof(hyp_spinlock_t) != sizeof(u32));

- return 0;
+ return smmu_init_pgt();

out_reclaim_smmu:
while (smmu != kvm_hyp_arm_smmu_v3_smmus)
@@ -789,8 +883,81 @@ static bool smmu_dabt_handler(struct user_pt_regs *regs, u64 esr, u64 addr)
return false;
}

+static size_t smmu_pgsize_idmap(size_t size, u64 paddr, size_t pgsize_bitmap)
+{
+ size_t pgsizes;
+
+ /* Remove page sizes that are larger than the current size */
+ pgsizes = pgsize_bitmap & GENMASK_ULL(__fls(size), 0);
+
+ /* Remove page sizes that the address is not aligned to. */
+ if (likely(paddr))
+ pgsizes &= GENMASK_ULL(__ffs(paddr), 0);
+
+ WARN_ON(!pgsizes);
+
+ /* Return the larget page size that fits. */
+ return BIT(__fls(pgsizes));
+}
+
static void smmu_host_stage2_idmap(phys_addr_t start, phys_addr_t end, int prot)
{
+ size_t size = end - start;
+ size_t pgsize = PAGE_SIZE, pgcount;
+ size_t mapped, unmapped;
+ int ret;
+ struct io_pgtable *pgtable = idmap_pgtable;
+
+ end = min(end, BIT(pgtable->cfg.oas));
+ if (start >= end)
+ return;
+
+ if (prot) {
+ if (!(prot & IOMMU_MMIO))
+ prot |= IOMMU_CACHE;
+
+ while (size) {
+ mapped = 0;
+ /*
+ * We handle pages size for memory and MMIO differently:
+ * - memory: Map everything with PAGE_SIZE, that is guaranteed to
+ * find memory as we allocated enough pages to cover the entire
+ * memory, we do that as io-pgtable-arm doesn't support
+ * split_blk_unmap logic any more, so we can't break blocks once
+ * mapped to tables.
+ * - MMIO: Unlike memory, pKVM allocate 1G to for all MMIO, while
+ * the MMIO space can be large, as it is assumed to cover the
+ * whole IAS that is not memory, we have to use block mappings,
+ * that is fine for MMIO as it is never donated at the moment,
+ * so we never need to unmap MMIO at the run time triggereing
+ * split block logic.
+ */
+ if (prot & IOMMU_MMIO)
+ pgsize = smmu_pgsize_idmap(size, start, pgtable->cfg.pgsize_bitmap);
+
+ pgcount = size / pgsize;
+ ret = pgtable->ops.map_pages(&pgtable->ops, start, start,
+ pgsize, pgcount, prot, 0, &mapped);
+ size -= mapped;
+ start += mapped;
+ if (!mapped || ret)
+ return;
+ }
+ } else {
+ /* Shouldn't happen. */
+ WARN_ON(prot & IOMMU_MMIO);
+ while (size) {
+ pgcount = size / pgsize;
+ unmapped = pgtable->ops.unmap_pages(&pgtable->ops, start,
+ pgsize, pgcount, NULL);
+ size -= unmapped;
+ start += unmapped;
+ if (!unmapped)
+ return;
+ }
+ /* Some memory were not unmapped. */
+ WARN_ON(size);
+ }
}

/* Shared with the kernel driver in EL1 */
--
2.51.0.rc1.167.g924127e9c0-goog