[PATCH v4 28/28] iommu/arm-smmu-v3-kvm: Enable nesting

Mostafa Saleh
2025-08-19T21:52:52Z

Now, as the hypervisor controls the command queue, stream table,
and shadows the stage-2 page table.
Enable stage-2 in case the host puts an STE in bypass or stage-1.

Signed-off-by: Mostafa Saleh <smostafa@google.com>
---
.../iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c | 72 ++++++++++++++++++-
1 file changed, 70 insertions(+), 2 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c
index 2d4ff21f83f9..5be44a37d581 100644
--- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c
@@ -336,6 +336,46 @@ static int smmu_init_cmdq(struct hyp_arm_smmu_v3_device *smmu)
return 0;
}

+static void smmu_attach_stage_2(struct hyp_arm_smmu_v3_device *smmu, struct arm_smmu_ste *ste)
+{
+ unsigned long vttbr;
+ unsigned long ts, sl, ic, oc, sh, tg, ps;
+ unsigned long cfg;
+ struct io_pgtable_cfg *pgt_cfg = &idmap_pgtable->cfg;
+
+ cfg = FIELD_GET(STRTAB_STE_0_CFG, ste->data[0]);
+ if (!FIELD_GET(STRTAB_STE_0_V, ste->data[0]) ||
+ (cfg == STRTAB_STE_0_CFG_ABORT))
+ return;
+ /* S2 is not advertised, that should never be attempted. */
+ if (WARN_ON(cfg == STRTAB_STE_0_CFG_NESTED))
+ return;
+ vttbr = pgt_cfg->arm_lpae_s2_cfg.vttbr;
+ ps = pgt_cfg->arm_lpae_s2_cfg.vtcr.ps;
+ tg = pgt_cfg->arm_lpae_s2_cfg.vtcr.tg;
+ sh = pgt_cfg->arm_lpae_s2_cfg.vtcr.sh;
+ oc = pgt_cfg->arm_lpae_s2_cfg.vtcr.orgn;
+ ic = pgt_cfg->arm_lpae_s2_cfg.vtcr.irgn;
+ sl = pgt_cfg->arm_lpae_s2_cfg.vtcr.sl;
+ ts = pgt_cfg->arm_lpae_s2_cfg.vtcr.tsz;
+
+ ste->data[1] |= FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING);
+ /* The host shouldn't write dwords 2 and 3, overwrite them. */
+ ste->data[2] = FIELD_PREP(STRTAB_STE_2_VTCR,
+ FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, ps) |
+ FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, tg) |
+ FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, sh) |
+ FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, oc) |
+ FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, ic) |
+ FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, sl) |
+ FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, ts)) |
+ FIELD_PREP(STRTAB_STE_2_S2VMID, 0) |
+ STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2R;
+ ste->data[3] = vttbr & STRTAB_STE_3_S2TTB_MASK;
+ /* Convert S1 => nested and bypass => S2 */
+ ste->data[0] |= FIELD_PREP(STRTAB_STE_0_CFG, cfg | BIT(1));
+}
+
/* Get an STE for a stream table base. */
static struct arm_smmu_ste *smmu_get_ste_ptr(struct hyp_arm_smmu_v3_device *smmu,
u32 sid, u64 *strtab)
@@ -394,6 +434,10 @@ static void smmu_reshadow_ste(struct hyp_arm_smmu_v3_device *smmu, u32 sid, bool
struct arm_smmu_ste *host_ste_ptr = smmu_get_ste_ptr(smmu, sid, host_ste_base);
struct arm_smmu_ste *hyp_ste_ptr = smmu_get_ste_ptr(smmu, sid, hyp_ste_base);
int i;
+ struct arm_smmu_ste target = {};
+ struct arm_smmu_cmdq_ent cfgi_cmd = {
+ .opcode = CMDQ_OP_CFGI_ALL,
+ };

/*
* Linux only uses leaf = 1, when leaf is 0, we need to verify that this
@@ -409,8 +453,32 @@ static void smmu_reshadow_ste(struct hyp_arm_smmu_v3_device *smmu, u32 sid, bool
hyp_ste_ptr = smmu_get_ste_ptr(smmu, sid, hyp_ste_base);
}

- for (i = 0; i < STRTAB_STE_DWORDS; i++)
- WRITE_ONCE(hyp_ste_ptr->data[i], host_ste_ptr->data[i]);
+ memcpy(target.data, host_ste_ptr->data, STRTAB_STE_DWORDS << 3);
+
+ /*
+ * Typically, STE update is done as the following
+ * 1- Write last 7 dwords, while STE is invalid
+ * 2- CFGI
+ * 3- Write first dword, making STE valid
+ * 4- CFGI
+ * As the SMMU has to least to load 64 bits atomically
+ * that gurantees that there is no race between writing
+ * the STE and the CFGI where the SMMU observes parts
+ * of the STE.
+ * In the shadow we update the STE to enable nested translation,
+ * which requires updating first 3 dwords.
+ * That is only done if the STE is valid and not in abort.
+ * Which means it happens at step 4)
+ * So we need to also write the last 7 dwords and send CFGI
+ * before writing the first dword.
+ * There is no need for last CFGI as the host will do that.
+ */
+ smmu_attach_stage_2(smmu, &target);
+ for (i = 1; i < STRTAB_STE_DWORDS; i++)
+ WRITE_ONCE(hyp_ste_ptr->data[i], target.data[i]);
+
+ WARN_ON(smmu_send_cmd(smmu, &cfgi_cmd));
+ WRITE_ONCE(hyp_ste_ptr->data[0], target.data[0]);
}

static int smmu_init_strtab(struct hyp_arm_smmu_v3_device *smmu)
--
2.51.0.rc1.167.g924127e9c0-goog