[PATCH 17/27] iommu/arm-smmu-v3: Thread SSID through the arm_smmu_attach_*() interface

[Jason Gunthorpe <jgg@nvidia.com>]

Allow creating and managing arm_smmu_mater_domain's with a non-zero SSID
through the arm_smmu_attach_*() family of functions. This triggers
ATC invalidation for the correct SSID in PASID cases and tracks the
per-attachment SSID in the struct arm_smmu_master_domain.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 59 ++++++++++++---------
 1 file changed, 35 insertions(+), 24 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index a3d2914bcc36ae..43e0c15432073f 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1937,13 +1937,14 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
 	cmd->atc.size	= log2_span;
 }
 
-static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
+static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
+				   ioasid_t ssid)
 {
 	int i;
 	struct arm_smmu_cmdq_ent cmd;
 	struct arm_smmu_cmdq_batch cmds;
 
-	arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
+	arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
 
 	cmds.num = 0;
 	for (i = 0; i < master->num_streams; i++) {
@@ -2427,7 +2428,7 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master)
 	/*
 	 * ATC invalidation of PASID 0 causes the entire ATC to be flushed.
 	 */
-	arm_smmu_atc_inv_master(master);
+	arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
 	if (pci_enable_ats(pdev, stu))
 		dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
 }
@@ -2498,14 +2499,14 @@ arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain,
 }
 
 static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
-					  struct arm_smmu_domain *smmu_domain)
+					  struct arm_smmu_domain *smmu_domain,
+					  ioasid_t ssid)
 {
 	struct arm_smmu_master_domain *master_domain;
 	unsigned long flags;
 
 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
-	master_domain = arm_smmu_find_master_domain(smmu_domain, master,
-						    IOMMU_NO_PASID);
+	master_domain = arm_smmu_find_master_domain(smmu_domain, master, ssid);
 	if (master_domain) {
 		list_del(&master_domain->devices_elm);
 		kfree(master_domain);
@@ -2526,7 +2527,7 @@ struct attach_state {
  */
 static int arm_smmu_attach_prepare(struct arm_smmu_master *master,
 				   struct arm_smmu_domain *smmu_domain,
-				   struct attach_state *state)
+				   ioasid_t ssid, struct attach_state *state)
 {
 	struct arm_smmu_master_domain *cur_master_domain;
 	struct arm_smmu_master_domain *master_domain;
@@ -2543,6 +2544,7 @@ static int arm_smmu_attach_prepare(struct arm_smmu_master *master,
 	if (!master_domain)
 		return -ENOMEM;
 	master_domain->master = master;
+	master_domain->ssid = ssid;
 
 	state->want_ats = arm_smmu_ats_supported(master);
 
@@ -2557,8 +2559,8 @@ static int arm_smmu_attach_prepare(struct arm_smmu_master *master,
 	 * domain, unrelated to ATS.
 	 */
 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
-	cur_master_domain = arm_smmu_find_master_domain(smmu_domain, master,
-							IOMMU_NO_PASID);
+	cur_master_domain =
+		arm_smmu_find_master_domain(smmu_domain, master, ssid);
 	if (cur_master_domain) {
 		kfree(master_domain);
 		state->existing_master_domain = true;
@@ -2577,8 +2579,9 @@ static int arm_smmu_attach_prepare(struct arm_smmu_master *master,
  * smmu_domain->devices list.
  */
 static void arm_smmu_attach_commit(struct arm_smmu_master *master,
-				   struct arm_smmu_domain *smmu_domain,
-				   struct attach_state *state)
+				   struct arm_smmu_domain *old_smmu_domain,
+				   struct arm_smmu_domain *new_smmu_domain,
+				   ioasid_t ssid, struct attach_state *state)
 {
 	lockdep_assert_held(&arm_smmu_asid_lock);
 
@@ -2593,7 +2596,7 @@ static void arm_smmu_attach_commit(struct arm_smmu_master *master,
 		 * SMMU is translating for the new domain and both the old&new
 		 * domain will issue invalidations.
 		 */
-		arm_smmu_atc_inv_master(master);
+		arm_smmu_atc_inv_master(master, ssid);
 	}
 
 	if (!state->existing_master_domain) {
@@ -2601,7 +2604,8 @@ static void arm_smmu_attach_commit(struct arm_smmu_master *master,
 			iommu_get_domain_for_dev(master->dev));
 
 		if (old_smmu_domain)
-			arm_smmu_remove_master_domain(master, old_smmu_domain);
+			arm_smmu_remove_master_domain(master, old_smmu_domain,
+						      ssid);
 	}
 }
 
@@ -2609,26 +2613,27 @@ static void arm_smmu_attach_commit(struct arm_smmu_master *master,
  * When an arm_smmu_master_domain is removed we have to turn off ATS as there is
  * no longer any tracking of invalidations.
  */
-static void arm_smmu_attach_remove(struct arm_smmu_master *master)
+static void arm_smmu_attach_remove(struct arm_smmu_master *master,
+				   struct arm_smmu_domain *smmu_domain,
+				   ioasid_t ssid)
 {
-	struct arm_smmu_domain *smmu_domain =
-		to_smmu_domain_safe(iommu_get_domain_for_dev(master->dev));
-
 	if (!smmu_domain)
 		return;
 
-	if (master->ats_enabled) {
+	if (ssid == IOMMU_NO_PASID && master->ats_enabled) {
 		pci_disable_ats(to_pci_dev(master->dev));
 		/*
 		 * Ensure ATS is disabled at the endpoint before we issue the
 		 * ATC invalidation via the SMMU.
 		 */
 		wmb();
-		arm_smmu_atc_inv_master(master);
+		arm_smmu_atc_inv_master(master, ssid);
 	}
 
-	arm_smmu_remove_master_domain(master, smmu_domain);
-	master->ats_enabled = false;
+	arm_smmu_remove_master_domain(master, smmu_domain, ssid);
+
+	if (ssid == IOMMU_NO_PASID)
+		master->ats_enabled = false;
 }
 
 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
@@ -2674,7 +2679,8 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	 */
 	mutex_lock(&arm_smmu_asid_lock);
 
-	ret = arm_smmu_attach_prepare(master, smmu_domain, &state);
+	ret = arm_smmu_attach_prepare(master, smmu_domain, IOMMU_NO_PASID,
+				      &state);
 	if (ret) {
 		mutex_unlock(&arm_smmu_asid_lock);
 		return ret;
@@ -2700,7 +2706,9 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 		break;
 	}
 
-	arm_smmu_attach_commit(master, smmu_domain, &state);
+	arm_smmu_attach_commit(
+		master, to_smmu_domain_safe(iommu_get_domain_for_dev(dev)),
+		smmu_domain, IOMMU_NO_PASID, &state);
 	mutex_unlock(&arm_smmu_asid_lock);
 	return 0;
 }
@@ -2750,7 +2758,10 @@ static int arm_smmu_attach_dev_ste(struct device *dev,
 	 * the stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and
 	 * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry).
 	 */
-	arm_smmu_attach_remove(master);
+	arm_smmu_attach_remove(
+		master,
+		to_smmu_domain_safe(iommu_get_domain_for_dev(master->dev)),
+		IOMMU_NO_PASID);
 
 	arm_smmu_install_ste_for_dev(master, ste);
 	mutex_unlock(&arm_smmu_asid_lock);
-- 
2.42.0