[PATCH v3 06/13] iommu/amd: Introduce per-device domain ID to workaround potential TLB aliasing issue

[Vasant Hegde <vasant.hegde@amd.com>]

With v1 page table (stage-2), the AMD IOMMU spec states that the hardware
must use the domain ID to tag its internal translation caches. I/O devices
with different v1 page tables must be given different domain IDs. I/O
devices that share the same v1 page table __may__ be given the same domain
ID. This domain ID management policy is currently implemented by the AMD
IOMMU driver. In this case, only the domain ID is needed when issuing the
INVALIDATE_IOMMU_PAGES command to invalidate the IOMMU translation cache
(TLB).

With v2 page table (stage-1), the hardware uses domain ID and PASID as
parameters to tag and issue the INVALIDATE_IOMMU_PAGES command. Since the
GCR3 table is setup per-device, and there is no guarantee for PASID to be
unique across multiple devices, the same PASID for different devices could
have different v2 page tables. In such case, if multiple devices share the
same domain ID, IOMMU translation cache for these devices would be polluted
due to TLB aliasing.

Hence, avoid the TLB aliasing issue by allocating unique domain ID for each
device even when multiple devices are sharing the same v1 page table.
Please note that this workaround would result in multiple
INVALIDATE_IOMMU_PAGES commands (one per domain id) when unmapping a
translation on the shared v1 page table.

Signed-off-by: Vasant Hegde <vasant.hegde@amd.com>
---
 drivers/iommu/amd/amd_iommu_types.h |  2 +
 drivers/iommu/amd/iommu.c           | 59 ++++++++++++++++++++++++++---
 2 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 3b1ef1030687..e836438a9318 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -831,6 +831,8 @@ struct iommu_dev_data {
 	u8 ppr          :1;		  /* Enable device PPR support */
 	bool use_vapic;			  /* Enable device to use vapic mode */
 	bool defer_attach;
+	/* Per device domain ID. Used with V2 page table */
+	u16 domid;
 
 	struct ratelimit_state rs;        /* Ratelimit IOPF messages */
 };
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 85d73555c1f9..09762332f681 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -102,6 +102,14 @@ static inline bool is_pasid_valid(ioasid_t pasid)
 	return (pasid != IOMMU_PASID_INVALID) ? true : false;
 }
 
+/*
+ * Allocate per device domain ID when protection domain is in V2 page table
+ */
+static inline bool domain_id_is_per_dev(struct protection_domain *pdom)
+{
+	return (pdom && pdom->pd_mode != PD_MODE_V1);
+}
+
 static inline int get_acpihid_device_id(struct device *dev,
 					struct acpihid_map_entry **entry)
 {
@@ -1418,6 +1426,19 @@ static int device_flush_iotlb_range(struct iommu_dev_data *dev_data,
 	return iommu_queue_command(iommu, &cmd);
 }
 
+/* Flush IOMMU TLB for the given device */
+static int device_flush_tlb_range(struct iommu_dev_data *dev_data,
+				  ioasid_t pasid, u64 address, size_t size)
+{
+	struct iommu_cmd cmd;
+	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
+	bool gn = is_pasid_valid(pasid);
+
+	build_inv_iommu_pages(&cmd, address, size, dev_data->domid, pasid, gn);
+
+	return iommu_queue_command(iommu, &cmd);
+}
+
 static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data)
 {
 	struct amd_iommu *iommu = data;
@@ -1523,11 +1544,25 @@ static int domain_flush_tlb_range(struct protection_domain *pdom,
 static void __domain_flush_pages(struct protection_domain *pdom,
 				 ioasid_t pasid, u64 address, size_t size)
 {
-	int ret;
+	struct iommu_dev_data *dev_data;
+	int ret = 0;
 
-	ret = domain_flush_tlb_range(pdom, pasid, address, size);
+	if (domain_id_is_per_dev(pdom)) {
+		list_for_each_entry(dev_data, &pdom->dev_list, list) {
+			ret |= device_flush_tlb_range(dev_data,
+						      pasid, address, size);
 
-	ret |= domain_flush_dev_iotlb_range(pdom, pasid, address, size);
+			if (!dev_data->ats_enabled)
+				continue;
+
+			ret |= device_flush_iotlb_range(dev_data,
+							pasid, address, size);
+		}
+	} else {
+		ret = domain_flush_tlb_range(pdom, pasid, address, size);
+
+		ret |= domain_flush_dev_iotlb_range(pdom, pasid, address, size);
+	}
 
 	WARN_ON(ret);
 }
@@ -1805,9 +1840,15 @@ static void set_dte_entry(struct amd_iommu *iommu,
 	u64 flags = 0;
 	u32 old_domid;
 	u16 devid = dev_data->devid;
+	u16 domid;
 	struct protection_domain *domain = dev_data->domain;
 	struct dev_table_entry *dev_table = get_dev_table(iommu);
 
+	if (domain_id_is_per_dev(domain))
+		domid = dev_data->domid;
+	else
+		domid = domain->id;
+
 	if (domain->iop.mode != PAGE_MODE_NONE)
 		pte_root = iommu_virt_to_phys(domain->iop.root);
 
@@ -1820,7 +1861,7 @@ static void set_dte_entry(struct amd_iommu *iommu,
 	 * When SNP is enabled, Only set TV bit when IOMMU
 	 * page translation is in use.
 	 */
-	if (!amd_iommu_snp_en || (domain->id != 0))
+	if (!amd_iommu_snp_en || (domid != 0))
 		pte_root |= DTE_FLAG_TV;
 
 	flags = dev_table[devid].data[1];
@@ -1866,7 +1907,7 @@ static void set_dte_entry(struct amd_iommu *iommu,
 	}
 
 	flags &= ~DEV_DOMID_MASK;
-	flags |= domain->id;
+	flags |= domid;
 
 	old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK;
 	dev_table[devid].data[1]  = flags;
@@ -1918,6 +1959,10 @@ static void do_attach(struct iommu_dev_data *dev_data,
 	domain->dev_iommu[iommu->index] += 1;
 	domain->dev_cnt                 += 1;
 
+	/* Allocate per device domain ID */
+	if (domain_id_is_per_dev(domain))
+		dev_data->domid = domain_id_alloc();
+
 	/* Update device table */
 	set_dte_entry(iommu, dev_data);
 	clone_aliases(iommu, dev_data->dev);
@@ -1949,6 +1994,10 @@ static void do_detach(struct iommu_dev_data *dev_data)
 	/* decrease reference counters - needs to happen after the flushes */
 	domain->dev_iommu[iommu->index] -= 1;
 	domain->dev_cnt                 -= 1;
+
+	/* Free per device domain ID */
+	if (domain_id_is_per_dev(domain))
+		domain_id_free(dev_data->domid);
 }
 
 /*
-- 
2.31.1