[PATCH 0/4] Improve the invalidation path in VT-d

[PATCH 0/4] Improve the invalidation path in VT-d

[Jason Gunthorpe]

Auditing shows a similar unnecessary complexity in the VT-d path as
AMD, simplify it.

- Don't intermix all and range invalidation flows at the command
  building phase
- Don't compute a size or pages for the gather range, stay in the
  start/last space
- Use the much simpler fls approach to compute the AM

Broadly this removes a bunch of unnecessary logic from the
invalidation path and makes it simpler and more of a straight line.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

Jason Gunthorpe (4):
  iommu/intel: Split piotlb invalidation into range and all
  iommu/vtd: Pass size_order to qi_desc_piotlb() not npages
  iommu/vtd: Remove the remaining pages along the invalidation path
  iommu/vt: Simplify calculate_psi_aligned_address()

 drivers/iommu/intel/cache.c | 100 +++++++++++++++---------------------
 drivers/iommu/intel/dmar.c  |  19 ++-----
 drivers/iommu/intel/iommu.h |  43 +++++++---------
 drivers/iommu/intel/pasid.c |   6 +--
 drivers/iommu/intel/prq.c   |   2 +-
 drivers/iommu/intel/trace.h |  18 +++----
 6 files changed, 76 insertions(+), 112 deletions(-)


base-commit: cc9c4e71a8dd4f8842b0b958deacc2410ce465bc
-- 
2.43.0

[PATCH 1/4] iommu/intel: Split piotlb invalidation into range and all

[Jason Gunthorpe]

Currently these call chains are muddled up by using npages=-1, but
only one caller has the possibility to do both options.

Simplify qi_flush_piotlb() to qi_flush_piotlb_all() since all
callers pass npages=-1.

Split qi_batch_add_piotlb() into qi_batch_add_piotlb_all() and
related helpers.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
---
 drivers/iommu/intel/cache.c | 20 +++++++++++++-------
 drivers/iommu/intel/dmar.c  | 19 ++++---------------
 drivers/iommu/intel/iommu.h | 37 ++++++++++++++++++-------------------
 drivers/iommu/intel/pasid.c |  6 +++---
 drivers/iommu/intel/prq.c   |  2 +-
 5 files changed, 39 insertions(+), 45 deletions(-)

diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c
index 249ab5886c739f..e08253980a6ee7 100644
--- a/drivers/iommu/intel/cache.c
+++ b/drivers/iommu/intel/cache.c
@@ -330,15 +330,17 @@ static void qi_batch_add_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid
 	qi_batch_increment_index(iommu, batch);
 }
 
+static void qi_batch_add_piotlb_all(struct intel_iommu *iommu, u16 did,
+				     u32 pasid, struct qi_batch *batch)
+{
+	qi_desc_piotlb_all(did, pasid, &batch->descs[batch->index]);
+	qi_batch_increment_index(iommu, batch);
+}
+
 static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,
 				u64 addr, unsigned long npages, bool ih,
 				struct qi_batch *batch)
 {
-	/*
-	 * npages == -1 means a PASID-selective invalidation, otherwise,
-	 * a positive value for Page-selective-within-PASID invalidation.
-	 * 0 is not a valid input.
-	 */
 	if (!npages)
 		return;
 
@@ -378,8 +380,12 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *
 	u64 type = DMA_TLB_PSI_FLUSH;
 
 	if (intel_domain_use_piotlb(domain)) {
-		qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr,
-				    pages, ih, domain->qi_batch);
+		if (pages == -1)
+			qi_batch_add_piotlb_all(iommu, tag->domain_id,
+						tag->pasid, domain->qi_batch);
+		else
+			qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid,
+					    addr, pages, ih, domain->qi_batch);
 		return;
 	}
 
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index 69222dbd2af0ea..6f496fc6a5ff1a 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -1550,23 +1550,12 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
 	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
-/* PASID-based IOTLB invalidation */
-void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
-		     unsigned long npages, bool ih)
+/* PASID-selective IOTLB invalidation */
+void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid)
 {
-	struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
+	struct qi_desc desc = {};
 
-	/*
-	 * npages == -1 means a PASID-selective invalidation, otherwise,
-	 * a positive value for Page-selective-within-PASID invalidation.
-	 * 0 is not a valid input.
-	 */
-	if (WARN_ON(!npages)) {
-		pr_err("Invalid input npages = %ld\n", npages);
-		return;
-	}
-
-	qi_desc_piotlb(did, pasid, addr, npages, ih, &desc);
+	qi_desc_piotlb_all(did, pasid, &desc);
 	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index 599913fb65d59e..40759587729953 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -1082,31 +1082,29 @@ static inline void qi_desc_dev_iotlb(u16 sid, u16 pfsid, u16 qdep, u64 addr,
 	desc->qw3 = 0;
 }
 
+/* PASID-selective IOTLB invalidation */
+static inline void qi_desc_piotlb_all(u16 did, u32 pasid, struct qi_desc *desc)
+{
+	desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
+		    QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | QI_EIOTLB_TYPE;
+	desc->qw1 = 0;
+}
+
+/* Page-selective-within-PASID IOTLB invalidation */
 static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr,
 				  unsigned long npages, bool ih,
 				  struct qi_desc *desc)
 {
-	if (npages == -1) {
-		desc->qw0 = QI_EIOTLB_PASID(pasid) |
-				QI_EIOTLB_DID(did) |
-				QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
-				QI_EIOTLB_TYPE;
-		desc->qw1 = 0;
-	} else {
-		int mask = ilog2(__roundup_pow_of_two(npages));
-		unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
+	int mask = ilog2(__roundup_pow_of_two(npages));
+	unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
 
-		if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
-			addr = ALIGN_DOWN(addr, align);
+	if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
+		addr = ALIGN_DOWN(addr, align);
 
-		desc->qw0 = QI_EIOTLB_PASID(pasid) |
-				QI_EIOTLB_DID(did) |
-				QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
-				QI_EIOTLB_TYPE;
-		desc->qw1 = QI_EIOTLB_ADDR(addr) |
-				QI_EIOTLB_IH(ih) |
-				QI_EIOTLB_AM(mask);
-	}
+	desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
+		    QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE;
+	desc->qw1 = QI_EIOTLB_ADDR(addr) | QI_EIOTLB_IH(ih) |
+		    QI_EIOTLB_AM(mask);
 }
 
 static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
@@ -1168,6 +1166,7 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
 			u16 qdep, u64 addr, unsigned mask);
 
+void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid);
 void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
 		     unsigned long npages, bool ih);
 
diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
index 9d30015b894057..89541b74ab8ca3 100644
--- a/drivers/iommu/intel/pasid.c
+++ b/drivers/iommu/intel/pasid.c
@@ -282,7 +282,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);
 
 	if (pgtt == PASID_ENTRY_PGTT_PT || pgtt == PASID_ENTRY_PGTT_FL_ONLY)
-		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
+		qi_flush_piotlb_all(iommu, did, pasid);
 	else
 		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
 
@@ -308,7 +308,7 @@ static void pasid_flush_caches(struct intel_iommu *iommu,
 
 	if (cap_caching_mode(iommu->cap)) {
 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);
-		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
+		qi_flush_piotlb_all(iommu, did, pasid);
 	} else {
 		iommu_flush_write_buffer(iommu);
 	}
@@ -342,7 +342,7 @@ static void intel_pasid_flush_present(struct intel_iommu *iommu,
 	 *      Addr[63:12]=0x7FFFFFFF_FFFFF) to affected functions
 	 */
 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);
-	qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
+	qi_flush_piotlb_all(iommu, did, pasid);
 
 	devtlb_invalidation_with_pasid(iommu, dev, pasid);
 }
diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c
index ff63c228e6e19d..ed98b1cb06fecf 100644
--- a/drivers/iommu/intel/prq.c
+++ b/drivers/iommu/intel/prq.c
@@ -113,7 +113,7 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
 		qi_desc_dev_iotlb(sid, info->pfsid, info->ats_qdep, 0,
 				  MAX_AGAW_PFN_WIDTH, &desc[2]);
 	} else {
-		qi_desc_piotlb(did, pasid, 0, -1, 0, &desc[1]);
+		qi_desc_piotlb_all(did, pasid, &desc[1]);
 		qi_desc_dev_iotlb_pasid(sid, info->pfsid, pasid, info->ats_qdep,
 					0, MAX_AGAW_PFN_WIDTH, &desc[2]);
 	}
-- 
2.43.0

[PATCH 2/4] iommu/vtd: Pass size_order to qi_desc_piotlb() not npages

[Jason Gunthorpe]

It doesn't make sense for the caller to compute mask, throw it away
and then have qi_desc_piotlb() compute it again.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
---
 drivers/iommu/intel/cache.c | 10 ++++------
 drivers/iommu/intel/iommu.h | 16 ++++++----------
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c
index e08253980a6ee7..74ee2002fb9c85 100644
--- a/drivers/iommu/intel/cache.c
+++ b/drivers/iommu/intel/cache.c
@@ -338,13 +338,11 @@ static void qi_batch_add_piotlb_all(struct intel_iommu *iommu, u16 did,
 }
 
 static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,
-				u64 addr, unsigned long npages, bool ih,
+				u64 addr, unsigned int size_order, bool ih,
 				struct qi_batch *batch)
 {
-	if (!npages)
-		return;
-
-	qi_desc_piotlb(did, pasid, addr, npages, ih, &batch->descs[batch->index]);
+	qi_desc_piotlb(did, pasid, addr, size_order, ih,
+		       &batch->descs[batch->index]);
 	qi_batch_increment_index(iommu, batch);
 }
 
@@ -385,7 +383,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *
 						tag->pasid, domain->qi_batch);
 		else
 			qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid,
-					    addr, pages, ih, domain->qi_batch);
+					    addr, mask, ih, domain->qi_batch);
 		return;
 	}
 
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index 40759587729953..7a92472985ee02 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -1092,19 +1092,16 @@ static inline void qi_desc_piotlb_all(u16 did, u32 pasid, struct qi_desc *desc)
 
 /* Page-selective-within-PASID IOTLB invalidation */
 static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr,
-				  unsigned long npages, bool ih,
+				  unsigned int size_order, bool ih,
 				  struct qi_desc *desc)
 {
-	int mask = ilog2(__roundup_pow_of_two(npages));
-	unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
-
-	if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
-		addr = ALIGN_DOWN(addr, align);
-
+	/*
+	 * calculate_psi_aligned_address() must be used for addr and size_order
+	 */
 	desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
 		    QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE;
 	desc->qw1 = QI_EIOTLB_ADDR(addr) | QI_EIOTLB_IH(ih) |
-		    QI_EIOTLB_AM(mask);
+		    QI_EIOTLB_AM(size_order);
 }
 
 static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
@@ -1167,8 +1164,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
 			u16 qdep, u64 addr, unsigned mask);
 
 void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid);
-void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
-		     unsigned long npages, bool ih);
+
 
 void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
 			      u32 pasid, u16 qdep, u64 addr,
-- 
2.43.0

[PATCH 3/4] iommu/vtd: Remove the remaining pages along the invalidation path

[Jason Gunthorpe]

This was only being used to signal that a flush all should be used.
Use mask/size_order >= 52 to signal this instead.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
---
 drivers/iommu/intel/cache.c | 27 +++++++++++----------------
 drivers/iommu/intel/trace.h | 18 ++++++++----------
 2 files changed, 19 insertions(+), 26 deletions(-)

diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c
index 74ee2002fb9c85..31d4bfba1842bf 100644
--- a/drivers/iommu/intel/cache.c
+++ b/drivers/iommu/intel/cache.c
@@ -255,7 +255,6 @@ void cache_tag_unassign_domain(struct dmar_domain *domain,
 
 static unsigned long calculate_psi_aligned_address(unsigned long start,
 						   unsigned long end,
-						   unsigned long *_pages,
 						   unsigned long *_mask)
 {
 	unsigned long pages = aligned_nrpages(start, end - start + 1);
@@ -281,10 +280,8 @@ static unsigned long calculate_psi_aligned_address(unsigned long start,
 		 */
 		shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
 		mask = shared_bits ? __ffs(shared_bits) : MAX_AGAW_PFN_WIDTH;
-		aligned_pages = 1UL << mask;
 	}
 
-	*_pages = aligned_pages;
 	*_mask = mask;
 
 	return ALIGN_DOWN(start, VTD_PAGE_SIZE << mask);
@@ -371,14 +368,13 @@ static bool intel_domain_use_piotlb(struct dmar_domain *domain)
 }
 
 static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *tag,
-				  unsigned long addr, unsigned long pages,
-				  unsigned long mask, int ih)
+				  unsigned long addr, unsigned long mask, int ih)
 {
 	struct intel_iommu *iommu = tag->iommu;
 	u64 type = DMA_TLB_PSI_FLUSH;
 
 	if (intel_domain_use_piotlb(domain)) {
-		if (pages == -1)
+		if (mask >= MAX_AGAW_PFN_WIDTH)
 			qi_batch_add_piotlb_all(iommu, tag->domain_id,
 						tag->pasid, domain->qi_batch);
 		else
@@ -392,7 +388,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *
 	 * is too big.
 	 */
 	if (!cap_pgsel_inv(iommu->cap) ||
-	    mask > cap_max_amask_val(iommu->cap) || pages == -1) {
+	    mask > cap_max_amask_val(iommu->cap)) {
 		addr = 0;
 		mask = 0;
 		ih = 0;
@@ -441,16 +437,15 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
 			   unsigned long end, int ih)
 {
 	struct intel_iommu *iommu = NULL;
-	unsigned long pages, mask, addr;
+	unsigned long mask, addr;
 	struct cache_tag *tag;
 	unsigned long flags;
 
 	if (start == 0 && end == ULONG_MAX) {
 		addr = 0;
-		pages = -1;
 		mask = MAX_AGAW_PFN_WIDTH;
 	} else {
-		addr = calculate_psi_aligned_address(start, end, &pages, &mask);
+		addr = calculate_psi_aligned_address(start, end, &mask);
 	}
 
 	spin_lock_irqsave(&domain->cache_lock, flags);
@@ -462,7 +457,7 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
 		switch (tag->type) {
 		case CACHE_TAG_IOTLB:
 		case CACHE_TAG_NESTING_IOTLB:
-			cache_tag_flush_iotlb(domain, tag, addr, pages, mask, ih);
+			cache_tag_flush_iotlb(domain, tag, addr, mask, ih);
 			break;
 		case CACHE_TAG_NESTING_DEVTLB:
 			/*
@@ -480,7 +475,7 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
 			break;
 		}
 
-		trace_cache_tag_flush_range(tag, start, end, addr, pages, mask);
+		trace_cache_tag_flush_range(tag, start, end, addr, mask);
 	}
 	qi_batch_flush_descs(iommu, domain->qi_batch);
 	spin_unlock_irqrestore(&domain->cache_lock, flags);
@@ -510,11 +505,11 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start,
 			      unsigned long end)
 {
 	struct intel_iommu *iommu = NULL;
-	unsigned long pages, mask, addr;
+	unsigned long mask, addr;
 	struct cache_tag *tag;
 	unsigned long flags;
 
-	addr = calculate_psi_aligned_address(start, end, &pages, &mask);
+	addr = calculate_psi_aligned_address(start, end, &mask);
 
 	spin_lock_irqsave(&domain->cache_lock, flags);
 	list_for_each_entry(tag, &domain->cache_tags, node) {
@@ -530,9 +525,9 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start,
 
 		if (tag->type == CACHE_TAG_IOTLB ||
 		    tag->type == CACHE_TAG_NESTING_IOTLB)
-			cache_tag_flush_iotlb(domain, tag, addr, pages, mask, 0);
+			cache_tag_flush_iotlb(domain, tag, addr, mask, 0);
 
-		trace_cache_tag_flush_range_np(tag, start, end, addr, pages, mask);
+		trace_cache_tag_flush_range_np(tag, start, end, addr, mask);
 	}
 	qi_batch_flush_descs(iommu, domain->qi_batch);
 	spin_unlock_irqrestore(&domain->cache_lock, flags);
diff --git a/drivers/iommu/intel/trace.h b/drivers/iommu/intel/trace.h
index 6311ba3f1691dd..9f0ab43539ea06 100644
--- a/drivers/iommu/intel/trace.h
+++ b/drivers/iommu/intel/trace.h
@@ -132,8 +132,8 @@ DEFINE_EVENT(cache_tag_log, cache_tag_unassign,
 
 DECLARE_EVENT_CLASS(cache_tag_flush,
 	TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end,
-		 unsigned long addr, unsigned long pages, unsigned long mask),
-	TP_ARGS(tag, start, end, addr, pages, mask),
+		 unsigned long addr, unsigned long mask),
+	TP_ARGS(tag, start, end, addr, mask),
 	TP_STRUCT__entry(
 		__string(iommu, tag->iommu->name)
 		__string(dev, dev_name(tag->dev))
@@ -143,7 +143,6 @@ DECLARE_EVENT_CLASS(cache_tag_flush,
 		__field(unsigned long, start)
 		__field(unsigned long, end)
 		__field(unsigned long, addr)
-		__field(unsigned long, pages)
 		__field(unsigned long, mask)
 	),
 	TP_fast_assign(
@@ -155,10 +154,9 @@ DECLARE_EVENT_CLASS(cache_tag_flush,
 		__entry->start = start;
 		__entry->end = end;
 		__entry->addr = addr;
-		__entry->pages = pages;
 		__entry->mask = mask;
 	),
-	TP_printk("%s %s[%d] type %s did %d [0x%lx-0x%lx] addr 0x%lx pages 0x%lx mask 0x%lx",
+	TP_printk("%s %s[%d] type %s did %d [0x%lx-0x%lx] addr 0x%lx mask 0x%lx",
 		  __get_str(iommu), __get_str(dev), __entry->pasid,
 		  __print_symbolic(__entry->type,
 			{ CACHE_TAG_IOTLB,		"iotlb" },
@@ -166,20 +164,20 @@ DECLARE_EVENT_CLASS(cache_tag_flush,
 			{ CACHE_TAG_NESTING_IOTLB,	"nesting_iotlb" },
 			{ CACHE_TAG_NESTING_DEVTLB,	"nesting_devtlb" }),
 		__entry->domain_id, __entry->start, __entry->end,
-		__entry->addr, __entry->pages, __entry->mask
+		__entry->addr, __entry->mask
 	)
 );
 
 DEFINE_EVENT(cache_tag_flush, cache_tag_flush_range,
 	TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end,
-		 unsigned long addr, unsigned long pages, unsigned long mask),
-	TP_ARGS(tag, start, end, addr, pages, mask)
+		 unsigned long addr, unsigned long mask),
+	TP_ARGS(tag, start, end, addr, mask)
 );
 
 DEFINE_EVENT(cache_tag_flush, cache_tag_flush_range_np,
 	TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end,
-		 unsigned long addr, unsigned long pages, unsigned long mask),
-	TP_ARGS(tag, start, end, addr, pages, mask)
+		 unsigned long addr, unsigned long mask),
+	TP_ARGS(tag, start, end, addr, mask)
 );
 #endif /* _TRACE_INTEL_IOMMU_H */
 
-- 
2.43.0

[PATCH 4/4] iommu/vt: Simplify calculate_psi_aligned_address()

[Jason Gunthorpe]

This is doing far too much math for the simple task of finding a
power of 2 that fully spans the given range. Use fls directly on
the xor which computes the common binary prefix.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
---
 drivers/iommu/intel/cache.c | 49 ++++++++++++-------------------------
 1 file changed, 16 insertions(+), 33 deletions(-)

diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c
index 31d4bfba1842bf..10402a0f0ed7da 100644
--- a/drivers/iommu/intel/cache.c
+++ b/drivers/iommu/intel/cache.c
@@ -254,37 +254,25 @@ void cache_tag_unassign_domain(struct dmar_domain *domain,
 }
 
 static unsigned long calculate_psi_aligned_address(unsigned long start,
-						   unsigned long end,
-						   unsigned long *_mask)
+						   unsigned long last,
+						   unsigned long *size_order)
 {
-	unsigned long pages = aligned_nrpages(start, end - start + 1);
-	unsigned long aligned_pages = __roundup_pow_of_two(pages);
-	unsigned long bitmask = aligned_pages - 1;
-	unsigned long mask = ilog2(aligned_pages);
-	unsigned long pfn = IOVA_PFN(start);
+	unsigned int sz_lg2;
 
-	/*
-	 * PSI masks the low order bits of the base address. If the
-	 * address isn't aligned to the mask, then compute a mask value
-	 * needed to ensure the target range is flushed.
-	 */
-	if (unlikely(bitmask & pfn)) {
-		unsigned long end_pfn = pfn + pages - 1, shared_bits;
-
-		/*
-		 * Since end_pfn <= pfn + bitmask, the only way bits
-		 * higher than bitmask can differ in pfn and end_pfn is
-		 * by carrying. This means after masking out bitmask,
-		 * high bits starting with the first set bit in
-		 * shared_bits are all equal in both pfn and end_pfn.
-		 */
-		shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
-		mask = shared_bits ? __ffs(shared_bits) : MAX_AGAW_PFN_WIDTH;
+	/* Compute a sz_lg2 that spans start and last */
+	start &= GENMASK(BITS_PER_LONG - 1, VTD_PAGE_SHIFT);
+	sz_lg2 = fls_long(start ^ last);
+	if (sz_lg2 <= 12) {
+		*size_order = 0;
+		return start;
+	}
+	if (unlikely(sz_lg2 >= MAX_AGAW_PFN_WIDTH)) {
+		*size_order = MAX_AGAW_PFN_WIDTH;
+		return 0;
 	}
 
-	*_mask = mask;
-
-	return ALIGN_DOWN(start, VTD_PAGE_SIZE << mask);
+	*size_order = sz_lg2 - VTD_PAGE_SHIFT;
+	return start & GENMASK(BITS_PER_LONG - 1, sz_lg2);
 }
 
 static void qi_batch_flush_descs(struct intel_iommu *iommu, struct qi_batch *batch)
@@ -441,12 +429,7 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
 	struct cache_tag *tag;
 	unsigned long flags;
 
-	if (start == 0 && end == ULONG_MAX) {
-		addr = 0;
-		mask = MAX_AGAW_PFN_WIDTH;
-	} else {
-		addr = calculate_psi_aligned_address(start, end, &mask);
-	}
+	addr = calculate_psi_aligned_address(start, end, &mask);
 
 	spin_lock_irqsave(&domain->cache_lock, flags);
 	list_for_each_entry(tag, &domain->cache_tags, node) {
-- 
2.43.0

Re: [PATCH 1/4] iommu/intel: Split piotlb invalidation into range and all

[Baolu Lu]

On 3/27/26 23:25, Jason Gunthorpe wrote:
> Currently these call chains are muddled up by using npages=-1, but
> only one caller has the possibility to do both options.
> 
> Simplify qi_flush_piotlb() to qi_flush_piotlb_all() since all
> callers pass npages=-1.
> 
> Split qi_batch_add_piotlb() into qi_batch_add_piotlb_all() and
> related helpers.
> 
> Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
> ---
>   drivers/iommu/intel/cache.c | 20 +++++++++++++-------
>   drivers/iommu/intel/dmar.c  | 19 ++++---------------
>   drivers/iommu/intel/iommu.h | 37 ++++++++++++++++++-------------------
>   drivers/iommu/intel/pasid.c |  6 +++---
>   drivers/iommu/intel/prq.c   |  2 +-
>   5 files changed, 39 insertions(+), 45 deletions(-)
> 
> diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c
> index 249ab5886c739f..e08253980a6ee7 100644
> --- a/drivers/iommu/intel/cache.c
> +++ b/drivers/iommu/intel/cache.c
> @@ -330,15 +330,17 @@ static void qi_batch_add_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid
>   	qi_batch_increment_index(iommu, batch);
>   }
>   
> +static void qi_batch_add_piotlb_all(struct intel_iommu *iommu, u16 did,
> +				     u32 pasid, struct qi_batch *batch)
> +{
> +	qi_desc_piotlb_all(did, pasid, &batch->descs[batch->index]);
> +	qi_batch_increment_index(iommu, batch);
> +}
> +
>   static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,
>   				u64 addr, unsigned long npages, bool ih,
>   				struct qi_batch *batch)
>   {
> -	/*
> -	 * npages == -1 means a PASID-selective invalidation, otherwise,
> -	 * a positive value for Page-selective-within-PASID invalidation.
> -	 * 0 is not a valid input.
> -	 */
>   	if (!npages)
>   		return;

The WARN_ON(!npages) logic is removed from the previous
qi_flush_piotlb() (see below ...). Could it be added here to keep the
"npages != 0" check?

>   
> @@ -378,8 +380,12 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *
>   	u64 type = DMA_TLB_PSI_FLUSH;
>   
>   	if (intel_domain_use_piotlb(domain)) {
> -		qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr,
> -				    pages, ih, domain->qi_batch);
> +		if (pages == -1)
> +			qi_batch_add_piotlb_all(iommu, tag->domain_id,
> +						tag->pasid, domain->qi_batch);
> +		else
> +			qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid,
> +					    addr, pages, ih, domain->qi_batch);
>   		return;
>   	}
>   
> diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
> index 69222dbd2af0ea..6f496fc6a5ff1a 100644
> --- a/drivers/iommu/intel/dmar.c
> +++ b/drivers/iommu/intel/dmar.c
> @@ -1550,23 +1550,12 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
>   	qi_submit_sync(iommu, &desc, 1, 0);
>   }
>   
> -/* PASID-based IOTLB invalidation */
> -void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
> -		     unsigned long npages, bool ih)
> +/* PASID-selective IOTLB invalidation */
> +void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid)
>   {
> -	struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
> +	struct qi_desc desc = {};
>   
> -	/*
> -	 * npages == -1 means a PASID-selective invalidation, otherwise,
> -	 * a positive value for Page-selective-within-PASID invalidation.
> -	 * 0 is not a valid input.
> -	 */
> -	if (WARN_ON(!npages)) {
> -		pr_err("Invalid input npages = %ld\n", npages);
> -		return;
> -	}
> -
> -	qi_desc_piotlb(did, pasid, addr, npages, ih, &desc);
> +	qi_desc_piotlb_all(did, pasid, &desc);
>   	qi_submit_sync(iommu, &desc, 1, 0);
>   }
>   
> diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
> index 599913fb65d59e..40759587729953 100644
> --- a/drivers/iommu/intel/iommu.h
> +++ b/drivers/iommu/intel/iommu.h
> @@ -1082,31 +1082,29 @@ static inline void qi_desc_dev_iotlb(u16 sid, u16 pfsid, u16 qdep, u64 addr,
>   	desc->qw3 = 0;
>   }
>   
> +/* PASID-selective IOTLB invalidation */
> +static inline void qi_desc_piotlb_all(u16 did, u32 pasid, struct qi_desc *desc)
> +{
> +	desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
> +		    QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | QI_EIOTLB_TYPE;
> +	desc->qw1 = 0;
> +}
> +
> +/* Page-selective-within-PASID IOTLB invalidation */
>   static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr,
>   				  unsigned long npages, bool ih,
>   				  struct qi_desc *desc)
>   {
> -	if (npages == -1) {
> -		desc->qw0 = QI_EIOTLB_PASID(pasid) |
> -				QI_EIOTLB_DID(did) |
> -				QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
> -				QI_EIOTLB_TYPE;
> -		desc->qw1 = 0;
> -	} else {
> -		int mask = ilog2(__roundup_pow_of_two(npages));
> -		unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
> +	int mask = ilog2(__roundup_pow_of_two(npages));
> +	unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
>   
> -		if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
> -			addr = ALIGN_DOWN(addr, align);
> +	if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
> +		addr = ALIGN_DOWN(addr, align);
>   
> -		desc->qw0 = QI_EIOTLB_PASID(pasid) |
> -				QI_EIOTLB_DID(did) |
> -				QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
> -				QI_EIOTLB_TYPE;
> -		desc->qw1 = QI_EIOTLB_ADDR(addr) |
> -				QI_EIOTLB_IH(ih) |
> -				QI_EIOTLB_AM(mask);
> -	}
> +	desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
> +		    QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE;
> +	desc->qw1 = QI_EIOTLB_ADDR(addr) | QI_EIOTLB_IH(ih) |
> +		    QI_EIOTLB_AM(mask);
>   }
>   
>   static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
> @@ -1168,6 +1166,7 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
>   void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
>   			u16 qdep, u64 addr, unsigned mask);
>   
> +void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid);

[...]

>   void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
>   		     unsigned long npages, bool ih);

qi_flush_piotlb() has been removed by this patch. Therefore above
declaration should also be cleaned.

>   
> diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
> index 9d30015b894057..89541b74ab8ca3 100644
> --- a/drivers/iommu/intel/pasid.c
> +++ b/drivers/iommu/intel/pasid.c
> @@ -282,7 +282,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
>   	pasid_cache_invalidation_with_pasid(iommu, did, pasid);
>   
>   	if (pgtt == PASID_ENTRY_PGTT_PT || pgtt == PASID_ENTRY_PGTT_FL_ONLY)
> -		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
> +		qi_flush_piotlb_all(iommu, did, pasid);
>   	else
>   		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
>   
> @@ -308,7 +308,7 @@ static void pasid_flush_caches(struct intel_iommu *iommu,
>   
>   	if (cap_caching_mode(iommu->cap)) {
>   		pasid_cache_invalidation_with_pasid(iommu, did, pasid);
> -		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
> +		qi_flush_piotlb_all(iommu, did, pasid);
>   	} else {
>   		iommu_flush_write_buffer(iommu);
>   	}
> @@ -342,7 +342,7 @@ static void intel_pasid_flush_present(struct intel_iommu *iommu,
>   	 *      Addr[63:12]=0x7FFFFFFF_FFFFF) to affected functions
>   	 */
>   	pasid_cache_invalidation_with_pasid(iommu, did, pasid);
> -	qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
> +	qi_flush_piotlb_all(iommu, did, pasid);
>   
>   	devtlb_invalidation_with_pasid(iommu, dev, pasid);
>   }
> diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c
> index ff63c228e6e19d..ed98b1cb06fecf 100644
> --- a/drivers/iommu/intel/prq.c
> +++ b/drivers/iommu/intel/prq.c
> @@ -113,7 +113,7 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid)
>   		qi_desc_dev_iotlb(sid, info->pfsid, info->ats_qdep, 0,
>   				  MAX_AGAW_PFN_WIDTH, &desc[2]);
>   	} else {
> -		qi_desc_piotlb(did, pasid, 0, -1, 0, &desc[1]);
> +		qi_desc_piotlb_all(did, pasid, &desc[1]);
>   		qi_desc_dev_iotlb_pasid(sid, info->pfsid, pasid, info->ats_qdep,
>   					0, MAX_AGAW_PFN_WIDTH, &desc[2]);
>   	}

Others look good to me. Thanks for the patch.

Thanks,
baolu

Re: [PATCH 2/4] iommu/vtd: Pass size_order to qi_desc_piotlb() not npages

[Baolu Lu]

On 3/27/26 23:25, Jason Gunthorpe wrote:
> It doesn't make sense for the caller to compute mask, throw it away
> and then have qi_desc_piotlb() compute it again.
> 
> Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
> ---
>   drivers/iommu/intel/cache.c | 10 ++++------
>   drivers/iommu/intel/iommu.h | 16 ++++++----------
>   2 files changed, 10 insertions(+), 16 deletions(-)
> 
> diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c
> index e08253980a6ee7..74ee2002fb9c85 100644
> --- a/drivers/iommu/intel/cache.c
> +++ b/drivers/iommu/intel/cache.c
> @@ -338,13 +338,11 @@ static void qi_batch_add_piotlb_all(struct intel_iommu *iommu, u16 did,
>   }
>   
>   static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,
> -				u64 addr, unsigned long npages, bool ih,
> +				u64 addr, unsigned int size_order, bool ih,
>   				struct qi_batch *batch)
>   {
> -	if (!npages)
> -		return;
> -
> -	qi_desc_piotlb(did, pasid, addr, npages, ih, &batch->descs[batch->index]);
> +	qi_desc_piotlb(did, pasid, addr, size_order, ih,
> +		       &batch->descs[batch->index]);
>   	qi_batch_increment_index(iommu, batch);
>   }
>   
> @@ -385,7 +383,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *
>   						tag->pasid, domain->qi_batch);
>   		else
>   			qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid,
> -					    addr, pages, ih, domain->qi_batch);
> +					    addr, mask, ih, domain->qi_batch);
>   		return;
>   	}
>   
> diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
> index 40759587729953..7a92472985ee02 100644
> --- a/drivers/iommu/intel/iommu.h
> +++ b/drivers/iommu/intel/iommu.h
> @@ -1092,19 +1092,16 @@ static inline void qi_desc_piotlb_all(u16 did, u32 pasid, struct qi_desc *desc)
>   
>   /* Page-selective-within-PASID IOTLB invalidation */
>   static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr,
> -				  unsigned long npages, bool ih,
> +				  unsigned int size_order, bool ih,
>   				  struct qi_desc *desc)
>   {
> -	int mask = ilog2(__roundup_pow_of_two(npages));
> -	unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
> -
> -	if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
> -		addr = ALIGN_DOWN(addr, align);
> -
> +	/*
> +	 * calculate_psi_aligned_address() must be used for addr and size_order
> +	 */
>   	desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
>   		    QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE;
>   	desc->qw1 = QI_EIOTLB_ADDR(addr) | QI_EIOTLB_IH(ih) |
> -		    QI_EIOTLB_AM(mask);
> +		    QI_EIOTLB_AM(size_order);
>   }
>   
>   static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
> @@ -1167,8 +1164,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
>   			u16 qdep, u64 addr, unsigned mask);
>   
>   void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid);
> -void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
> -		     unsigned long npages, bool ih);
> +

Could we move this cleanup to the previous patch?

Otherwise, looks good to me.

>   void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
>   			      u32 pasid, u16 qdep, u64 addr,

Thanks,
baolu

Re: [PATCH 1/4] iommu/intel: Split piotlb invalidation into range and all

[Jason Gunthorpe]

On Mon, Mar 30, 2026 at 02:39:50PM +0800, Baolu Lu wrote:
> >   static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,
> >   				u64 addr, unsigned long npages, bool ih,
> >   				struct qi_batch *batch)
> >   {
> > -	/*
> > -	 * npages == -1 means a PASID-selective invalidation, otherwise,
> > -	 * a positive value for Page-selective-within-PASID invalidation.
> > -	 * 0 is not a valid input.
> > -	 */
> >   	if (!npages)
> >   		return;
> 
> The WARN_ON(!npages) logic is removed from the previous
> qi_flush_piotlb() (see below ...). Could it be added here to keep the
> "npages != 0" check?

The later patches remove this check entirely so I don't think we
should add a WARN_ON then remove it again..

> > @@ -1168,6 +1166,7 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
> >   void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
> >   			u16 qdep, u64 addr, unsigned mask);
> > +void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid);
> 
> [...]
> 
> >   void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
> >   		     unsigned long npages, bool ih);
> 
> qi_flush_piotlb() has been removed by this patch. Therefore above
> declaration should also be cleaned.

Ooops that hunk got into a later patch, I shifted it back

Thanks,
Jason

Re: [PATCH 1/4] iommu/intel: Split piotlb invalidation into range and all

[Baolu Lu]

On 3/30/26 23:31, Jason Gunthorpe wrote:
>>> @@ -1168,6 +1166,7 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
>>>    void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
>>>    			u16 qdep, u64 addr, unsigned mask);
>>> +void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid);
>> [...]
>>
>>>    void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
>>>    		     unsigned long npages, bool ih);
>> qi_flush_piotlb() has been removed by this patch. Therefore above
>> declaration should also be cleaned.
> Ooops that hunk got into a later patch, I shifted it back

The rest of the series looks great to me. I have applied this change
manually and included the patches in my pull request to Joerg:

https://lore.kernel.org/linux-iommu/20260402065734.1687476-1-baolu.lu@linux.intel.com/

Please let me know if I missed anything.

Thanks,
baolu