[PATCH 2/5] iommu/amd: Pass last in through to build_inv_address()

Jason Gunthorpe
2026-03-27T15:24:13Z

This is the trivial call chain below amd_iommu_domain_flush_pages().

Cases that are doing a full invalidate will pass a last of U64_MAX.

This avoids converting between size and last, and type confusion with
size_t, unsigned long and u64 all being used in different places along
the driver's invalidation path. Consistently use u64 in the internals.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
---
drivers/iommu/amd/amd_iommu.h | 2 +-
drivers/iommu/amd/iommu.c | 56 +++++++++++++++++------------------
drivers/iommu/amd/pasid.c | 2 +-
3 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 1342e764a5486d..ef4ebecb003494 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -90,7 +90,7 @@ void amd_iommu_flush_all_caches(struct amd_iommu *iommu);
void amd_iommu_domain_flush_pages(struct protection_domain *domain,
u64 address, size_t size);
void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,
- ioasid_t pasid, u64 address, size_t size);
+ ioasid_t pasid, u64 address, u64 last);

#ifdef CONFIG_IRQ_REMAP
int amd_iommu_create_irq_domain(struct amd_iommu *iommu);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 28308759bb3e4d..caab941b531bfe 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1265,9 +1265,8 @@ static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
* Builds an invalidation address which is suitable for one page or multiple
* pages. Sets the size bit (S) as needed is more than one page is flushed.
*/
-static inline u64 build_inv_address(u64 address, size_t size)
+static inline u64 build_inv_address(u64 address, u64 last)
{
- u64 last = address + size - 1;
unsigned int sz_lg2;

address &= GENMASK_U64(63, 12);
@@ -1302,10 +1301,10 @@ static inline u64 build_inv_address(u64 address, size_t size)
}

static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
- size_t size, u16 domid,
+ u64 last, u16 domid,
ioasid_t pasid, bool gn)
{
- u64 inv_address = build_inv_address(address, size);
+ u64 inv_address = build_inv_address(address, last);

memset(cmd, 0, sizeof(*cmd));

@@ -1322,10 +1321,10 @@ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
}

static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
- u64 address, size_t size,
+ u64 address, u64 last,
ioasid_t pasid, bool gn)
{
- u64 inv_address = build_inv_address(address, size);
+ u64 inv_address = build_inv_address(address, last);

memset(cmd, 0, sizeof(*cmd));

@@ -1523,7 +1522,7 @@ static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)

for (dom_id = 0; dom_id <= last_bdf; ++dom_id) {
struct iommu_cmd cmd;
- build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
+ build_inv_iommu_pages(&cmd, 0, U64_MAX,
dom_id, IOMMU_NO_PASID, false);
iommu_queue_command(iommu, &cmd);
}
@@ -1535,14 +1534,14 @@ static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id)
{
struct iommu_cmd cmd;

- build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
+ build_inv_iommu_pages(&cmd, 0, U64_MAX,
dom_id, IOMMU_NO_PASID, false);
iommu_queue_command(iommu, &cmd);

iommu_completion_wait(iommu);
}

-static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 address, size_t size)
+static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 address, u64 last)
{
int ret = 0;
struct amd_iommu_viommu *aviommu;
@@ -1559,7 +1558,7 @@ static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 add

pr_debug("%s: iommu=%#x, hdom_id=%#x\n", __func__,
iommu->devid, gdom_info->hdom_id);
- build_inv_iommu_pages(&cmd, address, size, gdom_info->hdom_id,
+ build_inv_iommu_pages(&cmd, address, last, gdom_info->hdom_id,
IOMMU_NO_PASID, false);
ret |= iommu_queue_command(iommu, &cmd);
}
@@ -1616,14 +1615,14 @@ void amd_iommu_flush_all_caches(struct amd_iommu *iommu)
* Command send function for flushing on-device TLB
*/
static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address,
- size_t size, ioasid_t pasid, bool gn)
+ u64 last, ioasid_t pasid, bool gn)
{
struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
struct iommu_cmd cmd;
int qdep = dev_data->ats_qdep;

build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address,
- size, pasid, gn);
+ last, pasid, gn);

return iommu_queue_command(iommu, &cmd);
}
@@ -1667,7 +1666,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data)

if (dev_data->ats_enabled) {
/* Invalidate the entire contents of an IOTLB */
- ret = device_flush_iotlb(dev_data, 0, ~0UL,
+ ret = device_flush_iotlb(dev_data, 0, U64_MAX,
IOMMU_NO_PASID, false);
}

@@ -1675,7 +1674,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data)
}

static int domain_flush_pages_v2(struct protection_domain *pdom,
- u64 address, size_t size)
+ u64 address, u64 last)
{
struct iommu_dev_data *dev_data;
struct iommu_cmd cmd;
@@ -1686,7 +1685,7 @@ static int domain_flush_pages_v2(struct protection_domain *pdom,
struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
u16 domid = dev_data->gcr3_info.domid;

- build_inv_iommu_pages(&cmd, address, size,
+ build_inv_iommu_pages(&cmd, address, last,
domid, IOMMU_NO_PASID, true);

ret |= iommu_queue_command(iommu, &cmd);
@@ -1696,7 +1695,7 @@ static int domain_flush_pages_v2(struct protection_domain *pdom,
}

static int domain_flush_pages_v1(struct protection_domain *pdom,
- u64 address, size_t size)
+ u64 address, u64 last)
{
struct pdom_iommu_info *pdom_iommu_info;
struct iommu_cmd cmd;
@@ -1705,7 +1704,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom,

lockdep_assert_held(&pdom->lock);

- build_inv_iommu_pages(&cmd, address, size,
+ build_inv_iommu_pages(&cmd, address, last,
pdom->id, IOMMU_NO_PASID, false);

xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) {
@@ -1725,7 +1724,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom,
* See drivers/iommu/amd/nested.c: amd_iommu_alloc_domain_nested()
*/
if (!list_empty(&pdom->viommu_list))
- ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, size);
+ ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, last);

return ret;
}
@@ -1735,7 +1734,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom,
* It flushes range of PTEs of the domain.
*/
static void __domain_flush_pages(struct protection_domain *domain,
- u64 address, size_t size)
+ u64 address, u64 last)
{
struct iommu_dev_data *dev_data;
int ret = 0;
@@ -1746,9 +1745,9 @@ static void __domain_flush_pages(struct protection_domain *domain,

if (pdom_is_v2_pgtbl_mode(domain)) {
gn = true;
- ret = domain_flush_pages_v2(domain, address, size);
+ ret = domain_flush_pages_v2(domain, address, last);
} else {
- ret = domain_flush_pages_v1(domain, address, size);
+ ret = domain_flush_pages_v1(domain, address, last);
}

list_for_each_entry(dev_data, &domain->dev_list, list) {
@@ -1756,7 +1755,7 @@ static void __domain_flush_pages(struct protection_domain *domain,
if (!dev_data->ats_enabled)
continue;

- ret |= device_flush_iotlb(dev_data, address, size, pasid, gn);
+ ret |= device_flush_iotlb(dev_data, address, last, pasid, gn);
}

WARN_ON(ret);
@@ -1768,7 +1767,7 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain,
lockdep_assert_held(&domain->lock);

if (likely(!amd_iommu_np_cache)) {
- __domain_flush_pages(domain, address, size);
+ __domain_flush_pages(domain, address, address + size - 1);

/* Wait until IOMMU TLB and all device IOTLB flushes are complete */
domain_flush_complete(domain);
@@ -1805,7 +1804,7 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain,

flush_size = 1ul << min_alignment;

- __domain_flush_pages(domain, address, flush_size);
+ __domain_flush_pages(domain, address, address + flush_size - 1);
address += flush_size;
size -= flush_size;
}
@@ -1822,17 +1821,17 @@ static void amd_iommu_domain_flush_all(struct protection_domain *domain)
}

void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,
- ioasid_t pasid, u64 address, size_t size)
+ ioasid_t pasid, u64 address, u64 last)
{
struct iommu_cmd cmd;
struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);

- build_inv_iommu_pages(&cmd, address, size,
+ build_inv_iommu_pages(&cmd, address, last,
dev_data->gcr3_info.domid, pasid, true);
iommu_queue_command(iommu, &cmd);

if (dev_data->ats_enabled)
- device_flush_iotlb(dev_data, address, size, pasid, true);
+ device_flush_iotlb(dev_data, address, last, pasid, true);

iommu_completion_wait(iommu);
}
@@ -1840,8 +1839,7 @@ void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,
static void dev_flush_pasid_all(struct iommu_dev_data *dev_data,
ioasid_t pasid)
{
- amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0,
- CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
+ amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0, U64_MAX);
}

int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag)
diff --git a/drivers/iommu/amd/pasid.c b/drivers/iommu/amd/pasid.c
index 67eace9809f16a..d708c6532480a4 100644
--- a/drivers/iommu/amd/pasid.c
+++ b/drivers/iommu/amd/pasid.c
@@ -71,7 +71,7 @@ static void sva_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
for_each_pdom_dev_data(pdom_dev_data, sva_pdom) {
amd_iommu_dev_flush_pasid_pages(pdom_dev_data->dev_data,
pdom_dev_data->pasid,
- start, end - start);
+ start, end - 1);
}

spin_unlock_irqrestore(&sva_pdom->lock, flags);
--
2.43.0