[PATCH v2 1/7] iommufd/device: Move sw_msi_start from igroup to idev

Nicolin Chen
2025-10-15T00:31:20Z

At the IOMMU driver layer the IOMMU_RESV_SW_MSI region is per device. So,
storing the sw_msi_start per idev makes sense.

And looking at the iommufd_sw_msi design:
- The global ictx->sw_msi_list allocates an item for each different pair
of sw_msi_start and msi_addr. And the allocation is per msi_desc, i.e.
per idev.
- Each allocated list item will be added to the igroup->required_sw_msi
and the hwpt_paging->present_sw_msi.bitmap during a device attachment.

This makes it possible to move the sw_msi_start from struct iommufd_group
struct iommufd_device, giving a potential to support a new SET_OPTION uAPI
per idev for user space to configure the start_sw_msi for 2-stage mappings
of the MSI window.

Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
drivers/iommu/iommufd/iommufd_private.h | 2 +-
drivers/iommu/iommufd/device.c | 31 +++++++++++++------------
drivers/iommu/iommufd/driver.c | 4 ++--
3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index 627f9b78483a0..73e5cddad24e9 100644
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -472,7 +472,6 @@ struct iommufd_group {
struct iommu_group *group;
struct xarray pasid_attach;
struct iommufd_sw_msi_maps required_sw_msi;
- phys_addr_t sw_msi_start;
};

/*
@@ -490,6 +489,7 @@ struct iommufd_device {
bool enforce_cache_coherency;
struct iommufd_vdevice *vdev;
bool destroying;
+ phys_addr_t sw_msi_start;
};

static inline struct iommufd_device *
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index 4c842368289f0..ea7ed32bbaede 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -96,7 +96,6 @@ static struct iommufd_group *iommufd_get_group(struct iommufd_ctx *ictx,
kref_init(&new_igroup->ref);
mutex_init(&new_igroup->lock);
xa_init(&new_igroup->pasid_attach);
- new_igroup->sw_msi_start = PHYS_ADDR_MAX;
/* group reference moves into new_igroup */
new_igroup->group = group;

@@ -272,6 +271,7 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
refcount_inc(&idev->obj.users);
/* igroup refcount moves into iommufd_device */
idev->igroup = igroup;
+ idev->sw_msi_start = PHYS_ADDR_MAX;

/*
* If the caller fails after this success it must call
@@ -367,13 +367,13 @@ static unsigned int iommufd_group_device_num(struct iommufd_group *igroup,
}

#ifdef CONFIG_IRQ_MSI_IOMMU
-static int iommufd_group_setup_msi(struct iommufd_group *igroup,
- struct iommufd_hwpt_paging *hwpt_paging)
+static int iommufd_device_setup_msi(struct iommufd_device *idev,
+ struct iommufd_hwpt_paging *hwpt_paging)
{
- struct iommufd_ctx *ictx = igroup->ictx;
+ struct iommufd_ctx *ictx = idev->ictx;
struct iommufd_sw_msi_map *cur;

- if (igroup->sw_msi_start == PHYS_ADDR_MAX)
+ if (idev->sw_msi_start == PHYS_ADDR_MAX)
return 0;

/*
@@ -383,8 +383,8 @@ static int iommufd_group_setup_msi(struct iommufd_group *igroup,
list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) {
int rc;

- if (cur->sw_msi_start != igroup->sw_msi_start ||
- !test_bit(cur->id, igroup->required_sw_msi.bitmap))
+ if (cur->sw_msi_start != idev->sw_msi_start ||
+ !test_bit(cur->id, idev->igroup->required_sw_msi.bitmap))
continue;

rc = iommufd_sw_msi_install(ictx, hwpt_paging, cur);
@@ -395,8 +395,8 @@ static int iommufd_group_setup_msi(struct iommufd_group *igroup,
}
#else
static inline int
-iommufd_group_setup_msi(struct iommufd_group *igroup,
- struct iommufd_hwpt_paging *hwpt_paging)
+iommufd_device_setup_msi(struct iommufd_device *idev,
+ struct iommufd_hwpt_paging *hwpt_paging)
{
return 0;
}
@@ -420,12 +420,12 @@ iommufd_device_attach_reserved_iova(struct iommufd_device *idev,

rc = iopt_table_enforce_dev_resv_regions(&hwpt_paging->ioas->iopt,
idev->dev,
- &igroup->sw_msi_start);
+ &idev->sw_msi_start);
if (rc)
return rc;

if (iommufd_group_first_attach(igroup, IOMMU_NO_PASID)) {
- rc = iommufd_group_setup_msi(igroup, hwpt_paging);
+ rc = iommufd_device_setup_msi(idev, hwpt_paging);
if (rc) {
iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt,
idev->dev);
@@ -745,9 +745,10 @@ iommufd_group_remove_reserved_iova(struct iommufd_group *igroup,
}

static int
-iommufd_group_do_replace_reserved_iova(struct iommufd_group *igroup,
- struct iommufd_hwpt_paging *hwpt_paging)
+iommufd_device_do_replace_reserved_iova(struct iommufd_device *idev,
+ struct iommufd_hwpt_paging *hwpt_paging)
{
+ struct iommufd_group *igroup = idev->igroup;
struct iommufd_hwpt_paging *old_hwpt_paging;
struct iommufd_attach *attach;
struct iommufd_device *cur;
@@ -767,7 +768,7 @@ iommufd_group_do_replace_reserved_iova(struct iommufd_group *igroup,
}
}

- rc = iommufd_group_setup_msi(igroup, hwpt_paging);
+ rc = iommufd_device_setup_msi(idev, hwpt_paging);
if (rc)
goto err_unresv;
return 0;
@@ -813,7 +814,7 @@ iommufd_device_do_replace(struct iommufd_device *idev, ioasid_t pasid,
}

if (attach_resv) {
- rc = iommufd_group_do_replace_reserved_iova(igroup, hwpt_paging);
+ rc = iommufd_device_do_replace_reserved_iova(idev, hwpt_paging);
if (rc)
goto err_unlock;
}
diff --git a/drivers/iommu/iommufd/driver.c b/drivers/iommu/iommufd/driver.c
index 6f1010da221c9..35475937d069b 100644
--- a/drivers/iommu/iommufd/driver.c
+++ b/drivers/iommu/iommufd/driver.c
@@ -271,7 +271,7 @@ int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,

handle = to_iommufd_handle(raw_handle);
/* No IOMMU_RESV_SW_MSI means no change to the msi_msg */
- if (handle->idev->igroup->sw_msi_start == PHYS_ADDR_MAX)
+ if (handle->idev->sw_msi_start == PHYS_ADDR_MAX)
return 0;

ictx = handle->idev->ictx;
@@ -283,7 +283,7 @@ int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
*/
msi_map = iommufd_sw_msi_get_map(handle->idev->ictx,
msi_addr & PAGE_MASK,
- handle->idev->igroup->sw_msi_start);
+ handle->idev->sw_msi_start);
if (IS_ERR(msi_map))
return PTR_ERR(msi_map);

--
2.43.0