From: Baolu Lu <baolu.lu@linux.intel.com>
To: Samiullah Khawaja <skhawaja@google.com>,
Joerg Roedel <joro@8bytes.org>, Will Deacon <will@kernel.org>,
Jason Gunthorpe <jgg@ziepe.ca>,
David Woodhouse <dwmw2@infradead.org>
Cc: baolu.lu@linux.intel.com, Robin Murphy <robin.murphy@arm.com>,
Kevin Tian <kevin.tian@intel.com>,
Alex Williamson <alex@shazbot.org>, Shuah Khan <shuah@kernel.org>,
iommu@lists.linux.dev, linux-kernel@vger.kernel.org,
kvm@vger.kernel.org, Pratyush Yadav <pratyush@kernel.org>,
Pasha Tatashin <pasha.tatashin@soleen.com>,
David Matlack <dmatlack@google.com>,
Andrew Morton <akpm@linux-foundation.org>,
Pranjal Shrivastava <praan@google.com>,
Vipin Sharma <vipinsh@google.com>,
Ankit Soni <Ankit.Soni@amd.com>,
Nicolin Chen <nicolinc@nvidia.com>, John Starks <dev@jstarks.net>
Subject: Re: [PATCH v7 05/18] iommu: Implement device and IOMMU HW preservation
Date: Mon, 16 Nov 2026 18:19:11 +0800 [thread overview]
Message-ID: <5dc1b3a3-8e9d-4771-b70a-ace7d5b919df@linux.intel.com> (raw)
In-Reply-To: <20261110040828.1774343-6-skhawaja@google.com>
On 11/10/2026 12:08 PM, Samiullah Khawaja wrote:
> Add IOMMU ops to preserve/unpreserve the device state handled by the
> IOMMU driver. This is IOMMU specific state of the device that needs to
> be preserved for liveupdate. These ops can be implemented by the IOMMU
> drivers that support preservation of devices that have their IOMMU
> domains preserved. During device preservation the state of the
> associated IOMMU is also preserved as dependency.
>
> Signed-off-by: Samiullah Khawaja <skhawaja@google.com>
> ---
> drivers/iommu/iommu-priv.h | 2 +
> drivers/iommu/iommu.c | 32 +++++-
> drivers/iommu/liveupdate.c | 191 +++++++++++++++++++++++++++++++
> include/linux/iommu-liveupdate.h | 76 ++++++++++++
> include/linux/iommu.h | 21 ++++
> 5 files changed, 321 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h
> index 1f9f886b2c10..8a14d38a3e70 100644
> --- a/drivers/iommu/iommu-priv.h
> +++ b/drivers/iommu/iommu-priv.h
> @@ -21,6 +21,8 @@ static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
>
> void dev_iommu_free(struct device *dev);
>
> +size_t iommu_group_device_count(struct iommu_group *group);
> +
> const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode);
>
> static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwspec)
> diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
> index b486b8bbd1fc..932325e0af41 100644
> --- a/drivers/iommu/iommu.c
> +++ b/drivers/iommu/iommu.c
> @@ -18,6 +18,7 @@
> #include <linux/errno.h>
> #include <linux/host1x_context_bus.h>
> #include <linux/iommu.h>
> +#include <linux/iommu-liveupdate.h>
> #include <linux/iommufd.h>
> #include <linux/idr.h>
> #include <linux/err.h>
> @@ -626,8 +627,8 @@ DEFINE_MUTEX(iommu_probe_device_lock);
>
> static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
> {
> + struct group_device *gdev, *gdev2;
> struct iommu_group *group;
> - struct group_device *gdev;
> int ret;
>
> /*
> @@ -661,6 +662,13 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
> goto err_put_group;
> }
>
> + for_each_group_device(group, gdev2) {
> + if (dev_iommu_has_preserved_state(gdev2->dev)) {
> + ret = -EBUSY;
> + goto err_free_gdev;
> + }
> + }
> +
> /*
> * The gdev must be in the list before calling
> * iommu_setup_default_domain()
> @@ -697,6 +705,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
>
> err_remove_gdev:
> list_del(&gdev->list);
> +err_free_gdev:
> __iommu_group_free_device(group, gdev);
> err_put_group:
> iommu_deinit_device(dev);
> @@ -3543,6 +3552,27 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group)
> }
> EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
>
> +/**
> + * iommu_group_device_count() - Get the number of devices in a group
> + * @group: The group.
> + *
> + * The device count may change after this function returns, as devices can
> + * be added to or removed from the group. The caller must recheck it if it
> + * needs the count to stay the same.
> + *
> + * Return: the number of devices in the group.
> + */
> +size_t iommu_group_device_count(struct iommu_group *group)
> +{
> + size_t count;
> +
> + mutex_lock(&group->mutex);
> + count = list_count_nodes(&group->devices);
> + mutex_unlock(&group->mutex);
> +
> + return count;
> +}
> +
> static void iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
> struct iommu_domain *domain)
> {
[...]
> +/**
> + * iommu_preserve_device() - Preserve device state across live update
> + * @domain: Associated IOMMU domain
> + * @dev: Device to preserve
> + * @dma_owner_token: Token to identify DMA owner of this device
> + *
> + * Return: 0 on success, or negative error code.
> + */
> +int iommu_preserve_device(struct iommu_domain *domain,
> + struct device *dev, u64 dma_owner_token)
> +{
> + struct iommu_device_ser *device_ser;
> + struct iommu_flb_obj *flb_obj;
> + struct dev_iommu *iommu;
> + struct pci_dev *pdev;
> + int ret;
> +
> + if (!dev_is_pci(dev))
> + return -EOPNOTSUPP;
> +
> + if (!iommu_group_dma_owner_claimed(dev->iommu_group))
> + return -EINVAL;
> +
> + if (iommu_group_device_count(dev->iommu_group) != 1)
> + return -EOPNOTSUPP;
> +
> + if (iommu_get_domain_for_dev(dev) != domain)
> + return -EINVAL;
> +
> + pdev = to_pci_dev(dev);
> + iommu = dev->iommu;
> + if (!iommu->iommu_dev->ops->preserve_device ||
> + !iommu->iommu_dev->ops->unpreserve_device ||
> + !iommu->iommu_dev->ops->preserve ||
> + !iommu->iommu_dev->ops->unpreserve)
> + return -EOPNOTSUPP;
> +
> + ret = liveupdate_flb_get_outgoing(&iommu_flb, (void **)&flb_obj);
> + if (ret)
> + return ret;
> +
> + mutex_lock(&flb_obj->lock);
> + if (!domain->domain_ser) {
> + ret = -EINVAL;
> + goto out_unlock;
> + }
> +
> + if (dev_iommu_preserved_state(dev)) {
> + ret = -EBUSY;
> + goto out_unlock;
> + }
> +
> + device_ser = iommu_ser_set_alloc(&flb_obj->device_set);
> + if (IS_ERR(device_ser)) {
> + ret = PTR_ERR(device_ser);
> + goto out_unlock;
> + }
> +
> + ret = iommu_preserve_locked(iommu->iommu_dev, flb_obj);
> + if (ret) {
> + device_ser->hdr.flags |= IOMMU_SER_FLAG_DELETED;
> + goto out_unlock;
> + }
> +
> + device_ser->attachment.domain_phys = virt_to_phys(domain->domain_ser);
> + device_ser->iommu_phys = virt_to_phys(iommu->iommu_dev->iommu_ser);
> + device_ser->devid = pci_dev_id(pdev);
> + device_ser->pci_domain_nr = pci_domain_nr(pdev->bus);
> + device_ser->dma_owner_token = dma_owner_token;
> +
> + ret = iommu->iommu_dev->ops->preserve_device(dev, device_ser);
> + if (!ret) {
> + WRITE_ONCE(dev->iommu->device_ser, device_ser);
> +
> + /* Validate that no sibling device was added concurrently */
> + if (iommu_group_device_count(dev->iommu_group) != 1) {
> + iommu->iommu_dev->ops->unpreserve_device(dev, device_ser);
> + WRITE_ONCE(dev->iommu->device_ser, NULL);
> + ret = -EOPNOTSUPP;
> + }
> + }
> +
> + if (ret) {
> + device_ser->hdr.flags |= IOMMU_SER_FLAG_DELETED;
> + iommu_unpreserve_locked(iommu->iommu_dev, flb_obj);
> + goto out_unlock;
> + }
> +
> +out_unlock:
> + mutex_unlock(&flb_obj->lock);
> + liveupdate_flb_put_outgoing(&iommu_flb);
> + return ret;
> +}
> +EXPORT_SYMBOL_GPL(iommu_preserve_device);
Only devices in singleton iommu groups can be preserved, and no other
device may join the group while preservation is in progress or the
device remains preserved.
Could we simplify this by adding a flag to struct iommu_group, protected
by group->mutex? For example:
@@ -72,6 +72,9 @@ struct iommu_group {
*/
unsigned int recovery_cnt;
void *owner;
+#ifdef CONFIG_IOMMU_LIVEUPDATE
+ bool liveupdate_reserved;
+#endif
};
A helper could check that the group contains exactly one device and set
the flag atomically under group->mutex. Device probing would reject an
addition if the flag is set.
#ifdef CONFIG_IOMMU_LIVEUPDATE
int iommu_group_reserve_liveupdate(struct iommu_group *group)
{
int ret = 0;
mutex_lock(&group->mutex);
if (group->liveupdate_reserved)
ret = -EBUSY;
else if (list_count_nodes(&group->devices) != 1)
ret = -EOPNOTSUPP;
else
group->liveupdate_reserved = true;
mutex_unlock(&group->mutex);
return ret;
}
void iommu_group_unreserve_liveupdate(struct iommu_group *group)
{
mutex_lock(&group->mutex);
group->liveupdate_reserved = false;
mutex_unlock(&group->mutex);
}
#endif
The reservation could be taken before calling the driver's preserve
callback and cleared on every failure path and after unpreservation.
This would avoid the second device-count check.
Any thoughts?
Thanks,
baolu
next prev parent reply other threads:[~2026-11-16 10:19 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-10 4:08 [PATCH v7 00/18] iommu: Add live update state preservation Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 01/18] iommu: Implement IOMMU Live update FLB callbacks Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 02/18] iommu/pages: Add APIs to preserve/unpreserve/restore iommu pages Samiullah Khawaja
2026-11-12 16:31 ` Ankit Soni
2026-11-13 1:31 ` Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 03/18] iommupt: Implement preserve/unpreserve/restore callbacks Samiullah Khawaja
2026-11-16 15:22 ` Ankit Soni
2026-11-10 4:08 ` [PATCH v7 04/18] iommu: Implement IOMMU domain preservation Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 05/18] iommu: Implement device and IOMMU HW preservation Samiullah Khawaja
2026-11-16 10:19 ` Baolu Lu [this message]
2026-11-16 18:42 ` Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 06/18] iommu/vt-d: Implement device and iommu preserve/unpreserve ops Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 07/18] iommu/vt-d: Clear unpreserved context entries during shutdown Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 08/18] iommu: Add APIs to get iommu and device preserved state Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 09/18] iommu/vt-d: Combine disable_dmar_iommu() and free_dmar_iommu() Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 10/18] iommu/vt-d: Restore IOMMU state and reclaimed domain ids Samiullah Khawaja
2026-11-16 3:31 ` Baolu Lu
2026-11-10 4:08 ` [PATCH v7 11/18] iommu/vt-d: Preserve PASID table of preserved device Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 12/18] iommu: Restore and reattach preserved domains to devices Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 13/18] iommu/vt-d: Handle reattach of the restored domain Samiullah Khawaja
2026-11-16 7:38 ` Baolu Lu
2026-11-16 20:04 ` Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 14/18] iommufd: Implement ioctl to mark HWPT for preservation Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 15/18] iommufd: Persist iommu hardware pagetables for live update Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 16/18] iommufd: Add APIs to preserve/unpreserve a vfio cdev Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 17/18] vfio/pci: Preserve the iommufd state of the " Samiullah Khawaja
2026-11-10 4:08 ` [PATCH v7 18/18] iommufd/selftest: Add test to verify iommufd preservation Samiullah Khawaja
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=5dc1b3a3-8e9d-4771-b70a-ace7d5b919df@linux.intel.com \
--to=baolu.lu@linux.intel.com \
--cc=Ankit.Soni@amd.com \
--cc=akpm@linux-foundation.org \
--cc=alex@shazbot.org \
--cc=dev@jstarks.net \
--cc=dmatlack@google.com \
--cc=dwmw2@infradead.org \
--cc=iommu@lists.linux.dev \
--cc=jgg@ziepe.ca \
--cc=joro@8bytes.org \
--cc=kevin.tian@intel.com \
--cc=kvm@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=nicolinc@nvidia.com \
--cc=pasha.tatashin@soleen.com \
--cc=praan@google.com \
--cc=pratyush@kernel.org \
--cc=robin.murphy@arm.com \
--cc=shuah@kernel.org \
--cc=skhawaja@google.com \
--cc=vipinsh@google.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox