The Linux Kernel Mailing List
 help / color / mirror / Atom feed
From: Samiullah Khawaja <skhawaja@google.com>
To: Baolu Lu <baolu.lu@linux.intel.com>
Cc: Joerg Roedel <joro@8bytes.org>, Will Deacon <will@kernel.org>,
	 Jason Gunthorpe <jgg@ziepe.ca>,
	David Woodhouse <dwmw2@infradead.org>,
	 Robin Murphy <robin.murphy@arm.com>,
	Kevin Tian <kevin.tian@intel.com>,
	 Alex Williamson <alex@shazbot.org>,
	Shuah Khan <shuah@kernel.org>,
	iommu@lists.linux.dev,  linux-kernel@vger.kernel.org,
	kvm@vger.kernel.org, Pratyush Yadav <pratyush@kernel.org>,
	 Pasha Tatashin <pasha.tatashin@soleen.com>,
	David Matlack <dmatlack@google.com>,
	 Andrew Morton <akpm@linux-foundation.org>,
	Pranjal Shrivastava <praan@google.com>,
	 Vipin Sharma <vipinsh@google.com>,
	Ankit Soni <Ankit.Soni@amd.com>,
	 Nicolin Chen <nicolinc@nvidia.com>,
	John Starks <dev@jstarks.net>
Subject: Re: [PATCH v7 05/18] iommu: Implement device and IOMMU HW preservation
Date: Mon, 16 Nov 2026 18:42:54 +0000	[thread overview]
Message-ID: <avtMVRYk59ilQMlG@google.com> (raw)
In-Reply-To: <5dc1b3a3-8e9d-4771-b70a-ace7d5b919df@linux.intel.com>

On Mon, Nov 16, 2026 at 06:19:11PM +0800, Baolu Lu wrote:
>On 11/10/2026 12:08 PM, Samiullah Khawaja wrote:
>>Add IOMMU ops to preserve/unpreserve the device state handled by the
>>IOMMU driver. This is IOMMU specific state of the device that needs to
>>be preserved for liveupdate. These ops can be implemented by the IOMMU
>>drivers that support preservation of devices that have their IOMMU
>>domains preserved. During device preservation the state of the
>>associated IOMMU is also preserved as dependency.
>>
>>Signed-off-by: Samiullah Khawaja <skhawaja@google.com>
>>---
>>  drivers/iommu/iommu-priv.h       |   2 +
>>  drivers/iommu/iommu.c            |  32 +++++-
>>  drivers/iommu/liveupdate.c       | 191 +++++++++++++++++++++++++++++++
>>  include/linux/iommu-liveupdate.h |  76 ++++++++++++
>>  include/linux/iommu.h            |  21 ++++
>>  5 files changed, 321 insertions(+), 1 deletion(-)
>>
>>diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h
>>index 1f9f886b2c10..8a14d38a3e70 100644
>>--- a/drivers/iommu/iommu-priv.h
>>+++ b/drivers/iommu/iommu-priv.h
>>@@ -21,6 +21,8 @@ static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
>>  void dev_iommu_free(struct device *dev);
>>+size_t iommu_group_device_count(struct iommu_group *group);
>>+
>>  const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode);
>>  static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwspec)
>>diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
>>index b486b8bbd1fc..932325e0af41 100644
>>--- a/drivers/iommu/iommu.c
>>+++ b/drivers/iommu/iommu.c
>>@@ -18,6 +18,7 @@
>>  #include <linux/errno.h>
>>  #include <linux/host1x_context_bus.h>
>>  #include <linux/iommu.h>
>>+#include <linux/iommu-liveupdate.h>
>>  #include <linux/iommufd.h>
>>  #include <linux/idr.h>
>>  #include <linux/err.h>
>>@@ -626,8 +627,8 @@ DEFINE_MUTEX(iommu_probe_device_lock);
>>  static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
>>  {
>>+	struct group_device *gdev, *gdev2;
>>  	struct iommu_group *group;
>>-	struct group_device *gdev;
>>  	int ret;
>>  	/*
>>@@ -661,6 +662,13 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
>>  		goto err_put_group;
>>  	}
>>+	for_each_group_device(group, gdev2) {
>>+		if (dev_iommu_has_preserved_state(gdev2->dev)) {
>>+			ret = -EBUSY;
>>+			goto err_free_gdev;
>>+		}
>>+	}
>>+
>>  	/*
>>  	 * The gdev must be in the list before calling
>>  	 * iommu_setup_default_domain()
>>@@ -697,6 +705,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
>>  err_remove_gdev:
>>  	list_del(&gdev->list);
>>+err_free_gdev:
>>  	__iommu_group_free_device(group, gdev);
>>  err_put_group:
>>  	iommu_deinit_device(dev);
>>@@ -3543,6 +3552,27 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group)
>>  }
>>  EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
>>+/**
>>+ * iommu_group_device_count() - Get the number of devices in a group
>>+ * @group: The group.
>>+ *
>>+ * The device count may change after this function returns, as devices can
>>+ * be added to or removed from the group. The caller must recheck it if it
>>+ * needs the count to stay the same.
>>+ *
>>+ * Return: the number of devices in the group.
>>+ */
>>+size_t iommu_group_device_count(struct iommu_group *group)
>>+{
>>+	size_t count;
>>+
>>+	mutex_lock(&group->mutex);
>>+	count = list_count_nodes(&group->devices);
>>+	mutex_unlock(&group->mutex);
>>+
>>+	return count;
>>+}
>>+
>>  static void iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
>>  				   struct iommu_domain *domain)
>>  {
>
>[...]
>
>>+/**
>>+ * iommu_preserve_device() - Preserve device state across live update
>>+ * @domain: Associated IOMMU domain
>>+ * @dev: Device to preserve
>>+ * @dma_owner_token: Token to identify DMA owner of this device
>>+ *
>>+ * Return: 0 on success, or negative error code.
>>+ */
>>+int iommu_preserve_device(struct iommu_domain *domain,
>>+			  struct device *dev, u64 dma_owner_token)
>>+{
>>+	struct iommu_device_ser *device_ser;
>>+	struct iommu_flb_obj *flb_obj;
>>+	struct dev_iommu *iommu;
>>+	struct pci_dev *pdev;
>>+	int ret;
>>+
>>+	if (!dev_is_pci(dev))
>>+		return -EOPNOTSUPP;
>>+
>>+	if (!iommu_group_dma_owner_claimed(dev->iommu_group))
>>+		return -EINVAL;
>>+
>>+	if (iommu_group_device_count(dev->iommu_group) != 1)
>>+		return -EOPNOTSUPP;
>>+
>>+	if (iommu_get_domain_for_dev(dev) != domain)
>>+		return -EINVAL;
>>+
>>+	pdev = to_pci_dev(dev);
>>+	iommu = dev->iommu;
>>+	if (!iommu->iommu_dev->ops->preserve_device ||
>>+	    !iommu->iommu_dev->ops->unpreserve_device ||
>>+	    !iommu->iommu_dev->ops->preserve ||
>>+	    !iommu->iommu_dev->ops->unpreserve)
>>+		return -EOPNOTSUPP;
>>+
>>+	ret = liveupdate_flb_get_outgoing(&iommu_flb, (void **)&flb_obj);
>>+	if (ret)
>>+		return ret;
>>+
>>+	mutex_lock(&flb_obj->lock);
>>+	if (!domain->domain_ser) {
>>+		ret = -EINVAL;
>>+		goto out_unlock;
>>+	}
>>+
>>+	if (dev_iommu_preserved_state(dev)) {
>>+		ret = -EBUSY;
>>+		goto out_unlock;
>>+	}
>>+
>>+	device_ser = iommu_ser_set_alloc(&flb_obj->device_set);
>>+	if (IS_ERR(device_ser)) {
>>+		ret = PTR_ERR(device_ser);
>>+		goto out_unlock;
>>+	}
>>+
>>+	ret = iommu_preserve_locked(iommu->iommu_dev, flb_obj);
>>+	if (ret) {
>>+		device_ser->hdr.flags |= IOMMU_SER_FLAG_DELETED;
>>+		goto out_unlock;
>>+	}
>>+
>>+	device_ser->attachment.domain_phys = virt_to_phys(domain->domain_ser);
>>+	device_ser->iommu_phys = virt_to_phys(iommu->iommu_dev->iommu_ser);
>>+	device_ser->devid = pci_dev_id(pdev);
>>+	device_ser->pci_domain_nr = pci_domain_nr(pdev->bus);
>>+	device_ser->dma_owner_token = dma_owner_token;
>>+
>>+	ret = iommu->iommu_dev->ops->preserve_device(dev, device_ser);
>>+	if (!ret) {
>>+		WRITE_ONCE(dev->iommu->device_ser, device_ser);
>>+
>>+		/* Validate that no sibling device was added concurrently */
>>+		if (iommu_group_device_count(dev->iommu_group) != 1) {
>>+			iommu->iommu_dev->ops->unpreserve_device(dev, device_ser);
>>+			WRITE_ONCE(dev->iommu->device_ser, NULL);
>>+			ret = -EOPNOTSUPP;
>>+		}
>>+	}
>>+
>>+	if (ret) {
>>+		device_ser->hdr.flags |= IOMMU_SER_FLAG_DELETED;
>>+		iommu_unpreserve_locked(iommu->iommu_dev, flb_obj);
>>+		goto out_unlock;
>>+	}
>>+
>>+out_unlock:
>>+	mutex_unlock(&flb_obj->lock);
>>+	liveupdate_flb_put_outgoing(&iommu_flb);
>>+	return ret;
>>+}
>>+EXPORT_SYMBOL_GPL(iommu_preserve_device);
>Only devices in singleton iommu groups can be preserved, and no other
>device may join the group while preservation is in progress or the
>device remains preserved.
>
>Could we simplify this by adding a flag to struct iommu_group, protected
>by group->mutex? For example:
>
>@@ -72,6 +72,9 @@ struct iommu_group {
>         */
>        unsigned int recovery_cnt;
>        void *owner;
>+#ifdef CONFIG_IOMMU_LIVEUPDATE
>+       bool liveupdate_reserved;
>+#endif
> };
>
>A helper could check that the group contains exactly one device and set
>the flag atomically under group->mutex. Device probing would reject an
>addition if the flag is set.
>
>#ifdef CONFIG_IOMMU_LIVEUPDATE
>int iommu_group_reserve_liveupdate(struct iommu_group *group)
>{
>        int ret = 0;
>
>        mutex_lock(&group->mutex);
>        if (group->liveupdate_reserved)
>                ret = -EBUSY;
>        else if (list_count_nodes(&group->devices) != 1)
>                ret = -EOPNOTSUPP;
>        else
>                group->liveupdate_reserved = true;
>        mutex_unlock(&group->mutex);
>
>        return ret;
>}
>
>void iommu_group_unreserve_liveupdate(struct iommu_group *group)
>{
>        mutex_lock(&group->mutex);
>        group->liveupdate_reserved = false;
>        mutex_unlock(&group->mutex);
>}
>#endif
>
>The reservation could be taken before calling the driver's preserve
>callback and cleared on every failure path and after unpreservation.
>This would avoid the second device-count check.
>
>Any thoughts?

This looks good. I will use it in the next revision, and use the same
flag on the restore side as well.

Thanks,
Sami

  reply	other threads:[~2026-11-16 18:42 UTC|newest]

Thread overview: 26+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-10  4:08 [PATCH v7 00/18] iommu: Add live update state preservation Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 01/18] iommu: Implement IOMMU Live update FLB callbacks Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 02/18] iommu/pages: Add APIs to preserve/unpreserve/restore iommu pages Samiullah Khawaja
2026-11-12 16:31   ` Ankit Soni
2026-11-13  1:31     ` Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 03/18] iommupt: Implement preserve/unpreserve/restore callbacks Samiullah Khawaja
2026-11-16 15:22   ` Ankit Soni
2026-11-10  4:08 ` [PATCH v7 04/18] iommu: Implement IOMMU domain preservation Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 05/18] iommu: Implement device and IOMMU HW preservation Samiullah Khawaja
2026-11-16 10:19   ` Baolu Lu
2026-11-16 18:42     ` Samiullah Khawaja [this message]
2026-11-10  4:08 ` [PATCH v7 06/18] iommu/vt-d: Implement device and iommu preserve/unpreserve ops Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 07/18] iommu/vt-d: Clear unpreserved context entries during shutdown Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 08/18] iommu: Add APIs to get iommu and device preserved state Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 09/18] iommu/vt-d: Combine disable_dmar_iommu() and free_dmar_iommu() Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 10/18] iommu/vt-d: Restore IOMMU state and reclaimed domain ids Samiullah Khawaja
2026-11-16  3:31   ` Baolu Lu
2026-11-10  4:08 ` [PATCH v7 11/18] iommu/vt-d: Preserve PASID table of preserved device Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 12/18] iommu: Restore and reattach preserved domains to devices Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 13/18] iommu/vt-d: Handle reattach of the restored domain Samiullah Khawaja
2026-11-16  7:38   ` Baolu Lu
2026-11-10  4:08 ` [PATCH v7 14/18] iommufd: Implement ioctl to mark HWPT for preservation Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 15/18] iommufd: Persist iommu hardware pagetables for live update Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 16/18] iommufd: Add APIs to preserve/unpreserve a vfio cdev Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 17/18] vfio/pci: Preserve the iommufd state of the " Samiullah Khawaja
2026-11-10  4:08 ` [PATCH v7 18/18] iommufd/selftest: Add test to verify iommufd preservation Samiullah Khawaja

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=avtMVRYk59ilQMlG@google.com \
    --to=skhawaja@google.com \
    --cc=Ankit.Soni@amd.com \
    --cc=akpm@linux-foundation.org \
    --cc=alex@shazbot.org \
    --cc=baolu.lu@linux.intel.com \
    --cc=dev@jstarks.net \
    --cc=dmatlack@google.com \
    --cc=dwmw2@infradead.org \
    --cc=iommu@lists.linux.dev \
    --cc=jgg@ziepe.ca \
    --cc=joro@8bytes.org \
    --cc=kevin.tian@intel.com \
    --cc=kvm@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=nicolinc@nvidia.com \
    --cc=pasha.tatashin@soleen.com \
    --cc=praan@google.com \
    --cc=pratyush@kernel.org \
    --cc=robin.murphy@arm.com \
    --cc=shuah@kernel.org \
    --cc=vipinsh@google.com \
    --cc=will@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox