Re: [PATCH RFCv1 1/3] PCI: Allow ATS to be always on for CXL.cache capable devices

[Nicolin Chen <nicolinc@nvidia.com>]

On Wed, Jan 28, 2026 at 09:05:20AM -0400, Jason Gunthorpe wrote:
> On Tue, Jan 27, 2026 at 04:49:07PM -0800, dan.j.williams@intel.com wrote:
> > > Yes, ARM took the position that ATS should be left disabled for
> > > IDENTITY both because of SMMU constraints and also because it made
> > > some sense that you wouldn't want ATS overhead just to get a 1:1
> > > translation.
> > 
> > Does this mean that ARM already today does not enable ATS until driver
> > attach, or is incremental work needed for that capability?
> 
> All of the iommu drivers setup an iommu translation and enable ATS
> before any driver is bound.
> 
> We would need to do more work in the core to leave the translation
> blocked when there is no driver. I don't think it is that difficult

Hmm, not sure if we could use group->domain=NULL as "blocked..

Otherwise, I made a draft:
-----------------------------------------------------------------
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 349f31bedfa17..8ed15d5ea1f51 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -437,8 +437,6 @@ static int driver_sysfs_add(struct device *dev)
 {
 	int ret;
 
-	bus_notify(dev, BUS_NOTIFY_BIND_DRIVER);
-
 	ret = sysfs_create_link(&dev->driver->p->kobj, &dev->kobj,
 				kobject_name(&dev->kobj));
 	if (ret)
@@ -638,10 +636,12 @@ static int really_probe(struct device *dev, const struct device_driver *drv)
 	if (ret)
 		goto pinctrl_bind_failed;
 
+	bus_notify(dev, BUS_NOTIFY_BIND_DRIVER);
+
 	if (dev->bus->dma_configure) {
 		ret = dev->bus->dma_configure(dev);
 		if (ret)
-			goto pinctrl_bind_failed;
+			goto bus_notify_bind_failed;
 	}
 
 	ret = driver_sysfs_add(dev);
@@ -717,9 +717,10 @@ static int really_probe(struct device *dev, const struct device_driver *drv)
 probe_failed:
 	driver_sysfs_remove(dev);
 sysfs_failed:
-	bus_notify(dev, BUS_NOTIFY_DRIVER_NOT_BOUND);
 	if (dev->bus && dev->bus->dma_cleanup)
 		dev->bus->dma_cleanup(dev);
+bus_notify_bind_failed:
+	bus_notify(dev, BUS_NOTIFY_DRIVER_NOT_BOUND);
 pinctrl_bind_failed:
 	device_links_no_driver(dev);
 	device_unbind_cleanup(dev);
@@ -1275,8 +1276,6 @@ static void __device_release_driver(struct device *dev, struct device *parent)
 
 		driver_sysfs_remove(dev);
 
-		bus_notify(dev, BUS_NOTIFY_UNBIND_DRIVER);
-
 		pm_runtime_put_sync(dev);
 
 		device_remove(dev);
@@ -1284,6 +1283,8 @@ static void __device_release_driver(struct device *dev, struct device *parent)
 		if (dev->bus && dev->bus->dma_cleanup)
 			dev->bus->dma_cleanup(dev);
 
+		bus_notify(dev, BUS_NOTIFY_UNBIND_DRIVER);
+
 		device_unbind_cleanup(dev);
 		device_links_driver_cleanup(dev);
 
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 2ca990dfbb884..af53dce00e29b 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -106,6 +106,7 @@ static int __iommu_attach_group(struct iommu_domain *domain,
 static struct iommu_domain *__iommu_paging_domain_alloc_flags(struct device *dev,
 						       unsigned int type,
 						       unsigned int flags);
+static int __iommu_group_alloc_blocking_domain(struct iommu_group *group);
 
 enum {
 	IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
@@ -618,12 +619,6 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
 	ret = iommu_init_device(dev);
 	if (ret)
 		return ret;
-	/*
-	 * And if we do now see any replay calls, they would indicate someone
-	 * misusing the dma_configure path outside bus code.
-	 */
-	if (dev->driver)
-		dev_WARN(dev, "late IOMMU probe at driver bind, something fishy here!\n");
 
 	group = dev->iommu_group;
 	gdev = iommu_group_alloc_device(group, dev);
@@ -641,6 +636,15 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
 	WARN_ON(group->default_domain && !group->domain);
 	if (group->default_domain)
 		iommu_create_device_direct_mappings(group->default_domain, dev);
+
+	/* Block translation requests from a device without driver */
+	if (!dev->driver) {
+		ret = __iommu_group_alloc_blocking_domain(group);
+		if (ret)
+			goto err_remove_gdev;
+		group->domain = group->blocking_domain;
+	}
+
 	if (group->domain) {
 		ret = __iommu_device_set_domain(group, dev, group->domain, NULL,
 						0);
@@ -1781,19 +1785,70 @@ static int probe_iommu_group(struct device *dev, void *data)
 	return ret;
 }
 
+static int iommu_attach_default_domain(struct device *dev)
+{
+	struct iommu_group *group = iommu_group_get(dev);
+	int ret = 0;
+
+	if (!group)
+		return 0;
+
+	mutex_lock(&group->mutex);
+
+	if (group->blocking_domain) {
+		if (!group->default_domain) {
+			ret = iommu_setup_default_domain(group, 0);
+			if (!ret)
+				iommu_setup_dma_ops(dev);
+		} else if (group->domain == group->blocking_domain) {
+			ret = __iommu_group_set_domain(
+				group, group->default_domain);
+		}
+	}
+
+	mutex_unlock(&group->mutex);
+	iommu_group_put(group);
+	return ret;
+}
+
+static void iommu_detach_default_domain(struct device *dev)
+{
+	struct iommu_group *group = iommu_group_get(dev);
+
+	if (!group)
+		return;
+
+	mutex_lock(&group->mutex);
+
+	if (group->blocking_domain && group->domain != group->blocking_domain) {
+		__iommu_attach_device(group->blocking_domain, dev,
+				      group->domain);
+		group->domain = group->blocking_domain;
+	}
+
+	mutex_unlock(&group->mutex);
+	iommu_group_put(group);
+}
+
 static int iommu_bus_notifier(struct notifier_block *nb,
 			      unsigned long action, void *data)
 {
 	struct device *dev = data;
+	int ret;
 
 	if (action == BUS_NOTIFY_ADD_DEVICE) {
-		int ret;
-
 		ret = iommu_probe_device(dev);
 		return (ret) ? NOTIFY_DONE : NOTIFY_OK;
 	} else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
 		iommu_release_device(dev);
 		return NOTIFY_OK;
+	} else if (action == BUS_NOTIFY_BIND_DRIVER) {
+		ret = iommu_attach_default_domain(dev);
+		return ret ? NOTIFY_DONE : NOTIFY_OK;
+	} else if (action == BUS_NOTIFY_UNBOUND_DRIVER ||
+		   action == BUS_NOTIFY_DRIVER_NOT_BOUND) {
+		iommu_detach_default_domain(dev);
+		return NOTIFY_OK;
 	}
 
 	return 0;
-----------------------------------------------------------------

Thanks
Nicolin