Re: [PATCH v5 06/27] vfio/pci: Detect CXL devices and load the CXL provider on demand

[Richard Cheng <icheng@nvidia.com>]

On Thu, Sep 17, 2026 at 12:05:19AM +0800, mhonap@nvidia.com wrote:
> From: Manish Honap <mhonap@nvidia.com>
> 
> A CXL device needs the vfio-cxl callbacks, but pulling vfio-cxl and the
> CXL core in unconditionally would bloat every vfio-pci setup. Detect a
> CXL Type-2 device at bind and request_module("vfio-cxl") only then, then
> hand the device to the registered ops.
> 
> Each bound CXL device pins the provider through
> vfio_pci_get_cxl_ops() (try_module_get) and drops it with
> vfio_pci_put_cxl_ops() at release, so vfio-cxl can unload once no CXL
> device is bound.
> 
> If the provider is absent the device is driven as plain vfio-pci. A
> built-in provider whose initcall has not run yet is waited for with
> -EPROBE_DEFER; the deferred-probe machinery will retry the bind once the
> provider registers.
> 
> Assisted-by: LLM
> Signed-off-by: Manish Honap <mhonap@nvidia.com>
> ---
>  drivers/vfio/pci/vfio_pci_core.c | 84 ++++++++++++++++++++++++++++++++
>  include/linux/vfio_pci_core.h    |  3 ++
>  2 files changed, 87 insertions(+)
> 
> diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
> index 2cc5dd20396c..9eede1e56ab5 100644
> --- a/drivers/vfio/pci/vfio_pci_core.c
> +++ b/drivers/vfio/pci/vfio_pci_core.c
> @@ -2202,6 +2202,84 @@ void vfio_pci_core_unregister_cxl_ops(const struct vfio_cxl_ops *ops)
>  }
>  EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_cxl_ops);
>  
> +static const struct vfio_cxl_ops *vfio_pci_get_cxl_ops(void)
> +{
> +	guard(rwsem_read)(&vfio_pci_cxl_ops_rwsem);
> +
> +	if (vfio_pci_cxl_ops && try_module_get(vfio_pci_cxl_ops->owner))
> +		return vfio_pci_cxl_ops;
> +
> +	return NULL;
> +}
> +
> +static void vfio_pci_put_cxl_ops(const struct vfio_cxl_ops *ops)
> +{
> +	module_put(ops->owner);
> +}
> +

Anything guarantees that we can finish devres cleanup before dropping module ref ?

This function drops the module ref during vfio_pci_remove(), while devres might still pending.

I see in patch 18 it register vfio_cxl_release_hpa() as a devres callback, and it lives in vfio-cxl.
If this drops the last module ref, another process could unload vfio-cxl before devres invokes that callback.

Best regards,
Richard Cheng.




> +/*
> + * A CXL Type-2 device advertises both CXL.cache and CXL.mem in its CXL DVSEC.
> + * pcie_is_cxl() is also true for Type-1 (cache only) and Type-3 (mem only)
> + * devices, which the vfio-cxl provider does not handle, so confirm the Type-2
> + * identity before engaging it.
> + */
> +static bool vfio_pci_is_cxl_type2(struct pci_dev *pdev)
> +{
> +	u16 dvsec, cap;
> +
> +	if (!pcie_is_cxl(pdev))
> +		return false;
> +
> +	dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
> +					  PCI_DVSEC_CXL_DEVICE);
> +	if (!dvsec)
> +		return false;
> +
> +	if (pci_read_config_word(pdev, dvsec + PCI_DVSEC_CXL_CAP, &cap))
> +		return false;
> +
> +	return (cap & PCI_DVSEC_CXL_CACHE_CAPABLE) &&
> +		(cap & PCI_DVSEC_CXL_MEM_CAPABLE);
> +}
> +
> +/*
> + * Load vfio-cxl on demand for a CXL Type-2 device and hand the device to its
> + * ops. If the provider is absent the device is driven as plain vfio-pci; a
> + * built-in provider whose initcall has not run yet is waited for with
> + * -EPROBE_DEFER.
> + */
> +static int vfio_pci_core_cxl_init(struct vfio_pci_core_device *vdev)
> +{
> +	const struct vfio_cxl_ops *ops;
> +	int ret;
> +
> +	if (!vfio_pci_is_cxl_type2(vdev->pdev))
> +		return 0;
> +
> +	request_module("vfio-cxl");
> +	ops = vfio_pci_get_cxl_ops();
> +	if (!ops)
> +		return IS_BUILTIN(CONFIG_VFIO_CXL) ? -EPROBE_DEFER : 0;
> +
> +	ret = ops->init(vdev);
> +	if (ret) {
> +		vfio_pci_put_cxl_ops(ops);
> +		return ret;
> +	}
> +
> +	vdev->cxl_ops = ops;
> +	return 0;
> +}
> +
> +static void vfio_pci_core_cxl_release(struct vfio_pci_core_device *vdev)
> +{
> +	if (!vdev->cxl_ops)
> +		return;
> +
> +	vdev->cxl_ops->release(vdev);
> +	vfio_pci_put_cxl_ops(vdev->cxl_ops);
> +}
> +
>  int vfio_pci_core_init_dev(struct vfio_device *core_vdev)
>  {
>  	struct vfio_pci_core_device *vdev =
> @@ -2223,6 +2301,10 @@ int vfio_pci_core_init_dev(struct vfio_device *core_vdev)
>  	init_rwsem(&vdev->memory_lock);
>  	xa_init(&vdev->ctx);
>  
> +	ret = vfio_pci_core_cxl_init(vdev);
> +	if (ret)
> +		return ret;
> +
>  	return 0;
>  }
>  EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev);
> @@ -2232,6 +2314,8 @@ void vfio_pci_core_release_dev(struct vfio_device *core_vdev)
>  	struct vfio_pci_core_device *vdev =
>  		container_of(core_vdev, struct vfio_pci_core_device, vdev);
>  
> +	vfio_pci_core_cxl_release(vdev);
> +
>  	mutex_destroy(&vdev->igate);
>  	mutex_destroy(&vdev->ioeventfds_lock);
>  	kfree(vdev->region);
> diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
> index 9fe0d1a3a370..7f3a2bcb5830 100644
> --- a/include/linux/vfio_pci_core.h
> +++ b/include/linux/vfio_pci_core.h
> @@ -29,6 +29,7 @@ struct vfio_pci_core_device;
>  struct vfio_pci_region;
>  struct p2pdma_provider;
>  struct dma_buf_attachment;
> +struct vfio_cxl_state;
>  
>  struct vfio_pci_eventfd {
>  	struct eventfd_ctx	*ctx;
> @@ -109,6 +110,8 @@ struct vfio_pci_core_device {
>  	struct vfio_device	vdev;
>  	struct pci_dev		*pdev;
>  	const struct vfio_pci_device_ops *pci_ops;
> +	const struct vfio_cxl_ops *cxl_ops;
> +	struct vfio_cxl_state   *cxl;
>  	void __iomem		*barmap[PCI_STD_NUM_BARS];
>  	bool			bar_mmap_supported[PCI_STD_NUM_BARS];
>  	/* Flags modified at runtime - dedicated storage unit */
> -- 
> 2.25.1
> 
>