[RFC v2 14/15] vfio/cxl: VFIO variant driver for QEMU CXL accel device

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

To demonstrate the VFIO CXL core, a VFIO variant driver for QEMU CXL
accel device is introduced, so that people to test can try the patches.

This patch is not meant to be merged.

Co-developed-by: Zhi Wang <zhiw@nvidia.com>
Signed-off-by: Zhi Wang <zhiw@nvidia.com>
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 drivers/vfio/pci/Kconfig            |   2 +
 drivers/vfio/pci/Makefile           |   2 +
 drivers/vfio/pci/cxl-accel/Kconfig  |   9 ++
 drivers/vfio/pci/cxl-accel/Makefile |   4 +
 drivers/vfio/pci/cxl-accel/main.c   | 143 ++++++++++++++++++++++++++++
 5 files changed, 160 insertions(+)
 create mode 100644 drivers/vfio/pci/cxl-accel/Kconfig
 create mode 100644 drivers/vfio/pci/cxl-accel/Makefile
 create mode 100644 drivers/vfio/pci/cxl-accel/main.c

diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig
index 2f441d118f1c..441ded7ea035 100644
--- a/drivers/vfio/pci/Kconfig
+++ b/drivers/vfio/pci/Kconfig
@@ -77,4 +77,6 @@ source "drivers/vfio/pci/nvgrace-gpu/Kconfig"
 
 source "drivers/vfio/pci/qat/Kconfig"
 
+source "drivers/vfio/pci/cxl-accel/Kconfig"
+
 endmenu
diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile
index 452b7387f9fb..1b81d75b8ef7 100644
--- a/drivers/vfio/pci/Makefile
+++ b/drivers/vfio/pci/Makefile
@@ -22,3 +22,5 @@ obj-$(CONFIG_VIRTIO_VFIO_PCI) += virtio/
 obj-$(CONFIG_NVGRACE_GPU_VFIO_PCI) += nvgrace-gpu/
 
 obj-$(CONFIG_QAT_VFIO_PCI) += qat/
+
+obj-$(CONFIG_CXL_ACCEL_VFIO_PCI) += cxl-accel/
diff --git a/drivers/vfio/pci/cxl-accel/Kconfig b/drivers/vfio/pci/cxl-accel/Kconfig
new file mode 100644
index 000000000000..9a8884ded049
--- /dev/null
+++ b/drivers/vfio/pci/cxl-accel/Kconfig
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config CXL_ACCEL_VFIO_PCI
+	tristate "VFIO support for the QEMU CXL accel device"
+	select VFIO_CXL_CORE
+	help
+	  VFIO support for the CXL devices is needed for assigning the CXL
+	  devices to userspace using KVM/qemu/etc.
+
+	  If you don't know what to do here, say N.
diff --git a/drivers/vfio/pci/cxl-accel/Makefile b/drivers/vfio/pci/cxl-accel/Makefile
new file mode 100644
index 000000000000..8d0e076f405f
--- /dev/null
+++ b/drivers/vfio/pci/cxl-accel/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+obj-$(CONFIG_CXL_ACCEL_VFIO_PCI) += cxl-accel-vfio-pci.o
+cxl-accel-vfio-pci-y := main.o
diff --git a/drivers/vfio/pci/cxl-accel/main.c b/drivers/vfio/pci/cxl-accel/main.c
new file mode 100644
index 000000000000..3e5001ed5e2a
--- /dev/null
+++ b/drivers/vfio/pci/cxl-accel/main.c
@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved
+ */
+
+#include <linux/sizes.h>
+#include <linux/vfio_pci_core.h>
+
+struct cxl_device {
+	struct vfio_pci_core_device core_device;
+};
+
+static int cxl_open_device(struct vfio_device *vdev)
+{
+	struct vfio_cxl_core_device *cxl =
+		container_of(vdev, struct vfio_cxl_core_device, pci_core.vdev);
+	struct vfio_cxl *cxl_core = cxl->cxl_core;
+	struct vfio_cxl_dev_info info = {0};
+	int ret;
+
+	/* Driver reports the device DPA and RAM size */
+	info.dpa_res = DEFINE_RES_MEM(0, SZ_256M);
+	info.ram_res = DEFINE_RES_MEM_NAMED(0, SZ_256M, "ram");
+
+	/* Initialize the CXL device and enable the vfio-pci-core */
+	ret = vfio_cxl_core_enable(cxl, &info);
+	if (ret)
+		return ret;
+
+	vfio_cxl_core_finish_enable(cxl);
+
+	cxl_core = cxl->cxl_core;
+
+	/* No precommitted region, create one. */
+	if (!cxl_core->region.region) {
+		/*
+		 * Driver can choose to create cxl region at a certain time
+		 * E.g. at driver initialization or later
+		 */
+		ret = vfio_cxl_core_create_cxl_region(cxl, SZ_256M);
+		if (ret)
+			goto fail_create_cxl_region;
+	}
+
+	ret = vfio_cxl_core_register_cxl_region(cxl);
+	if (ret)
+		goto fail_register_cxl_region;
+
+	return 0;
+
+fail_register_cxl_region:
+	if (cxl_core->region.region)
+		vfio_cxl_core_destroy_cxl_region(cxl);
+fail_create_cxl_region:
+	vfio_cxl_core_disable(cxl);
+	return ret;
+}
+
+static void cxl_close_device(struct vfio_device *vdev)
+{
+	struct vfio_cxl_core_device *cxl =
+		container_of(vdev, struct vfio_cxl_core_device, pci_core.vdev);
+
+	vfio_cxl_core_unregister_cxl_region(cxl);
+	vfio_cxl_core_destroy_cxl_region(cxl);
+	vfio_cxl_core_close_device(vdev);
+}
+
+static const struct vfio_device_ops cxl_core_ops = {
+	.name             = "cxl-vfio-pci",
+	.init             = vfio_pci_core_init_dev,
+	.release          = vfio_pci_core_release_dev,
+	.open_device      = cxl_open_device,
+	.close_device     = cxl_close_device,
+	.ioctl            = vfio_cxl_core_ioctl,
+	.device_feature   = vfio_pci_core_ioctl_feature,
+	.read             = vfio_cxl_core_read,
+	.write            = vfio_cxl_core_write,
+	.mmap             = vfio_pci_core_mmap,
+	.request          = vfio_pci_core_request,
+	.match            = vfio_pci_core_match,
+	.match_token_uuid = vfio_pci_core_match_token_uuid,
+	.bind_iommufd     = vfio_iommufd_physical_bind,
+	.unbind_iommufd   = vfio_iommufd_physical_unbind,
+	.attach_ioas      = vfio_iommufd_physical_attach_ioas,
+	.detach_ioas      = vfio_iommufd_physical_detach_ioas,
+};
+
+static int cxl_probe(struct pci_dev *pdev,
+		     const struct pci_device_id *id)
+{
+	const struct vfio_device_ops *ops = &cxl_core_ops;
+	struct vfio_cxl_core_device *cxl_device;
+	int ret;
+
+	cxl_device = vfio_alloc_device(vfio_cxl_core_device, pci_core.vdev,
+				       &pdev->dev, ops);
+	if (IS_ERR(cxl_device))
+		return PTR_ERR(cxl_device);
+
+	dev_set_drvdata(&pdev->dev, &cxl_device->pci_core);
+
+	ret = vfio_pci_core_register_device(&cxl_device->pci_core);
+	if (ret)
+		goto out_put_vdev;
+
+	return ret;
+
+out_put_vdev:
+	vfio_put_device(&cxl_device->pci_core.vdev);
+	return ret;
+}
+
+static void cxl_remove(struct pci_dev *pdev)
+{
+	struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
+
+	vfio_pci_core_unregister_device(core_device);
+	vfio_put_device(&core_device->vdev);
+}
+
+static const struct pci_device_id cxl_vfio_pci_table[] = {
+	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0xd94) },
+	{}
+};
+
+MODULE_DEVICE_TABLE(pci, cxl_vfio_pci_table);
+
+static struct pci_driver cxl_vfio_pci_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = cxl_vfio_pci_table,
+	.probe = cxl_probe,
+	.remove = cxl_remove,
+	.err_handler = &vfio_pci_core_err_handlers,
+	.driver_managed_dma = true,
+};
+
+module_pci_driver(cxl_vfio_pci_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Zhi Wang <zhiw@nvidia.com>");
+MODULE_DESCRIPTION("VFIO variant driver for QEMU CXL accel device");
+MODULE_IMPORT_NS("CXL");
-- 
2.25.1