[PATCH v6 21/31] vfio/cxl: Expose the HDM memory region to the guest

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

A CXL Type-2 guest maps the device HDM memory to use its coherent
CXL.mem. The HDM memory is a host physical range with no struct page,
so the guest and KVM need a write-back mapping of it.

Register the range as a region under VFIO_REGION_TYPE_PCI_VENDOR_TYPE
with the CXL vendor id rather than a bespoke region type, per open
because vfio-pci-core tears down all dynamic regions on close and on a
failed open. This patch serves read and write on the region fd; mmap
follows.

Own the resolved host physical range exclusively (IORESOURCE_EXCLUSIVE)
so nothing, /dev/mem included, can map a conflicting cacheable alias
that would fault the host once the range is mapped write-back. Release
it in the provider .release op rather than through a devm action on the
PCI device, which could run after vfio-cxl has unloaded. vfio-cxl now
holds the range, so its reset op runs CXL Reset with
pci_cxl_reset_owned().

Host access to the range is safe only while the committed decoder is in
place. Track that in hdm_valid, set at bind once the decoder is verified
committed. Low power loses the decoder, so close the gate in power_down
and restore the decoder with cxl_restore_state_after_pci_reset() in
power_up, reopening the gate on success.

Device memory can stay unavailable for seconds after the decoder is back:
the device clears Memory_Active in DVSEC CXL Range 1 Size Low until the
memory is usable (CXL r4.0 sec 8.1.3.8.2).

At bind, wait up to the Memory_Active_Timeout the device reports for
Memory_Active, since a reset just before the bind also clears it. A device
whose memory then reads active brings it up by itself, and power_up waits
for Memory_Active, up to the same timeout, before it reopens the gate.

A device whose memory stays inactive relies on its driver and is not
waited for.

CXL.mem decode follows the committed decoder and Mem_Enable, not Memory
Space Enable (CXL r4.0 sec 8.2.4.20.2, 8.2.4.20.7 and 8.1.3.2), so the gate
does not check the Command register. Read and write copy through a
page-sized kernel buffer, so memory_lock is never held across a user copy.

Assisted-by: LLM
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 drivers/vfio/pci/cxl/vfio_cxl_core.c | 285 ++++++++++++++++++++++++++-
 include/uapi/linux/vfio.h            |   4 +
 2 files changed, 279 insertions(+), 10 deletions(-)

diff --git a/drivers/vfio/pci/cxl/vfio_cxl_core.c b/drivers/vfio/pci/cxl/vfio_cxl_core.c
index 052be27bc46f..e6e633902ecd 100644
--- a/drivers/vfio/pci/cxl/vfio_cxl_core.c
+++ b/drivers/vfio/pci/cxl/vfio_cxl_core.c
@@ -5,11 +5,15 @@
  * Copyright (c) 2026 NVIDIA Corporation & Affiliates
  */
 
+#include <linux/bitfield.h>
 #include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/io.h>
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/range.h>
 #include <linux/slab.h>
+#include <linux/uaccess.h>
 #include <linux/vfio_pci_core.h>
 #include <cxl/cxl.h>
 #include <cxl/hdm.h>
@@ -20,13 +24,114 @@
  * @cxlds: CXL device state; kept first for devm_cxl_dev_state_create()
  * @cxlmd: memory device joined to the CXL topology at bind
  * @hpa_range: host physical range of the HDM region
+ * @hpa_res: exclusive reservation of @hpa_range, held for the bind
  * @hdm: copy of the HDM state cached at PCI enumeration, taken at bind
+ * @hdm_valid: true when host CPU access to the HDM range is safe; under memory_lock
+ * @await_mem_active: the device reported its memory active at bind, so wait
+ *	for Memory_Active before host access reopens after a reset or low power
  */
 struct vfio_cxl_state {
 	struct cxl_dev_state cxlds;
 	struct cxl_memdev *cxlmd;
 	struct range hpa_range;
+	struct resource *hpa_res;
 	struct cxl_hdm_info *hdm;
+	bool hdm_valid;
+	bool await_mem_active;
+};
+
+/*
+ * Copy one chunk between the bounce buffer and the HDM range under
+ * memory_lock. The user copy stays outside the lock: a user buffer that is
+ * itself an mmap of this device faults through a handler that takes
+ * memory_lock for read, and a writer queued in between would deadlock the
+ * two readers.
+ */
+static int vfio_cxl_mem_copy(struct vfio_pci_core_device *vdev, void *bounce,
+			     u64 pos, size_t len, bool iswrite)
+{
+	struct vfio_cxl_state *cxl = vdev->cxl;
+	void *mem;
+
+	guard(rwsem_read)(&vdev->memory_lock);
+
+	/*
+	 * Only touch the HDM range with the decoder in a known-good state, or
+	 * a host CPU access aborts as a fatal SError. CXL.mem decode follows
+	 * the committed decoder and Mem_Enable, not Memory Space Enable (CXL
+	 * r4.0 sec 8.2.4.20.2, 8.1.3.2).
+	 */
+	if (!cxl->hdm_valid || !cxl->cxlds.media_ready)
+		return -EIO;
+
+	mem = memremap(cxl->hpa_range.start + pos, len, MEMREMAP_WB);
+	if (!mem)
+		return -ENOMEM;
+	if (iswrite)
+		memcpy(mem, bounce, len);
+	else
+		memcpy(bounce, mem, len);
+	memunmap(mem);
+
+	return 0;
+}
+
+static ssize_t vfio_cxl_mem_rw(struct vfio_pci_core_device *vdev,
+			       char __user *buf, size_t count, loff_t *ppos,
+			       bool iswrite)
+{
+	struct vfio_cxl_state *cxl = vdev->cxl;
+	u64 pos = *ppos & VFIO_PCI_OFFSET_MASK;
+	ssize_t done = 0;
+	void *bounce;
+	int ret = 0;
+
+	if (pos >= range_len(&cxl->hpa_range))
+		return -EINVAL;
+	count = min_t(u64, count, range_len(&cxl->hpa_range) - pos);
+
+	bounce = kmalloc(min_t(size_t, count, PAGE_SIZE), GFP_KERNEL);
+	if (!bounce)
+		return -ENOMEM;
+
+	while (count) {
+		size_t len = min_t(size_t, count, PAGE_SIZE);
+
+		if (iswrite && copy_from_user(bounce, buf + done, len)) {
+			ret = -EFAULT;
+			break;
+		}
+		ret = vfio_cxl_mem_copy(vdev, bounce, pos + done, len, iswrite);
+		if (ret)
+			break;
+		if (!iswrite && copy_to_user(buf + done, bounce, len)) {
+			ret = -EFAULT;
+			break;
+		}
+		done += len;
+		count -= len;
+	}
+	kfree(bounce);
+
+	if (!done)
+		return ret;
+
+	*ppos += done;
+	return done;
+}
+
+/*
+ * The CXL regions carry no per-region state (region->data is the shared,
+ * devm-managed vfio_cxl_state), so releasing a region is a no-op.
+ */
+static void vfio_cxl_region_release(struct vfio_pci_core_device *vdev,
+				    struct vfio_pci_region *region)
+{
+}
+
+static const struct vfio_pci_regops vfio_cxl_mem_regops = {
+	.rw = vfio_cxl_mem_rw,
+	.release = vfio_cxl_region_release,
 };
 
 /*
@@ -94,12 +199,75 @@ vfio_cxl_create_memdev(struct pci_dev *pdev, const struct cxl_hdm_info *hdm,
 	return cxl;
 }
 
+/* CXL r4.0 sec 8.1.3.8.2: Range 1 Size Low Memory_Active_Timeout encodings */
+static const unsigned int vfio_cxl_mem_active_timeout_s[] = { 1, 4, 16, 64, 256 };
+
+/*
+ * Poll Memory_Active in DVSEC CXL Range 1 Size Low (CXL r4.0 sec 8.1.3.8.2)
+ * for up to the Memory_Active_Timeout the device reports, which is stored in
+ * @timeout. Return 0 once it is set, -ETIMEDOUT, or -ENODEV if config space
+ * does not respond.
+ */
+static int vfio_cxl_poll_mem_active(struct pci_dev *pdev, u16 dvsec,
+				    unsigned int *timeout)
+{
+	int pos = dvsec + PCI_DVSEC_CXL_RANGE_SIZE_LOW(0);
+	unsigned long deadline;
+	u32 size_lo;
+
+	if (pci_read_config_dword(pdev, pos, &size_lo) ||
+	    PCI_POSSIBLE_ERROR(size_lo))
+		return -ENODEV;
+
+	*timeout = FIELD_GET(PCI_DVSEC_CXL_MEM_ACTIVE_TIMEOUT, size_lo);
+	*timeout = vfio_cxl_mem_active_timeout_s[min_t(unsigned int, *timeout,
+				ARRAY_SIZE(vfio_cxl_mem_active_timeout_s) - 1)];
+	deadline = jiffies + secs_to_jiffies(*timeout);
+
+	while (!(size_lo & PCI_DVSEC_CXL_MEM_ACTIVE)) {
+		if (time_after(jiffies, deadline))
+			return -ETIMEDOUT;
+		msleep(20);
+		if (pci_read_config_dword(pdev, pos, &size_lo) ||
+		    PCI_POSSIBLE_ERROR(size_lo))
+			return -ENODEV;
+	}
+
+	return 0;
+}
+
+/*
+ * A reset or a return from low power leaves device memory unavailable until
+ * the device sets Memory_Active. That can take seconds after the reset
+ * completes and the decoder is restored, and a host access to the HDM range
+ * before then can abort. Wait for it on a device that brings its memory up
+ * by itself; a device that relies on its driver for that was found at bind
+ * and is not waited for.
+ */
+static int vfio_cxl_await_mem_active(struct vfio_pci_core_device *vdev)
+{
+	struct vfio_cxl_state *cxl = vdev->cxl;
+	unsigned int timeout = 0;
+	int ret;
+
+	if (!cxl->await_mem_active)
+		return 0;
+
+	ret = vfio_cxl_poll_mem_active(vdev->pdev, cxl->hdm->dvsec, &timeout);
+	if (ret == -ETIMEDOUT)
+		pci_err(vdev->pdev, "vfio-cxl: device memory not active after %us\n",
+			timeout);
+	return ret;
+}
+
 static int vfio_cxl_init_device(struct vfio_pci_core_device *vdev)
 {
 	struct pci_dev *pdev = vdev->pdev;
 	struct cxl_hdm_info *hdm __free(kfree) = NULL;
 	struct vfio_cxl_state *cxl;
+	unsigned int timeout = 0;
 	u64 hdm_size;
+	int ret;
 
 	/*
 	 * pdev->hdm is cached at PCI enumeration, before any driver binds, so a
@@ -150,9 +318,44 @@ static int vfio_cxl_init_device(struct vfio_pci_core_device *vdev)
 		devres_release_group(&pdev->dev, NULL);
 		return PTR_ERR(cxl);
 	}
+
+	/*
+	 * Claim the range IORESOURCE_EXCLUSIVE so no conflicting cacheable
+	 * alias can fault the host once it is mapped write-back. Release it
+	 * explicitly in vfio_cxl_release_device() rather than through a devm
+	 * action on pdev->dev: the provider module ref is dropped as soon as
+	 * that .release op returns, so a pdev-devres callback could run after
+	 * vfio-cxl has unloaded and jump into freed module text.
+	 */
+	cxl->hpa_res = request_mem_region_exclusive(cxl->hpa_range.start,
+						    range_len(&cxl->hpa_range),
+						    "vfio-cxl-hdm");
+	if (!cxl->hpa_res) {
+		devres_release_group(&pdev->dev, NULL);
+		return -EBUSY;
+	}
 	devres_close_group(&pdev->dev, NULL);
 
 	cxl->hdm = no_free_ptr(hdm);
+	/*
+	 * Wait for Memory_Active after a reset only on a device that brings its
+	 * memory up by itself; see vfio_cxl_await_mem_active(). Memory_Active
+	 * also reads clear for seconds after a reset, for example one the host
+	 * ran just before this bind, so give the device its Memory_Active_Timeout
+	 * before deciding. A device that relies on its driver times out here
+	 * once.
+	 */
+	ret = vfio_cxl_poll_mem_active(pdev, cxl->hdm->dvsec, &timeout);
+	cxl->await_mem_active = !ret;
+	if (ret == -ETIMEDOUT)
+		pci_info(pdev, "vfio-cxl: device memory not active after %us, left to its driver\n",
+			 timeout);
+	/*
+	 * The decoder decodes a non-empty range, so it is committed, and host
+	 * access to the HDM range is safe. Reset, bus reset and low power
+	 * close this gate until the decoder is restored.
+	 */
+	cxl->hdm_valid = true;
 	vdev->cxl = cxl;
 
 	return 0;
@@ -160,13 +363,40 @@ static int vfio_cxl_init_device(struct vfio_pci_core_device *vdev)
 
 static void vfio_cxl_release_device(struct vfio_pci_core_device *vdev)
 {
-	kfree(vdev->cxl->hdm);
+	struct vfio_cxl_state *cxl = vdev->cxl;
+
+	/*
+	 * Drop the exclusive HDM range claim before vfio_pci_core_cxl_release()
+	 * drops the provider module ref; see vfio_cxl_init_device().
+	 */
+	release_mem_region(cxl->hpa_range.start, range_len(&cxl->hpa_range));
+	kfree(cxl->hdm);
 	vdev->cxl = NULL;
 }
 
+static int vfio_cxl_add_region(struct vfio_pci_core_device *vdev, u32 subtype,
+			       const struct vfio_pci_regops *ops, size_t size,
+			       u32 flags)
+{
+	u32 type = VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_CXL;
+
+	return vfio_pci_core_register_dev_region(vdev, type, subtype, ops,
+						 size, flags, vdev->cxl);
+}
+
 static int vfio_cxl_open_device(struct vfio_pci_core_device *vdev)
 {
-	return 0;
+	struct vfio_cxl_state *cxl = vdev->cxl;
+
+	/*
+	 * vfio-pci-core frees all dynamic regions on close and on a failed
+	 * open, so register them here per open rather than at bind.
+	 */
+	return vfio_cxl_add_region(vdev, VFIO_REGION_SUBTYPE_CXL_MEM,
+				   &vfio_cxl_mem_regops,
+				   range_len(&cxl->hpa_range),
+				   VFIO_REGION_INFO_FLAG_READ |
+				   VFIO_REGION_INFO_FLAG_WRITE);
 }
 
 static void vfio_cxl_close_device(struct vfio_pci_core_device *vdev)
@@ -183,21 +413,54 @@ static int vfio_cxl_reset_done(struct vfio_pci_core_device *vdev)
 }
 
 /*
- * Until vfio-cxl serves the device memory, reset through the PCI core's
- * reset methods, which run CXL Reset ahead of FLR. Restore the PCI state
- * here rather than through pci_reset_function_locked(): its reset_prepare
- * and reset_done callbacks take memory_lock, which the caller already holds.
- * The caller has saved the state, or on close loaded the open-time copy.
+ * vfio-cxl holds the HDM range, so the PCI core's cxl_reset method would
+ * fail to request it. Run the same CXL Reset with that claim.
  */
 static int vfio_cxl_reset(struct vfio_pci_core_device *vdev)
 {
+	/*
+	 * The reset restores PCI state from the saved copy. The caller provides
+	 * it: the try-reset helper saves the current state, and the close path
+	 * keeps the open-time state that vfio_pci_core_disable() loaded.
+	 */
+	return pci_cxl_reset_owned(vdev->pdev, vdev->cxl->hpa_res);
+}
+
+/* Low power loses the committed decoder: close host access until restored. */
+static void vfio_cxl_power_down(struct vfio_pci_core_device *vdev)
+{
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	vdev->cxl->hdm_valid = false;
+}
+
+static void vfio_cxl_power_up(struct vfio_pci_core_device *vdev)
+{
+	struct vfio_cxl_state *cxl = vdev->cxl;
 	struct pci_dev *pdev = vdev->pdev;
 	int ret;
 
-	ret = __pci_reset_function_locked(pdev);
-	pci_restore_state(pdev);
+	lockdep_assert_held_write(&vdev->memory_lock);
 
-	return ret;
+	/*
+	 * Restore the decoder that low power lost. The restore needs the device
+	 * lock; if it is contended, keep host access closed until a reset
+	 * restores the decoder. The restore refreshes the decoders in cxl->hdm
+	 * from the cache; global and DVSEC control stay as taken at bind, when
+	 * pci_cxl_reset_capable() required HDM decode and Mem_Enable on.
+	 */
+	if (!pci_dev_trylock(pdev))
+		return;
+
+	ret = pci_save_state(pdev);
+	if (!ret)
+		ret = cxl_restore_state_after_pci_reset(pdev, cxl->hdm);
+	pci_dev_unlock(pdev);
+
+	if (ret)
+		pci_warn(pdev, "vfio-cxl: HDM decoder restore failed: %d\n", ret);
+	else if (!vfio_cxl_await_mem_active(vdev))
+		cxl->hdm_valid = true;
 }
 
 static const struct vfio_cxl_ops vfio_cxl_ops = {
@@ -208,6 +471,8 @@ static const struct vfio_cxl_ops vfio_cxl_ops = {
 	.reset_prepare	= vfio_cxl_reset_prepare,
 	.reset_done	= vfio_cxl_reset_done,
 	.reset		= vfio_cxl_reset,
+	.power_down	= vfio_cxl_power_down,
+	.power_up	= vfio_cxl_power_up,
 	.owner		= THIS_MODULE,
 };
 
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index e41437fa17ad..883b51521e6a 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -370,6 +370,10 @@ struct vfio_region_info_cap_type {
  */
 #define VFIO_REGION_SUBTYPE_IBM_NVLINK2_ATSD	(1)
 
+/* CXL Type-2 device (0x1e98) sub-types for VFIO_REGION_TYPE_PCI_VENDOR_TYPE */
+/* CXL.mem HDM region of a Type-2 device, read and write */
+#define VFIO_REGION_SUBTYPE_CXL_MEM		(1)
+
 /* sub-types for VFIO_REGION_TYPE_GFX */
 #define VFIO_REGION_SUBTYPE_GFX_EDID            (1)
 
-- 
2.53.0