[PATCH 05/10] hw/vfio/pci: Back the CXL memory with a RAM-device region

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

mmap the kernel's HDM memory region so its host physical pages back a
RAM-device MemoryRegion. It stays out of the guest address space until
the guest commits its decoder and QEMU learns the GPA to place it at.

Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 hw/vfio/pci.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
 hw/vfio/pci.h |  1 +
 2 files changed, 46 insertions(+)

diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 1f21da9513..066333d52d 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -3221,8 +3221,11 @@ bool vfio_pci_populate_device(VFIOPCIDevice *vdev, Error **errp)
     return true;
 }
 
+static void vfio_cxl_teardown(VFIOPCIDevice *vdev);
+
 void vfio_pci_put_device(VFIOPCIDevice *vdev)
 {
+    vfio_cxl_teardown(vdev);
     vfio_display_finalize(vdev);
     vfio_bars_finalize(vdev);
     vfio_cpr_pci_unregister_device(vdev);
@@ -3666,6 +3669,19 @@ static bool vfio_cxl_setup(VFIOPCIDevice *vdev, Error **errp)
         return true;
     }
 
+    /*
+     * The HDM memory is exposed only as an mmap-backed RAM-device region; the
+     * kernel rejects fd read/write on it. With x-no-mmap the region would fall
+     * back to that always-failing fd path, so every guest access to the HDM
+     * range would fault. Reject it here rather than start an unusable VM.
+     */
+    if (vbasedev->no_mmap) {
+        error_setg(errp, "vfio-cxl: %s: x-no-mmap is not supported for a CXL "
+                   "device; its HDM memory has no fd read/write fallback",
+                   vbasedev->name);
+        return false;
+    }
+
     if (vfio_device_get_region_info_type(vbasedev, VFIO_REGION_TYPE_CXL,
                                          VFIO_REGION_SUBTYPE_CXL_MEM,
                                          &mem_info)) {
@@ -3703,11 +3719,40 @@ static bool vfio_cxl_setup(VFIOPCIDevice *vdev, Error **errp)
         return false;
     }
 
+    /*
+     * The HDM memory is host physical. mmap it now; it is added to the guest
+     * address space only once the guest commits its endpoint decoder.
+     */
+    if (vfio_region_setup(OBJECT(vdev), vbasedev, &cxl->mem_region,
+                          cxl->mem_region_index, "cxl-mem", errp)) {
+        return false;
+    }
+    if (vfio_region_mmap(&cxl->mem_region)) {
+        error_setg(errp, "vfio-cxl: %s: failed to mmap the HDM memory region",
+                   vbasedev->name);
+        vfio_region_exit(&cxl->mem_region);
+        vfio_region_finalize(&cxl->mem_region);
+        return false;
+    }
+
     cxl->enabled = true;
 
     return true;
 }
 
+static void vfio_cxl_teardown(VFIOPCIDevice *vdev)
+{
+    VFIOCXL *cxl = &vdev->cxl;
+
+    if (!cxl->enabled) {
+        return;
+    }
+    if (cxl->mem_region.mem) {
+        vfio_region_exit(&cxl->mem_region);
+        vfio_region_finalize(&cxl->mem_region);
+    }
+}
+
 static void vfio_pci_realize(PCIDevice *pdev, Error **errp)
 {
     ERRP_GUARD();
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index 7fdd695704..06d15e807d 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -134,6 +134,7 @@ typedef struct VFIOCXL {
     uint32_t comp_bar;               /* component BAR carrying that block */
     uint64_t hdm_offset;             /* block offset within the component BAR */
     uint64_t dpa_size;               /* size of the HDM memory region */
+    VFIORegion mem_region;           /* HDM memory, mapped at committed GPA */
 } VFIOCXL;
 
 struct VFIOPCIDevice {
-- 
2.25.1