[PATCH v6 22/31] vfio/cxl: Allow mmap of the HDM memory region

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

A CXL Type-2 guest maps the device HDM memory to use its coherent
CXL.mem, so the guest and KVM need a write-back mapping of the HDM
memory region. Make the region mmap-able. The range has no struct page,
so the mapping is VM_PFNMAP and its fault handler inserts the pfn, at
huge-page order where the alignment allows.

The fault handler inserts a pfn only while the decoder is known good and
the media is ready, and returns SIGBUS otherwise. It inserts with
__vfio_pci_vmf_insert_pfn(), so a fault does not depend on memory
enable; low power already closes the gate through power_down. When the
gate closes, in low power and later in reset and bus reset, revoke the
region with vfio_pci_core_revoke_region() so no live mapping outlives
the decoder, and re-arm it once the decoder is restored. The BARs keep
following the memory enable bit; this region does not.

Assisted-by: LLM
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 drivers/vfio/pci/cxl/vfio_cxl_core.c | 131 +++++++++++++++++++++++++--
 include/uapi/linux/vfio.h            |   2 +-
 2 files changed, 123 insertions(+), 10 deletions(-)

diff --git a/drivers/vfio/pci/cxl/vfio_cxl_core.c b/drivers/vfio/pci/cxl/vfio_cxl_core.c
index e6e633902ecd..977d0c9c747a 100644
--- a/drivers/vfio/pci/cxl/vfio_cxl_core.c
+++ b/drivers/vfio/pci/cxl/vfio_cxl_core.c
@@ -9,6 +9,7 @@
 #include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/io.h>
+#include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/range.h>
@@ -29,6 +30,7 @@
  * @hdm_valid: true when host CPU access to the HDM range is safe; under memory_lock
  * @await_mem_active: the device reported its memory active at bind, so wait
  *	for Memory_Active before host access reopens after a reset or low power
+ * @mem_region_index: vfio region index of the HDM memory region while open, else 0
  */
 struct vfio_cxl_state {
 	struct cxl_dev_state cxlds;
@@ -38,8 +40,82 @@ struct vfio_cxl_state {
 	struct cxl_hdm_info *hdm;
 	bool hdm_valid;
 	bool await_mem_active;
+	unsigned int mem_region_index;
 };
 
+static unsigned long vfio_cxl_mem_pgoff(struct vm_area_struct *vma,
+					unsigned long addr)
+{
+	unsigned long mask = (1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1;
+
+	return (vma->vm_pgoff & mask) + ((addr - vma->vm_start) >> PAGE_SHIFT);
+}
+
+static vm_fault_t vfio_cxl_mem_huge_fault(struct vm_fault *vmf,
+					  unsigned int order)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct vfio_pci_core_device *vdev = vma->vm_private_data;
+	struct vfio_cxl_state *cxl = vdev->cxl;
+	unsigned long addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
+	unsigned long pfn = PHYS_PFN(cxl->hpa_range.start) +
+			    vfio_cxl_mem_pgoff(vma, addr);
+	vm_fault_t ret = VM_FAULT_FALLBACK;
+
+	if (is_aligned_for_order(vma, addr, pfn, order)) {
+		scoped_guard(rwsem_read, &vdev->memory_lock) {
+			/*
+			 * Insert a PFN only for a known-good decoder whose
+			 * media is ready. CXL.mem decode follows the committed
+			 * decoder and Mem_Enable, not Memory Space Enable (CXL
+			 * r4.0 sec 8.2.4.20.2, 8.1.3.2), so skip the BAR memory
+			 * enable check; low power closes hdm_valid through
+			 * power_down.
+			 */
+			if (cxl->hdm_valid && cxl->cxlds.media_ready)
+				ret = __vfio_pci_vmf_insert_pfn(vdev, vmf, pfn,
+								order);
+			else
+				ret = VM_FAULT_SIGBUS;
+		}
+	}
+
+	return ret;
+}
+
+static vm_fault_t vfio_cxl_mem_fault(struct vm_fault *vmf)
+{
+	return vfio_cxl_mem_huge_fault(vmf, 0);
+}
+
+static const struct vm_operations_struct vfio_cxl_mem_vm_ops = {
+	.fault = vfio_cxl_mem_fault,
+#ifdef CONFIG_ARCH_SUPPORTS_HUGE_PFNMAP
+	.huge_fault = vfio_cxl_mem_huge_fault,
+#endif
+};
+
+static int vfio_cxl_mem_mmap(struct vfio_pci_core_device *vdev,
+			     struct vfio_pci_region *region,
+			     struct vm_area_struct *vma)
+{
+	unsigned long mask = (1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1;
+	u64 req_start = (vma->vm_pgoff & mask) << PAGE_SHIFT;
+	u64 req_len = vma->vm_end - vma->vm_start;
+
+	if (req_start + req_len > region->size)
+		return -EINVAL;
+
+	/*
+	 * CXL.mem is coherent memory, so leave the mapping write-back cacheable.
+	 */
+	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
+	vma->vm_ops = &vfio_cxl_mem_vm_ops;
+	vma->vm_private_data = vdev;
+
+	return 0;
+}
+
 /*
  * Copy one chunk between the bounce buffer and the HDM range under
  * memory_lock. The user copy stays outside the lock: a user buffer that is
@@ -131,6 +207,7 @@ static void vfio_cxl_region_release(struct vfio_pci_core_device *vdev,
 
 static const struct vfio_pci_regops vfio_cxl_mem_regops = {
 	.rw = vfio_cxl_mem_rw,
+	.mmap = vfio_cxl_mem_mmap,
 	.release = vfio_cxl_region_release,
 };
 
@@ -387,20 +464,31 @@ static int vfio_cxl_add_region(struct vfio_pci_core_device *vdev, u32 subtype,
 static int vfio_cxl_open_device(struct vfio_pci_core_device *vdev)
 {
 	struct vfio_cxl_state *cxl = vdev->cxl;
+	int ret;
 
 	/*
 	 * vfio-pci-core frees all dynamic regions on close and on a failed
 	 * open, so register them here per open rather than at bind.
 	 */
-	return vfio_cxl_add_region(vdev, VFIO_REGION_SUBTYPE_CXL_MEM,
-				   &vfio_cxl_mem_regops,
-				   range_len(&cxl->hpa_range),
-				   VFIO_REGION_INFO_FLAG_READ |
-				   VFIO_REGION_INFO_FLAG_WRITE);
+	ret = vfio_cxl_add_region(vdev, VFIO_REGION_SUBTYPE_CXL_MEM,
+				  &vfio_cxl_mem_regops,
+				  range_len(&cxl->hpa_range),
+				  VFIO_REGION_INFO_FLAG_READ |
+				  VFIO_REGION_INFO_FLAG_WRITE |
+				  VFIO_REGION_INFO_FLAG_MMAP);
+	if (ret)
+		return ret;
+
+	/* Record where the HDM memory region landed, to revoke it later. */
+	cxl->mem_region_index = VFIO_PCI_NUM_REGIONS + vdev->num_regions - 1;
+
+	return 0;
 }
 
 static void vfio_cxl_close_device(struct vfio_pci_core_device *vdev)
 {
+	/* vfio-pci-core frees the regions after this returns. */
+	vdev->cxl->mem_region_index = 0;
 }
 
 static void vfio_cxl_reset_prepare(struct vfio_pci_core_device *vdev)
@@ -412,6 +500,33 @@ static int vfio_cxl_reset_done(struct vfio_pci_core_device *vdev)
 	return 0;
 }
 
+/*
+ * Take the HDM memory region out of use: zap its mmaps and revoke its
+ * dma-bufs. A fault then gets SIGBUS from vfio_cxl_mem_huge_fault() until
+ * hdm_valid is set again.
+ */
+static void vfio_cxl_revoke_hdm(struct vfio_pci_core_device *vdev)
+{
+	struct vfio_cxl_state *cxl = vdev->cxl;
+
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	cxl->hdm_valid = false;
+	if (cxl->mem_region_index)
+		vfio_pci_core_revoke_region(vdev, cxl->mem_region_index);
+}
+
+static void vfio_cxl_rearm_hdm(struct vfio_pci_core_device *vdev)
+{
+	struct vfio_cxl_state *cxl = vdev->cxl;
+
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	cxl->hdm_valid = true;
+	if (cxl->mem_region_index)
+		vfio_pci_core_rearm_region(vdev, cxl->mem_region_index);
+}
+
 /*
  * vfio-cxl holds the HDM range, so the PCI core's cxl_reset method would
  * fail to request it. Run the same CXL Reset with that claim.
@@ -429,9 +544,7 @@ static int vfio_cxl_reset(struct vfio_pci_core_device *vdev)
 /* Low power loses the committed decoder: close host access until restored. */
 static void vfio_cxl_power_down(struct vfio_pci_core_device *vdev)
 {
-	lockdep_assert_held_write(&vdev->memory_lock);
-
-	vdev->cxl->hdm_valid = false;
+	vfio_cxl_revoke_hdm(vdev);
 }
 
 static void vfio_cxl_power_up(struct vfio_pci_core_device *vdev)
@@ -460,7 +573,7 @@ static void vfio_cxl_power_up(struct vfio_pci_core_device *vdev)
 	if (ret)
 		pci_warn(pdev, "vfio-cxl: HDM decoder restore failed: %d\n", ret);
 	else if (!vfio_cxl_await_mem_active(vdev))
-		cxl->hdm_valid = true;
+		vfio_cxl_rearm_hdm(vdev);
 }
 
 static const struct vfio_cxl_ops vfio_cxl_ops = {
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 883b51521e6a..359fed40eb1b 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -371,7 +371,7 @@ struct vfio_region_info_cap_type {
 #define VFIO_REGION_SUBTYPE_IBM_NVLINK2_ATSD	(1)
 
 /* CXL Type-2 device (0x1e98) sub-types for VFIO_REGION_TYPE_PCI_VENDOR_TYPE */
-/* CXL.mem HDM region of a Type-2 device, read and write */
+/* CXL.mem HDM region of a Type-2 device, read, write and mmap */
 #define VFIO_REGION_SUBTYPE_CXL_MEM		(1)
 
 /* sub-types for VFIO_REGION_TYPE_GFX */
-- 
2.53.0