[PATCH v6 28/31] vfio/cxl: Export the HDM memory region as a dma-buf

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

A Type-2 accelerator issues ATS-translated DMA to addresses inside its
own HDM window, so that coherent host range must be present in the
guest's IOAS (the iommufd IOAS backing the nested SMMU stage-2). iommufd
maps a struct-page-less range only by fd, via IOMMU_IOAS_MAP_FILE over a
dma-buf; a userspace-VA IOMMU_IOAS_MAP of the HDM mmap is rejected
because the VMA is VM_IO | VM_PFNMAP. Without a dma-buf the range could
only be mapped through an out-of-tree PFNMAP work-around.

vfio-pci already exports BAR memory as a P2P dma-buf, but the exporter
is BAR-only: vfio_pci_core_feature_dma_buf() rejects any region index at
or above the ROM index, and vfio_pci_core_get_dmabuf_phys() resolves the
physical range from a PCI BAR. The HDM memory region is a dynamic
device-specific region, not a BAR.

Let a device-specific region reach the device's get_dmabuf_phys(): a
region index at or above VFIO_PCI_NUM_REGIONS skips the BAR-resource
check and is validated by the driver instead, bounded to the regions
that exist. Install a CXL-aware get_dmabuf_phys() in the vfio-cxl
provider that returns cxl->hpa_range for the HDM memory region and
delegates real BARs to the core, keeping the BAR path unchanged and the
core free of CXL knowledge.

The HDM window is coherent host memory with no p2pdma provider of its
own, so borrow BAR 0's, matching nvgrace-gpu's handling of its non-BAR
device memory. The iommufd importer does not consume the provider; the
scatterlist map path (real peer DMA) is left to a follow-up once
upstream grows a negotiated interconnect for coherent CXL memory. Mark a
device-specific dma-buf so vfio_pci_dma_buf_map() refuses that path,
which would translate the range with the borrowed provider's bus offset.

The HDM dma-buf belongs to its region, not to the BARs: a memory enable
change leaves it alone, and vfio-cxl revokes and re-arms it with the
region when the decoder state changes. A new dma-buf of a
device-specific region starts in the state its region reports through a
new optional revoked() region op, so an HDM export made with memory
disabled starts armed, and one made after a failed reset starts revoked.

Assisted-by: LLM
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 drivers/vfio/pci/cxl/vfio_cxl_core.c | 76 +++++++++++++++++++++++++++-
 drivers/vfio/pci/vfio_pci_dmabuf.c   | 73 +++++++++++++++++++++++---
 include/linux/vfio_pci_core.h        |  9 ++++
 3 files changed, 151 insertions(+), 7 deletions(-)

diff --git a/drivers/vfio/pci/cxl/vfio_cxl_core.c b/drivers/vfio/pci/cxl/vfio_cxl_core.c
index 9ef052a3745e..518f0b093652 100644
--- a/drivers/vfio/pci/cxl/vfio_cxl_core.c
+++ b/drivers/vfio/pci/cxl/vfio_cxl_core.c
@@ -13,6 +13,7 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/pci-p2pdma.h>
 #include <linux/range.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
@@ -206,6 +207,21 @@ static ssize_t vfio_cxl_mem_rw(struct vfio_pci_core_device *vdev,
 	return done;
 }
 
+/*
+ * The HDM region follows hdm_valid, not Memory Space Enable, so a dma-buf
+ * exported from it starts revoked while host access is unsafe and is
+ * re-armed with the region in vfio_cxl_rearm_hdm().
+ */
+static bool vfio_cxl_mem_revoked(struct vfio_pci_core_device *vdev,
+				 struct vfio_pci_region *region)
+{
+	struct vfio_cxl_state *cxl = vdev->cxl;
+
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	return !cxl->hdm_valid || !cxl->cxlds.media_ready;
+}
+
 /*
  * The CXL regions carry no per-region state (region->data is the shared,
  * devm-managed vfio_cxl_state), so releasing a region is a no-op.
@@ -219,6 +235,7 @@ static const struct vfio_pci_regops vfio_cxl_mem_regops = {
 	.rw = vfio_cxl_mem_rw,
 	.mmap = vfio_cxl_mem_mmap,
 	.release = vfio_cxl_region_release,
+	.revoked = vfio_cxl_mem_revoked,
 };
 
 static ssize_t vfio_cxl_comp_rw(struct vfio_pci_core_device *vdev,
@@ -443,6 +460,50 @@ vfio_cxl_create_memdev(struct pci_dev *pdev, const struct cxl_hdm_info *hdm,
 	return cxl;
 }
 
+/*
+ * Resolve the physical range that backs a dma-buf export. The core exporter
+ * only knows BARs; teach it the HDM memory region so a guest IOAS can map the
+ * coherent window by fd (IOMMU_IOAS_MAP_FILE) instead of the removed PFNMAP
+ * work-around. Real BARs stay on the byte-identical core path.
+ */
+static int vfio_cxl_get_dmabuf_phys(struct vfio_pci_core_device *vdev,
+				    struct p2pdma_provider **provider,
+				    unsigned int region_index,
+				    struct phys_vec *phys_vec,
+				    struct vfio_region_dma_range *dma_ranges,
+				    size_t nr_ranges)
+{
+	struct vfio_cxl_state *cxl = vdev->cxl;
+
+	/* Real BARs go through the core P2P exporter unchanged. */
+	if (region_index < VFIO_PCI_NUM_REGIONS)
+		return vfio_pci_core_get_dmabuf_phys(vdev, provider,
+						     region_index, phys_vec,
+						     dma_ranges, nr_ranges);
+
+	/* Of the device regions, only the HDM memory window is exportable. */
+	if (region_index != cxl->mem_region_index)
+		return -EINVAL;
+
+	/*
+	 * The HDM window is coherent host memory, not BAR MMIO, so it has no
+	 * p2pdma provider of its own. Borrow BAR 0's: the P2P properties match
+	 * and the iommufd importer does not consume the provider. The sgt map
+	 * path (real peer DMA) is not supported for the HDM window.
+	 */
+	*provider = pcim_p2pdma_provider(vdev->pdev, 0);
+	if (!*provider)
+		return -EINVAL;
+
+	return vfio_pci_core_fill_phys_vec(phys_vec, dma_ranges, nr_ranges,
+					  cxl->hpa_range.start,
+					  range_len(&cxl->hpa_range));
+}
+
+static const struct vfio_pci_device_ops vfio_cxl_pci_dev_ops = {
+	.get_dmabuf_phys = vfio_cxl_get_dmabuf_phys,
+};
+
 /* CXL r4.0 sec 8.1.3.8.2: Range 1 Size Low Memory_Active_Timeout encodings */
 static const unsigned int vfio_cxl_mem_active_timeout_s[] = { 1, 4, 16, 64, 256 };
 
@@ -662,9 +723,22 @@ static int vfio_cxl_open_device(struct vfio_pci_core_device *vdev)
 	if (ret)
 		return ret;
 
-	/* Record where the HDM memory region landed, to revoke it later. */
+	/*
+	 * Record where the HDM memory region landed, to revoke it and to
+	 * resolve its dma-buf export.
+	 */
 	cxl->mem_region_index = VFIO_PCI_NUM_REGIONS + vdev->num_regions - 1;
 
+	/*
+	 * Override the device ops so a dma-buf export of the HDM memory region
+	 * resolves to the coherent host range. This is done at open, not init:
+	 * vfio_pci_probe() resets pci_ops after vfio_alloc_device() returns, so
+	 * an override installed during init would be clobbered. Only a CXL device
+	 * reaches this hook (cxl_ops is set on init success), so a device bound
+	 * without CXL support keeps the core ops.
+	 */
+	vdev->pci_ops = &vfio_cxl_pci_dev_ops;
+
 	ret = vfio_cxl_add_region(vdev, VFIO_REGION_SUBTYPE_CXL_COMP_REGS,
 				  &vfio_cxl_comp_regops, cxl->hdm_len,
 				  VFIO_REGION_INFO_FLAG_READ |
diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c
index d0f7ec5487de..237f7e055dd8 100644
--- a/drivers/vfio/pci/vfio_pci_dmabuf.c
+++ b/drivers/vfio/pci/vfio_pci_dmabuf.c
@@ -21,6 +21,13 @@ struct vfio_pci_dma_buf {
 	struct completion comp;
 	u32 region_index;
 	u8 revoked : 1;
+	/*
+	 * Set for a device-specific region (e.g. the CXL HDM window) that
+	 * borrows a BAR's p2pdma provider: mapping by physical address through
+	 * iommufd is safe, but a peer-DMA sgt map would translate with the
+	 * wrong provider bus offset.
+	 */
+	u8 no_peer2peer : 1;
 };
 
 static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,
@@ -60,6 +67,15 @@ vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment,
 	if (priv->revoked)
 		return ERR_PTR(-ENODEV);
 
+	/*
+	 * A borrowed-provider export can only be mapped by physical address
+	 * through iommufd, not as a peer-DMA sgt: dma_buf_phys_vec_to_sgt()
+	 * would translate the physical range with the borrowed provider's bus
+	 * offset and hand the peer a wrong address. Refuse it.
+	 */
+	if (priv->no_peer2peer)
+		return ERR_PTR(-EOPNOTSUPP);
+
 	ret = dma_buf_phys_vec_to_sgt(attachment, priv->provider,
 				      priv->phys_vec, priv->nr_ranges,
 				      priv->size, dir);
@@ -179,6 +195,15 @@ int vfio_pci_core_get_dmabuf_phys(struct vfio_pci_core_device *vdev,
 {
 	struct pci_dev *pdev = vdev->pdev;
 
+	/*
+	 * This resolver only handles PCI BARs. A device-specific region index
+	 * (>= PCI_STD_NUM_BARS) would index the per-BAR provider array
+	 * (p2p->mem[]) out of bounds in pcim_p2pdma_provider(), so reject it; a
+	 * driver that exports such a region installs its own get_dmabuf_phys.
+	 */
+	if (region_index >= PCI_STD_NUM_BARS)
+		return -EINVAL;
+
 	*provider = pcim_p2pdma_provider(pdev, region_index);
 	if (!*provider)
 		return -EINVAL;
@@ -219,6 +244,26 @@ static int validate_dmabuf_input(struct vfio_device_feature_dma_buf *dma_buf,
 	return 0;
 }
 
+/*
+ * A BAR dma-buf follows the memory enable bit. A device-specific region
+ * follows its own state, which its provider moves with
+ * vfio_pci_core_revoke_region() and vfio_pci_core_rearm_region(); start a
+ * new dma-buf in that state, since a memory enable change never moves it.
+ */
+static bool vfio_pci_dma_buf_start_revoked(struct vfio_pci_core_device *vdev,
+					   u32 index)
+{
+	struct vfio_pci_region *region;
+
+	lockdep_assert_held_write(&vdev->memory_lock);
+
+	if (index < VFIO_PCI_NUM_REGIONS)
+		return !__vfio_pci_memory_enabled(vdev);
+
+	region = &vdev->region[index - VFIO_PCI_NUM_REGIONS];
+	return region->ops->revoked && region->ops->revoked(vdev, region);
+}
+
 int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 				  struct vfio_device_feature_dma_buf __user *arg,
 				  size_t argsz)
@@ -228,6 +273,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 	struct vfio_pci_dma_buf *priv;
 	size_t length;
+	u32 index;
 	int ret;
 
 	if (!vdev->pci_ops || !vdev->pci_ops->get_dmabuf_phys)
@@ -244,13 +290,22 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	if (!get_dma_buf.nr_ranges || get_dma_buf.flags)
 		return -EINVAL;
 
+	index = get_dma_buf.region_index;
+
 	/*
-	 * For PCI the region_index is the BAR number like everything
-	 * else.  Check that PCI resources have been claimed for it.
+	 * A fixed region index is the BAR number; only a BAR can be exported
+	 * and its PCI resource must be claimed. A device-specific region (index
+	 * >= VFIO_PCI_NUM_REGIONS) has no BAR resource and is validated by the
+	 * device's get_dmabuf_phys instead, but the index must name a region
+	 * that exists.
 	 */
-	if (get_dma_buf.region_index >= VFIO_PCI_ROM_REGION_INDEX ||
-	    IS_ERR(vfio_pci_core_get_iomap(vdev, get_dma_buf.region_index)))
+	if (index < VFIO_PCI_NUM_REGIONS) {
+		if (index >= VFIO_PCI_ROM_REGION_INDEX ||
+		    IS_ERR(vfio_pci_core_get_iomap(vdev, index)))
+			return -ENODEV;
+	} else if (index - VFIO_PCI_NUM_REGIONS >= vdev->num_regions) {
 		return -ENODEV;
+	}
 
 	dma_ranges = memdup_array_user(&arg->dma_ranges, get_dma_buf.nr_ranges,
 				       sizeof(*dma_ranges));
@@ -266,7 +321,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	 * BAR keeps off mmap, so refuse such a range here as mmap does.
 	 */
 	for (u32 i = 0; i < get_dma_buf.nr_ranges; i++) {
-		if (vfio_pci_bar_range_excluded(vdev, get_dma_buf.region_index,
+		if (vfio_pci_bar_range_excluded(vdev, index,
 						dma_ranges[i].offset,
 						dma_ranges[i].length,
 						VFIO_PCI_EXCLUDE_MMAP)) {
@@ -290,6 +345,12 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	priv->nr_ranges = get_dma_buf.nr_ranges;
 	priv->size = length;
 	priv->region_index = get_dma_buf.region_index;
+	/*
+	 * A device-specific region borrows a BAR's p2pdma provider, so it can
+	 * only be mapped by physical address (iommufd), never as a peer-DMA
+	 * sgt. See vfio_pci_dma_buf_map().
+	 */
+	priv->no_peer2peer = index >= VFIO_PCI_NUM_REGIONS;
 	ret = vdev->pci_ops->get_dmabuf_phys(vdev, &priv->provider,
 					     get_dma_buf.region_index,
 					     priv->phys_vec, dma_ranges,
@@ -323,7 +384,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	INIT_LIST_HEAD(&priv->dmabufs_elm);
 	down_write(&vdev->memory_lock);
 	dma_resv_lock(priv->dmabuf->resv, NULL);
-	priv->revoked = !__vfio_pci_memory_enabled(vdev);
+	priv->revoked = vfio_pci_dma_buf_start_revoked(vdev, index);
 	list_add_tail(&priv->dmabufs_elm, &vdev->dmabufs);
 	dma_resv_unlock(priv->dmabuf->resv);
 	up_write(&vdev->memory_lock);
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index ec1660e0d6d2..6069a2db5dd6 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -47,6 +47,15 @@ struct vfio_pci_regops {
 	int	(*add_capability)(struct vfio_pci_core_device *vdev,
 				  struct vfio_pci_region *region,
 				  struct vfio_info_cap *caps);
+	/*
+	 * Optional, for a device-specific region exported as a dma-buf.
+	 * Called with memory_lock held for write; returns true while the
+	 * region is revoked, so a new dma-buf starts in the state that
+	 * vfio_pci_core_revoke_region() and vfio_pci_core_rearm_region()
+	 * maintain. Without it, the dma-buf starts armed.
+	 */
+	bool	(*revoked)(struct vfio_pci_core_device *vdev,
+			   struct vfio_pci_region *region);
 };
 
 struct vfio_pci_region {
-- 
2.53.0