Re: [PATCH v4 12/27] vfio/pci: Let a provider exclude a BAR sub-range from mmap

[Alex Williamson <alex@shazbot.org>]

On Thu, 13 Aug 2026 15:06:16 +0530
<mhonap@nvidia.com> wrote:

> From: Manish Honap <mhonap@nvidia.com>
> 
> Some devices expose registers in a BAR that must be reached only through
> a trap, not a direct guest mapping. A CXL Type-2 device's HDM decoder
> block is one: mapping it would let userspace reprogram the physical
> decoder that governs host memory decode. Give a provider a way to mark a
> BAR sub-range off-limits to mmap; it is advertised as a sparse-mmap
> region and refused in the mmap path, while the provider's own region
> still serves it.
> 
> Signed-off-by: Manish Honap <mhonap@nvidia.com>
> ---
>  drivers/vfio/pci/vfio_pci_core.c   | 72 ++++++++++++++++++++++++++++++
>  drivers/vfio/pci/vfio_pci_dmabuf.c | 13 ++++++
>  drivers/vfio/pci/vfio_pci_priv.h   | 13 ++++++
>  include/linux/vfio_pci_core.h      |  6 +++
>  4 files changed, 104 insertions(+)
> 
> diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
> index 0f9b5dfeea66..49dfbdaf3f05 100644
> --- a/drivers/vfio/pci/vfio_pci_core.c
> +++ b/drivers/vfio/pci/vfio_pci_core.c
> @@ -1009,6 +1009,67 @@ static int msix_mmappable_cap(struct vfio_pci_core_device *vdev,
>  	return vfio_info_add_capability(caps, &header, sizeof(header));
>  }
>  
> +/*
> + * A provider can keep a BAR sub-range off mmap (for example a CXL device's
> + * trapped HDM decoder block). Callers hold the resource so /dev/mem is already
> + * blocked; this only governs the vfio mmap path.
> + */
> +void vfio_pci_core_set_mmap_exclude(struct vfio_pci_core_device *vdev, int bar,
> +				    u64 start, u64 len)
> +{
> +	vdev->mmap_exclude_bar = bar;
> +	vdev->mmap_exclude_start = start;
> +	vdev->mmap_exclude_len = len;
> +}
> +EXPORT_SYMBOL_GPL(vfio_pci_core_set_mmap_exclude);

If we're going to go to the trouble of creating vfio-pci-core
infrastructure for handling excluded ranges, I'd rather see it handled
more generically.  One excluded "mmap" range per device is limited,
leaves MSI-X vector table existing as a separate implementation,
doesn't accurately describe what it does since it's excluded for both
mmap, read/write, and ioeventfds, and doesn't make use of the existing
x_start/x_end infrastructure we already have in read/write paths.

I think we should probably create a list of excluded ranges, each
containing a BAR index, start, size, and flags.  The flags are
necessary to manage mmap vs read vs write exclusions, where MSI-X only
excludes read/write, but this feature wants to exclude them all.

All existing use cases of msix_start/size would be migrated to this new
interface.  Handling in vfio_pci_bar_rw() would also need to account
for multiple excluded ranges per BAR (HDM exclusion adds that as a
possibility), iterating for any access extending beyond the intersecting
exclusion. mmap would generically fail any intersecting range with the
mmap exclusion flag set and region info would iterate the same set of
exclusions in generating the sparse mmap capability.

This would also correct the behavior of the next patch that
intersecting read/write accesses generate errors rather than fill reads
with -1 and drop writes.  Thanks,

Alex

> +
> +/* Advertise the BAR as mmappable minus the excluded sub-range. */
> +static int vfio_pci_mmap_exclude_cap(struct vfio_pci_core_device *vdev,
> +				     int index, struct vfio_info_cap *caps)
> +{
> +	u64 bar_len = pci_resource_len(vdev->pdev, index);
> +	u64 excl_start = ALIGN_DOWN(vdev->mmap_exclude_start, PAGE_SIZE);
> +	u64 excl_end = ALIGN(vdev->mmap_exclude_start + vdev->mmap_exclude_len,
> +			     PAGE_SIZE);
> +	struct vfio_region_info_cap_sparse_mmap *sparse;
> +	int nr_areas = 0, i = 0, ret;
> +	size_t size;
> +
> +	/*
> +	 * mmap is page granular, so the mmappable areas must stop at the page
> +	 * boundaries enclosing the excluded sub-range. The byte-granular
> +	 * exclusion still governs the fault and read/write paths; only the
> +	 * advertised mmap areas round out to whole pages.
> +	 */
> +	if (excl_start > 0)
> +		nr_areas++;
> +	if (excl_end < bar_len)
> +		nr_areas++;
> +
> +	size = struct_size(sparse, areas, nr_areas);
> +	sparse = kzalloc(size, GFP_KERNEL);
> +	if (!sparse)
> +		return -ENOMEM;
> +
> +	sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
> +	sparse->header.version = 1;
> +	sparse->nr_areas = nr_areas;
> +
> +	if (excl_start > 0) {
> +		sparse->areas[i].offset = 0;
> +		sparse->areas[i].size = excl_start;
> +		i++;
> +	}
> +	if (excl_end < bar_len) {
> +		sparse->areas[i].offset = excl_end;
> +		sparse->areas[i].size = bar_len - excl_end;
> +	}
> +
> +	ret = vfio_info_add_capability(caps, &sparse->header, size);
> +	kfree(sparse);
> +	return ret;
> +}
> +
>  int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,
>  				      unsigned int type, unsigned int subtype,
>  				      const struct vfio_pci_regops *ops,
> @@ -1157,6 +1218,13 @@ int vfio_pci_ioctl_get_region_info(struct vfio_device *core_vdev,
>  				if (ret)
>  					return ret;
>  			}
> +			if (vdev->mmap_exclude_len &&
> +			    info->index == vdev->mmap_exclude_bar) {
> +				ret = vfio_pci_mmap_exclude_cap(vdev, info->index,
> +								caps);
> +				if (ret)
> +					return ret;
> +			}
>  		}
>  
>  		break;
> @@ -1851,6 +1919,10 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
>  	if (req_start + req_len > phys_len)
>  		return -EINVAL;
>  
> +	/* An excluded sub-range is reachable only through its trap, not mmap. */
> +	if (vfio_pci_bar_is_excluded(vdev, index, req_start, req_len))
> +		return -EINVAL;
> +
>  	/*
>  	 * Ensure the BAR resource region is reserved for use.
>  	 */
> diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c
> index c16f460c01d6..51983105d38b 100644
> --- a/drivers/vfio/pci/vfio_pci_dmabuf.c
> +++ b/drivers/vfio/pci/vfio_pci_dmabuf.c
> @@ -177,11 +177,24 @@ int vfio_pci_core_get_dmabuf_phys(struct vfio_pci_core_device *vdev,
>  				  size_t nr_ranges)
>  {
>  	struct pci_dev *pdev = vdev->pdev;
> +	unsigned int i;
>  
>  	*provider = pcim_p2pdma_provider(pdev, region_index);
>  	if (!*provider)
>  		return -EINVAL;
>  
> +	/*
> +	 * A provider (e.g. vfio-cxl) can exclude a BAR sub-range that must be
> +	 * reached only through its trap. The mmap and read/write paths already
> +	 * refuse it; reject a DMA-BUF export overlapping it too, so a device fd
> +	 * holder cannot map the excluded registers to a peer and bypass the trap.
> +	 */
> +	for (i = 0; i < nr_ranges; i++)
> +		if (vfio_pci_bar_is_excluded(vdev, region_index,
> +					     dma_ranges[i].offset,
> +					     dma_ranges[i].length))
> +			return -EINVAL;
> +
>  	return vfio_pci_core_fill_phys_vec(
>  		phys_vec, dma_ranges, nr_ranges,
>  		pci_resource_start(pdev, region_index),
> diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h
> index fca9d0dfac90..902d17815ab6 100644
> --- a/drivers/vfio/pci/vfio_pci_priv.h
> +++ b/drivers/vfio/pci/vfio_pci_priv.h
> @@ -44,6 +44,19 @@ ssize_t vfio_pci_config_rw_single(struct vfio_pci_core_device *vdev,
>  ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
>  			size_t count, loff_t *ppos, bool iswrite);
>  
> +/*
> + * A provider (e.g. vfio-cxl) can carve a sub-range out of a BAR that must be
> + * reached only through its trap, never the direct BAR.  Returns true when
> + * [start, start + len) on this BAR overlaps that excluded range.
> + */
> +static inline bool vfio_pci_bar_is_excluded(struct vfio_pci_core_device *vdev,
> +					    int bar, u64 start, u64 len)
> +{
> +	return vdev->mmap_exclude_len && bar == vdev->mmap_exclude_bar &&
> +	       start < vdev->mmap_exclude_start + vdev->mmap_exclude_len &&
> +	       start + len > vdev->mmap_exclude_start;
> +}
> +
>  #ifdef CONFIG_VFIO_PCI_VGA
>  ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
>  			size_t count, loff_t *ppos, bool iswrite);
> diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
> index 117cd67995d8..43755b91880f 100644
> --- a/include/linux/vfio_pci_core.h
> +++ b/include/linux/vfio_pci_core.h
> @@ -162,6 +162,10 @@ struct vfio_pci_core_device {
>  	struct notifier_block	nb;
>  	struct rw_semaphore	memory_lock;
>  	struct list_head	dmabufs;
> +	/* BAR sub-range a provider keeps off mmap, reached only through a trap */
> +	int			mmap_exclude_bar;
> +	u64			mmap_exclude_start;
> +	u64			mmap_exclude_len;
>  };
>  
>  enum vfio_pci_io_width {
> @@ -176,6 +180,8 @@ int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,
>  				      unsigned int type, unsigned int subtype,
>  				      const struct vfio_pci_regops *ops,
>  				      size_t size, u32 flags, void *data);
> +void vfio_pci_core_set_mmap_exclude(struct vfio_pci_core_device *vdev, int bar,
> +				    u64 start, u64 len);
>  void vfio_pci_core_close_device(struct vfio_device *core_vdev);
>  int vfio_pci_core_init_dev(struct vfio_device *core_vdev);
>  void vfio_pci_core_release_dev(struct vfio_device *core_vdev);