Re: [PATCH v6 11/11] vfio/nvgrace: Support get_dmabuf_phys

Re: [PATCH v6 11/11] vfio/nvgrace: Support get_dmabuf_phys

[kernel test robot]

BCC: lkp@intel.com
CC: oe-kbuild-all@lists.linux.dev
In-Reply-To: <20251102-dmabuf-vfio-v6-11-d773cff0db9f@nvidia.com>
References: <20251102-dmabuf-vfio-v6-11-d773cff0db9f@nvidia.com>
TO: Leon Romanovsky <leon@kernel.org>

Hi Leon,

kernel test robot noticed the following build warnings:

[auto build test WARNING on 3a8660878839faadb4f1a6dd72c3179c1df56787]

url:    https://github.com/intel-lab-lkp/linux/commits/Leon-Romanovsky/PCI-P2PDMA-Separate-the-mmap-support-from-the-core-logic/20251102-160355
base:   3a8660878839faadb4f1a6dd72c3179c1df56787
patch link:    https://lore.kernel.org/r/20251102-dmabuf-vfio-v6-11-d773cff0db9f%40nvidia.com
patch subject: [PATCH v6 11/11] vfio/nvgrace: Support get_dmabuf_phys
:::::: branch date: 7 days ago
:::::: commit date: 7 days ago
config: arm64-randconfig-r072-20251109 (https://download.01.org/0day-ci/archive/20251110/202511100152.SyE9Mhps-lkp@intel.com/config)
compiler: clang version 22.0.0git (https://github.com/llvm/llvm-project b9ea93cd5c37fb6d606502fd01208dd48330549d)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Reported-by: Dan Carpenter <error27@gmail.com>
| Closes: https://lore.kernel.org/r/202511100152.SyE9Mhps-lkp@intel.com/

New smatch warnings:
drivers/vfio/pci/nvgrace-gpu/main.c:706 nvgrace_get_dmabuf_phys() warn: '*provider' is an error pointer or valid

Old smatch warnings:
drivers/vfio/pci/nvgrace-gpu/main.c:720 nvgrace_get_dmabuf_phys() warn: '*provider' is an error pointer or valid
drivers/vfio/pci/nvgrace-gpu/main.c:794 nvgrace_gpu_fetch_memory_property() warn: impossible condition '(*pmemphys > ((((1 << (8 * 8 - 1 - (((-1)) < 1))) - 1) + (1 << (8 * 8 - 1 - (((-1)) < 1)))))) => (0-u64max > u64max)'
drivers/vfio/pci/nvgrace-gpu/main.c:802 nvgrace_gpu_fetch_memory_property() warn: impossible condition '(*pmemlength > ((((1 << (8 * 8 - 1 - (((-1)) < 1))) - 1) + (1 << (8 * 8 - 1 - (((-1)) < 1)))))) => (0-u64max > u64max)'

vim +706 drivers/vfio/pci/nvgrace-gpu/main.c

701ab935859fcfd Ankit Agrawal   2024-02-20  686  
d7ddf63944708fb Jason Gunthorpe 2025-11-02  687  static int nvgrace_get_dmabuf_phys(struct vfio_pci_core_device *core_vdev,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  688  				   struct p2pdma_provider **provider,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  689  				   unsigned int region_index,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  690  				   struct dma_buf_phys_vec *phys_vec,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  691  				   struct vfio_region_dma_range *dma_ranges,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  692  				   size_t nr_ranges)
d7ddf63944708fb Jason Gunthorpe 2025-11-02  693  {
d7ddf63944708fb Jason Gunthorpe 2025-11-02  694  	struct nvgrace_gpu_pci_core_device *nvdev = container_of(
d7ddf63944708fb Jason Gunthorpe 2025-11-02  695  		core_vdev, struct nvgrace_gpu_pci_core_device, core_device);
d7ddf63944708fb Jason Gunthorpe 2025-11-02  696  	struct pci_dev *pdev = core_vdev->pdev;
d7ddf63944708fb Jason Gunthorpe 2025-11-02  697  
d7ddf63944708fb Jason Gunthorpe 2025-11-02  698  	if (nvdev->resmem.memlength && region_index == RESMEM_REGION_INDEX) {
d7ddf63944708fb Jason Gunthorpe 2025-11-02  699  		/*
d7ddf63944708fb Jason Gunthorpe 2025-11-02  700  		 * The P2P properties of the non-BAR memory is the same as the
d7ddf63944708fb Jason Gunthorpe 2025-11-02  701  		 * BAR memory, so just use the provider for index 0. Someday
d7ddf63944708fb Jason Gunthorpe 2025-11-02  702  		 * when CXL gets P2P support we could create CXLish providers
d7ddf63944708fb Jason Gunthorpe 2025-11-02  703  		 * for the non-BAR memory.
d7ddf63944708fb Jason Gunthorpe 2025-11-02  704  		 */
d7ddf63944708fb Jason Gunthorpe 2025-11-02  705  		*provider = pcim_p2pdma_provider(pdev, 0);
d7ddf63944708fb Jason Gunthorpe 2025-11-02 @706  		if (!*provider)
d7ddf63944708fb Jason Gunthorpe 2025-11-02  707  			return -EINVAL;
d7ddf63944708fb Jason Gunthorpe 2025-11-02  708  		return vfio_pci_core_fill_phys_vec(phys_vec, dma_ranges,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  709  						   nr_ranges,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  710  						   nvdev->resmem.memphys,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  711  						   nvdev->resmem.memlength);
d7ddf63944708fb Jason Gunthorpe 2025-11-02  712  	} else if (region_index == USEMEM_REGION_INDEX) {
d7ddf63944708fb Jason Gunthorpe 2025-11-02  713  		/*
d7ddf63944708fb Jason Gunthorpe 2025-11-02  714  		 * This is actually cachable memory and isn't treated as P2P in
d7ddf63944708fb Jason Gunthorpe 2025-11-02  715  		 * the chip. For now we have no way to push cachable memory
d7ddf63944708fb Jason Gunthorpe 2025-11-02  716  		 * through everything and the Grace HW doesn't care what caching
d7ddf63944708fb Jason Gunthorpe 2025-11-02  717  		 * attribute is programmed into the SMMU. So use BAR 0.
d7ddf63944708fb Jason Gunthorpe 2025-11-02  718  		 */
d7ddf63944708fb Jason Gunthorpe 2025-11-02  719  		*provider = pcim_p2pdma_provider(pdev, 0);
d7ddf63944708fb Jason Gunthorpe 2025-11-02  720  		if (!*provider)
d7ddf63944708fb Jason Gunthorpe 2025-11-02  721  			return -EINVAL;
d7ddf63944708fb Jason Gunthorpe 2025-11-02  722  		return vfio_pci_core_fill_phys_vec(phys_vec, dma_ranges,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  723  						   nr_ranges,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  724  						   nvdev->usemem.memphys,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  725  						   nvdev->usemem.memlength);
d7ddf63944708fb Jason Gunthorpe 2025-11-02  726  	}
d7ddf63944708fb Jason Gunthorpe 2025-11-02  727  	return vfio_pci_core_get_dmabuf_phys(core_vdev, provider, region_index,
d7ddf63944708fb Jason Gunthorpe 2025-11-02  728  					     phys_vec, dma_ranges, nr_ranges);
d7ddf63944708fb Jason Gunthorpe 2025-11-02  729  }
d7ddf63944708fb Jason Gunthorpe 2025-11-02  730  

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

[PATCH v6 00/11] vfio/pci: Allow MMIO regions to be exported through dma-buf

[Leon Romanovsky]

Changelog:
v6:
 * Fixed wrong error check from pcim_p2pdma_init().
 * Documented pcim_p2pdma_provider() function.
 * Improved commit messages.
 * Added VFIO DMA-BUF selftest.
 * Added __counted_by(nr_ranges) annotation to struct vfio_device_feature_dma_buf.
 * Fixed error unwind when dma_buf_fd() fails.
 * Document latest changes to p2pmem.
 * Removed EXPORT_SYMBOL_GPL from pci_p2pdma_map_type.
 * Moved DMA mapping logic to DMA-BUF.
 * Removed types patch to avoid dependencies between subsystems.
 * Moved vfio_pci_dma_buf_move() in err_undo block.
 * Added nvgrace patch.
v5: https://lore.kernel.org/all/cover.1760368250.git.leon@kernel.org
 * Rebased on top of v6.18-rc1.
 * Added more validation logic to make sure that DMA-BUF length doesn't
   overflow in various scenarios.
 * Hide kernel config from the users.
 * Fixed type conversion issue. DMA ranges are exposed with u64 length,
   but DMA-BUF uses "unsigned int" as a length for SG entries.
 * Added check to prevent from VFIO drivers which reports BAR size
   different from PCI, do not use DMA-BUF functionality.
v4: https://lore.kernel.org/all/cover.1759070796.git.leon@kernel.org
 * Split pcim_p2pdma_provider() to two functions, one that initializes
   array of providers and another to return right provider pointer.
v3: https://lore.kernel.org/all/cover.1758804980.git.leon@kernel.org
 * Changed pcim_p2pdma_enable() to be pcim_p2pdma_provider().
 * Cache provider in vfio_pci_dma_buf struct instead of BAR index.
 * Removed misleading comment from pcim_p2pdma_provider().
 * Moved MMIO check to be in pcim_p2pdma_provider().
v2: https://lore.kernel.org/all/cover.1757589589.git.leon@kernel.org/
 * Added extra patch which adds new CONFIG, so next patches can reuse
 * it.
 * Squashed "PCI/P2PDMA: Remove redundant bus_offset from map state"
   into the other patch.
 * Fixed revoke calls to be aligned with true->false semantics.
 * Extended p2pdma_providers to be per-BAR and not global to whole
 * device.
 * Fixed possible race between dmabuf states and revoke.
 * Moved revoke to PCI BAR zap block.
v1: https://lore.kernel.org/all/cover.1754311439.git.leon@kernel.org
 * Changed commit messages.
 * Reused DMA_ATTR_MMIO attribute.
 * Returned support for multiple DMA ranges per-dMABUF.
v0: https://lore.kernel.org/all/cover.1753274085.git.leonro@nvidia.com

---------------------------------------------------------------------------
Based on "[PATCH v6 00/16] dma-mapping: migrate to physical address-based API"
https://lore.kernel.org/all/cover.1757423202.git.leonro@nvidia.com/ series.
---------------------------------------------------------------------------

This series extends the VFIO PCI subsystem to support exporting MMIO
regions from PCI device BARs as dma-buf objects, enabling safe sharing of
non-struct page memory with controlled lifetime management. This allows RDMA
and other subsystems to import dma-buf FDs and build them into memory regions
for PCI P2P operations.

The series supports a use case for SPDK where a NVMe device will be
owned by SPDK through VFIO but interacting with a RDMA device. The RDMA
device may directly access the NVMe CMB or directly manipulate the NVMe
device's doorbell using PCI P2P.

However, as a general mechanism, it can support many other scenarios with
VFIO. This dmabuf approach can be usable by iommufd as well for generic
and safe P2P mappings.

In addition to the SPDK use-case mentioned above, the capability added
in this patch series can also be useful when a buffer (located in device
memory such as VRAM) needs to be shared between any two dGPU devices or
instances (assuming one of them is bound to VFIO PCI) as long as they
are P2P DMA compatible.

The implementation provides a revocable attachment mechanism using dma-buf
move operations. MMIO regions are normally pinned as BARs don't change
physical addresses, but access is revoked when the VFIO device is closed
or a PCI reset is issued. This ensures kernel self-defense against
potentially hostile userspace.

The series includes significant refactoring of the PCI P2PDMA subsystem
to separate core P2P functionality from memory allocation features,
making it more modular and suitable for VFIO use cases that don't need
struct page support.

-----------------------------------------------------------------------
The series is based originally on
https://lore.kernel.org/all/20250307052248.405803-1-vivek.kasireddy@intel.com/
but heavily rewritten to be based on DMA physical API.
-----------------------------------------------------------------------
The WIP branch can be found here:
https://git.kernel.org/pub/scm/linux/kernel/git/leon/linux-rdma.git/log/?h=dmabuf-vfio-v6

Thanks

---
Jason Gunthorpe (2):
      PCI/P2PDMA: Document DMABUF model
      vfio/nvgrace: Support get_dmabuf_phys

Leon Romanovsky (7):
      PCI/P2PDMA: Separate the mmap() support from the core logic
      PCI/P2PDMA: Simplify bus address mapping API
      PCI/P2PDMA: Refactor to separate core P2P functionality from memory allocation
      PCI/P2PDMA: Provide an access to pci_p2pdma_map_type() function
      dma-buf: provide phys_vec to scatter-gather mapping routine
      vfio/pci: Enable peer-to-peer DMA transactions by default
      vfio/pci: Add dma-buf export support for MMIO regions

Vivek Kasireddy (2):
      vfio: Export vfio device get and put registration helpers
      vfio/pci: Share the core device pointer while invoking feature functions

 Documentation/driver-api/pci/p2pdma.rst |  95 +++++++---
 block/blk-mq-dma.c                      |   2 +-
 drivers/dma-buf/dma-buf.c               | 235 ++++++++++++++++++++++++
 drivers/iommu/dma-iommu.c               |   4 +-
 drivers/pci/p2pdma.c                    | 182 +++++++++++++-----
 drivers/vfio/pci/Kconfig                |   3 +
 drivers/vfio/pci/Makefile               |   1 +
 drivers/vfio/pci/nvgrace-gpu/main.c     |  56 ++++++
 drivers/vfio/pci/vfio_pci.c             |   5 +
 drivers/vfio/pci/vfio_pci_config.c      |  22 ++-
 drivers/vfio/pci/vfio_pci_core.c        |  56 ++++--
 drivers/vfio/pci/vfio_pci_dmabuf.c      | 315 ++++++++++++++++++++++++++++++++
 drivers/vfio/pci/vfio_pci_priv.h        |  23 +++
 drivers/vfio/vfio_main.c                |   2 +
 include/linux/dma-buf.h                 |  18 ++
 include/linux/pci-p2pdma.h              | 120 +++++++-----
 include/linux/vfio.h                    |   2 +
 include/linux/vfio_pci_core.h           |  42 +++++
 include/uapi/linux/vfio.h               |  27 +++
 kernel/dma/direct.c                     |   4 +-
 mm/hmm.c                                |   2 +-
 21 files changed, 1077 insertions(+), 139 deletions(-)
---
base-commit: 3a8660878839faadb4f1a6dd72c3179c1df56787
change-id: 20251016-dmabuf-vfio-6cef732adf5a

Best regards,
--  
Leon Romanovsky <leonro@nvidia.com>

[PATCH v6 11/11] vfio/nvgrace: Support get_dmabuf_phys

[Leon Romanovsky]

From: Jason Gunthorpe <jgg@nvidia.com>

Call vfio_pci_core_fill_phys_vec() with the proper physical ranges for the
synthetic BAR 2 and BAR 4 regions. Otherwise use the normal flow based on
the PCI bar.

This demonstrates a DMABUF that follows the region info report to only
allow mapping parts of the region that are mmapable. Since the BAR is
power of two sized and the "CXL" region is just page aligned the there can
be a padding region at the end that is not mmaped or passed into the
DMABUF.

The "CXL" ranges that are remapped into BAR 2 and BAR 4 areas are not PCI
MMIO, they actually run over the CXL-like coherent interconnect and for
the purposes of DMA behave identically to DRAM. We don't try to model this
distinction between true PCI BAR memory that takes a real PCI path and the
"CXL" memory that takes a different path in the p2p framework for now.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 drivers/vfio/pci/nvgrace-gpu/main.c | 56 +++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c
index e346392b72f6..7d7ab2c84018 100644
--- a/drivers/vfio/pci/nvgrace-gpu/main.c
+++ b/drivers/vfio/pci/nvgrace-gpu/main.c
@@ -7,6 +7,7 @@
 #include <linux/vfio_pci_core.h>
 #include <linux/delay.h>
 #include <linux/jiffies.h>
+#include <linux/pci-p2pdma.h>
 
 /*
  * The device memory usable to the workloads running in the VM is cached
@@ -683,6 +684,54 @@ nvgrace_gpu_write(struct vfio_device *core_vdev,
 	return vfio_pci_core_write(core_vdev, buf, count, ppos);
 }
 
+static int nvgrace_get_dmabuf_phys(struct vfio_pci_core_device *core_vdev,
+				   struct p2pdma_provider **provider,
+				   unsigned int region_index,
+				   struct dma_buf_phys_vec *phys_vec,
+				   struct vfio_region_dma_range *dma_ranges,
+				   size_t nr_ranges)
+{
+	struct nvgrace_gpu_pci_core_device *nvdev = container_of(
+		core_vdev, struct nvgrace_gpu_pci_core_device, core_device);
+	struct pci_dev *pdev = core_vdev->pdev;
+
+	if (nvdev->resmem.memlength && region_index == RESMEM_REGION_INDEX) {
+		/*
+		 * The P2P properties of the non-BAR memory is the same as the
+		 * BAR memory, so just use the provider for index 0. Someday
+		 * when CXL gets P2P support we could create CXLish providers
+		 * for the non-BAR memory.
+		 */
+		*provider = pcim_p2pdma_provider(pdev, 0);
+		if (!*provider)
+			return -EINVAL;
+		return vfio_pci_core_fill_phys_vec(phys_vec, dma_ranges,
+						   nr_ranges,
+						   nvdev->resmem.memphys,
+						   nvdev->resmem.memlength);
+	} else if (region_index == USEMEM_REGION_INDEX) {
+		/*
+		 * This is actually cachable memory and isn't treated as P2P in
+		 * the chip. For now we have no way to push cachable memory
+		 * through everything and the Grace HW doesn't care what caching
+		 * attribute is programmed into the SMMU. So use BAR 0.
+		 */
+		*provider = pcim_p2pdma_provider(pdev, 0);
+		if (!*provider)
+			return -EINVAL;
+		return vfio_pci_core_fill_phys_vec(phys_vec, dma_ranges,
+						   nr_ranges,
+						   nvdev->usemem.memphys,
+						   nvdev->usemem.memlength);
+	}
+	return vfio_pci_core_get_dmabuf_phys(core_vdev, provider, region_index,
+					     phys_vec, dma_ranges, nr_ranges);
+}
+
+static const struct vfio_pci_device_ops nvgrace_gpu_pci_dev_ops = {
+	.get_dmabuf_phys = nvgrace_get_dmabuf_phys,
+};
+
 static const struct vfio_device_ops nvgrace_gpu_pci_ops = {
 	.name		= "nvgrace-gpu-vfio-pci",
 	.init		= vfio_pci_core_init_dev,
@@ -703,6 +752,10 @@ static const struct vfio_device_ops nvgrace_gpu_pci_ops = {
 	.detach_ioas	= vfio_iommufd_physical_detach_ioas,
 };
 
+static const struct vfio_pci_device_ops nvgrace_gpu_pci_dev_core_ops = {
+	.get_dmabuf_phys = vfio_pci_core_get_dmabuf_phys,
+};
+
 static const struct vfio_device_ops nvgrace_gpu_pci_core_ops = {
 	.name		= "nvgrace-gpu-vfio-pci-core",
 	.init		= vfio_pci_core_init_dev,
@@ -965,6 +1018,9 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev,
 						    memphys, memlength);
 		if (ret)
 			goto out_put_vdev;
+		nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_ops;
+	} else {
+		nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_core_ops;
 	}
 
 	ret = vfio_pci_core_register_device(&nvdev->core_device);

-- 
2.51.0