Re: [PATCH] drm/xe: Add force_p2p module/configfs parameter

[Rodrigo Vivi <rodrigo.vivi@intel.com>]

On Tue, Aug 18, 2026 at 07:08:51PM +0200, Thomas Hellström wrote:
> On Tue, 2026-08-18 at 08:40 -0700, Nathan Ciobanu wrote:
> > Add force_p2p module and configfs parameter to give users an option
> > to
> > allow P2P DMA between GPU cards that are connected off of the root
> > bus
> > with no switch/bridge in between, as seen in some virtual machine
> > topologies (Q35/QEMU, VMware):
> > 
> >  \-[0000:03]-+-00.0  Intel Corporation [Intel Graphics]
> >              \-01.0  Intel Corporation [Intel Graphics]
> > 
> > or
> > 
> >  -[0000:00]-+-01.6-[07]----00.0  Intel Graphics
> >             \-01.7-[08]----00.0  Intel Graphics
> > 
> > In these situations the p2pdma distance calculation API would fail
> > the
> > host-bridge case
> 
> Why does it fail. Typically there is a very good reason, like the host
> bridge is not whitelisted?
> 
> 
> >  and not give a chance to users to take advantage of P2P
> > DMA.
> > 
> > This module/config parameter should be used with caution after
> > proving
> > that the memory mapped between the two devices is not getting
> > corrupted.
> > Users need to verify that the platform's physical topology
> > allows/facilitates P2P and tests such as the IGT xe_peer2peer pass.
> > 
> > Assisted-by: Claude:claude-3.5-sonnet
> > Signed-off-by: Nathan Ciobanu <nathan.d.ciobanu@linux.intel.com>
> > Cc: Rodrigo Vivi <rodrigo.vivi@intel.com>
> > Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
> > Cc: Matthew Brost <matthew.brost@intel.com>
> 
> I don't think we should do this, but rather fix upstream the cases that
> should work (probably by whitelisting the host bridges?)

Right, this looks like a good patch to be in some downstream tree for VMWare
since they don't have ways to create the virtual bridge for the proper whitelist
handling.

Or get this case and take to the PCI core kernel folks and see if there's
anything that could be done in the whitelist for that case. For qemu this
is not an issue because they can setup the virtual bridge, so they might
just push back on that as well though ;/

For-Downstream-Acked-by: Rodrigo Vivi <rodrigo.vivi@intel.com>

> 
> Thanks,
> Thomas
> 
> 
> 
> > 
> > diff --git a/drivers/gpu/drm/xe/xe_configfs.c
> > b/drivers/gpu/drm/xe/xe_configfs.c
> > index 052cce962161..187352a8b419 100644
> > --- a/drivers/gpu/drm/xe/xe_configfs.c
> > +++ b/drivers/gpu/drm/xe/xe_configfs.c
> > @@ -255,6 +255,39 @@
> >   * The created device directories can be removed using ``rmdir``::
> >   *
> >   *	# rmdir /sys/kernel/config/xe/0000:03:00.0/
> > + *
> > + * Force P2P DMA
> > + * -------------
> > + *
> > + * This option is useful in virtual machines when the PCIe emulated
> > topology
> > + * doesn't mimic the physical one and the p2pdma API fails to
> > calculate a
> > + * distance between nodes due to whitelist checks. In such cases,
> > the user can
> > + * set the force_p2p module parameter globally or through configfs,
> > if needed,
> > + * have the option to granularly control that setting on a per-
> > device basis.
> > + *
> > + * By default, force_p2p will have the same value as the
> > xe_modparam.force_p2p
> > + * which by default is 0.
> > + *
> > + * The symptoms that point to the P2P issue that this option is
> > trying to
> > + * address is:
> > + * - dmesg errors such as "xe 0000:03:01.0: cannot be used for peer-
> > to-peer DMA
> > + *   as the client and provider (0000:03:00.0) do not share an
> > upstream bridge
> > + *   or whitelisted host bridge"
> > + * - Running the IGT xe_peer2peer tests result in skips.
> > + * - UMD libraries fail with p2p related errors.
> > + *
> > + * Use these steps if the global xe.force_p2p parameter is
> > inconvenient
> > + * for some GPU devices in your system. NOTE that this option needs
> > to be
> > + * set on both endpoints of a pair to take effect.
> > + *
> > + * 1. Unbind the driver from device:
> > + *	# echo 0000:03:00.0 > /sys/bus/pci/drivers/xe/unbind
> > + *
> > + * 2. Set the force_p2p option on/off:
> > + *	# echo [0|1] >
> > /sys/kernel/config/xe/0000\:03\:00.0/force_p2p
> > + *
> > + * 3. Rebind the driver to device:
> > + *	# echo 0000:03:00.0 > /sys/bus/pci/drivers/xe/bind
> >   */
> >  
> >  /* Similar to struct xe_bb, but not tied to HW (yet) */
> > @@ -275,6 +308,7 @@ struct xe_config_group_device {
> >  		bool survivability_mode;
> >  		bool enable_psmi;
> >  		bool enable_multi_queue;
> > +		bool force_p2p;
> >  		struct {
> >  			unsigned int max_vfs;
> >  			bool admin_only_pf;
> > @@ -295,6 +329,7 @@ static const struct xe_config_device
> > device_defaults = {
> >  	.survivability_mode = false,
> >  	.enable_psmi = false,
> >  	.enable_multi_queue = true,
> > +	.force_p2p = XE_DEFAULT_FORCE_P2P,
> >  	.sriov = {
> >  		.max_vfs = XE_DEFAULT_MAX_VFS,
> >  		.admin_only_pf = XE_DEFAULT_ADMIN_ONLY_PF,
> > @@ -304,6 +339,7 @@ static const struct xe_config_device
> > device_defaults = {
> >  static void set_device_defaults(struct xe_config_device *config)
> >  {
> >  	*config = device_defaults;
> > +	config->force_p2p = xe_modparam.force_p2p;
> >  #ifdef CONFIG_PCI_IOV
> >  	config->sriov.max_vfs = xe_modparam.max_vfs;
> >  #endif
> > @@ -563,6 +599,37 @@ static ssize_t engines_allowed_store(struct
> > config_item *item, const char *page,
> >  	return len;
> >  }
> >  
> > +static ssize_t force_p2p_show(struct config_item *item, char *page)
> > +{
> > +	struct xe_config_group_device *dev =
> > to_xe_config_group_device(item);
> > +
> > +	guard(mutex)(&dev->lock);
> > +
> > +	return sprintf(page, "%d\n", dev->config.force_p2p);
> > +}
> > +
> > +static ssize_t force_p2p_store(struct config_item *item, const char
> > *page, size_t len)
> > +{
> > +	struct xe_config_group_device *dev =
> > to_xe_config_group_device(item);
> > +	bool force_p2p;
> > +	int ret;
> > +
> > +	guard(mutex)(&dev->lock);
> > +
> > +	if (is_bound(dev))
> > +		return -EBUSY;
> > +
> > +	ret = kstrtobool(page, &force_p2p);
> > +	if (ret)
> > +		return ret;
> > +
> > +	dev->config.force_p2p = force_p2p;
> > +
> > +	add_taint(TAINT_USER, LOCKDEP_STILL_OK);
> > +
> > +	return len;
> > +}
> > +
> >  static ssize_t enable_psmi_show(struct config_item *item, char
> > *page)
> >  {
> >  	struct xe_config_device *dev = to_xe_config_device(item);
> > @@ -856,6 +923,7 @@ CONFIGFS_ATTR(, ctx_restore_post_bb);
> >  CONFIGFS_ATTR(, enable_multi_queue);
> >  CONFIGFS_ATTR(, enable_psmi);
> >  CONFIGFS_ATTR(, engines_allowed);
> > +CONFIGFS_ATTR(, force_p2p);
> >  CONFIGFS_ATTR(, gt_types_allowed);
> >  CONFIGFS_ATTR(, survivability_mode);
> >  
> > @@ -865,6 +933,7 @@ static struct configfs_attribute
> > *xe_config_device_attrs[] = {
> >  	&attr_enable_multi_queue,
> >  	&attr_enable_psmi,
> >  	&attr_engines_allowed,
> > +	&attr_force_p2p,
> >  	&attr_gt_types_allowed,
> >  	&attr_survivability_mode,
> >  	NULL,
> > @@ -1269,6 +1338,25 @@ bool xe_configfs_get_psmi_enabled(struct
> > pci_dev *pdev)
> >  	return ret;
> >  }
> >  
> > +/** xe_configfs_get_force_p2p - get configfs force_p2p setting
> > + * @pdev: pci device
> > + *
> > + * Return: force_p2p setting in configfs
> > + */
> > +bool xe_configfs_get_force_p2p(struct pci_dev *pdev)
> > +{
> > +	struct xe_config_group_device *dev =
> > find_xe_config_group_device(pdev);
> > +	bool ret;
> > +
> > +	if (!dev)
> > +		return xe_modparam.force_p2p;
> > +
> > +	ret = dev->config.force_p2p;
> > +
> > +	config_group_put(&dev->group);
> > +	return ret;
> > +}
> > +
> >  /**
> >   * xe_configfs_get_enable_multi_queue - get configfs
> > enable_multi_queue setting
> >   * @pdev: pci device
> > diff --git a/drivers/gpu/drm/xe/xe_configfs.h
> > b/drivers/gpu/drm/xe/xe_configfs.h
> > index 4fbbeafba473..45af69cb74b1 100644
> > --- a/drivers/gpu/drm/xe/xe_configfs.h
> > +++ b/drivers/gpu/drm/xe/xe_configfs.h
> > @@ -30,6 +30,7 @@ u32 xe_configfs_get_ctx_restore_mid_bb(struct
> > pci_dev *pdev,
> >  u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev,
> >  					enum xe_engine_class class,
> >  					const u32 **cs);
> > +bool xe_configfs_get_force_p2p(struct pci_dev *pdev);
> >  #ifdef CONFIG_PCI_IOV
> >  unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev);
> >  bool xe_configfs_admin_only_pf(struct pci_dev *pdev);
> > @@ -50,6 +51,7 @@ static inline u32
> > xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
> >  static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev
> > *pdev,
> >  						      enum
> > xe_engine_class class,
> >  						      const u32
> > **cs) { return 0; }
> > +static inline bool xe_configfs_get_force_p2p(struct pci_dev *pdev) {
> > return xe_modparam.force_p2p; }
> >  #ifdef CONFIG_PCI_IOV
> >  static inline unsigned int xe_configfs_get_max_vfs(struct pci_dev
> > *pdev)
> >  {
> > diff --git a/drivers/gpu/drm/xe/xe_defaults.h
> > b/drivers/gpu/drm/xe/xe_defaults.h
> > index 0884224ef7c7..3881e832566a 100644
> > --- a/drivers/gpu/drm/xe/xe_defaults.h
> > +++ b/drivers/gpu/drm/xe/xe_defaults.h
> > @@ -13,6 +13,7 @@
> >  #define XE_DEFAULT_GUC_LOG_LEVEL		1
> >  #endif
> >  
> > +#define XE_DEFAULT_FORCE_P2P			0
> >  #define
> > XE_DEFAULT_PROBE_DISPLAY		IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
> >  #define XE_DEFAULT_VRAM_BAR_SIZE		0
> >  #define
> > XE_DEFAULT_FORCE_PROBE			CONFIG_DRM_XE_FORCE_PROBE
> > diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c
> > b/drivers/gpu/drm/xe/xe_dma_buf.c
> > index bf0728838ead..bd8f8ad70bcd 100644
> > --- a/drivers/gpu/drm/xe/xe_dma_buf.c
> > +++ b/drivers/gpu/drm/xe/xe_dma_buf.c
> > @@ -15,6 +15,7 @@
> >  
> >  #include "tests/xe_test.h"
> >  #include "xe_bo.h"
> > +#include "xe_configfs.h"
> >  #include "xe_device.h"
> >  #include "xe_pm.h"
> >  #include "xe_ttm_vram_mgr.h"
> > @@ -26,9 +27,14 @@ static int xe_dma_buf_attach(struct dma_buf
> > *dmabuf,
> >  			     struct dma_buf_attachment *attach)
> >  {
> >  	struct drm_gem_object *obj = attach->dmabuf->priv;
> > +	struct pci_dev *a_pdev = to_pci_dev(obj->dev->dev);
> > +	struct pci_dev *b_pdev = dev_is_pci(attach->dev) ?
> > +				 to_pci_dev(attach->dev) : NULL;
> >  
> >  	if (attach->peer2peer &&
> > -	    pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach-
> > >dev, false) < 0)
> > +	    !(b_pdev && xe_configfs_get_force_p2p(a_pdev) &&
> > +	      xe_configfs_get_force_p2p(b_pdev)) &&
> > +	    pci_p2pdma_distance(a_pdev, attach->dev, false) < 0)
> >  		attach->peer2peer = false;
> >  
> >  	if (!attach->peer2peer &&
> > !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
> > diff --git a/drivers/gpu/drm/xe/xe_module.c
> > b/drivers/gpu/drm/xe/xe_module.c
> > index 4bc28dfc1992..99cb2a455c6e 100644
> > --- a/drivers/gpu/drm/xe/xe_module.c
> > +++ b/drivers/gpu/drm/xe/xe_module.c
> > @@ -22,6 +22,7 @@
> >  #include "xe_sched_job.h"
> >  
> >  struct xe_modparam xe_modparam = {
> > +	.force_p2p =		XE_DEFAULT_FORCE_P2P,
> >  	.probe_display =	XE_DEFAULT_PROBE_DISPLAY,
> >  	.guc_log_level =	XE_DEFAULT_GUC_LOG_LEVEL,
> >  	.force_probe =		XE_DEFAULT_FORCE_PROBE,
> > @@ -38,6 +39,10 @@ module_param_named(svm_notifier_size,
> > xe_modparam.svm_notifier_size, uint, 0600)
> >  MODULE_PARM_DESC(svm_notifier_size, "Set the svm notifier size in
> > MiB, must be power of 2 "
> >  		 "[default="
> > __stringify(XE_DEFAULT_SVM_NOTIFIER_SIZE) "]");
> >  
> > +module_param_named_unsafe(force_p2p, xe_modparam.force_p2p, bool,
> > 0400);
> > +MODULE_PARM_DESC(force_p2p, "Force P2P DMA between GPUs (useful in
> > virtual machines), bypassing p2pdma checks. "
> > +		 "[default=" __stringify(XE_DEFAULT_FORCE_P2P) "]");
> > +
> >  #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
> >  module_param_named(probe_display, xe_modparam.probe_display, bool,
> > 0444);
> >  MODULE_PARM_DESC(probe_display, "Probe display HW, otherwise it's
> > left untouched "
> > diff --git a/drivers/gpu/drm/xe/xe_module.h
> > b/drivers/gpu/drm/xe/xe_module.h
> > index 6272d9e41207..bb154c7b095a 100644
> > --- a/drivers/gpu/drm/xe/xe_module.h
> > +++ b/drivers/gpu/drm/xe/xe_module.h
> > @@ -12,6 +12,7 @@ struct work_struct;
> >  
> >  /* Module modprobe variables */
> >  struct xe_modparam {
> > +	bool force_p2p;
> >  	bool probe_display;
> >  	int force_vram_bar_size;
> >  	int guc_log_level;
> > diff --git a/drivers/gpu/drm/xe/xe_svm.c
> > b/drivers/gpu/drm/xe/xe_svm.c
> > index 627a741293d5..3cd53bb166af 100644
> > --- a/drivers/gpu/drm/xe/xe_svm.c
> > +++ b/drivers/gpu/drm/xe/xe_svm.c
> > @@ -12,6 +12,7 @@
> >  
> >  #include "xe_bo.h"
> >  #include "xe_exec_queue_types.h"
> > +#include "xe_configfs.h"
> >  #include "xe_gt_stats.h"
> >  #include "xe_migrate.h"
> >  #include "xe_module.h"
> > @@ -892,6 +893,10 @@ static bool xe_has_interconnect(struct
> > drm_pagemap_peer *peer1,
> >  	if (dev1 == dev2)
> >  		return true;
> >  
> > +	if (xe_configfs_get_force_p2p(to_pci_dev(dev1)) &&
> > +	    xe_configfs_get_force_p2p(to_pci_dev(dev2)))
> > +		return true;
> > +
> >  	return pci_p2pdma_distance(to_pci_dev(dev1), dev2, true) >=
> > 0;
> >  }
> >