Re: [PATCH i-g-t v7 03/10] tests/intel/xe_multi_gpusvm: Add SVM multi-GPU cross-GPU memory access test

["Hellstrom, Thomas" <thomas.hellstrom@intel.com>]

On Mon, 2025-11-17 at 21:19 +0530, Sharma, Nishit wrote:
> 
> On 11/17/2025 6:30 PM, Hellstrom, Thomas wrote:
> > On Thu, 2025-11-13 at 16:33 +0000, nishit.sharma@intel.com wrote:
> > > From: Nishit Sharma <nishit.sharma@intel.com>
> > > 
> > > This test allocates a buffer in SVM, writes data to it from src
> > > GPU ,
> > > and reads/verifies
> > > the data from dst GPU. Optionally, the CPU also reads or modifies
> > > the
> > > buffer and both
> > > GPUs verify the results, ensuring correct cross-GPU and CPU
> > > memory
> > > access in a
> > > multi-GPU environment.
> > > 
> > > Signed-off-by: Nishit Sharma <nishit.sharma@intel.com>
> > > Acked-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
> > > ---
> > >   tests/intel/xe_multi_gpusvm.c | 373
> > > ++++++++++++++++++++++++++++++++++
> > >   tests/meson.build             |   1 +
> > >   2 files changed, 374 insertions(+)
> > >   create mode 100644 tests/intel/xe_multi_gpusvm.c
> > > 
> > > diff --git a/tests/intel/xe_multi_gpusvm.c
> > > b/tests/intel/xe_multi_gpusvm.c
> > > new file mode 100644
> > > index 000000000..6614ea3d1
> > > --- /dev/null
> > > +++ b/tests/intel/xe_multi_gpusvm.c
> > > @@ -0,0 +1,373 @@
> > > +// SPDX-License-Identifier: MIT
> > > +/*
> > > + * Copyright © 2023 Intel Corporation
> > > + */
> > > +
> > > +#include <unistd.h>
> > > +
> > > +#include "drmtest.h"
> > > +#include "igt.h"
> > > +#include "igt_multigpu.h"
> > > +
> > > +#include "intel_blt.h"
> > > +#include "intel_mocs.h"
> > > +#include "intel_reg.h"
> > > +
> > > +#include "xe/xe_ioctl.h"
> > > +#include "xe/xe_query.h"
> > > +#include "xe/xe_util.h"
> > > +
> > > +/**
> > > + * TEST: Basic multi-gpu SVM testing
> > > + * Category: SVM
> > > + * Mega feature: Compute
> > > + * Sub-category: Compute tests
> > > + * Functionality: SVM p2p access, madvise and prefetch.
> > > + * Test category: functionality test
> > > + *
> > > + * SUBTEST: cross-gpu-mem-access
> > > + * Description:
> > > + *      This test creates two malloced regions, places the
> > > destination
> > > + *      region both remotely and locally and copies to it. Reads
> > > back to
> > > + *      system memory and checks the result.
> > > + *
> > > + */
> > > +
> > > +#define MAX_XE_REGIONS	8
> > > +#define MAX_XE_GPUS 8
> > > +#define NUM_LOOPS 1
> > > +#define BATCH_SIZE(_fd) ALIGN(SZ_8K,
> > > xe_get_default_alignment(_fd))
> > > +#define BIND_SYNC_VAL 0x686868
> > > +#define EXEC_SYNC_VAL 0x676767
> > > +#define COPY_SIZE SZ_64M
> > > +
> > > +struct xe_svm_gpu_info {
> > > +	bool supports_faults;
> > > +	int vram_regions[MAX_XE_REGIONS];
> > > +	unsigned int num_regions;
> > > +	unsigned int va_bits;
> > > +	int fd;
> > > +};
> > > +
> > > +struct multigpu_ops_args {
> > > +	bool prefetch_req;
> > > +	bool op_mod;
> > > +};
> > > +
> > > +typedef void (*gpu_pair_fn) (
> > > +		struct xe_svm_gpu_info *src,
> > > +		struct xe_svm_gpu_info *dst,
> > > +		struct drm_xe_engine_class_instance *eci,
> > > +		void *extra_args
> > > +);
> > > +
> > > +static void for_each_gpu_pair(int num_gpus,
> > > +			      struct xe_svm_gpu_info *gpus,
> > > +			      struct
> > > drm_xe_engine_class_instance
> > > *eci,
> > > +			      gpu_pair_fn fn,
> > > +			      void *extra_args);
> > > +
> > > +static void gpu_mem_access_wrapper(struct xe_svm_gpu_info *src,
> > > +				   struct xe_svm_gpu_info *dst,
> > > +				   struct
> > > drm_xe_engine_class_instance *eci,
> > > +				   void *extra_args);
> > > +
> > > +static void open_pagemaps(int fd, struct xe_svm_gpu_info *info);
> > > +
> > > +static void
> > > +create_vm_and_queue(struct xe_svm_gpu_info *gpu, struct
> > > drm_xe_engine_class_instance *eci,
> > > +		    uint32_t *vm, uint32_t *exec_queue)
> > > +{
> > > +	*vm = xe_vm_create(gpu->fd,
> > > +			   DRM_XE_VM_CREATE_FLAG_LR_MODE |
> > > DRM_XE_VM_CREATE_FLAG_FAULT_MODE, 0);
> > > +	*exec_queue = xe_exec_queue_create(gpu->fd, *vm, eci,
> > > 0);
> > > +	xe_vm_bind_lr_sync(gpu->fd, *vm, 0, 0, 0, 1ull << gpu-
> > > > va_bits,
> > > +			   DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR);
> > > +}
> > > +
> > > +static void
> > > +setup_sync(struct drm_xe_sync *sync, volatile uint64_t
> > > **sync_addr,
> > > uint64_t timeline_value)
> > > +{
> > > +	*sync_addr = malloc(sizeof(**sync_addr));
> > > +	igt_assert(*sync_addr);
> > > +	sync->flags = DRM_XE_SYNC_FLAG_SIGNAL;
> > > +	sync->type = DRM_XE_SYNC_TYPE_USER_FENCE;
> > > +	sync->addr = to_user_pointer((uint64_t *)*sync_addr);
> > > +	sync->timeline_value = timeline_value;
> > > +	**sync_addr = 0;
> > > +}
> > > +
> > > +static void
> > > +cleanup_vm_and_queue(struct xe_svm_gpu_info *gpu, uint32_t vm,
> > > uint32_t exec_queue)
> > > +{
> > > +	xe_vm_unbind_lr_sync(gpu->fd, vm, 0, 0, 1ull << gpu-
> > > > va_bits);
> > > +	xe_exec_queue_destroy(gpu->fd, exec_queue);
> > > +	xe_vm_destroy(gpu->fd, vm);
> > > +}
> > > +
> > > +static void xe_multigpu_madvise(int src_fd, uint32_t vm,
> > > uint64_t
> > > addr, uint64_t size,
> > > +				uint64_t ext, uint32_t type, int
> > > dst_fd, uint16_t policy,
> > > +				uint16_t instance, uint32_t
> > > exec_queue, int local_fd,
> > > +				uint16_t local_vram)
> > > +{
> > > +	int ret;
> > > +
> > > +#define SYSTEM_MEMORY	0
> > Please use DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM.
> > A new define isn't necessary and it's also incorrect.
> Sure, will use that at required places.
> > 
> > > +	if (src_fd != dst_fd) {
> > > +		ret = xe_vm_madvise(src_fd, vm, addr, size, ext,
> > > type, dst_fd, policy, instance);
> > > +		if (ret == -ENOLINK) {
> > > +			igt_info("No fast interconnect between
> > > GPU0
> > > and GPU1, falling back to local VRAM\n");
> > > +			ret = xe_vm_madvise(src_fd, vm, addr,
> > > size,
> > > ext, type, local_fd,
> > > +					    policy, local_vram);
> > Please use DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE

Note that this also means you can skip the last two parameters to the
function AFAICT.

/Thomas