Re: [PATCH i-g-t 1/2] lib/xe/xe_sriov_debugfs: Add VF save/restore related functions

[Michal Wajdeczko <michal.wajdeczko@intel.com>]

On 19.02.2025 11:25, Adam Miszczak wrote:
> Provide helpers required to exercise VF (fake) migration:
> - control VF state (stop/pause/resume/restore)
> - read/write GuC, GGTT and LMEM state
> - initiate GGTT address relocation
> 
> Signed-off-by: Adam Miszczak <adam.miszczak@linux.intel.com>
> ---
>  lib/xe/xe_sriov_debugfs.c | 252 ++++++++++++++++++++++++++++++++++++++
>  lib/xe/xe_sriov_debugfs.h |  31 +++++
>  2 files changed, 283 insertions(+)
> 
> diff --git a/lib/xe/xe_sriov_debugfs.c b/lib/xe/xe_sriov_debugfs.c
> index 8f30fa312..2e33871ad 100644
> --- a/lib/xe/xe_sriov_debugfs.c
> +++ b/lib/xe/xe_sriov_debugfs.c
> @@ -15,6 +15,10 @@
>  #include "xe/xe_sriov_provisioning.h"
>  
>  #define SRIOV_DEBUGFS_PATH_MAX 96
> +/* Maximum size of buffers used to read GuC, GGTT, LMEM state */
> +#define SRIOV_GUC_STATE_BUF_SIZE_MAX SZ_4M
> +#define SRIOV_GGTT_RAW_BUF_SIZE_MAX SZ_4M
> +#define SRIOV_LMEM_STATE_BUF_SIZE_MAX SZ_512M
>  
>  static char *xe_sriov_pf_debugfs_path(int pf, unsigned int vf_num, unsigned int gt_num, char *path,
>  				      int pathlen)
> @@ -285,6 +289,254 @@ static int validate_vf_ids(enum xe_sriov_shared_res res,
>  	return 0;
>  }
>  
> +static const char *xe_sriov_debugfs_control_value(enum xe_sriov_vf_control operation)
> +{
> +	switch (operation) {
> +	case XE_SRIOV_VF_CONTROL_STOP:
> +		return "stop";
> +	case XE_SRIOV_VF_CONTROL_PAUSE:
> +		return "pause";
> +	case XE_SRIOV_VF_CONTROL_RESUME:
> +		return "resume";
> +	case XE_SRIOV_VF_CONTROL_RESTORE:
> +		return "restore!";

this opcode is only supported by the PF driver compiled with
CONFIG_DRM_XE_DEBUG_SRIOV enabled, which is not set for the CI builds
and never will be enabled (it is purely for low level debug/bringup)

so was this taken into consideration?

> +	}
> +
> +	return NULL;
> +}
> +
> +/**
> + * xe_sriov_set_vf_control - Controls VF's state.
> + * @pf_fd: PF device file descriptor
> + * @vf_num: VF number
> + * @gt_num: GT number
> + * @operation: VF control command to perform
> + *
> + * Allows to send control VF state commands: pause, resume and stop.
> + * Additionally, restore! action is provided to restore previosusly

typo

> + * saved GuC state for VF migration enabling and testing purposes.
> + *
> + * Return: 0 on success, negative error code on failure.
> + */
> +int xe_sriov_set_vf_control(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						enum xe_sriov_vf_control operation)
> +{
> +	char path[PATH_MAX];
> +	const char *op_name;
> +
> +	igt_assert(igt_sriov_is_pf(pf_fd) && is_xe_device(pf_fd));
> +	igt_assert(gt_num < xe_number_gt(pf_fd));
> +	igt_assert(vf_num > 0);
> +
> +	if (!xe_sriov_pf_debugfs_path(pf_fd, vf_num, gt_num, path, sizeof(path)))
> +		return -ENOENT;
> +
> +	op_name = xe_sriov_debugfs_control_value(operation);
> +	if (!op_name)
> +		return -EINVAL;
> +
> +	strncat(path, "control", sizeof(path) - strlen(path));
> +
> +	igt_debug("Set VF%d control: %s (%s)\n", vf_num, op_name, path);
> +	igt_debugfs_write(pf_fd, path, op_name);
> +
> +	return 0;
> +}
> +
> +/**
> + * xe_sriov_relocate_ggtt - Enforce VF's GGTT address relaction.

typo

> + * @pf_fd: PF device file descriptor
> + * @vf_num: VF number
> + * @gt_num: GT number
> + *
> + * Triggers move of the existing GGTT allocation to other location.
> + */
> +void xe_sriov_relocate_ggtt(int pf_fd, unsigned int vf_num, unsigned int gt_num)
> +{
> +	char path[PATH_MAX];
> +
> +	igt_assert(vf_num > 0);
> +
> +	sprintf(path, "/sys/kernel/debug/dri/0/gt%u/vf%u/relocate_ggtt", gt_num, vf_num);

do we really have support for the "relocate_ggtt"

> +	__igt_debugfs_write(pf_fd, path, "1", 1);
> +
> +	igt_debug("Set VF%d GGTT relocate (%s)\n", vf_num, path);
> +}
> +
> +/**
> + * xe_sriov_get_guc_state - Read VF's GuC state data.
> + * @pf_fd: PF device file descriptor
> + * @vf_num: VF number
> + * @gt_num: GT number
> + * @lmem_state_size: Pointer to store the size of a returned buffer
> + *
> + * Reads the GuC state of given VF device @vf on GT @gt_num.
> + * Allocates an output buffer with a size limited to SRIOV_GUC_STATE_BUF_SIZE_MAX.
> + * The caller should free the allocated space.
> + *
> + * Return: pointer to the GuC state buffer on success, negative error code on failure.
> + */
> +void *xe_sriov_get_guc_state(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						int *guc_state_size)
> +{
> +	char path[PATH_MAX];
> +	int dir;
> +	void *buf;
> +
> +	igt_assert(vf_num > 0);
> +
> +	sprintf(path, "/sys/kernel/debug/dri/0/gt%u/vf%u/guc_state", gt_num, vf_num);
> +
> +	buf = malloc(SRIOV_GUC_STATE_BUF_SIZE_MAX);
> +	dir = igt_debugfs_dir(pf_fd);
> +
> +	*guc_state_size = igt_debugfs_simple_read(dir, path, buf, SRIOV_GUC_STATE_BUF_SIZE_MAX);
> +	close(dir);
> +
> +	igt_debug("Read VF%d GuC state: %d B (%s)\n", vf_num, *guc_state_size, path);
> +
> +	return buf;
> +}
> +
> +/**
> + * xe_sriov_set_guc_state - Write VF's GuC state data.
> + * @pf_fd: PF device file descriptor
> + * @vf_num: VF number
> + * @gt_num: GT number
> + * @guc_state: Pointer to a buffer to write
> + * @guc_state_size: Size of a buffer to write
> + *
> + * Writes the GuC state of given VF device @vf on GT @gt_num.
> + */
> +void xe_sriov_set_guc_state(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						void *guc_state, int guc_state_size)
> +{
> +	char path[PATH_MAX];
> +
> +	igt_assert(vf_num > 0);
> +
> +	sprintf(path, "/sys/kernel/debug/dri/0/gt%u/vf%u/guc_state", gt_num, vf_num);
> +	__igt_debugfs_write(pf_fd, path, guc_state, guc_state_size);
> +
> +	igt_debug("Write VF%d GuC state: %d B (%s)\n", vf_num, guc_state_size, path);
> +}
> +
> +/**
> + * xe_sriov_get_ggtt_raw - Read VF's GGTT state data.
> + * @pf_fd: PF device file descriptor
> + * @vf_num: VF number
> + * @gt_num: GT number
> + * @ggtt_raw_size: Pointer to store the size of a returned buffer
> + *
> + * Reads the GGTT state of given VF device @vf on GT @gt_num.
> + * Allocates an output buffer with a size limited to SRIOV_GGTT_RAW_BUF_SIZE_MAX.
> + * The caller should free the allocated space.
> + *
> + * Return: pointer to the GGTT state buffer on success, negative error code on failure.
> + */
> +void *xe_sriov_get_ggtt_raw(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						int *ggtt_raw_size)
> +{
> +	char path[PATH_MAX];
> +	int dir;
> +	void *buf;
> +
> +	igt_assert(vf_num > 0);
> +
> +	sprintf(path, "/sys/kernel/debug/dri/0/gt%u/vf%u/ggtt_raw", gt_num, vf_num);

this was also not merged upstream ..
and even if it will be exposed, likely only under DEBUG_SRIOV config

> +
> +	buf = malloc(SRIOV_GGTT_RAW_BUF_SIZE_MAX);
> +	dir = igt_debugfs_dir(pf_fd);
> +
> +	*ggtt_raw_size = igt_debugfs_simple_read(dir, path, buf, SRIOV_GGTT_RAW_BUF_SIZE_MAX);
> +	close(dir);
> +
> +	igt_debug("Read VF%d GGTT raw: %d B (%s)\n", vf_num, *ggtt_raw_size, path);
> +
> +	return buf;
> +}
> +
> +/**
> + * xe_sriov_set_ggtt_raw - Write VF's GGTT state data.
> + * @pf_fd: PF device file descriptor
> + * @vf_num: VF number
> + * @gt_num: GT number
> + * @ggtt_raw: Pointer to a buffer to write
> + * @ggtt_raw_size: Size of a buffer to write
> + *
> + * Writes the GGTT state of given VF device @vf on GT @gt_num.
> + */
> +void xe_sriov_set_ggtt_raw(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						void *ggtt_raw, int ggtt_raw_size)
> +{
> +	char path[PATH_MAX];
> +
> +	igt_assert(vf_num > 0);
> +
> +	sprintf(path, "/sys/kernel/debug/dri/0/gt%u/vf%u/ggtt_raw", gt_num, vf_num);
> +	__igt_debugfs_write(pf_fd, path, ggtt_raw, ggtt_raw_size);
> +
> +	igt_debug("Write VF%d GGTT raw: %d B (%s)\n", vf_num, ggtt_raw_size, path);
> +}
> +
> +/**
> + * xe_sriov_get_lmem_state - Read VF's LMEM state data.
> + * @pf_fd: PF device file descriptor
> + * @vf_num: VF number
> + * @gt_num: GT number
> + * @lmem_state_size: Pointer to store the size of a returned buffer
> + *
> + * Reads the LMEM state of given VF device @vf on GT @gt_num.
> + * Allocates an output buffer with a size limited to SRIOV_LMEM_STATE_BUF_SIZE_MAX.
> + * The caller should free the allocated space.
> + *
> + * Return: pointer to the LMEM state buffer on success, negative error code on failure.
> + */
> +void *xe_sriov_get_lmem_state(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						int *lmem_state_size)
> +{
> +	char path[PATH_MAX];
> +	int dir;
> +	void *buf;
> +
> +	igt_assert(vf_num > 0);
> +
> +	sprintf(path, "/sys/kernel/debug/dri/0/gt%u/vf%u/lmem_state", gt_num, vf_num);
> +
> +	buf = malloc(SRIOV_LMEM_STATE_BUF_SIZE_MAX);
> +	dir = igt_debugfs_dir(pf_fd);
> +
> +	*lmem_state_size = igt_debugfs_simple_read(dir, path, buf, SRIOV_LMEM_STATE_BUF_SIZE_MAX);
> +	close(dir);
> +
> +	igt_debug("Read VF%d LMEM state: %d B (%s)\n", vf_num, *lmem_state_size,  path);
> +
> +	return buf;
> +}
> +
> +/**
> + * xe_sriov_set_lmem_state - Write VF's LMEM state data.
> + * @pf_fd: PF device file descriptor
> + * @vf_num: VF number
> + * @gt_num: GT number
> + * @lmem_state: Pointer to a buffer to write
> + * @lmem_state_size: Size of a buffer to write
> + *
> + * Writes the LMEM state of given VF device @vf on GT @gt_num.
> + */
> +void xe_sriov_set_lmem_state(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						void *lmem_state, int lmem_state_size)
> +{
> +	char path[PATH_MAX];
> +
> +	igt_assert(vf_num > 0);
> +
> +	sprintf(path, "/sys/kernel/debug/dri/0/gt%u/vf%u/lmem_state", gt_num, vf_num);
> +	__igt_debugfs_write(pf_fd, path, lmem_state, lmem_state_size);
> +
> +	igt_debug("Write VF%d LMEM state: %d B (%s)\n", vf_num, lmem_state_size, path);
> +}
> +
>  /**
>   * xe_sriov_pf_debugfs_read_check_ranges:
>   * @pf_fd: PF device file descriptor
> diff --git a/lib/xe/xe_sriov_debugfs.h b/lib/xe/xe_sriov_debugfs.h
> index 4983afbb3..d801084a9 100644
> --- a/lib/xe/xe_sriov_debugfs.h
> +++ b/lib/xe/xe_sriov_debugfs.h
> @@ -9,6 +9,20 @@
>  enum xe_sriov_shared_res;
>  struct xe_sriov_provisioned_range;
>  
> +/**
> + * enum xe_sriov_vf_control - VF control
> + * @XE_SRIOV_VF_CONTROL_STOP: stop VF
> + * @XE_SRIOV_VF_CONTROL_PAUSE: pause VF
> + * @XE_SRIOV_VF_CONTROL_RESUME: resume VF
> + * @XE_SRIOV_VF_CONTROL_RESTORE: restore VF GuC state
> + */
> +enum xe_sriov_vf_control {
> +	XE_SRIOV_VF_CONTROL_STOP,
> +	XE_SRIOV_VF_CONTROL_PAUSE,
> +	XE_SRIOV_VF_CONTROL_RESUME,
> +	XE_SRIOV_VF_CONTROL_RESTORE,
> +};
> +
>  int xe_sriov_pf_debugfs_attr_open(int pf, unsigned int vf_num, unsigned int gt_num,
>  				  const char *attr, int mode);
>  const char *xe_sriov_debugfs_provisioned_attr_name(enum xe_sriov_shared_res res);
> @@ -20,6 +34,23 @@ int xe_sriov_pf_debugfs_read_check_ranges(int pf_fd, enum xe_sriov_shared_res re
>  					  unsigned int gt_id,
>  					  struct xe_sriov_provisioned_range **ranges,
>  					  unsigned int expected_num_vfs);
> +
> +int xe_sriov_set_vf_control(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						enum xe_sriov_vf_control operation);
> +void xe_sriov_relocate_ggtt(int pf_fd, unsigned int vf_num, unsigned int gt_num);
> +void *xe_sriov_get_guc_state(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						int *guc_state_size);
> +void xe_sriov_set_guc_state(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						void *guc_state, int guc_state_size);
> +void *xe_sriov_get_ggtt_raw(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						int *ggtt_raw_size);
> +void xe_sriov_set_ggtt_raw(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						void *ggtt_raw, int ggtt_raw_size);
> +void *xe_sriov_get_lmem_state(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						int *lmem_state_size);
> +void xe_sriov_set_lmem_state(int pf_fd, unsigned int vf_num, unsigned int gt_num,
> +						void *lmem_state, int lmem_state_size);
> +
>  int __xe_sriov_pf_debugfs_get_u32(int pf, unsigned int vf_num,
>  				  unsigned int gt_num, const char *attr,
>  				  uint32_t *value);