[PATCH 0/2] Inject errors during xe_guc_ct_send_recv & xe_guc_mmio_send_recv

[PATCH 0/2] Inject errors during xe_guc_ct_send_recv & xe_guc_mmio_send_recv

[Satyanarayana K V P]

Use the kernel fault injection infrastructure to test error handling of xe
at probe time when executing xe_guc_mmio_send_recv() and at enabling of VFs
stage when executing xe_guc_ct_send_recv().

Error can be injected using:
    ./xe_fault_injection --run-subtest inject-fault-probe-function-xe_guc_mmio_send_recv
	./xe_fault_injection --run-subtest guc-fail-xe_guc_ct_send_recv


Satyanarayana K V P (2):
  tests/intel/xe_fault_injection: Inject errors during
    xe_guc_mmio_send_recv
  tests/intel/xe_fault_injection: Inject errors during
    xe_guc_ct_send_recv

 tests/intel/xe_fault_injection.c | 61 ++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Michał Wajdeczko <michal.wajdeczko@intel.com>
Cc: Francois Dugast <francois.dugast@intel.com>
Signed-off-by: Satyanarayana K V P <satyanarayana.k.v.p@intel.com>

-- 
2.35.3

[PATCH 1/2] tests/intel/xe_fault_injection: Inject errors during xe_guc_mmio_send_recv

[Satyanarayana K V P]

Use the kernel fault injection infrastructure to test error handling
of xe at probe time when executing xe_guc_mmio_send_recv() so that
more code paths are tested, such as error handling and unwinding.

Error can be injected using:
./xe_fault_injection --run-subtest inject-fault-probe-function-xe_guc_mmio_send_recv

Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Michał Wajdeczko <michal.wajdeczko@intel.com>
Cc: Francois Dugast <francois.dugast@intel.com>
Signed-off-by: Satyanarayana K V P <satyanarayana.k.v.p@intel.com>
---
 tests/intel/xe_fault_injection.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/intel/xe_fault_injection.c b/tests/intel/xe_fault_injection.c
index 7ae941367..ef3bbb7ec 100644
--- a/tests/intel/xe_fault_injection.c
+++ b/tests/intel/xe_fault_injection.c
@@ -170,6 +170,7 @@ static void set_retval(const char function_name[], long long retval)
  * @xe_uc_fw_init:		xe_uc_fw_init
  * @xe_wa_init:			xe_wa_init
  * @xe_wopcm_init:		xe_wopcm_init
+ * @xe_guc_mmio_send_recv:     xe_guc_mmio_send_recv
  */
 static void
 inject_fault_probe(int fd, char pci_slot[], const char function_name[])
@@ -301,6 +302,7 @@ igt_main
 		{ "xe_uc_fw_init" },
 		{ "xe_wa_init" },
 		{ "xe_wopcm_init" },
+		{ "xe_guc_mmio_send_recv" },
 		{ }
 	};
 	const struct section vm_create_fail_functions[] = {
-- 
2.35.3

[PATCH 2/2] tests/intel/xe_fault_injection: Inject errors during xe_guc_ct_send_recv

[Satyanarayana K V P]

Use the kernel fault injection infrastructure to test error handling
of xe at enabling of VFs stage when executing xe_guc_ct_send_recv()
so that more code paths are tested, such as error handling and unwinding.

Error can be injected using:
./xe_fault_injection --run-subtest guc-fail-xe_guc_ct_send_recv

Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Michał Wajdeczko <michal.wajdeczko@intel.com>
Cc: Francois Dugast <francois.dugast@intel.com>
Signed-off-by: Satyanarayana K V P <satyanarayana.k.v.p@intel.com>
---
 tests/intel/xe_fault_injection.c | 59 ++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/tests/intel/xe_fault_injection.c b/tests/intel/xe_fault_injection.c
index ef3bbb7ec..fce8d2846 100644
--- a/tests/intel/xe_fault_injection.c
+++ b/tests/intel/xe_fault_injection.c
@@ -19,12 +19,14 @@
 #include "igt_sysfs.h"
 #include "lib/igt_syncobj.h"
 #include "lib/intel_pat.h"
+#include "lib/igt_sriov_device.h"
 #include "xe/xe_ioctl.h"
 #include "xe/xe_query.h"
 
 #define INJECT_ERRNO	-ENOMEM
 #define BO_ADDR		0x1a0000
 #define BO_SIZE		(1024*1024)
+#define NUM_VFS		1
 
 enum injection_list_action {
 	INJECTION_LIST_ADD,
@@ -281,6 +283,55 @@ vm_bind_fail(int fd, const char function_name[])
 	igt_assert_eq(simple_vm_bind(fd, vm), 0);
 }
 
+static int sriov_enable_vfs(int fd, int num_vfs)
+{
+	int sysfs;
+	bool ret;
+
+	sysfs = igt_sysfs_open(fd);
+	igt_assert_fd(sysfs);
+
+	ret = __igt_sysfs_set_u32(sysfs, "device/sriov_numvfs", num_vfs);
+	close(sysfs);
+
+	return ret;
+}
+
+/**
+ * SUBTEST: guc-fail-%s
+ * Description: inject an error in function %arg[1] used when xe interacts with guc to make it fail
+ * Functionality: fault
+ *
+ * arg[1]:
+ * @xe_guc_ct_send_recv:               xe_guc_ct_send_recv
+ */
+
+static void
+guc_fail(int fd, int num_vfs, const char function_name[])
+{
+	bool autoprobe_en = 0;
+
+	ignore_faults_in_dmesg(function_name);
+	injection_list_do(INJECTION_LIST_ADD, function_name);
+	set_retval(function_name, INJECT_ERRNO);
+
+	autoprobe_en = igt_sriov_is_driver_autoprobe_enabled(fd);
+
+	if (autoprobe_en)
+		igt_sriov_disable_driver_autoprobe(fd);
+
+	/* igt_sriov_enable_vfs can't be used here as it is causing abort on any error.
+	 * Since error in this test is expected, we have written our own static function here.
+	 */
+	sriov_enable_vfs(fd, num_vfs);
+
+	igt_assert_eq(-errno, INJECT_ERRNO);
+	injection_list_do(INJECTION_LIST_REMOVE, function_name);
+
+	if (autoprobe_en)
+		igt_sriov_enable_driver_autoprobe(fd);
+}
+
 igt_main
 {
 	int fd;
@@ -319,6 +370,10 @@ igt_main
 		{ "xe_vma_ops_alloc" },
 		{ }
 	};
+	const struct section guc_fail_functions[] = {
+		{ "xe_guc_ct_send_recv" },
+		{ }
+	};
 
 	igt_fixture {
 		igt_require(fail_function_injection_enabled());
@@ -335,6 +390,10 @@ igt_main
 		igt_subtest_f("vm-bind-fail-%s", s->name)
 			vm_bind_fail(fd, s->name);
 
+	for (const struct section *s = guc_fail_functions; s->name; s++)
+		igt_subtest_f("guc-fail-%s", s->name)
+			guc_fail(fd, NUM_VFS, s->name);
+
 	igt_fixture {
 		xe_sysfs_driver_do(fd, pci_slot, XE_SYSFS_DRIVER_UNBIND);
 	}
-- 
2.35.3

Re: [PATCH 1/2] tests/intel/xe_fault_injection: Inject errors during xe_guc_mmio_send_recv

[Francois Dugast]

Hi,

On Mon, Dec 30, 2024 at 02:14:50PM +0530, Satyanarayana K V P wrote:
> Use the kernel fault injection infrastructure to test error handling
> of xe at probe time when executing xe_guc_mmio_send_recv() so that
> more code paths are tested, such as error handling and unwinding.
> 
> Error can be injected using:
> ./xe_fault_injection --run-subtest inject-fault-probe-function-xe_guc_mmio_send_recv
> 
> Cc: Matthew Brost <matthew.brost@intel.com>
> Cc: Michał Wajdeczko <michal.wajdeczko@intel.com>
> Cc: Francois Dugast <francois.dugast@intel.com>
> Signed-off-by: Satyanarayana K V P <satyanarayana.k.v.p@intel.com>
> ---
>  tests/intel/xe_fault_injection.c | 2 ++
>  1 file changed, 2 insertions(+)
> 
> diff --git a/tests/intel/xe_fault_injection.c b/tests/intel/xe_fault_injection.c
> index 7ae941367..ef3bbb7ec 100644
> --- a/tests/intel/xe_fault_injection.c
> +++ b/tests/intel/xe_fault_injection.c
> @@ -170,6 +170,7 @@ static void set_retval(const char function_name[], long long retval)
>   * @xe_uc_fw_init:		xe_uc_fw_init
>   * @xe_wa_init:			xe_wa_init
>   * @xe_wopcm_init:		xe_wopcm_init
> + * @xe_guc_mmio_send_recv:     xe_guc_mmio_send_recv

Please keep this list sorted.

>   */
>  static void
>  inject_fault_probe(int fd, char pci_slot[], const char function_name[])
> @@ -301,6 +302,7 @@ igt_main
>  		{ "xe_uc_fw_init" },
>  		{ "xe_wa_init" },
>  		{ "xe_wopcm_init" },
> +		{ "xe_guc_mmio_send_recv" },

Please also keep this list sorted.

Francois

>  		{ }
>  	};
>  	const struct section vm_create_fail_functions[] = {
> -- 
> 2.35.3
> 

Re: [PATCH 2/2] tests/intel/xe_fault_injection: Inject errors during xe_guc_ct_send_recv

[Francois Dugast]

Hi,

On Mon, Dec 30, 2024 at 02:14:51PM +0530, Satyanarayana K V P wrote:
> Use the kernel fault injection infrastructure to test error handling
> of xe at enabling of VFs stage when executing xe_guc_ct_send_recv()
> so that more code paths are tested, such as error handling and unwinding.
> 
> Error can be injected using:
> ./xe_fault_injection --run-subtest guc-fail-xe_guc_ct_send_recv
> 
> Cc: Matthew Brost <matthew.brost@intel.com>
> Cc: Michał Wajdeczko <michal.wajdeczko@intel.com>
> Cc: Francois Dugast <francois.dugast@intel.com>
> Signed-off-by: Satyanarayana K V P <satyanarayana.k.v.p@intel.com>
> ---
>  tests/intel/xe_fault_injection.c | 59 ++++++++++++++++++++++++++++++++
>  1 file changed, 59 insertions(+)
> 
> diff --git a/tests/intel/xe_fault_injection.c b/tests/intel/xe_fault_injection.c
> index ef3bbb7ec..fce8d2846 100644
> --- a/tests/intel/xe_fault_injection.c
> +++ b/tests/intel/xe_fault_injection.c
> @@ -19,12 +19,14 @@
>  #include "igt_sysfs.h"
>  #include "lib/igt_syncobj.h"
>  #include "lib/intel_pat.h"
> +#include "lib/igt_sriov_device.h"
>  #include "xe/xe_ioctl.h"
>  #include "xe/xe_query.h"
>  
>  #define INJECT_ERRNO	-ENOMEM
>  #define BO_ADDR		0x1a0000
>  #define BO_SIZE		(1024*1024)
> +#define NUM_VFS		1
>  
>  enum injection_list_action {
>  	INJECTION_LIST_ADD,
> @@ -281,6 +283,55 @@ vm_bind_fail(int fd, const char function_name[])
>  	igt_assert_eq(simple_vm_bind(fd, vm), 0);
>  }
>  
> +static int sriov_enable_vfs(int fd, int num_vfs)
> +{
> +	int sysfs;
> +	bool ret;
> +
> +	sysfs = igt_sysfs_open(fd);
> +	igt_assert_fd(sysfs);
> +
> +	ret = __igt_sysfs_set_u32(sysfs, "device/sriov_numvfs", num_vfs);
> +	close(sysfs);
> +
> +	return ret;
> +}
> +
> +/**
> + * SUBTEST: guc-fail-%s
> + * Description: inject an error in function %arg[1] used when xe interacts with guc to make it fail
> + * Functionality: fault
> + *
> + * arg[1]:
> + * @xe_guc_ct_send_recv:               xe_guc_ct_send_recv
> + */
> +
> +static void
> +guc_fail(int fd, int num_vfs, const char function_name[])
> +{
> +	bool autoprobe_en = 0;
> +
> +	ignore_faults_in_dmesg(function_name);
> +	injection_list_do(INJECTION_LIST_ADD, function_name);
> +	set_retval(function_name, INJECT_ERRNO);
> +
> +	autoprobe_en = igt_sriov_is_driver_autoprobe_enabled(fd);
> +
> +	if (autoprobe_en)
> +		igt_sriov_disable_driver_autoprobe(fd);
> +
> +	/* igt_sriov_enable_vfs can't be used here as it is causing abort on any error.
> +	 * Since error in this test is expected, we have written our own static function here.
> +	 */
> +	sriov_enable_vfs(fd, num_vfs);
> +
> +	igt_assert_eq(-errno, INJECT_ERRNO);
> +	injection_list_do(INJECTION_LIST_REMOVE, function_name);
> +
> +	if (autoprobe_en)
> +		igt_sriov_enable_driver_autoprobe(fd);
> +}
> +
>  igt_main
>  {
>  	int fd;
> @@ -319,6 +370,10 @@ igt_main
>  		{ "xe_vma_ops_alloc" },
>  		{ }
>  	};
> +	const struct section guc_fail_functions[] = {
> +		{ "xe_guc_ct_send_recv" },
> +		{ }
> +	};
>  
>  	igt_fixture {
>  		igt_require(fail_function_injection_enabled());

On my setup other SRIOV tests are just skipped but
guc-fail-xe_guc_ct_send_recv crashes instead of skipping. Maybe we are
missing some igt_require for SRIOV?

> @@ -335,6 +390,10 @@ igt_main
>  		igt_subtest_f("vm-bind-fail-%s", s->name)
>  			vm_bind_fail(fd, s->name);
>  
> +	for (const struct section *s = guc_fail_functions; s->name; s++)
> +		igt_subtest_f("guc-fail-%s", s->name)
> +			guc_fail(fd, NUM_VFS, s->name);

To be consistent with other tests and also the commit message, I would
emphasize what is being tested from the user's perspective, which is
enabling VFs, so rename:

    guc_fail_functions -> enable_vfs_fail_functions
    guc-fail-%s -> enable-vfs-fail-%s
    guc_fail -> enable_vfs_fail

This way if we have more fault injection points to exercise while
enabling VFs, they can be added to enable_vfs_fail_functions, whether
related to GuC or not.

Francois

> +
>  	igt_fixture {
>  		xe_sysfs_driver_do(fd, pci_slot, XE_SYSFS_DRIVER_UNBIND);
>  	}
> -- 
> 2.35.3
>