Re: [PATCH v3 07/10] tests/intel/xe_err_injection: Add GT UC Unicast GAM Walker Command Parity Error Injection

["Koppuravuri, Ravi Kishore" <ravi.kishore.koppuravuri@intel.com>]

Hi Soham,

On 26-08-2026 14:48, Purkait, Soham wrote:
>
> On 24-08-2026 22:11, Ravi Kishore Koppuravuri wrote:
>> MMIO based GT Uncorrectable Unicast GAM Walker command parity error
>> injection to verify the Xe driver error handling and recovery flows with
>> the help of DRM Netlink API suite
>>
>> Signed-off-by: Ravi Kishore Koppuravuri 
>> <ravi.kishore.koppuravuri@intel.com>
>> ---
>>   tests/intel/xe_err_injection.c | 321 +++++++++++++++++++++++++++++++++
>>   tests/intel/xe_err_injection.h |  18 ++
>>   tests/meson.build              |   1 +
>>   3 files changed, 340 insertions(+)
>>   create mode 100644 tests/intel/xe_err_injection.c
>>   create mode 100644 tests/intel/xe_err_injection.h
>>
>> diff --git a/tests/intel/xe_err_injection.c 
>> b/tests/intel/xe_err_injection.c
>> new file mode 100644
>> index 000000000..ded5f558f
>> --- /dev/null
>> +++ b/tests/intel/xe_err_injection.c
>> @@ -0,0 +1,321 @@
>> +// SPDX-License-Identifier: MIT
>> +/*
>> + * Copyright © 2026 Intel Corporation
>> + */
>> +
>> +/**
>> + * TEST: MMIO based Error Injection
>> + * Category: RAS
>> + * Mega feature: Telemetry
>> + * Sub-category: Driver
>> + * Test category: Error Injection
>> + */
>> +
>> +#include <stdio.h>
>> +#include <stdlib.h>
>> +#include <string.h>
>> +#include <time.h>
>> +#include <fcntl.h>
>> +#include <unistd.h>
>> +
>> +#include "igt.h"
>> +#include "lib/igt_drm_netlink.h"
>> +#include "lib/intel_reg.h"
>> +#include "lib/intel_compute.h"
>> +
>> +#include "xe_drm.h"
>> +#include "xe/xe_ioctl.h"
>> +#include "xe/xe_mmio.h"
>> +#include "xe/xe_query.h"
>> +#include "xe_err_injection.h"
>> +
>> +enum {
>> +    RECOVERY_SUCCESS = 1,
>> +    RECOVERY_FAILED,
>> +    RECOVERY_TIMEOUT,
>> +};
>> +
>> +static void run_xe_compute_on_all_engines(int fd, bool state)
>> +{
>> +    struct drm_xe_engine_class_instance *hwe;
>> +
>> +    if (state == POST_ARMING_INJECTION_WL)
>> +        igt_info("Running compute-square on all engines post 
>> injection\n");
>> +    else if (state == POST_RECOVERY_WL)
>> +        igt_info("Running compute-square on all engines post 
>> recovery\n");
>> +
>> +    xe_for_each_engine(fd, hwe) {
>> +        if (hwe->engine_class != DRM_XE_ENGINE_CLASS_COMPUTE)
>> +            continue;
>> +
>> +        igt_require_f(xe_run_intel_compute_kernel_on_engine(fd, hwe, 
>> NULL,
>> +                                    EXECENV_PREF_SYSTEM),
>> +                                    "GPU does not support "
>> +                                    "compute on engine\n");
>> +    }
>> +}
>> +
>> +static void write_reg(struct xe_mmio *mmio, uint32_t reg, uint32_t 
>> value)
>> +{
>> +    xe_mmio_write32(mmio, reg, value);
>> +}
>> +
>> +static void modify_reg_bit(struct xe_mmio *mmio, uint32_t reg_addr,
>> +               uint32_t bit_mask, bool set)
>> +{
>> +    uint32_t regval;
>> +
>> +    regval = xe_mmio_read32(mmio, reg_addr);
>> +    if (set)
>> +        regval |= bit_mask;
>> +    else
>> +        regval &= ~bit_mask;
>> +
>> +    xe_mmio_write32(mmio, reg_addr, regval);
>> +}
>> +
>> +static int acquire_forcewake(int fd)
>> +{
>> +    int fw_handle;
>> +
>> +    fw_handle = igt_debugfs_open(fd, "forcewake_all", O_RDONLY);
>> +    igt_assert_lte(0, fw_handle);
>> +    return fw_handle;
>> +}
>> +
>> +static void release_forcewake(int fw_handle)
>> +{
>> +    if (fw_handle >= 0)
>> +        close(fw_handle);
>> +}
>> +
>> +static uint32_t get_counter(uint32_t node_id, uint32_t error_id)
>> +{
>> +    struct app_context ctx;
>> +    int ret;
>> +    uint32_t error_value = UINT32_MAX;
>> +
>> +    ret = igt_init_nl_socket(&ctx);
>> +    if (ret < 0) {
>> +        igt_warn("Failed to initialize netlink socket for error 
>> command (ret=%d)\n", ret);
>> +        return error_value;
>> +    }
>> +
>> +    ctx.node_id = node_id;
>> +    ctx.error_id = error_id;
>> +
>> +    ret = igt_get_error_counter(&ctx);
>> +    if (ret < 0) {
>> +        igt_warn("get_error_counter failed from error command path 
>> (ret=%d)\n", ret);
>> +    } else {
>> +        error_value = ctx.error_value;
>> +        igt_info("get_error_counter: node_id=%u error_id=%u 
>> value=%u\n",
>> +             ctx.node_id, ctx.error_id, ctx.error_value);
>> +    }
>> +
>> +    igt_cleanup_nl_socket(&ctx);
>> +
>> +    return error_value;
>> +}
>> +
>> +static int check_dmesg_for_recovery(const char *marker, int 
>> elapsed_secs)
>> +{
>> +    char *buff = NULL;
>> +    size_t buff_size = 0;
>> +    ssize_t line_len;
>> +    FILE *fp;
>> +    const char *success = "AER: device recovery successful";
>> +    const char *failed = "AER: device recovery failed";
>> +    bool marker_seen = false;
>> +
>> +    fp = popen("dmesg", "r");
>> +    if (!fp) {
>> +        igt_warn("Unable to open dmesg to check recovery status\n");
>> +        return -1;
>> +    }
>> +
>> +    while ((line_len = getline(&buff, &buff_size, fp)) != -1) {
>> +        (void)line_len;
>> +        if (!marker_seen) {
>> +            if (strstr(buff, marker))
>> +                marker_seen = true;
>> +            continue;
>> +        }
>> +
>> +        if (strstr(buff, success)) {
>> +            igt_info("Found \"%s\" in dmesg after %d secs\n", 
>> success, elapsed_secs);
>> +            free(buff);
>> +            pclose(fp);
>> +            return RECOVERY_SUCCESS;
>> +        }
>> +        if (strstr(buff, failed)) {
>> +            igt_info("Found \"%s\" in dmesg after %d secs\n", 
>> failed, elapsed_secs);
>> +            free(buff);
>> +            pclose(fp);
>> +            return RECOVERY_FAILED;
>> +        }
>> +    }
>> +
>> +    free(buff);
>> +    pclose(fp);
>> +    return RECOVERY_TIMEOUT;
>> +}
>> +
>> +static int check_err_recovery(void)
>> +{
>> +    time_t start_time = time(NULL);
>> +    time_t timeout = 5 * 60;
>> +    int time_interval = 30;
>> +    time_t elapsed_time;
>> +    int status;
>> +    char marker[128];
>> +
>> +    snprintf(marker, sizeof(marker),
>> +         "IGT xe_err_injection recovery marker pid=%d start=%lld",
>> +         getpid(), (long long)start_time);
>> +    igt_kmsg(KMSG_INFO "%s\n", marker);
>> +
>> +    while (1) {
>> +        elapsed_time = time(NULL) - start_time;
>> +        status = check_dmesg_for_recovery(marker, elapsed_time);
>> +        if (status < 0) {
>> +            igt_warn("Failed to query dmesg for recovery status\n");
>> +            return RECOVERY_FAILED;
>> +        }
>> +        if (status == RECOVERY_SUCCESS || status == RECOVERY_FAILED)
>> +            return status;
>> +
>> +        if (elapsed_time >= timeout) {
>> +            igt_warn("Timed out while waiting for error recovery\n");
>> +            return RECOVERY_TIMEOUT;
>> +        }
>> +
>> +        sleep(time_interval);
>> +    }
>> +}
>> +
>> +static void print_aer_recovery_status(void)
>> +{
>> +    int recovery_ret;
>> +
>> +    recovery_ret = check_err_recovery();
>> +    igt_assert_f(recovery_ret != RECOVERY_TIMEOUT,
>> +             "AER error recovery timed out\n");
>> +    igt_assert_f(recovery_ret != RECOVERY_FAILED,
>> +             "AER device recovery failed\n");
>> +    if (recovery_ret == RECOVERY_SUCCESS)
>> +        igt_info("AER device recovery successful\n");
>> +}
>> +
>> +static void log_status(bool val, const char *err_name)
>> +{
>> +    if (val)
>> +        igt_info("Injection: %s Success\n", err_name);
>> +    else
>> +        igt_info("Injection: %s Failed\n", err_name);
>> +}
>> +
>> +static bool gt_uc_wkr_parity_recovered;
>> +
>> +/**
>> + * SUBTEST: test_GT-UC-unicast-wkr-cmd-parity-err
>> + * Description: GT Uncorrectable Unicast Walker Command parity error 
>> injection
>> + * Functionality: error injection
>> + */
>> +static void wkr_cmd_parity_err_injection(struct xe_mmio *mmio, int fd)
>> +{
>> +    int fw_handle;
>> +    uint32_t error_counter_before_inj;
>> +    uint32_t error_counter_after_inj;
>> +    /** TODO
>> +     *  Fetching node_id and error_id dynamically is pending to 
>> implement.
>> +     *  For now, using
>> +     *  node_id=1 (uncorrectable_errors) and
>> +     *  error_id=1 (core_compute)
>> +     */
>> +    uint32_t node_id = 1;
>> +    uint32_t error_id = 1;
>> +
>> +    fw_handle = acquire_forcewake(fd);
>> +
>> +    error_counter_before_inj = get_counter(node_id, error_id);
>> +
>> +    /* Arm the injection sequence. */
>> +    write_reg(mmio, MC_PKT_CTRL_MGSR_3D_ADDRESS, 0x0);
>> +    modify_reg_bit(mmio,
>> +               WKR_FABRIC_ERR_INJ_GAMWALK_3D_ADDRESS,
>> +               WKR_FABRIC_ERR_INJ_GAMWALK_3D_VALUE,
>> +               true);
>> +    modify_reg_bit(mmio,
>> +               MC_PKT_CTRL_MGSR_3D_ADDRESS,
>> +               MC_PKT_CTRL_MGSR_3D_VALUE,
>> +               true);
>
> Is steering semaphore needed to be acquired before these operations?
>
> Thanks,
> Soham
>
Yes, steering semaphore is required while operating on multicast register.

I will add semaphore support in the next version.

Thanks,

Ravi Kishore K.

>> +    igt_info("Injected GT Uncorrectable Unicast Walker Cmd parity 
>> error\n");
>> +
>> +    run_xe_compute_on_all_engines(fd, POST_ARMING_INJECTION_WL);
>> +
>> +    release_forcewake(fw_handle);
>> +    print_aer_recovery_status();
>> +    error_counter_after_inj = get_counter(node_id, error_id);
>> +
>> +    igt_assert_f(error_counter_before_inj != UINT32_MAX &&
>> +             error_counter_after_inj != UINT32_MAX,
>> +             "Failed to fetch valid error counters: before=%u 
>> after=%u\n",
>> +             error_counter_before_inj, error_counter_after_inj);
>> +
>> +    igt_info("error counter: before injection=%u after injection=%u\n",
>> +         error_counter_before_inj, error_counter_after_inj);
>> +
>> +    igt_assert_f(error_counter_after_inj > error_counter_before_inj,
>> +             "GT Uncorrectable Unicast Walker Cmd parity error 
>> injection "
>> +             "failed: before injection=%u after injection=%u\n",
>> +             error_counter_before_inj, error_counter_after_inj);
>> +    igt_info("GT Uncorrectable Unicast Walker Cmd parity error 
>> injection successful\n");
>> +    gt_uc_wkr_parity_recovered = true;
>> +}
>> +
>> +static void inject_error(const char *injection, struct xe_mmio 
>> *mmio, int fd)
>> +{
>> +    igt_info("Starting Error Injection test: %s\n", injection);
>> +    if (strcmp(injection, "test_GT-UC-unicast-wkr-cmd-parity-err") 
>> == 0)
>> +        wkr_cmd_parity_err_injection(mmio, fd);
>> +    else if (strcmp(injection, 
>> "test_GT-UC-unicast-wkr-cmd-parity-err-post-recovery-wl") == 0)
>> +        run_xe_compute_on_all_engines(fd, POST_RECOVERY_WL);
>> +    else
>> +        igt_info("Invalid Error Injection specified\n");
>> +}
>> +
>> +/**
>> + * SUBTEST: test_GT-UC-unicast-wkr-cmd-parity-err-post-recovery-wl
>> + * Description: Run a post-recovery Xe workload after parity error 
>> injection.
>> + * Functionality: workload validation
>> + */
>> +
>> +int igt_main()
>> +{
>> +    int fd;
>> +    struct xe_mmio mmio;
>> +
>> +    igt_fixture() {
>> +        fd = drm_open_driver(DRIVER_XE);
>> +        igt_require(igt_debugfs_exists(fd, "forcewake_all", O_RDONLY));
>> +        xe_mmio_access_init(fd, &mmio);
>> +        igt_require(xe_mmio_is_initialized(&mmio));
>> +    }
>> +
>> +    igt_describe("Inject GT uncorrectable unicast worker command 
>> parity error.");
>> +    igt_subtest("test_GT-UC-unicast-wkr-cmd-parity-err")
>> +        inject_error("test_GT-UC-unicast-wkr-cmd-parity-err", &mmio, 
>> fd);
>> +
>> +    igt_describe("Run post-recovery workload after GT parity error 
>> injection test.");
>> + 
>> igt_subtest("test_GT-UC-unicast-wkr-cmd-parity-err-post-recovery-wl") {
>> +        igt_require_f(gt_uc_wkr_parity_recovered,
>> +                  "Run GT-UC-unicast-wkr-cmd-parity-err first\n");
>> + 
>> inject_error("test_GT-UC-unicast-wkr-cmd-parity-err-post-recovery-wl", 
>> &mmio, fd);
>> +    }
>> +
>> +    igt_fixture() {
>> +        xe_mmio_access_fini(&mmio);
>> +        drm_close_driver(fd);
>> +    }
>> +}
>> diff --git a/tests/intel/xe_err_injection.h 
>> b/tests/intel/xe_err_injection.h
>> new file mode 100644
>> index 000000000..d7efc2fd2
>> --- /dev/null
>> +++ b/tests/intel/xe_err_injection.h
>> @@ -0,0 +1,18 @@
>> +/* SPDX-License-Identifier: MIT */
>> +/*
>> + * Copyright © 2026 Intel Corporation
>> + */
>> +
>> +#ifndef XE_ERR_INJECTION_H
>> +#define XE_ERR_INJECTION_H
>> +
>> +/* GT Uncorrectable unicast worker command parity error injection */
>> +#define MC_PKT_CTRL_MGSR_3D_ADDRESS 0x00FD4
>> +#define MC_PKT_CTRL_MGSR_3D_VALUE 0x80000000
>> +#define WKR_FABRIC_ERR_INJ_GAMWALK_3D_ADDRESS 0xF310
>> +#define WKR_FABRIC_ERR_INJ_GAMWALK_3D_VALUE 0x1
>> +
>> +#define POST_RECOVERY_WL 1
>> +#define POST_ARMING_INJECTION_WL 0
>> +
>> +#endif /* XE_ERR_INJECTION_H */
>> diff --git a/tests/meson.build b/tests/meson.build
>> index a62f447df..0f090dcee 100644
>> --- a/tests/meson.build
>> +++ b/tests/meson.build
>> @@ -292,6 +292,7 @@ intel_xe_progs = [
>>       'xe_debugfs',
>>       'xe_dma_buf_sync',
>>       'xe_drm_fdinfo',
>> +    'xe_err_injection',
>>       'xe_eu_stall',
>>       'xe_evict',
>>       'xe_evict_ccs',