Re: [PATCH i-g-t v3 2/3] tests/intel/xe_pmu: Add engine activity test for all functions

[Riana Tauro <riana.tauro@intel.com>]

Hi Umesh

On 3/22/2025 2:15 AM, Umesh Nerlige Ramappa wrote:
> On Fri, Mar 21, 2025 at 04:39:57PM +0530, Riana Tauro wrote:
>> Provision and enable 2 VFs with execution quantum and scheduling policy
>> set. Add a test that runs workload on all functions simultaneously and
>> validates that all engines are equally and fully loaded for the
>> entire execution quantum of the function
>>
>> v2: add a different function for function config
>>    add function details to log
>>    move pmu_fd to struct
>>    enable and provision VF's (Umesh)
>>
>> v3: split patches
>>    move exec_quantum to variables
>>    skip test if vfs are already provisioned
>>    fix assert (Umesh)
>>
>> Signed-off-by: Riana Tauro <riana.tauro@intel.com>
>> ---
>> tests/intel/xe_pmu.c | 129 +++++++++++++++++++++++++++++++++++++++++++
>> 1 file changed, 129 insertions(+)
>>
>> diff --git a/tests/intel/xe_pmu.c b/tests/intel/xe_pmu.c
>> index f0959abe2..9a1f9b39e 100644
>> --- a/tests/intel/xe_pmu.c
>> +++ b/tests/intel/xe_pmu.c
>> @@ -14,11 +14,13 @@
>>
>> #include "igt.h"
>> #include "igt_perf.h"
>> +#include "igt_sriov_device.h"
>> #include "igt_sysfs.h"
>>
>> #include "xe/xe_gt.h"
>> #include "xe/xe_ioctl.h"
>> #include "xe/xe_spin.h"
>> +#include "xe/xe_sriov_provisioning.h"
>>
>> #define SLEEP_DURATION 2 /* in seconds */
>> /* flag masks */
>> @@ -124,6 +126,12 @@ static uint64_t get_event_config(unsigned int gt, 
>> struct drm_xe_engine_class_ins
>>     return pmu_config;
>> }
>>
>> +static uint64_t get_event_config_fn(unsigned int gt, int function,
>> +                    struct drm_xe_engine_class_instance *eci, const 
>> char *event)
>> +{
>> +    return get_event_config(gt, eci, event) | 
>> add_format_config("function", function);
>> +}
>> +
>> /**
>>  * SUBTEST: engine-activity-idle
>>  * Description: Test to validate engine activity shows no load when idle
>> @@ -187,6 +195,73 @@ static void engine_activity(int fd, struct 
>> drm_xe_engine_class_instance *eci, un
>>         igt_assert(!engine_active_ticks);
>> }
>>
>> +/**
>> + * SUBTEST: all-fn-engine-activity-load
>> + * Description: Test to validate engine activity by running load on 
>> all functions simultaneously
>> + */
>> +static void engine_activity_all_fn(int fd, struct 
>> drm_xe_engine_class_instance *eci, int num_fns)
>> +{
>> +    uint64_t config, engine_active_ticks, engine_total_ticks;
>> +    uint64_t after[2 * num_fns], before[2 * num_fns];
>> +    struct pmu_function {
>> +        struct xe_cork *cork;
>> +        uint32_t vm;
>> +        uint64_t pmu_fd[2];
>> +        int fd;
>> +    } fn[num_fns];
>> +    int i;
>> +
>> +    fn[0].pmu_fd[0] = -1;
>> +    for (i = 0; i < num_fns; i++) {
> 
> nit:
> In the for loops,
> struct pmu_function *f = &fn[i];
> and then you can use f-> below instead of fn[i].
will add this>
>> +        config = get_event_config_fn(eci->gt_id, i, eci, "engine- 
>> active-ticks");
>> +        fn[i].pmu_fd[0] = open_group(fd, config, fn[0].pmu_fd[0]);
>> +
>> +        config = get_event_config_fn(eci->gt_id, i, eci, "engine- 
>> total-ticks");
>> +        fn[i].pmu_fd[1] = open_group(fd, config, fn[0].pmu_fd[0]);
>> +
>> +        if (i > 0)
>> +            fn[i].fd = igt_sriov_open_vf_drm_device(fd, i);
>> +        else
>> +            fn[i].fd = fd;
>> +
>> +        igt_assert_fd(fn[i].fd);
>> +
>> +        fn[i].vm = xe_vm_create(fn[i].fd, 0, 0);
>> +        fn[i].cork = xe_cork_create_opts(fn[i].fd, eci, fn[i].vm, 1, 1);
>> +        xe_cork_sync_start(fn[i].fd, fn[i].cork);
>> +    }
>> +
>> +    pmu_read_multi(fn[0].pmu_fd[0], 2 * num_fns, before);
>> +    usleep(SLEEP_DURATION * USEC_PER_SEC);
>> +    pmu_read_multi(fn[0].pmu_fd[0], 2 * num_fns, after);
>> +
>> +    for (i = 0; i < num_fns; i++) {
>> +        int idx = i * 2;
>> +
>> +        xe_cork_sync_end(fn[i].fd, fn[i].cork);
>> +        engine_active_ticks = after[idx] - before[idx];
>> +        engine_total_ticks = after[idx + 1] - before[idx + 1];
>> +
>> +        igt_debug("[%d] Engine active ticks: after %ld, before %ld 
>> delta %ld\n", i,
>> +              after[idx], before[idx], engine_active_ticks);
>> +        igt_debug("[%d] Engine total ticks: after %ld, before %ld 
>> delta %ld\n", i,
>> +              after[idx + 1], before[idx + 1], engine_total_ticks);
>> +
>> +        if (fn[i].cork)
>> +            xe_cork_destroy(fn[i].fd, fn[i].cork);
>> +
>> +        xe_vm_destroy(fn[i].fd, fn[i].vm);
>> +
>> +        close(fn[i].pmu_fd[0]);
>> +        close(fn[i].pmu_fd[1]);
>> +
>> +        if (i > 0)
>> +            close(fn[i].fd);
>> +
>> +        assert_within_epsilon(engine_active_ticks, 
>> engine_total_ticks, tolerance);
>> +    }
>> +}
>> +
>> /**
>>  * SUBTEST: gt-c6-idle
>>  * Description: Basic residency test to validate idle residency
>> @@ -225,6 +300,33 @@ static void test_gt_c6_idle(int xe, unsigned int gt)
>>     close(pmu_fd);
>> }
>>
>> +static unsigned int enable_and_provision_vfs(int fd)
>> +{
>> +    unsigned int gt, num_vfs;
>> +    int pf_exec_quantum = 64, vf_exec_quantum = 32, vf;
>> +
>> +    /* Enable VF's */
>> +    igt_sriov_disable_driver_autoprobe(fd);
>> +    igt_sriov_enable_vfs(fd, 2);
>> +    num_vfs = igt_sriov_get_enabled_vfs(fd);
>> +    igt_require(num_vfs == 2);
>> +
>> +    /* Set 32ms for VF execution quantum and 64ms for PF execution 
>> quantum */
>> +    xe_for_each_gt(fd, gt) {
>> +        xe_sriov_set_sched_if_idle(fd, gt, 0);
> 
> Looking at patch 3, for the new test, the sched if idle is set to true 
> after binding the VFs. That's a slightly different use case compared to 
> the sched_if_idle = false case where vfs are bound later.
> 
Even if sched if idle is set after vfs are bound, test is working as 
expected. And the test which runs simulatenously on all engines doesn't 
require the sched-if-idle case as there is no difference wrt to 
scheduling policy

So added it only for one and set it after provisioning

> I was imagining that the provisioning would be separate for the 2 tests
> that depend on sched_if_idle. Each test (with different provisioning 
> params)
> may need to provision the VFs separately.
> 
> More details in the next comment.
> 
>> +        for (int fn = 0; fn <= num_vfs; fn++)
>> +            xe_sriov_set_exec_quantum_ms(fd, fn, gt, fn ? 
>> vf_exec_quantum :
>> +                             pf_exec_quantum);
>> +    }
>> +
>> +    /* probe VFs */
>> +    igt_sriov_enable_driver_autoprobe(fd);
>> +    for (vf = 1; vf <= num_vfs; vf++)
>> +        igt_sriov_bind_vf_drm_driver(fd, vf);
>> +
>> +    return num_vfs;
>> +}
>> +
>> igt_main
>> {
>>     int fd, gt;
>> @@ -254,6 +356,33 @@ igt_main
>>     test_each_engine("engine-activity-load", fd, eci)
>>         engine_activity(fd, eci, TEST_LOAD);
>>
>> +    igt_subtest_group {
>> +        bool autoprobe;
>> +        unsigned int num_fns;
>> +
>> +        igt_fixture {
>> +            igt_require(igt_sriov_is_pf(fd));
>> +            igt_require(igt_sriov_get_enabled_vfs(fd) == 0);
>> +            autoprobe = igt_sriov_is_driver_autoprobe_enabled(fd);
>> +            num_fns = enable_and_provision_vfs(fd) + 1;
> 
> 0) Move the igt_require lines and autoprobe line to
> enable_and_provision_vfs(). Maybe autoprobe can be global.
> 
Will add this> 1) the enable_and_provision_vfs can take params - 
num_vfs, pf_quanta,
> vf_quanta and sched_if_idle.
> 
> 2) Since we just want to modify the sched_if_idle param, we could have 2
> subtest groups.The subtest fixture runs for every non-sriov test from the file in CI.
So even if i run the native subtest, all the provisioning and disabling 
VF's will run on sriov supported machines.

Will check what difference will it be if we set sched-if-idle after 
enabling.

Thanks
Riana>
>> +        }
>> +
>> +        igt_describe("Validate engine activity on all functions");
>> +        test_each_engine("all-fn-engine-activity-load", fd, eci)
>> +            engine_activity_all_fn(fd, eci, num_fns);
>> +
>> +        igt_fixture {
>> +            igt_sriov_disable_vfs(fd);
>> +            /* abort to avoid execution of next tests with enabled 
>> VFs */
>> +            igt_abort_on_f(igt_sriov_get_enabled_vfs(fd) > 0,
>> +                       "Failed to disable VF(s)");
>> +            autoprobe ? igt_sriov_enable_driver_autoprobe(fd) :
>> +                    igt_sriov_disable_driver_autoprobe(fd);
>> +            igt_abort_on_f(autoprobe != 
>> igt_sriov_is_driver_autoprobe_enabled(fd),
>> +                       "Failed to restore sriov_drivers_autoprobe 
>> value\n");
> 
> everything in this fixture could just be part of a helper, for example - 
> disable_provisioned_vfs()
> I imagine things would look like this:
> 
> igt_subtest_group { /* sched_if_idle = 0 */
>      unsigned int num_fns;
> 
>      igt_fixture num_fns = enable_and_provision_vfs(... sched_if_idle = 0);
> 
>      igt_describe("Validate engine activity on all functions");
>      test_each_engine("all-fn-engine-activity-load", fd, eci)
>          engine_activity_all_fn(fd, eci, num_fns);
> 
>      igt_describe("Validate per-function engine activity");
>      test_each_engine("fn-engine-activity-load", fd, eci)
>          for (int fn = 0; fn < num_fns; fn++)
>              engine_activity_fn(fd, eci, fn);
> 
>      igt_fixture disable_provisioned_vfs(...);
> }
> 
> igt_subtest_group { /* sched_if_idle = 1 */
>      unsigned int num_fns;
> 
>      igt_fixture num_fns = enable_and_provision_vfs(... sched_if_idle = 1);
> 
>      igt_describe("Validate per-function engine activity when sched-if- 
> idle is set");
>      test_each_engine("fn-engine-activity-sched-if-idle", fd, eci) {
>          for (int fn = 0; fn < num_fns; fn++)
>              engine_activity_fn(fd, eci, fn);
> 
>      igt_fixture disable_provisioned_vfs(...);
> }
> 
> Optionally, you could have the all-fn-engine-activity-load-sched-if-idle 
> in the
> second group as well.
> 
> Thanks,
> Umesh
> 
>> +        }
>> +    }
>> +
>>     igt_fixture {
>>         close(fd);
>>     }
>> -- 
>> 2.47.1
>>