* [PATCH v5 01/10] tools/lib: Add list_is_first()
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info Swapnil Sapkal
` (11 subsequent siblings)
12 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users
Add list_is_first() to check whether @list is the first entry in list @head
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/include/linux/list.h | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/tools/include/linux/list.h b/tools/include/linux/list.h
index a4dfb6a7cc6a..a692ff7aed5c 100644
--- a/tools/include/linux/list.h
+++ b/tools/include/linux/list.h
@@ -169,6 +169,16 @@ static inline void list_move_tail(struct list_head *list,
list_add_tail(list, head);
}
+/**
+ * list_is_first -- tests whether @list is the first entry in list @head
+ * @list: the entry to test
+ * @head: the head of the list
+ */
+static inline int list_is_first(const struct list_head *list, const struct list_head *head)
+{
+ return list->prev == head;
+}
+
/**
* list_is_last - tests whether @list is the last entry in list @head
* @list: the entry to test
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 01/10] tools/lib: Add list_is_first() Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-21 17:18 ` Shrikanth Hegde
2026-01-26 14:46 ` kernel test robot
2026-01-19 17:58 ` [PATCH v5 03/10] perf sched stats: Add record and rawdump support Swapnil Sapkal
` (10 subsequent siblings)
12 siblings, 2 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users
'/proc/schedstat' gives the info about load balancing statistics within
a given domain. It also contains the cpu_mask giving information about
the sibling cpus and domain names after schedstat version 17. Storing
this information in perf header will help tools like `perf sched stats`
for better analysis.
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
.../Documentation/perf.data-file-format.txt | 17 ++
tools/perf/builtin-inject.c | 1 +
tools/perf/util/env.c | 29 ++
tools/perf/util/env.h | 17 ++
| 286 ++++++++++++++++++
| 1 +
tools/perf/util/util.c | 42 +++
tools/perf/util/util.h | 3 +
8 files changed, 396 insertions(+)
diff --git a/tools/perf/Documentation/perf.data-file-format.txt b/tools/perf/Documentation/perf.data-file-format.txt
index c9d4dec65344..0e4d0ecc9e12 100644
--- a/tools/perf/Documentation/perf.data-file-format.txt
+++ b/tools/perf/Documentation/perf.data-file-format.txt
@@ -447,6 +447,23 @@ struct {
} [nr_pmu];
};
+ HEADER_CPU_DOMAIN_INFO = 32,
+
+List of cpu-domain relation info. The format of the data is as below.
+
+struct domain_info {
+ int domain;
+ char dname[];
+ char cpumask[];
+ char cpulist[];
+};
+
+struct cpu_domain_info {
+ int cpu;
+ int nr_domains;
+ struct domain_info domains[];
+};
+
other bits are reserved and should ignored for now
HEADER_FEAT_BITS = 256,
diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c
index 6080afec537d..587c180035b2 100644
--- a/tools/perf/builtin-inject.c
+++ b/tools/perf/builtin-inject.c
@@ -2047,6 +2047,7 @@ static bool keep_feat(struct perf_inject *inject, int feat)
case HEADER_CLOCK_DATA:
case HEADER_HYBRID_TOPOLOGY:
case HEADER_PMU_CAPS:
+ case HEADER_CPU_DOMAIN_INFO:
return true;
/* Information that can be updated */
case HEADER_BUILD_ID:
diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c
index f1626d2032cd..93d475a80f14 100644
--- a/tools/perf/util/env.c
+++ b/tools/perf/util/env.c
@@ -216,6 +216,34 @@ static void perf_env__purge_bpf(struct perf_env *env __maybe_unused)
}
#endif // HAVE_LIBBPF_SUPPORT
+void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, u32 nr)
+{
+ if (!cd_map)
+ return;
+
+ for (u32 i = 0; i < nr; i++) {
+ if (!cd_map[i])
+ continue;
+
+ for (u32 j = 0; j < cd_map[i]->nr_domains; j++) {
+ struct domain_info *d_info = cd_map[i]->domains[j];
+
+ if (!d_info)
+ continue;
+
+ if (schedstat_version >= 17)
+ zfree(&d_info->dname);
+
+ zfree(&d_info->cpumask);
+ zfree(&d_info->cpulist);
+ zfree(&d_info);
+ }
+ zfree(&cd_map[i]->domains);
+ zfree(&cd_map[i]);
+ }
+ zfree(&cd_map);
+}
+
void perf_env__exit(struct perf_env *env)
{
int i, j;
@@ -265,6 +293,7 @@ void perf_env__exit(struct perf_env *env)
zfree(&env->pmu_caps[i].pmu_name);
}
zfree(&env->pmu_caps);
+ free_cpu_domain_info(env->cpu_domain, env->schedstat_version, env->nr_cpus_avail);
}
void perf_env__init(struct perf_env *env)
diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h
index 9977b85523a8..76ba1a36e9ff 100644
--- a/tools/perf/util/env.h
+++ b/tools/perf/util/env.h
@@ -54,6 +54,19 @@ struct pmu_caps {
char *pmu_name;
};
+struct domain_info {
+ u32 domain;
+ char *dname;
+ char *cpumask;
+ char *cpulist;
+};
+
+struct cpu_domain_map {
+ u32 cpu;
+ u32 nr_domains;
+ struct domain_info **domains;
+};
+
typedef const char *(arch_syscalls__strerrno_t)(int err);
struct perf_env {
@@ -70,6 +83,8 @@ struct perf_env {
unsigned int max_branches;
unsigned int br_cntr_nr;
unsigned int br_cntr_width;
+ unsigned int schedstat_version;
+ unsigned int max_sched_domains;
int kernel_is_64_bit;
int nr_cmdline;
@@ -92,6 +107,7 @@ struct perf_env {
char **cpu_pmu_caps;
struct cpu_topology_map *cpu;
struct cpu_cache_level *caches;
+ struct cpu_domain_map **cpu_domain;
int caches_cnt;
u32 comp_ratio;
u32 comp_ver;
@@ -151,6 +167,7 @@ struct bpf_prog_info_node;
struct btf_node;
int perf_env__read_core_pmu_caps(struct perf_env *env);
+void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, u32 nr);
void perf_env__exit(struct perf_env *env);
int perf_env__kernel_is_64_bit(struct perf_env *env);
--git a/tools/perf/util/header.c b/tools/perf/util/header.c
index f5cad377c99e..673d53bb2a2c 100644
--- a/tools/perf/util/header.c
+++ b/tools/perf/util/header.c
@@ -1614,6 +1614,162 @@ static int write_pmu_caps(struct feat_fd *ff,
return 0;
}
+static struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains,
+ u32 nr)
+{
+ struct domain_info *domain_info;
+ struct cpu_domain_map **cd_map;
+ char dname[16], cpumask[256];
+ char cpulist[1024];
+ char *line = NULL;
+ u32 cpu, domain;
+ u32 dcount = 0;
+ size_t len;
+ FILE *fp;
+
+ fp = fopen("/proc/schedstat", "r");
+ if (!fp) {
+ pr_err("Failed to open /proc/schedstat\n");
+ return NULL;
+ }
+
+ cd_map = zalloc(sizeof(*cd_map) * nr);
+ if (!cd_map)
+ goto out;
+
+ while (getline(&line, &len, fp) > 0) {
+ int retval;
+
+ if (strncmp(line, "version", 7) == 0) {
+ retval = sscanf(line, "version %d\n", schedstat_version);
+ if (retval != 1)
+ continue;
+
+ } else if (strncmp(line, "cpu", 3) == 0) {
+ retval = sscanf(line, "cpu%u %*s", &cpu);
+ if (retval == 1) {
+ cd_map[cpu] = zalloc(sizeof(*cd_map[cpu]));
+ if (!cd_map[cpu])
+ goto out_free_line;
+ cd_map[cpu]->cpu = cpu;
+ } else
+ continue;
+
+ dcount = 0;
+ } else if (strncmp(line, "domain", 6) == 0) {
+ struct domain_info **temp_domains;
+
+ dcount++;
+ temp_domains = realloc(cd_map[cpu]->domains, dcount * sizeof(domain_info));
+ if (!temp_domains)
+ goto out_free_line;
+ else
+ cd_map[cpu]->domains = temp_domains;
+
+ domain_info = zalloc(sizeof(*domain_info));
+ if (!domain_info)
+ goto out_free_line;
+
+ cd_map[cpu]->domains[dcount - 1] = domain_info;
+
+ if (*schedstat_version >= 17) {
+ retval = sscanf(line, "domain%u %s %s %*s", &domain, dname,
+ cpumask);
+ if (retval != 3)
+ continue;
+
+ domain_info->dname = strdup(dname);
+ if (!domain_info->dname)
+ goto out_free_line;
+ } else {
+ retval = sscanf(line, "domain%u %s %*s", &domain, cpumask);
+ if (retval != 2)
+ continue;
+ }
+
+ domain_info->domain = domain;
+ if (domain > *max_sched_domains)
+ *max_sched_domains = domain;
+
+ domain_info->cpumask = strdup(cpumask);
+ if (!domain_info->cpumask)
+ goto out_free_line;
+
+ cpumask_to_cpulist(cpumask, cpulist);
+ domain_info->cpulist = strdup(cpulist);
+ if (!domain_info->cpulist)
+ goto out_free_line;
+
+ cd_map[cpu]->nr_domains = dcount;
+ }
+ }
+
+out_free_line:
+ free(line);
+out:
+ fclose(fp);
+ return cd_map;
+}
+
+static int write_cpu_domain_info(struct feat_fd *ff,
+ struct evlist *evlist __maybe_unused)
+{
+ u32 max_sched_domains = 0, schedstat_version = 0;
+ struct cpu_domain_map **cd_map;
+ u32 i, j, nr, ret;
+
+ nr = cpu__max_present_cpu().cpu;
+
+ cd_map = build_cpu_domain_map(&schedstat_version, &max_sched_domains, nr);
+ if (!cd_map)
+ return -1;
+
+ ret = do_write(ff, &schedstat_version, sizeof(u32));
+ if (ret < 0)
+ goto out;
+
+ max_sched_domains += 1;
+ ret = do_write(ff, &max_sched_domains, sizeof(u32));
+ if (ret < 0)
+ goto out;
+
+ for (i = 0; i < nr; i++) {
+ if (!cd_map[i])
+ continue;
+
+ ret = do_write(ff, &cd_map[i]->cpu, sizeof(u32));
+ if (ret < 0)
+ goto out;
+
+ ret = do_write(ff, &cd_map[i]->nr_domains, sizeof(u32));
+ if (ret < 0)
+ goto out;
+
+ for (j = 0; j < cd_map[i]->nr_domains; j++) {
+ ret = do_write(ff, &cd_map[i]->domains[j]->domain, sizeof(u32));
+ if (ret < 0)
+ goto out;
+ if (schedstat_version >= 17) {
+ ret = do_write_string(ff, cd_map[i]->domains[j]->dname);
+ if (ret < 0)
+ goto out;
+ }
+
+ ret = do_write_string(ff, cd_map[i]->domains[j]->cpumask);
+ if (ret < 0)
+ goto out;
+
+ ret = do_write_string(ff, cd_map[i]->domains[j]->cpulist);
+ if (ret < 0)
+ goto out;
+ }
+ }
+
+out:
+ free_cpu_domain_info(cd_map, schedstat_version, nr);
+ return ret;
+}
+
static void print_hostname(struct feat_fd *ff, FILE *fp)
{
fprintf(fp, "# hostname : %s\n", ff->ph->env.hostname);
@@ -2247,6 +2403,39 @@ static void print_mem_topology(struct feat_fd *ff, FILE *fp)
}
}
+static void print_cpu_domain_info(struct feat_fd *ff, FILE *fp)
+{
+ struct cpu_domain_map **cd_map = ff->ph->env.cpu_domain;
+ u32 nr = ff->ph->env.nr_cpus_avail;
+ struct domain_info *d_info;
+ u32 i, j;
+
+ fprintf(fp, "# schedstat version : %u\n", ff->ph->env.schedstat_version);
+ fprintf(fp, "# Maximum sched domains : %u\n", ff->ph->env.max_sched_domains);
+
+ for (i = 0; i < nr; i++) {
+ if (!cd_map[i])
+ continue;
+
+ fprintf(fp, "# cpu : %u\n", cd_map[i]->cpu);
+ fprintf(fp, "# nr_domains : %u\n", cd_map[i]->nr_domains);
+
+ for (j = 0; j < cd_map[i]->nr_domains; j++) {
+ d_info = cd_map[i]->domains[j];
+ if (!d_info)
+ continue;
+
+ fprintf(fp, "# Domain : %u\n", d_info->domain);
+
+ if (ff->ph->env.schedstat_version >= 17)
+ fprintf(fp, "# Domain name : %s\n", d_info->dname);
+
+ fprintf(fp, "# Domain cpu map : %s\n", d_info->cpumask);
+ fprintf(fp, "# Domain cpu list : %s\n", d_info->cpulist);
+ }
+ }
+}
+
static int __event_process_build_id(struct perf_record_header_build_id *bev,
char *filename,
struct perf_session *session)
@@ -3388,6 +3577,102 @@ static int process_pmu_caps(struct feat_fd *ff, void *data __maybe_unused)
return ret;
}
+static int process_cpu_domain_info(struct feat_fd *ff, void *data __maybe_unused)
+{
+ u32 schedstat_version, max_sched_domains, cpu, domain, nr_domains;
+ struct perf_env *env = &ff->ph->env;
+ char *dname, *cpumask, *cpulist;
+ struct cpu_domain_map **cd_map;
+ struct domain_info *d_info;
+ u32 nra, nr, i, j;
+ int ret;
+
+ nra = env->nr_cpus_avail;
+ nr = env->nr_cpus_online;
+
+ cd_map = zalloc(sizeof(*cd_map) * nra);
+ if (!cd_map)
+ return -1;
+
+ env->cpu_domain = cd_map;
+
+ ret = do_read_u32(ff, &schedstat_version);
+ if (ret)
+ return ret;
+
+ env->schedstat_version = schedstat_version;
+
+ ret = do_read_u32(ff, &max_sched_domains);
+ if (ret)
+ return ret;
+
+ env->max_sched_domains = max_sched_domains;
+
+ for (i = 0; i < nr; i++) {
+ if (do_read_u32(ff, &cpu))
+ return -1;
+
+ cd_map[cpu] = zalloc(sizeof(*cd_map[cpu]));
+ if (!cd_map[cpu])
+ return -1;
+
+ cd_map[cpu]->cpu = cpu;
+
+ if (do_read_u32(ff, &nr_domains))
+ return -1;
+
+ cd_map[cpu]->nr_domains = nr_domains;
+
+ cd_map[cpu]->domains = zalloc(sizeof(*d_info) * max_sched_domains);
+ if (!cd_map[cpu]->domains)
+ return -1;
+
+ for (j = 0; j < nr_domains; j++) {
+ if (do_read_u32(ff, &domain))
+ return -1;
+
+ d_info = zalloc(sizeof(*d_info));
+ if (!d_info)
+ return -1;
+
+ cd_map[cpu]->domains[domain] = d_info;
+ d_info->domain = domain;
+
+ if (schedstat_version >= 17) {
+ dname = do_read_string(ff);
+ if (!dname)
+ return -1;
+
+ d_info->dname = zalloc(strlen(dname) + 1);
+ if (!d_info->dname)
+ return -1;
+
+ d_info->dname = strdup(dname);
+ }
+
+ cpumask = do_read_string(ff);
+ if (!cpumask)
+ return -1;
+
+ d_info->cpumask = zalloc(strlen(cpumask) + 1);
+ if (!d_info->cpumask)
+ return -1;
+ d_info->cpumask = strdup(cpumask);
+
+ cpulist = do_read_string(ff);
+ if (!cpulist)
+ return -1;
+
+ d_info->cpulist = zalloc(strlen(cpulist) + 1);
+ if (!d_info->cpulist)
+ return -1;
+ d_info->cpulist = strdup(cpulist);
+ }
+ }
+
+ return ret;
+}
+
#define FEAT_OPR(n, func, __full_only) \
[HEADER_##n] = { \
.name = __stringify(n), \
@@ -3453,6 +3738,7 @@ const struct perf_header_feature_ops feat_ops[HEADER_LAST_FEATURE] = {
FEAT_OPR(CLOCK_DATA, clock_data, false),
FEAT_OPN(HYBRID_TOPOLOGY, hybrid_topology, true),
FEAT_OPR(PMU_CAPS, pmu_caps, false),
+ FEAT_OPR(CPU_DOMAIN_INFO, cpu_domain_info, true),
};
struct header_print_data {
--git a/tools/perf/util/header.h b/tools/perf/util/header.h
index c058021c3150..c62f3275a80f 100644
--- a/tools/perf/util/header.h
+++ b/tools/perf/util/header.h
@@ -53,6 +53,7 @@ enum {
HEADER_CLOCK_DATA,
HEADER_HYBRID_TOPOLOGY,
HEADER_PMU_CAPS,
+ HEADER_CPU_DOMAIN_INFO,
HEADER_LAST_FEATURE,
HEADER_FEAT_BITS = 256,
};
diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c
index 0f031eb80b4c..b87ff96a9f45 100644
--- a/tools/perf/util/util.c
+++ b/tools/perf/util/util.c
@@ -257,6 +257,48 @@ static int rm_rf_kcore_dir(const char *path)
return 0;
}
+void cpumask_to_cpulist(char *cpumask, char *cpulist)
+{
+ int i, j, bm_size, nbits;
+ int len = strlen(cpumask);
+ unsigned long *bm;
+ char cpus[1024];
+
+ for (i = 0; i < len; i++) {
+ if (cpumask[i] == ',') {
+ for (j = i; j < len; j++)
+ cpumask[j] = cpumask[j + 1];
+ }
+ }
+
+ len = strlen(cpumask);
+ bm_size = (len + 15) / 16;
+ nbits = bm_size * 64;
+ if (nbits <= 0)
+ return;
+
+ bm = calloc(bm_size, sizeof(unsigned long));
+ if (!cpumask)
+ goto free_bm;
+
+ for (i = 0; i < bm_size; i++) {
+ char blk[17];
+ int blklen = len > 16 ? 16 : len;
+
+ strncpy(blk, cpumask + len - blklen, blklen);
+ blk[blklen] = '\0';
+ bm[i] = strtoul(blk, NULL, 16);
+ cpumask[len - blklen] = '\0';
+ len = strlen(cpumask);
+ }
+
+ bitmap_scnprintf(bm, nbits, cpus, sizeof(cpus));
+ strcpy(cpulist, cpus);
+
+free_bm:
+ free(bm);
+}
+
int rm_rf_perf_data(const char *path)
{
const char *pat[] = {
diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h
index 3423778e39a5..1572c8cf04e5 100644
--- a/tools/perf/util/util.h
+++ b/tools/perf/util/util.h
@@ -11,6 +11,7 @@
#include <stdbool.h>
#include <stddef.h>
#include <linux/compiler.h>
+#include <linux/bitmap.h>
#include <sys/types.h>
#ifndef __cplusplus
#include <internal/cpumap.h>
@@ -48,6 +49,8 @@ bool sysctl__nmi_watchdog_enabled(void);
int perf_tip(char **strp, const char *dirpath);
+void cpumask_to_cpulist(char *cpumask, char *cpulist);
+
#ifndef HAVE_SCHED_GETCPU_SUPPORT
int sched_getcpu(void);
#endif
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* Re: [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
2026-01-19 17:58 ` [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info Swapnil Sapkal
@ 2026-01-21 17:18 ` Shrikanth Hegde
2026-01-23 15:19 ` Swapnil Sapkal
2026-01-26 14:46 ` kernel test robot
1 sibling, 1 reply; 36+ messages in thread
From: Shrikanth Hegde @ 2026-01-21 17:18 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: ravi.bangoria, yu.c.chen, mark.rutland, alexander.shishkin, jolsa,
rostedt, vincent.guittot, adrian.hunter, kan.liang,
gautham.shenoy, kprateek.nayak, juri.lelli, yangjihong, void, tj,
ctshao, quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, peterz, mingo, acme, namhyung, irogers,
james.clark
Hi Swapnil.
On 1/19/26 11:28 PM, Swapnil Sapkal wrote:
> '/proc/schedstat' gives the info about load balancing statistics within
> a given domain. It also contains the cpu_mask giving information about
> the sibling cpus and domain names after schedstat version 17. Storing
> this information in perf header will help tools like `perf sched stats`
> for better analysis.
>
> Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
> ---
> .../Documentation/perf.data-file-format.txt | 17 ++
> tools/perf/builtin-inject.c | 1 +
> tools/perf/util/env.c | 29 ++
> tools/perf/util/env.h | 17 ++
> tools/perf/util/header.c | 286 ++++++++++++++++++
> tools/perf/util/header.h | 1 +
> tools/perf/util/util.c | 42 +++
> tools/perf/util/util.h | 3 +
> 8 files changed, 396 insertions(+)
>
> diff --git a/tools/perf/Documentation/perf.data-file-format.txt b/tools/perf/Documentation/perf.data-file-format.txt
> index c9d4dec65344..0e4d0ecc9e12 100644
> --- a/tools/perf/Documentation/perf.data-file-format.txt
> +++ b/tools/perf/Documentation/perf.data-file-format.txt
> @@ -447,6 +447,23 @@ struct {
> } [nr_pmu];
> };
>
> + HEADER_CPU_DOMAIN_INFO = 32,
> +
> +List of cpu-domain relation info. The format of the data is as below.
> +
> +struct domain_info {
> + int domain;
> + char dname[];
> + char cpumask[];
> + char cpulist[];
> +};
> +
> +struct cpu_domain_info {
> + int cpu;
> + int nr_domains;
> + struct domain_info domains[];
> +};
> +
> other bits are reserved and should ignored for now
> HEADER_FEAT_BITS = 256,
>
> diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c
> index 6080afec537d..587c180035b2 100644
> --- a/tools/perf/builtin-inject.c
> +++ b/tools/perf/builtin-inject.c
> @@ -2047,6 +2047,7 @@ static bool keep_feat(struct perf_inject *inject, int feat)
> case HEADER_CLOCK_DATA:
> case HEADER_HYBRID_TOPOLOGY:
> case HEADER_PMU_CAPS:
> + case HEADER_CPU_DOMAIN_INFO:
> return true;
> /* Information that can be updated */
> case HEADER_BUILD_ID:
> diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c
> index f1626d2032cd..93d475a80f14 100644
> --- a/tools/perf/util/env.c
> +++ b/tools/perf/util/env.c
> @@ -216,6 +216,34 @@ static void perf_env__purge_bpf(struct perf_env *env __maybe_unused)
> }
> #endif // HAVE_LIBBPF_SUPPORT
>
> +void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, u32 nr)
> +{
> + if (!cd_map)
> + return;
> +
> + for (u32 i = 0; i < nr; i++) {
> + if (!cd_map[i])
> + continue;
> +
> + for (u32 j = 0; j < cd_map[i]->nr_domains; j++) {
> + struct domain_info *d_info = cd_map[i]->domains[j];
> +
> + if (!d_info)
> + continue;
> +
> + if (schedstat_version >= 17)
> + zfree(&d_info->dname);
> +
> + zfree(&d_info->cpumask);
> + zfree(&d_info->cpulist);
> + zfree(&d_info);
> + }
> + zfree(&cd_map[i]->domains);
> + zfree(&cd_map[i]);
> + }
> + zfree(&cd_map);
> +}
> +
> void perf_env__exit(struct perf_env *env)
> {
> int i, j;
> @@ -265,6 +293,7 @@ void perf_env__exit(struct perf_env *env)
> zfree(&env->pmu_caps[i].pmu_name);
> }
> zfree(&env->pmu_caps);
> + free_cpu_domain_info(env->cpu_domain, env->schedstat_version, env->nr_cpus_avail);
> }
>
> void perf_env__init(struct perf_env *env)
> diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h
> index 9977b85523a8..76ba1a36e9ff 100644
> --- a/tools/perf/util/env.h
> +++ b/tools/perf/util/env.h
> @@ -54,6 +54,19 @@ struct pmu_caps {
> char *pmu_name;
> };
>
> +struct domain_info {
> + u32 domain;
> + char *dname;
> + char *cpumask;
> + char *cpulist;
> +};
> +
> +struct cpu_domain_map {
> + u32 cpu;
> + u32 nr_domains;
> + struct domain_info **domains;
> +};
> +
> typedef const char *(arch_syscalls__strerrno_t)(int err);
>
> struct perf_env {
> @@ -70,6 +83,8 @@ struct perf_env {
> unsigned int max_branches;
> unsigned int br_cntr_nr;
> unsigned int br_cntr_width;
> + unsigned int schedstat_version;
> + unsigned int max_sched_domains;
> int kernel_is_64_bit;
>
> int nr_cmdline;
> @@ -92,6 +107,7 @@ struct perf_env {
> char **cpu_pmu_caps;
> struct cpu_topology_map *cpu;
> struct cpu_cache_level *caches;
> + struct cpu_domain_map **cpu_domain;
> int caches_cnt;
> u32 comp_ratio;
> u32 comp_ver;
> @@ -151,6 +167,7 @@ struct bpf_prog_info_node;
> struct btf_node;
>
> int perf_env__read_core_pmu_caps(struct perf_env *env);
> +void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, u32 nr);
> void perf_env__exit(struct perf_env *env);
>
> int perf_env__kernel_is_64_bit(struct perf_env *env);
> diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c
> index f5cad377c99e..673d53bb2a2c 100644
> --- a/tools/perf/util/header.c
> +++ b/tools/perf/util/header.c
> @@ -1614,6 +1614,162 @@ static int write_pmu_caps(struct feat_fd *ff,
> return 0;
> }
>
> +static struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains,
> + u32 nr)
> +{
> + struct domain_info *domain_info;
> + struct cpu_domain_map **cd_map;
> + char dname[16], cpumask[256];
You should likely make cpumask and cpulist as NR_CPUS to be
safe.
256 will work till 1024 CPUs no? These days there are systems with
more CPUs than that.
Making it NR_CPUS will likely cover all crazy cases/configurations.
> + char cpulist[1024];
> + char *line = NULL;
> + u32 cpu, domain;
> + u32 dcount = 0;
> + size_t len;
> + FILE *fp;
> +
> + fp = fopen("/proc/schedstat", "r");
> + if (!fp) {
> + pr_err("Failed to open /proc/schedstat\n");
> + return NULL;
> + }
> +
> + cd_map = zalloc(sizeof(*cd_map) * nr);
> + if (!cd_map)
> + goto out;
> +
> + while (getline(&line, &len, fp) > 0) {
> + int retval;
> +
> + if (strncmp(line, "version", 7) == 0) {
> + retval = sscanf(line, "version %d\n", schedstat_version);
> + if (retval != 1)
> + continue;
> +
> + } else if (strncmp(line, "cpu", 3) == 0) {
> + retval = sscanf(line, "cpu%u %*s", &cpu);
> + if (retval == 1) {
> + cd_map[cpu] = zalloc(sizeof(*cd_map[cpu]));
> + if (!cd_map[cpu])
> + goto out_free_line;
> + cd_map[cpu]->cpu = cpu;
> + } else
> + continue;
> +
> + dcount = 0;
> + } else if (strncmp(line, "domain", 6) == 0) {
> + struct domain_info **temp_domains;
> +
> + dcount++;
> + temp_domains = realloc(cd_map[cpu]->domains, dcount * sizeof(domain_info));
> + if (!temp_domains)
> + goto out_free_line;
> + else
> + cd_map[cpu]->domains = temp_domains;
> +
> + domain_info = zalloc(sizeof(*domain_info));
> + if (!domain_info)
> + goto out_free_line;
> +
> + cd_map[cpu]->domains[dcount - 1] = domain_info;
> +
> + if (*schedstat_version >= 17) {
> + retval = sscanf(line, "domain%u %s %s %*s", &domain, dname,
> + cpumask);
> + if (retval != 3)
> + continue;
> +
> + domain_info->dname = strdup(dname);
> + if (!domain_info->dname)
> + goto out_free_line;
> + } else {
> + retval = sscanf(line, "domain%u %s %*s", &domain, cpumask);
> + if (retval != 2)
> + continue;
> + }
> +
> + domain_info->domain = domain;
> + if (domain > *max_sched_domains)
> + *max_sched_domains = domain;
> +
> + domain_info->cpumask = strdup(cpumask);
> + if (!domain_info->cpumask)
> + goto out_free_line;
> +
> + cpumask_to_cpulist(cpumask, cpulist);
> + domain_info->cpulist = strdup(cpulist);
> + if (!domain_info->cpulist)
> + goto out_free_line;
> +
> + cd_map[cpu]->nr_domains = dcount;
> + }
> + }
> +
> +out_free_line:
> + free(line);
> +out:
> + fclose(fp);
> + return cd_map;
> +}
> +
> +static int write_cpu_domain_info(struct feat_fd *ff,
> + struct evlist *evlist __maybe_unused)
> +{
> + u32 max_sched_domains = 0, schedstat_version = 0;
> + struct cpu_domain_map **cd_map;
> + u32 i, j, nr, ret;
> +
> + nr = cpu__max_present_cpu().cpu;
> +
> + cd_map = build_cpu_domain_map(&schedstat_version, &max_sched_domains, nr);
> + if (!cd_map)
> + return -1;
> +
> + ret = do_write(ff, &schedstat_version, sizeof(u32));
> + if (ret < 0)
> + goto out;
> +
> + max_sched_domains += 1;
> + ret = do_write(ff, &max_sched_domains, sizeof(u32));
> + if (ret < 0)
> + goto out;
> +
> + for (i = 0; i < nr; i++) {
> + if (!cd_map[i])
> + continue;
> +
> + ret = do_write(ff, &cd_map[i]->cpu, sizeof(u32));
> + if (ret < 0)
> + goto out;
> +
> + ret = do_write(ff, &cd_map[i]->nr_domains, sizeof(u32));
> + if (ret < 0)
> + goto out;
> +
> + for (j = 0; j < cd_map[i]->nr_domains; j++) {
> + ret = do_write(ff, &cd_map[i]->domains[j]->domain, sizeof(u32));
> + if (ret < 0)
> + goto out;
> + if (schedstat_version >= 17) {
> + ret = do_write_string(ff, cd_map[i]->domains[j]->dname);
> + if (ret < 0)
> + goto out;
> + }
> +
> + ret = do_write_string(ff, cd_map[i]->domains[j]->cpumask);
> + if (ret < 0)
> + goto out;
> +
> + ret = do_write_string(ff, cd_map[i]->domains[j]->cpulist);
> + if (ret < 0)
> + goto out;
> + }
> + }
> +
> +out:
> + free_cpu_domain_info(cd_map, schedstat_version, nr);
> + return ret;
> +}
> +
> static void print_hostname(struct feat_fd *ff, FILE *fp)
> {
> fprintf(fp, "# hostname : %s\n", ff->ph->env.hostname);
> @@ -2247,6 +2403,39 @@ static void print_mem_topology(struct feat_fd *ff, FILE *fp)
> }
> }
>
> +static void print_cpu_domain_info(struct feat_fd *ff, FILE *fp)
> +{
> + struct cpu_domain_map **cd_map = ff->ph->env.cpu_domain;
> + u32 nr = ff->ph->env.nr_cpus_avail;
> + struct domain_info *d_info;
> + u32 i, j;
> +
> + fprintf(fp, "# schedstat version : %u\n", ff->ph->env.schedstat_version);
> + fprintf(fp, "# Maximum sched domains : %u\n", ff->ph->env.max_sched_domains);
> +
> + for (i = 0; i < nr; i++) {
> + if (!cd_map[i])
> + continue;
> +
> + fprintf(fp, "# cpu : %u\n", cd_map[i]->cpu);
> + fprintf(fp, "# nr_domains : %u\n", cd_map[i]->nr_domains);
> +
> + for (j = 0; j < cd_map[i]->nr_domains; j++) {
> + d_info = cd_map[i]->domains[j];
> + if (!d_info)
> + continue;
> +
> + fprintf(fp, "# Domain : %u\n", d_info->domain);
> +
> + if (ff->ph->env.schedstat_version >= 17)
> + fprintf(fp, "# Domain name : %s\n", d_info->dname);
> +
> + fprintf(fp, "# Domain cpu map : %s\n", d_info->cpumask);
> + fprintf(fp, "# Domain cpu list : %s\n", d_info->cpulist);
> + }
> + }
> +}
> +
> static int __event_process_build_id(struct perf_record_header_build_id *bev,
> char *filename,
> struct perf_session *session)
> @@ -3388,6 +3577,102 @@ static int process_pmu_caps(struct feat_fd *ff, void *data __maybe_unused)
> return ret;
> }
>
> +static int process_cpu_domain_info(struct feat_fd *ff, void *data __maybe_unused)
> +{
> + u32 schedstat_version, max_sched_domains, cpu, domain, nr_domains;
> + struct perf_env *env = &ff->ph->env;
> + char *dname, *cpumask, *cpulist;
> + struct cpu_domain_map **cd_map;
> + struct domain_info *d_info;
> + u32 nra, nr, i, j;
> + int ret;
> +
> + nra = env->nr_cpus_avail;
> + nr = env->nr_cpus_online;
> +
> + cd_map = zalloc(sizeof(*cd_map) * nra);
> + if (!cd_map)
> + return -1;
> +
> + env->cpu_domain = cd_map;
> +
> + ret = do_read_u32(ff, &schedstat_version);
> + if (ret)
> + return ret;
> +
> + env->schedstat_version = schedstat_version;
> +
> + ret = do_read_u32(ff, &max_sched_domains);
> + if (ret)
> + return ret;
> +
> + env->max_sched_domains = max_sched_domains;
> +
> + for (i = 0; i < nr; i++) {
> + if (do_read_u32(ff, &cpu))
> + return -1;
> +
> + cd_map[cpu] = zalloc(sizeof(*cd_map[cpu]));
> + if (!cd_map[cpu])
> + return -1;
> +
> + cd_map[cpu]->cpu = cpu;
> +
> + if (do_read_u32(ff, &nr_domains))
> + return -1;
> +
> + cd_map[cpu]->nr_domains = nr_domains;
> +
> + cd_map[cpu]->domains = zalloc(sizeof(*d_info) * max_sched_domains);
> + if (!cd_map[cpu]->domains)
> + return -1;
> +
> + for (j = 0; j < nr_domains; j++) {
> + if (do_read_u32(ff, &domain))
> + return -1;
> +
> + d_info = zalloc(sizeof(*d_info));
> + if (!d_info)
> + return -1;
> +
> + cd_map[cpu]->domains[domain] = d_info;
> + d_info->domain = domain;
> +
> + if (schedstat_version >= 17) {
> + dname = do_read_string(ff);
> + if (!dname)
> + return -1;
> +
> + d_info->dname = zalloc(strlen(dname) + 1);
> + if (!d_info->dname)
> + return -1;
> +
> + d_info->dname = strdup(dname);
> + }
> +
> + cpumask = do_read_string(ff);
> + if (!cpumask)
> + return -1;
> +
> + d_info->cpumask = zalloc(strlen(cpumask) + 1);
> + if (!d_info->cpumask)
> + return -1;
> + d_info->cpumask = strdup(cpumask);
> +
> + cpulist = do_read_string(ff);
> + if (!cpulist)
> + return -1;
> +
> + d_info->cpulist = zalloc(strlen(cpulist) + 1);
> + if (!d_info->cpulist)
> + return -1;
> + d_info->cpulist = strdup(cpulist);
> + }
> + }
> +
> + return ret;
> +}
> +
> #define FEAT_OPR(n, func, __full_only) \
> [HEADER_##n] = { \
> .name = __stringify(n), \
> @@ -3453,6 +3738,7 @@ const struct perf_header_feature_ops feat_ops[HEADER_LAST_FEATURE] = {
> FEAT_OPR(CLOCK_DATA, clock_data, false),
> FEAT_OPN(HYBRID_TOPOLOGY, hybrid_topology, true),
> FEAT_OPR(PMU_CAPS, pmu_caps, false),
> + FEAT_OPR(CPU_DOMAIN_INFO, cpu_domain_info, true),
> };
>
> struct header_print_data {
> diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h
> index c058021c3150..c62f3275a80f 100644
> --- a/tools/perf/util/header.h
> +++ b/tools/perf/util/header.h
> @@ -53,6 +53,7 @@ enum {
> HEADER_CLOCK_DATA,
> HEADER_HYBRID_TOPOLOGY,
> HEADER_PMU_CAPS,
> + HEADER_CPU_DOMAIN_INFO,
> HEADER_LAST_FEATURE,
> HEADER_FEAT_BITS = 256,
> };
> diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c
> index 0f031eb80b4c..b87ff96a9f45 100644
> --- a/tools/perf/util/util.c
> +++ b/tools/perf/util/util.c
> @@ -257,6 +257,48 @@ static int rm_rf_kcore_dir(const char *path)
> return 0;
> }
>
> +void cpumask_to_cpulist(char *cpumask, char *cpulist)
> +{
> + int i, j, bm_size, nbits;
> + int len = strlen(cpumask);
> + unsigned long *bm;
> + char cpus[1024];
> +
> + for (i = 0; i < len; i++) {
> + if (cpumask[i] == ',') {
> + for (j = i; j < len; j++)
> + cpumask[j] = cpumask[j + 1];
> + }
> + }
> +
> + len = strlen(cpumask);
> + bm_size = (len + 15) / 16;
> + nbits = bm_size * 64;
> + if (nbits <= 0)
> + return;
> +
> + bm = calloc(bm_size, sizeof(unsigned long));
> + if (!cpumask)
> + goto free_bm;
> +
> + for (i = 0; i < bm_size; i++) {
> + char blk[17];
> + int blklen = len > 16 ? 16 : len;
> +
> + strncpy(blk, cpumask + len - blklen, blklen);
> + blk[blklen] = '\0';
> + bm[i] = strtoul(blk, NULL, 16);
> + cpumask[len - blklen] = '\0';
> + len = strlen(cpumask);
> + }
> +
> + bitmap_scnprintf(bm, nbits, cpus, sizeof(cpus));
> + strcpy(cpulist, cpus);
> +
> +free_bm:
> + free(bm);
> +}
> +
> int rm_rf_perf_data(const char *path)
> {
> const char *pat[] = {
> diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h
> index 3423778e39a5..1572c8cf04e5 100644
> --- a/tools/perf/util/util.h
> +++ b/tools/perf/util/util.h
> @@ -11,6 +11,7 @@
> #include <stdbool.h>
> #include <stddef.h>
> #include <linux/compiler.h>
> +#include <linux/bitmap.h>
> #include <sys/types.h>
> #ifndef __cplusplus
> #include <internal/cpumap.h>
> @@ -48,6 +49,8 @@ bool sysctl__nmi_watchdog_enabled(void);
>
> int perf_tip(char **strp, const char *dirpath);
>
> +void cpumask_to_cpulist(char *cpumask, char *cpulist);
> +
> #ifndef HAVE_SCHED_GETCPU_SUPPORT
> int sched_getcpu(void);
> #endif
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
2026-01-21 17:18 ` Shrikanth Hegde
@ 2026-01-23 15:19 ` Swapnil Sapkal
2026-01-23 16:31 ` Arnaldo Carvalho de Melo
0 siblings, 1 reply; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-23 15:19 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: ravi.bangoria, yu.c.chen, mark.rutland, alexander.shishkin, jolsa,
rostedt, vincent.guittot, adrian.hunter, kan.liang,
gautham.shenoy, kprateek.nayak, juri.lelli, yangjihong, void, tj,
ctshao, quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, peterz, mingo, acme, namhyung, irogers,
james.clark
Hi Shrikanth,
Thanks for the review.
On 21-01-2026 22:48, Shrikanth Hegde wrote:
> Hi Swapnil.
>
>
[...]
>> }
>> +static struct cpu_domain_map **build_cpu_domain_map(u32
>> *schedstat_version, u32 *max_sched_domains,
>> + u32 nr)
>> +{
>> + struct domain_info *domain_info;
>> + struct cpu_domain_map **cd_map;
>> + char dname[16], cpumask[256];
>
> You should likely make cpumask and cpulist as NR_CPUS to be
> safe.
>
> 256 will work till 1024 CPUs no? These days there are systems with
> more CPUs than that.
>
> Making it NR_CPUS will likely cover all crazy cases/configurations.
>
This was a miss at my end. This should be NR_CPUS.
--
Thanks and Regards,
Swapnil
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
2026-01-23 15:19 ` Swapnil Sapkal
@ 2026-01-23 16:31 ` Arnaldo Carvalho de Melo
0 siblings, 0 replies; 36+ messages in thread
From: Arnaldo Carvalho de Melo @ 2026-01-23 16:31 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: Shrikanth Hegde, ravi.bangoria, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, peterz, mingo, namhyung, irogers, james.clark
On Fri, Jan 23, 2026 at 08:49:40PM +0530, Swapnil Sapkal wrote:
> On 21-01-2026 22:48, Shrikanth Hegde wrote:
> > > +static struct cpu_domain_map **build_cpu_domain_map(u32
> > > *schedstat_version, u32 *max_sched_domains,
> > > + u32 nr)
> > > +{
> > > + struct domain_info *domain_info;
> > > + struct cpu_domain_map **cd_map;
> > > + char dname[16], cpumask[256];
> > You should likely make cpumask and cpulist as NR_CPUS to be
> > safe.
> > 256 will work till 1024 CPUs no? These days there are systems with
> > more CPUs than that.
> > Making it NR_CPUS will likely cover all crazy cases/configurations.
> This was a miss at my end. This should be NR_CPUS.
Please continue from what is in perf-tools-next.
- Arnaldo
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
2026-01-19 17:58 ` [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info Swapnil Sapkal
2026-01-21 17:18 ` Shrikanth Hegde
@ 2026-01-26 14:46 ` kernel test robot
2026-01-26 17:12 ` Ian Rogers
1 sibling, 1 reply; 36+ messages in thread
From: kernel test robot @ 2026-01-26 14:46 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: oe-lkp, lkp, linux-perf-users, linux-kernel, peterz, mingo, acme,
namhyung, irogers, james.clark, ravi.bangoria, swapnil.sapkal,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, oliver.sang
Hello,
kernel test robot noticed "kmsg.sanitizer.direct_leak/calloc/zalloc/process_cpu_domain_info/perf_file_section__process/perf_header__process_sections/perf_session__read_header" on:
commit: 6f993818d20c79b874f546cec1dcce40a9927c7b ("[PATCH v5 02/10] perf header: Support CPU DOMAIN relation info")
url: https://github.com/intel-lab-lkp/linux/commits/Swapnil-Sapkal/tools-lib-Add-list_is_first/20260120-021007
base: https://git.kernel.org/cgit/linux/kernel/git/perf/perf-tools-next.git perf-tools-next
patch link: https://lore.kernel.org/all/20260119175833.340369-3-swapnil.sapkal@amd.com/
patch subject: [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
in testcase: perf-sanity-tests
version:
with following parameters:
perf_compiler: gcc
group: group-02
config: x86_64-rhel-9.4-bpf
compiler: gcc-14
test machine: 256 threads 4 sockets INTEL(R) XEON(R) PLATINUM 8592+ (Emerald Rapids) with 256G memory
(please refer to attached dmesg/kmsg for entire log/backtrace)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <oliver.sang@intel.com>
| Closes: https://lore.kernel.org/oe-lkp/202601262235.ee84b79e-lkp@intel.com
we found below tests impacted.
=========================================================================================
tbox_group/testcase/rootfs/kconfig/compiler/perf_compiler/group:
lkp-emr-2sp1/perf-sanity-tests/debian-13-x86_64-20250902.cgz/x86_64-rhel-9.4-bpf/gcc-14/gcc/group-02
4d991caa3f1e36c3 6f993818d20c79b874f546cec1d
---------------- ---------------------------
fail:runs %reproduction fail:runs
| | |
:6 100% 6:6 perf-sanity-tests.build_id_cache_operations.fail
:6 100% 6:6 perf-sanity-tests.perf_evlist_tests.fail
:6 100% 6:6 perf-sanity-tests.perf_header_tests.fail
2026-01-24 13:53:34 sudo ASAN_OPTIONS=fast_unwind_on_malloc=0 /usr/src/linux-perf-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf test 80 -v
80: build id cache operations : Running (1 active)
--- start ---
test child forked, pid 14171
WARNING: perf not built with libbfd. PE binaries will not be tested.
WARNING: wine not found. PE binaries will not be run.
test binaries: /tmp/perf_buildid_test.ex.SHA1.fdj /tmp/perf_buildid_test.ex.MD5.U8m /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/tests/shell/../pe-file.exe
DEBUGINFOD_URLS=
Downloading debug info with build id 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
Adding 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11 /tmp/perf_buildid_test.ex.SHA1.fdj: Ok
build id: 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
link: /tmp/perf_buildid_test.debug.PIE/.build-id/89/e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
file: /tmp/perf_buildid_test.debug.PIE/.build-id/89/../../tmp/perf_buildid_test.ex.SHA1.fdj/89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11/elf
OK for /tmp/perf_buildid_test.ex.SHA1.fdj
DEBUGINFOD_URLS=
Downloading debug info with build id ee26502b2cb816e82d08fd203ebf46b4
Adding ee26502b2cb816e82d08fd203ebf46b4 /tmp/perf_buildid_test.ex.MD5.U8m: Ok
build id: ee26502b2cb816e82d08fd203ebf46b4
link: /tmp/perf_buildid_test.debug.dem/.build-id/ee/26502b2cb816e82d08fd203ebf46b4
file: /tmp/perf_buildid_test.debug.dem/.build-id/ee/../../tmp/perf_buildid_test.ex.MD5.U8m/ee26502b2cb816e82d08fd203ebf46b4/elf
OK for /tmp/perf_buildid_test.ex.MD5.U8m
running: perf record /tmp/perf_buildid_test.ex.SHA1.fdj
build id: 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
link: /tmp/perf_buildid_test.debug.T5w/.build-id/89/e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
file: /tmp/perf_buildid_test.debug.T5w/.build-id/89/../../tmp/perf_buildid_test.ex.SHA1.fdj/89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11/elf
=================================================================
==14272==ERROR: LeakSanitizer: detected memory leaks
Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
#0 0x7f3a7fd98c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x563053942c76 in do_read_string util/header.c:261
#2 0x563053967c30 in process_cpu_domain_info util/header.c:3653
#3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
#4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
#5 0x563053971ef1 in perf_session__read_header util/header.c:4620
#6 0x5630539cd802 in perf_session__open util/session.c:54
#7 0x5630539cea98 in __perf_session__new util/session.c:168
#8 0x5630532b6d43 in perf_session__new util/session.h:116
#9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
#10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
#11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
#0 0x7f3a7fd98610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x56305353dd0c in zalloc ../../lib/zalloc.c:8
#2 0x563053967c7f in process_cpu_domain_info util/header.c:3657
#3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
#4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
#5 0x563053971ef1 in perf_session__read_header util/header.c:4620
#6 0x5630539cd802 in perf_session__open util/session.c:54
#7 0x5630539cea98 in __perf_session__new util/session.c:168
#8 0x5630532b6d43 in perf_session__new util/session.h:116
#9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
#10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
#11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7f3a7fd98c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x563053942c76 in do_read_string util/header.c:261
#2 0x563053967a1c in process_cpu_domain_info util/header.c:3642
#3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
#4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
#5 0x563053971ef1 in perf_session__read_header util/header.c:4620
#6 0x5630539cd802 in perf_session__open util/session.c:54
#7 0x5630539cea98 in __perf_session__new util/session.c:168
#8 0x5630532b6d43 in perf_session__new util/session.h:116
#9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
#10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
#11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7f3a7fd98c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x563053942c76 in do_read_string util/header.c:261
#2 0x563053967e44 in process_cpu_domain_info util/header.c:3662
#3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
#4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
#5 0x563053971ef1 in perf_session__read_header util/header.c:4620
#6 0x5630539cd802 in perf_session__open util/session.c:54
#7 0x5630539cea98 in __perf_session__new util/session.c:168
#8 0x5630532b6d43 in perf_session__new util/session.h:116
#9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
#10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
#11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
#0 0x7f3a7fd98610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x56305353dd0c in zalloc ../../lib/zalloc.c:8
#2 0x563053967e93 in process_cpu_domain_info util/header.c:3666
#3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
#4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
#5 0x563053971ef1 in perf_session__read_header util/header.c:4620
#6 0x5630539cd802 in perf_session__open util/session.c:54
#7 0x5630539cea98 in __perf_session__new util/session.c:168
#8 0x5630532b6d43 in perf_session__new util/session.h:116
#9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
#10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
#11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
#0 0x7f3a7fd98610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x56305353dd0c in zalloc ../../lib/zalloc.c:8
#2 0x563053967a6b in process_cpu_domain_info util/header.c:3646
#3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
#4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
#5 0x563053971ef1 in perf_session__read_header util/header.c:4620
#6 0x5630539cd802 in perf_session__open util/session.c:54
#7 0x5630539cea98 in __perf_session__new util/session.c:168
#8 0x5630532b6d43 in perf_session__new util/session.h:116
#9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
#10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
#11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
failed: 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11 is not reported by "perf buildid-list -i /tmp/perf_buildid_test.data.9pp"
Unexpected signal in check
---- end(-1) ----
80: build id cache operations : FAILED!
...
2026-01-24 13:55:59 sudo ASAN_OPTIONS=fast_unwind_on_malloc=0 /usr/src/linux-perf-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf test 85 -v
85: perf evlist tests : Running (1 active)
--- start ---
test child forked, pid 15479
Simple evlist test
=================================================================
==15499==ERROR: LeakSanitizer: detected memory leaks
Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
#0 0x7f3c9bf85c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x55c6557f9c76 in do_read_string util/header.c:261
#2 0x55c65581ec30 in process_cpu_domain_info util/header.c:3653
#3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
#4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
#5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
#6 0x55c655884802 in perf_session__open util/session.c:54
#7 0x55c655885a98 in __perf_session__new util/session.c:168
#8 0x55c6551528d3 in perf_session__new util/session.h:116
#9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
#0 0x7f3c9bf85610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x55c6553f4d0c in zalloc ../../lib/zalloc.c:8
#2 0x55c65581ec7f in process_cpu_domain_info util/header.c:3657
#3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
#4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
#5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
#6 0x55c655884802 in perf_session__open util/session.c:54
#7 0x55c655885a98 in __perf_session__new util/session.c:168
#8 0x55c6551528d3 in perf_session__new util/session.h:116
#9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7f3c9bf85c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x55c6557f9c76 in do_read_string util/header.c:261
#2 0x55c65581ea1c in process_cpu_domain_info util/header.c:3642
#3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
#4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
#5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
#6 0x55c655884802 in perf_session__open util/session.c:54
#7 0x55c655885a98 in __perf_session__new util/session.c:168
#8 0x55c6551528d3 in perf_session__new util/session.h:116
#9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7f3c9bf85c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x55c6557f9c76 in do_read_string util/header.c:261
#2 0x55c65581ee44 in process_cpu_domain_info util/header.c:3662
#3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
#4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
#5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
#6 0x55c655884802 in perf_session__open util/session.c:54
#7 0x55c655885a98 in __perf_session__new util/session.c:168
#8 0x55c6551528d3 in perf_session__new util/session.h:116
#9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
#0 0x7f3c9bf85610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x55c6553f4d0c in zalloc ../../lib/zalloc.c:8
#2 0x55c65581ee93 in process_cpu_domain_info util/header.c:3666
#3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
#4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
#5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
#6 0x55c655884802 in perf_session__open util/session.c:54
#7 0x55c655885a98 in __perf_session__new util/session.c:168
#8 0x55c6551528d3 in perf_session__new util/session.h:116
#9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
#0 0x7f3c9bf85610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x55c6553f4d0c in zalloc ../../lib/zalloc.c:8
#2 0x55c65581ea6b in process_cpu_domain_info util/header.c:3646
#3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
#4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
#5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
#6 0x55c655884802 in perf_session__open util/session.c:54
#7 0x55c655885a98 in __perf_session__new util/session.c:168
#8 0x55c6551528d3 in perf_session__new util/session.h:116
#9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
Simple evlist [Failed to list event]
Group evlist test
=================================================================
==15528==ERROR: LeakSanitizer: detected memory leaks
Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
#0 0x7efcde5dfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x56040444bc76 in do_read_string util/header.c:261
#2 0x560404470c30 in process_cpu_domain_info util/header.c:3653
#3 0x5604044789f3 in perf_file_section__process util/header.c:4386
#4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
#5 0x56040447aef1 in perf_session__read_header util/header.c:4620
#6 0x5604044d6802 in perf_session__open util/session.c:54
#7 0x5604044d7a98 in __perf_session__new util/session.c:168
#8 0x560403da48d3 in perf_session__new util/session.h:116
#9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
#0 0x7efcde5df610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x560404046d0c in zalloc ../../lib/zalloc.c:8
#2 0x560404470c7f in process_cpu_domain_info util/header.c:3657
#3 0x5604044789f3 in perf_file_section__process util/header.c:4386
#4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
#5 0x56040447aef1 in perf_session__read_header util/header.c:4620
#6 0x5604044d6802 in perf_session__open util/session.c:54
#7 0x5604044d7a98 in __perf_session__new util/session.c:168
#8 0x560403da48d3 in perf_session__new util/session.h:116
#9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7efcde5dfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x56040444bc76 in do_read_string util/header.c:261
#2 0x560404470a1c in process_cpu_domain_info util/header.c:3642
#3 0x5604044789f3 in perf_file_section__process util/header.c:4386
#4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
#5 0x56040447aef1 in perf_session__read_header util/header.c:4620
#6 0x5604044d6802 in perf_session__open util/session.c:54
#7 0x5604044d7a98 in __perf_session__new util/session.c:168
#8 0x560403da48d3 in perf_session__new util/session.h:116
#9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7efcde5dfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x56040444bc76 in do_read_string util/header.c:261
#2 0x560404470e44 in process_cpu_domain_info util/header.c:3662
#3 0x5604044789f3 in perf_file_section__process util/header.c:4386
#4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
#5 0x56040447aef1 in perf_session__read_header util/header.c:4620
#6 0x5604044d6802 in perf_session__open util/session.c:54
#7 0x5604044d7a98 in __perf_session__new util/session.c:168
#8 0x560403da48d3 in perf_session__new util/session.h:116
#9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
#0 0x7efcde5df610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x560404046d0c in zalloc ../../lib/zalloc.c:8
#2 0x560404470e93 in process_cpu_domain_info util/header.c:3666
#3 0x5604044789f3 in perf_file_section__process util/header.c:4386
#4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
#5 0x56040447aef1 in perf_session__read_header util/header.c:4620
#6 0x5604044d6802 in perf_session__open util/session.c:54
#7 0x5604044d7a98 in __perf_session__new util/session.c:168
#8 0x560403da48d3 in perf_session__new util/session.h:116
#9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
#0 0x7efcde5df610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x560404046d0c in zalloc ../../lib/zalloc.c:8
#2 0x560404470a6b in process_cpu_domain_info util/header.c:3646
#3 0x5604044789f3 in perf_file_section__process util/header.c:4386
#4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
#5 0x56040447aef1 in perf_session__read_header util/header.c:4620
#6 0x5604044d6802 in perf_session__open util/session.c:54
#7 0x5604044d7a98 in __perf_session__new util/session.c:168
#8 0x560403da48d3 in perf_session__new util/session.h:116
#9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
Group evlist [Failed to list event group]
Event configuration evlist test
=================================================================
==15547==ERROR: LeakSanitizer: detected memory leaks
Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
#0 0x7f6169abfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x559f7b345c76 in do_read_string util/header.c:261
#2 0x559f7b36ac30 in process_cpu_domain_info util/header.c:3653
#3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
#4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
#5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
#6 0x559f7b3d0802 in perf_session__open util/session.c:54
#7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
#8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
#9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
#0 0x7f6169abf610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x559f7af40d0c in zalloc ../../lib/zalloc.c:8
#2 0x559f7b36ac7f in process_cpu_domain_info util/header.c:3657
#3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
#4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
#5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
#6 0x559f7b3d0802 in perf_session__open util/session.c:54
#7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
#8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
#9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7f6169abfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x559f7b345c76 in do_read_string util/header.c:261
#2 0x559f7b36aa1c in process_cpu_domain_info util/header.c:3642
#3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
#4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
#5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
#6 0x559f7b3d0802 in perf_session__open util/session.c:54
#7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
#8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
#9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7f6169abfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x559f7b345c76 in do_read_string util/header.c:261
#2 0x559f7b36ae44 in process_cpu_domain_info util/header.c:3662
#3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
#4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
#5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
#6 0x559f7b3d0802 in perf_session__open util/session.c:54
#7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
#8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
#9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
#0 0x7f6169abf610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x559f7af40d0c in zalloc ../../lib/zalloc.c:8
#2 0x559f7b36ae93 in process_cpu_domain_info util/header.c:3666
#3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
#4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
#5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
#6 0x559f7b3d0802 in perf_session__open util/session.c:54
#7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
#8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
#9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
#0 0x7f6169abf610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x559f7af40d0c in zalloc ../../lib/zalloc.c:8
#2 0x559f7b36aa6b in process_cpu_domain_info util/header.c:3646
#3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
#4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
#5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
#6 0x559f7b3d0802 in perf_session__open util/session.c:54
#7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
#8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
#9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
#10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
#11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
Event configuration evlist [Failed to list verbose info]
---- end(-1) ----
85: perf evlist tests : FAILED!
...
2026-01-24 13:56:27 sudo ASAN_OPTIONS=fast_unwind_on_malloc=0 /usr/src/linux-perf-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf test 87 -v
87: perf header tests : Running (1 active)
--- start ---
test child forked, pid 15636
Test perf header file
[ perf record: Woken up 2 times to write data ]
[ perf record: Captured and wrote 0.213 MB /tmp/__perf_test_header.perf.data.LAgPK (4662 samples) ]
=================================================================
==15661==ERROR: LeakSanitizer: detected memory leaks
Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
#0 0x7f5bd6234c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x55684cf14c76 in do_read_string util/header.c:261
#2 0x55684cf39c30 in process_cpu_domain_info util/header.c:3653
#3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
#4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
#5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
#6 0x55684cf9f802 in perf_session__open util/session.c:54
#7 0x55684cfa0a98 in __perf_session__new util/session.c:168
#8 0x55684c8cf199 in perf_session__new util/session.h:116
#9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
#10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
#0 0x7f5bd6234610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x55684cb0fd0c in zalloc ../../lib/zalloc.c:8
#2 0x55684cf39c7f in process_cpu_domain_info util/header.c:3657
#3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
#4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
#5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
#6 0x55684cf9f802 in perf_session__open util/session.c:54
#7 0x55684cfa0a98 in __perf_session__new util/session.c:168
#8 0x55684c8cf199 in perf_session__new util/session.h:116
#9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
#10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7f5bd6234c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x55684cf14c76 in do_read_string util/header.c:261
#2 0x55684cf39a1c in process_cpu_domain_info util/header.c:3642
#3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
#4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
#5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
#6 0x55684cf9f802 in perf_session__open util/session.c:54
#7 0x55684cfa0a98 in __perf_session__new util/session.c:168
#8 0x55684c8cf199 in perf_session__new util/session.h:116
#9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
#10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
#0 0x7f5bd6234c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
#1 0x55684cf14c76 in do_read_string util/header.c:261
#2 0x55684cf39e44 in process_cpu_domain_info util/header.c:3662
#3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
#4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
#5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
#6 0x55684cf9f802 in perf_session__open util/session.c:54
#7 0x55684cfa0a98 in __perf_session__new util/session.c:168
#8 0x55684c8cf199 in perf_session__new util/session.h:116
#9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
#10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
#0 0x7f5bd6234610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x55684cb0fd0c in zalloc ../../lib/zalloc.c:8
#2 0x55684cf39e93 in process_cpu_domain_info util/header.c:3666
#3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
#4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
#5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
#6 0x55684cf9f802 in perf_session__open util/session.c:54
#7 0x55684cfa0a98 in __perf_session__new util/session.c:168
#8 0x55684c8cf199 in perf_session__new util/session.h:116
#9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
#10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
#0 0x7f5bd6234610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
#1 0x55684cb0fd0c in zalloc ../../lib/zalloc.c:8
#2 0x55684cf39a6b in process_cpu_domain_info util/header.c:3646
#3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
#4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
#5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
#6 0x55684cf9f802 in perf_session__open util/session.c:54
#7 0x55684cfa0a98 in __perf_session__new util/session.c:168
#8 0x55684c8cf199 in perf_session__new util/session.h:116
#9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
#10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
#11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
#12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
#13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
#14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
#16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
Unexpected signal in test_file
---- end(-1) ----
87: perf header tests : FAILED!
The kernel config and materials to reproduce are available at:
https://download.01.org/0day-ci/archive/20260126/202601262235.ee84b79e-lkp@intel.com
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
2026-01-26 14:46 ` kernel test robot
@ 2026-01-26 17:12 ` Ian Rogers
2026-01-27 2:46 ` Oliver Sang
0 siblings, 1 reply; 36+ messages in thread
From: Ian Rogers @ 2026-01-26 17:12 UTC (permalink / raw)
To: kernel test robot
Cc: Swapnil Sapkal, oe-lkp, lkp, linux-perf-users, linux-kernel,
peterz, mingo, acme, namhyung, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das
On Mon, Jan 26, 2026 at 6:47 AM kernel test robot <oliver.sang@intel.com> wrote:
>
>
>
> Hello,
>
> kernel test robot noticed "kmsg.sanitizer.direct_leak/calloc/zalloc/process_cpu_domain_info/perf_file_section__process/perf_header__process_sections/perf_session__read_header" on:
>
> commit: 6f993818d20c79b874f546cec1dcce40a9927c7b ("[PATCH v5 02/10] perf header: Support CPU DOMAIN relation info")
> url: https://github.com/intel-lab-lkp/linux/commits/Swapnil-Sapkal/tools-lib-Add-list_is_first/20260120-021007
> base: https://git.kernel.org/cgit/linux/kernel/git/perf/perf-tools-next.git perf-tools-next
> patch link: https://lore.kernel.org/all/20260119175833.340369-3-swapnil.sapkal@amd.com/
> patch subject: [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
>
> in testcase: perf-sanity-tests
> version:
> with following parameters:
>
> perf_compiler: gcc
> group: group-02
>
>
>
> config: x86_64-rhel-9.4-bpf
> compiler: gcc-14
> test machine: 256 threads 4 sockets INTEL(R) XEON(R) PLATINUM 8592+ (Emerald Rapids) with 256G memory
>
> (please refer to attached dmesg/kmsg for entire log/backtrace)
>
>
>
>
> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> the same patch/commit), kindly add following tags
> | Reported-by: kernel test robot <oliver.sang@intel.com>
> | Closes: https://lore.kernel.org/oe-lkp/202601262235.ee84b79e-lkp@intel.com
As always thanks for the testing! I believe this issue is addressed by:
https://lore.kernel.org/lkml/20260122213516.671089-2-irogers@google.com/
which is merged into the perf-tools-next tree:
https://web.git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git/commit/?h=perf-tools-next&id=d84f24c898864facc13412a58b78964f6a769d76
Thanks,
Ian
> we found below tests impacted.
>
> =========================================================================================
> tbox_group/testcase/rootfs/kconfig/compiler/perf_compiler/group:
> lkp-emr-2sp1/perf-sanity-tests/debian-13-x86_64-20250902.cgz/x86_64-rhel-9.4-bpf/gcc-14/gcc/group-02
>
> 4d991caa3f1e36c3 6f993818d20c79b874f546cec1d
> ---------------- ---------------------------
> fail:runs %reproduction fail:runs
> | | |
> :6 100% 6:6 perf-sanity-tests.build_id_cache_operations.fail
> :6 100% 6:6 perf-sanity-tests.perf_evlist_tests.fail
> :6 100% 6:6 perf-sanity-tests.perf_header_tests.fail
>
>
>
>
> 2026-01-24 13:53:34 sudo ASAN_OPTIONS=fast_unwind_on_malloc=0 /usr/src/linux-perf-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf test 80 -v
> 80: build id cache operations : Running (1 active)
> --- start ---
> test child forked, pid 14171
> WARNING: perf not built with libbfd. PE binaries will not be tested.
> WARNING: wine not found. PE binaries will not be run.
> test binaries: /tmp/perf_buildid_test.ex.SHA1.fdj /tmp/perf_buildid_test.ex.MD5.U8m /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/tests/shell/../pe-file.exe
> DEBUGINFOD_URLS=
> Downloading debug info with build id 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
> Adding 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11 /tmp/perf_buildid_test.ex.SHA1.fdj: Ok
> build id: 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
> link: /tmp/perf_buildid_test.debug.PIE/.build-id/89/e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
> file: /tmp/perf_buildid_test.debug.PIE/.build-id/89/../../tmp/perf_buildid_test.ex.SHA1.fdj/89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11/elf
> OK for /tmp/perf_buildid_test.ex.SHA1.fdj
> DEBUGINFOD_URLS=
> Downloading debug info with build id ee26502b2cb816e82d08fd203ebf46b4
> Adding ee26502b2cb816e82d08fd203ebf46b4 /tmp/perf_buildid_test.ex.MD5.U8m: Ok
> build id: ee26502b2cb816e82d08fd203ebf46b4
> link: /tmp/perf_buildid_test.debug.dem/.build-id/ee/26502b2cb816e82d08fd203ebf46b4
> file: /tmp/perf_buildid_test.debug.dem/.build-id/ee/../../tmp/perf_buildid_test.ex.MD5.U8m/ee26502b2cb816e82d08fd203ebf46b4/elf
> OK for /tmp/perf_buildid_test.ex.MD5.U8m
> running: perf record /tmp/perf_buildid_test.ex.SHA1.fdj
> build id: 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
> link: /tmp/perf_buildid_test.debug.T5w/.build-id/89/e03056b3c150ac13d49ad72e7ccfe9e8e1fb11
> file: /tmp/perf_buildid_test.debug.T5w/.build-id/89/../../tmp/perf_buildid_test.ex.SHA1.fdj/89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11/elf
>
> =================================================================
> ==14272==ERROR: LeakSanitizer: detected memory leaks
>
> Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3a7fd98c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x563053942c76 in do_read_string util/header.c:261
> #2 0x563053967c30 in process_cpu_domain_info util/header.c:3653
> #3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
> #4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
> #5 0x563053971ef1 in perf_session__read_header util/header.c:4620
> #6 0x5630539cd802 in perf_session__open util/session.c:54
> #7 0x5630539cea98 in __perf_session__new util/session.c:168
> #8 0x5630532b6d43 in perf_session__new util/session.h:116
> #9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
> #10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
> #11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3a7fd98610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x56305353dd0c in zalloc ../../lib/zalloc.c:8
> #2 0x563053967c7f in process_cpu_domain_info util/header.c:3657
> #3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
> #4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
> #5 0x563053971ef1 in perf_session__read_header util/header.c:4620
> #6 0x5630539cd802 in perf_session__open util/session.c:54
> #7 0x5630539cea98 in __perf_session__new util/session.c:168
> #8 0x5630532b6d43 in perf_session__new util/session.h:116
> #9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
> #10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
> #11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3a7fd98c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x563053942c76 in do_read_string util/header.c:261
> #2 0x563053967a1c in process_cpu_domain_info util/header.c:3642
> #3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
> #4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
> #5 0x563053971ef1 in perf_session__read_header util/header.c:4620
> #6 0x5630539cd802 in perf_session__open util/session.c:54
> #7 0x5630539cea98 in __perf_session__new util/session.c:168
> #8 0x5630532b6d43 in perf_session__new util/session.h:116
> #9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
> #10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
> #11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3a7fd98c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x563053942c76 in do_read_string util/header.c:261
> #2 0x563053967e44 in process_cpu_domain_info util/header.c:3662
> #3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
> #4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
> #5 0x563053971ef1 in perf_session__read_header util/header.c:4620
> #6 0x5630539cd802 in perf_session__open util/session.c:54
> #7 0x5630539cea98 in __perf_session__new util/session.c:168
> #8 0x5630532b6d43 in perf_session__new util/session.h:116
> #9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
> #10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
> #11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3a7fd98610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x56305353dd0c in zalloc ../../lib/zalloc.c:8
> #2 0x563053967e93 in process_cpu_domain_info util/header.c:3666
> #3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
> #4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
> #5 0x563053971ef1 in perf_session__read_header util/header.c:4620
> #6 0x5630539cd802 in perf_session__open util/session.c:54
> #7 0x5630539cea98 in __perf_session__new util/session.c:168
> #8 0x5630532b6d43 in perf_session__new util/session.h:116
> #9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
> #10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
> #11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3a7fd98610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x56305353dd0c in zalloc ../../lib/zalloc.c:8
> #2 0x563053967a6b in process_cpu_domain_info util/header.c:3646
> #3 0x56305396f9f3 in perf_file_section__process util/header.c:4386
> #4 0x56305396cfb5 in perf_header__process_sections util/header.c:4144
> #5 0x563053971ef1 in perf_session__read_header util/header.c:4620
> #6 0x5630539cd802 in perf_session__open util/session.c:54
> #7 0x5630539cea98 in __perf_session__new util/session.c:168
> #8 0x5630532b6d43 in perf_session__new util/session.h:116
> #9 0x5630532b798b in perf_session__list_build_ids /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:113
> #10 0x5630532b8891 in cmd_buildid_list /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-buildid-list.c:175
> #11 0x5630535113bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x563053511b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x5630535120e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x563053512851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3a75ac0ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3a75ac0d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x563053277310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
> failed: 89e03056b3c150ac13d49ad72e7ccfe9e8e1fb11 is not reported by "perf buildid-list -i /tmp/perf_buildid_test.data.9pp"
> Unexpected signal in check
> ---- end(-1) ----
> 80: build id cache operations : FAILED!
>
>
> ...
>
>
> 2026-01-24 13:55:59 sudo ASAN_OPTIONS=fast_unwind_on_malloc=0 /usr/src/linux-perf-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf test 85 -v
> 85: perf evlist tests : Running (1 active)
> --- start ---
> test child forked, pid 15479
> Simple evlist test
>
> =================================================================
> ==15499==ERROR: LeakSanitizer: detected memory leaks
>
> Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3c9bf85c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x55c6557f9c76 in do_read_string util/header.c:261
> #2 0x55c65581ec30 in process_cpu_domain_info util/header.c:3653
> #3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
> #4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
> #5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
> #6 0x55c655884802 in perf_session__open util/session.c:54
> #7 0x55c655885a98 in __perf_session__new util/session.c:168
> #8 0x55c6551528d3 in perf_session__new util/session.h:116
> #9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3c9bf85610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x55c6553f4d0c in zalloc ../../lib/zalloc.c:8
> #2 0x55c65581ec7f in process_cpu_domain_info util/header.c:3657
> #3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
> #4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
> #5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
> #6 0x55c655884802 in perf_session__open util/session.c:54
> #7 0x55c655885a98 in __perf_session__new util/session.c:168
> #8 0x55c6551528d3 in perf_session__new util/session.h:116
> #9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3c9bf85c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x55c6557f9c76 in do_read_string util/header.c:261
> #2 0x55c65581ea1c in process_cpu_domain_info util/header.c:3642
> #3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
> #4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
> #5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
> #6 0x55c655884802 in perf_session__open util/session.c:54
> #7 0x55c655885a98 in __perf_session__new util/session.c:168
> #8 0x55c6551528d3 in perf_session__new util/session.h:116
> #9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3c9bf85c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x55c6557f9c76 in do_read_string util/header.c:261
> #2 0x55c65581ee44 in process_cpu_domain_info util/header.c:3662
> #3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
> #4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
> #5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
> #6 0x55c655884802 in perf_session__open util/session.c:54
> #7 0x55c655885a98 in __perf_session__new util/session.c:168
> #8 0x55c6551528d3 in perf_session__new util/session.h:116
> #9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3c9bf85610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x55c6553f4d0c in zalloc ../../lib/zalloc.c:8
> #2 0x55c65581ee93 in process_cpu_domain_info util/header.c:3666
> #3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
> #4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
> #5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
> #6 0x55c655884802 in perf_session__open util/session.c:54
> #7 0x55c655885a98 in __perf_session__new util/session.c:168
> #8 0x55c6551528d3 in perf_session__new util/session.h:116
> #9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
> #0 0x7f3c9bf85610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x55c6553f4d0c in zalloc ../../lib/zalloc.c:8
> #2 0x55c65581ea6b in process_cpu_domain_info util/header.c:3646
> #3 0x55c6558269f3 in perf_file_section__process util/header.c:4386
> #4 0x55c655823fb5 in perf_header__process_sections util/header.c:4144
> #5 0x55c655828ef1 in perf_session__read_header util/header.c:4620
> #6 0x55c655884802 in perf_session__open util/session.c:54
> #7 0x55c655885a98 in __perf_session__new util/session.c:168
> #8 0x55c6551528d3 in perf_session__new util/session.h:116
> #9 0x55c655152d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x55c655153e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x55c6553c83bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x55c6553c8b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x55c6553c90e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x55c6553c9851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f3c91cadca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f3c91cadd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x55c65512e310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
> Simple evlist [Failed to list event]
> Group evlist test
>
> =================================================================
> ==15528==ERROR: LeakSanitizer: detected memory leaks
>
> Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
> #0 0x7efcde5dfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x56040444bc76 in do_read_string util/header.c:261
> #2 0x560404470c30 in process_cpu_domain_info util/header.c:3653
> #3 0x5604044789f3 in perf_file_section__process util/header.c:4386
> #4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
> #5 0x56040447aef1 in perf_session__read_header util/header.c:4620
> #6 0x5604044d6802 in perf_session__open util/session.c:54
> #7 0x5604044d7a98 in __perf_session__new util/session.c:168
> #8 0x560403da48d3 in perf_session__new util/session.h:116
> #9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
> #0 0x7efcde5df610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x560404046d0c in zalloc ../../lib/zalloc.c:8
> #2 0x560404470c7f in process_cpu_domain_info util/header.c:3657
> #3 0x5604044789f3 in perf_file_section__process util/header.c:4386
> #4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
> #5 0x56040447aef1 in perf_session__read_header util/header.c:4620
> #6 0x5604044d6802 in perf_session__open util/session.c:54
> #7 0x5604044d7a98 in __perf_session__new util/session.c:168
> #8 0x560403da48d3 in perf_session__new util/session.h:116
> #9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7efcde5dfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x56040444bc76 in do_read_string util/header.c:261
> #2 0x560404470a1c in process_cpu_domain_info util/header.c:3642
> #3 0x5604044789f3 in perf_file_section__process util/header.c:4386
> #4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
> #5 0x56040447aef1 in perf_session__read_header util/header.c:4620
> #6 0x5604044d6802 in perf_session__open util/session.c:54
> #7 0x5604044d7a98 in __perf_session__new util/session.c:168
> #8 0x560403da48d3 in perf_session__new util/session.h:116
> #9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7efcde5dfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x56040444bc76 in do_read_string util/header.c:261
> #2 0x560404470e44 in process_cpu_domain_info util/header.c:3662
> #3 0x5604044789f3 in perf_file_section__process util/header.c:4386
> #4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
> #5 0x56040447aef1 in perf_session__read_header util/header.c:4620
> #6 0x5604044d6802 in perf_session__open util/session.c:54
> #7 0x5604044d7a98 in __perf_session__new util/session.c:168
> #8 0x560403da48d3 in perf_session__new util/session.h:116
> #9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
> #0 0x7efcde5df610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x560404046d0c in zalloc ../../lib/zalloc.c:8
> #2 0x560404470e93 in process_cpu_domain_info util/header.c:3666
> #3 0x5604044789f3 in perf_file_section__process util/header.c:4386
> #4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
> #5 0x56040447aef1 in perf_session__read_header util/header.c:4620
> #6 0x5604044d6802 in perf_session__open util/session.c:54
> #7 0x5604044d7a98 in __perf_session__new util/session.c:168
> #8 0x560403da48d3 in perf_session__new util/session.h:116
> #9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
> #0 0x7efcde5df610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x560404046d0c in zalloc ../../lib/zalloc.c:8
> #2 0x560404470a6b in process_cpu_domain_info util/header.c:3646
> #3 0x5604044789f3 in perf_file_section__process util/header.c:4386
> #4 0x560404475fb5 in perf_header__process_sections util/header.c:4144
> #5 0x56040447aef1 in perf_session__read_header util/header.c:4620
> #6 0x5604044d6802 in perf_session__open util/session.c:54
> #7 0x5604044d7a98 in __perf_session__new util/session.c:168
> #8 0x560403da48d3 in perf_session__new util/session.h:116
> #9 0x560403da4d1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x560403da5e1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x56040401a3bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x56040401ab8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x56040401b0e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x56040401b851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7efcd4307ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7efcd4307d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x560403d80310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
> Group evlist [Failed to list event group]
> Event configuration evlist test
>
> =================================================================
> ==15547==ERROR: LeakSanitizer: detected memory leaks
>
> Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
> #0 0x7f6169abfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x559f7b345c76 in do_read_string util/header.c:261
> #2 0x559f7b36ac30 in process_cpu_domain_info util/header.c:3653
> #3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
> #4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
> #5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
> #6 0x559f7b3d0802 in perf_session__open util/session.c:54
> #7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
> #8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
> #9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
> #0 0x7f6169abf610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x559f7af40d0c in zalloc ../../lib/zalloc.c:8
> #2 0x559f7b36ac7f in process_cpu_domain_info util/header.c:3657
> #3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
> #4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
> #5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
> #6 0x559f7b3d0802 in perf_session__open util/session.c:54
> #7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
> #8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
> #9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7f6169abfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x559f7b345c76 in do_read_string util/header.c:261
> #2 0x559f7b36aa1c in process_cpu_domain_info util/header.c:3642
> #3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
> #4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
> #5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
> #6 0x559f7b3d0802 in perf_session__open util/session.c:54
> #7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
> #8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
> #9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7f6169abfc57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x559f7b345c76 in do_read_string util/header.c:261
> #2 0x559f7b36ae44 in process_cpu_domain_info util/header.c:3662
> #3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
> #4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
> #5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
> #6 0x559f7b3d0802 in perf_session__open util/session.c:54
> #7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
> #8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
> #9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
> #0 0x7f6169abf610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x559f7af40d0c in zalloc ../../lib/zalloc.c:8
> #2 0x559f7b36ae93 in process_cpu_domain_info util/header.c:3666
> #3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
> #4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
> #5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
> #6 0x559f7b3d0802 in perf_session__open util/session.c:54
> #7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
> #8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
> #9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
> #0 0x7f6169abf610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x559f7af40d0c in zalloc ../../lib/zalloc.c:8
> #2 0x559f7b36aa6b in process_cpu_domain_info util/header.c:3646
> #3 0x559f7b3729f3 in perf_file_section__process util/header.c:4386
> #4 0x559f7b36ffb5 in perf_header__process_sections util/header.c:4144
> #5 0x559f7b374ef1 in perf_session__read_header util/header.c:4620
> #6 0x559f7b3d0802 in perf_session__open util/session.c:54
> #7 0x559f7b3d1a98 in __perf_session__new util/session.c:168
> #8 0x559f7ac9e8d3 in perf_session__new util/session.h:116
> #9 0x559f7ac9ed1f in __cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:46
> #10 0x559f7ac9fe1a in cmd_evlist /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-evlist.c:101
> #11 0x559f7af143bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #12 0x559f7af14b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #13 0x559f7af150e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #14 0x559f7af15851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #15 0x7f615f7e7ca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x7f615f7e7d64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #17 0x559f7ac7a310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
> Event configuration evlist [Failed to list verbose info]
> ---- end(-1) ----
> 85: perf evlist tests : FAILED!
>
>
> ...
>
>
> 2026-01-24 13:56:27 sudo ASAN_OPTIONS=fast_unwind_on_malloc=0 /usr/src/linux-perf-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf test 87 -v
> 87: perf header tests : Running (1 active)
> --- start ---
> test child forked, pid 15636
> Test perf header file
> [ perf record: Woken up 2 times to write data ]
> [ perf record: Captured and wrote 0.213 MB /tmp/__perf_test_header.perf.data.LAgPK (4662 samples) ]
>
> =================================================================
> ==15661==ERROR: LeakSanitizer: detected memory leaks
>
> Direct leak of 131072 byte(s) in 1024 object(s) allocated from:
> #0 0x7f5bd6234c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x55684cf14c76 in do_read_string util/header.c:261
> #2 0x55684cf39c30 in process_cpu_domain_info util/header.c:3653
> #3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
> #4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
> #5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
> #6 0x55684cf9f802 in perf_session__open util/session.c:54
> #7 0x55684cfa0a98 in __perf_session__new util/session.c:168
> #8 0x55684c8cf199 in perf_session__new util/session.h:116
> #9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
> #10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 73728 byte(s) in 1024 object(s) allocated from:
> #0 0x7f5bd6234610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x55684cb0fd0c in zalloc ../../lib/zalloc.c:8
> #2 0x55684cf39c7f in process_cpu_domain_info util/header.c:3657
> #3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
> #4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
> #5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
> #6 0x55684cf9f802 in perf_session__open util/session.c:54
> #7 0x55684cfa0a98 in __perf_session__new util/session.c:168
> #8 0x55684c8cf199 in perf_session__new util/session.h:116
> #9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
> #10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7f5bd6234c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x55684cf14c76 in do_read_string util/header.c:261
> #2 0x55684cf39a1c in process_cpu_domain_info util/header.c:3642
> #3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
> #4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
> #5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
> #6 0x55684cf9f802 in perf_session__open util/session.c:54
> #7 0x55684cfa0a98 in __perf_session__new util/session.c:168
> #8 0x55684c8cf199 in perf_session__new util/session.h:116
> #9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
> #10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 65536 byte(s) in 1024 object(s) allocated from:
> #0 0x7f5bd6234c57 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
> #1 0x55684cf14c76 in do_read_string util/header.c:261
> #2 0x55684cf39e44 in process_cpu_domain_info util/header.c:3662
> #3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
> #4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
> #5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
> #6 0x55684cf9f802 in perf_session__open util/session.c:54
> #7 0x55684cfa0a98 in __perf_session__new util/session.c:168
> #8 0x55684c8cf199 in perf_session__new util/session.h:116
> #9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
> #10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 10532 byte(s) in 1024 object(s) allocated from:
> #0 0x7f5bd6234610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x55684cb0fd0c in zalloc ../../lib/zalloc.c:8
> #2 0x55684cf39e93 in process_cpu_domain_info util/header.c:3666
> #3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
> #4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
> #5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
> #6 0x55684cf9f802 in perf_session__open util/session.c:54
> #7 0x55684cfa0a98 in __perf_session__new util/session.c:168
> #8 0x55684c8cf199 in perf_session__new util/session.h:116
> #9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
> #10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> Direct leak of 4352 byte(s) in 1024 object(s) allocated from:
> #0 0x7f5bd6234610 in calloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:77
> #1 0x55684cb0fd0c in zalloc ../../lib/zalloc.c:8
> #2 0x55684cf39a6b in process_cpu_domain_info util/header.c:3646
> #3 0x55684cf419f3 in perf_file_section__process util/header.c:4386
> #4 0x55684cf3efb5 in perf_header__process_sections util/header.c:4144
> #5 0x55684cf43ef1 in perf_session__read_header util/header.c:4620
> #6 0x55684cf9f802 in perf_session__open util/session.c:54
> #7 0x55684cfa0a98 in __perf_session__new util/session.c:168
> #8 0x55684c8cf199 in perf_session__new util/session.h:116
> #9 0x55684c8e5d65 in cmd_report /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/builtin-report.c:1629
> #10 0x55684cae33bf in run_builtin /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:348
> #11 0x55684cae3b8f in handle_internal_command /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:398
> #12 0x55684cae40e5 in run_argv /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:442
> #13 0x55684cae4851 in main /usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf.c:549
> #14 0x7f5bcbf5cca7 (/lib/x86_64-linux-gnu/libc.so.6+0x29ca7) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #15 0x7f5bcbf5cd64 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29d64) (BuildId: def5460e3cee00bfee25b429c97bcc4853e5b3a8)
> #16 0x55684c849310 in _start (/usr/src/perf_selftests-x86_64-rhel-9.4-bpf-6f993818d20c79b874f546cec1dcce40a9927c7b/tools/perf/perf+0x1048310) (BuildId: 07c75edf5764b1417259ec0a6cb5fc6de1859c88)
>
> SUMMARY: AddressSanitizer: 350756 byte(s) leaked in 6144 allocation(s).
> Unexpected signal in test_file
> ---- end(-1) ----
> 87: perf header tests : FAILED!
>
>
>
> The kernel config and materials to reproduce are available at:
> https://download.01.org/0day-ci/archive/20260126/202601262235.ee84b79e-lkp@intel.com
>
>
>
> --
> 0-DAY CI Kernel Test Service
> https://github.com/intel/lkp-tests/wiki
>
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info
2026-01-26 17:12 ` Ian Rogers
@ 2026-01-27 2:46 ` Oliver Sang
0 siblings, 0 replies; 36+ messages in thread
From: Oliver Sang @ 2026-01-27 2:46 UTC (permalink / raw)
To: Ian Rogers
Cc: Swapnil Sapkal, oe-lkp, lkp, linux-perf-users, linux-kernel,
peterz, mingo, acme, namhyung, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das
hi, Ian,
On Mon, Jan 26, 2026 at 09:12:58AM -0800, Ian Rogers wrote:
[...]
>
> As always thanks for the testing! I believe this issue is addressed by:
> https://lore.kernel.org/lkml/20260122213516.671089-2-irogers@google.com/
> which is merged into the perf-tools-next tree:
> https://web.git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git/commit/?h=perf-tools-next&id=d84f24c898864facc13412a58b78964f6a769d76
thanks a lot for information!
>
> Thanks,
> Ian
^ permalink raw reply [flat|nested] 36+ messages in thread
* [PATCH v5 03/10] perf sched stats: Add record and rawdump support
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 01/10] tools/lib: Add list_is_first() Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 02/10] perf header: Support CPU DOMAIN relation info Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 04/10] perf sched stats: Add schedstat v16 support Swapnil Sapkal
` (9 subsequent siblings)
12 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
Define new, perf tool only, sample types and their layouts. Add logic
to parse /proc/schedstat, convert it to perf sample format and save
samples to perf.data file with `perf sched stats record` command. Also
add logic to read perf.data file, interpret schedstat samples and
print rawdump of samples with `perf script -D`.
Note that, /proc/schedstat file output is standardized with version
number. The patch supports v15 but older or newer version can be added
easily.
Co-developed-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
Tested-by: James Clark <james.clark@linaro.org>
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/lib/perf/Documentation/libperf.txt | 2 +
tools/lib/perf/Makefile | 1 +
tools/lib/perf/include/perf/event.h | 41 ++++
tools/lib/perf/include/perf/schedstat-v15.h | 146 +++++++++++++
tools/perf/builtin-inject.c | 2 +
tools/perf/builtin-sched.c | 222 +++++++++++++++++++-
tools/perf/util/event.c | 40 ++++
tools/perf/util/event.h | 2 +
tools/perf/util/session.c | 22 ++
tools/perf/util/synthetic-events.c | 179 ++++++++++++++++
tools/perf/util/synthetic-events.h | 3 +
tools/perf/util/tool.c | 20 ++
tools/perf/util/tool.h | 4 +-
13 files changed, 682 insertions(+), 2 deletions(-)
create mode 100644 tools/lib/perf/include/perf/schedstat-v15.h
diff --git a/tools/lib/perf/Documentation/libperf.txt b/tools/lib/perf/Documentation/libperf.txt
index 4072bc9b7670..576ecc5fc312 100644
--- a/tools/lib/perf/Documentation/libperf.txt
+++ b/tools/lib/perf/Documentation/libperf.txt
@@ -211,6 +211,8 @@ SYNOPSIS
struct perf_record_header_feature;
struct perf_record_compressed;
struct perf_record_compressed2;
+ struct perf_record_schedstat_cpu;
+ struct perf_record_schedstat_domain;
--
DESCRIPTION
diff --git a/tools/lib/perf/Makefile b/tools/lib/perf/Makefile
index 7fbb50b74c00..9fa28e512ca8 100644
--- a/tools/lib/perf/Makefile
+++ b/tools/lib/perf/Makefile
@@ -179,6 +179,7 @@ install_lib: libs
cp -fpR $(LIBPERF_ALL) $(DESTDIR)$(libdir_SQ)
HDRS := bpf_perf.h core.h cpumap.h threadmap.h evlist.h evsel.h event.h mmap.h
+HDRS += schedstat-v15.h
INTERNAL_HDRS := cpumap.h evlist.h evsel.h lib.h mmap.h rc_check.h threadmap.h xyarray.h
INSTALL_HDRS_PFX := $(DESTDIR)$(prefix)/include/perf
diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h
index 43a8cb04994f..ce04fed7cefc 100644
--- a/tools/lib/perf/include/perf/event.h
+++ b/tools/lib/perf/include/perf/event.h
@@ -496,6 +496,43 @@ struct perf_record_bpf_metadata {
struct perf_record_bpf_metadata_entry entries[];
};
+struct perf_record_schedstat_cpu_v15 {
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) _type _name
+#include "schedstat-v15.h"
+#undef CPU_FIELD
+};
+
+struct perf_record_schedstat_cpu {
+ struct perf_event_header header;
+ __u64 timestamp;
+ __u32 cpu;
+ __u16 version;
+ /* Padding */
+ char __pad[2];
+ union {
+ struct perf_record_schedstat_cpu_v15 v15;
+ };
+};
+
+struct perf_record_schedstat_domain_v15 {
+#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) _type _name
+#include "schedstat-v15.h"
+#undef DOMAIN_FIELD
+};
+
+#define DOMAIN_NAME_LEN 16
+
+struct perf_record_schedstat_domain {
+ struct perf_event_header header;
+ __u64 timestamp;
+ __u32 cpu;
+ __u16 version;
+ __u16 domain;
+ union {
+ struct perf_record_schedstat_domain_v15 v15;
+ };
+};
+
enum perf_user_event_type { /* above any possible kernel type */
PERF_RECORD_USER_TYPE_START = 64,
PERF_RECORD_HEADER_ATTR = 64,
@@ -519,6 +556,8 @@ enum perf_user_event_type { /* above any possible kernel type */
PERF_RECORD_FINISHED_INIT = 82,
PERF_RECORD_COMPRESSED2 = 83,
PERF_RECORD_BPF_METADATA = 84,
+ PERF_RECORD_SCHEDSTAT_CPU = 85,
+ PERF_RECORD_SCHEDSTAT_DOMAIN = 86,
PERF_RECORD_HEADER_MAX
};
@@ -562,6 +601,8 @@ union perf_event {
struct perf_record_compressed pack;
struct perf_record_compressed2 pack2;
struct perf_record_bpf_metadata bpf_metadata;
+ struct perf_record_schedstat_cpu schedstat_cpu;
+ struct perf_record_schedstat_domain schedstat_domain;
};
#endif /* __LIBPERF_EVENT_H */
diff --git a/tools/lib/perf/include/perf/schedstat-v15.h b/tools/lib/perf/include/perf/schedstat-v15.h
new file mode 100644
index 000000000000..639458df05f8
--- /dev/null
+++ b/tools/lib/perf/include/perf/schedstat-v15.h
@@ -0,0 +1,146 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifdef CPU_FIELD
+CPU_FIELD(__u32, yld_count, "sched_yield() count",
+ "%11u", false, yld_count, v15);
+CPU_FIELD(__u32, array_exp, "Legacy counter can be ignored",
+ "%11u", false, array_exp, v15);
+CPU_FIELD(__u32, sched_count, "schedule() called",
+ "%11u", false, sched_count, v15);
+CPU_FIELD(__u32, sched_goidle, "schedule() left the processor idle",
+ "%11u", true, sched_count, v15);
+CPU_FIELD(__u32, ttwu_count, "try_to_wake_up() was called",
+ "%11u", false, ttwu_count, v15);
+CPU_FIELD(__u32, ttwu_local, "try_to_wake_up() was called to wake up the local cpu",
+ "%11u", true, ttwu_count, v15);
+CPU_FIELD(__u64, rq_cpu_time, "total runtime by tasks on this processor (in jiffies)",
+ "%11llu", false, rq_cpu_time, v15);
+CPU_FIELD(__u64, run_delay, "total waittime by tasks on this processor (in jiffies)",
+ "%11llu", true, rq_cpu_time, v15);
+CPU_FIELD(__u64, pcount, "total timeslices run on this cpu",
+ "%11llu", false, pcount, v15);
+#endif
+
+#ifdef DOMAIN_FIELD
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category idle> ");
+#endif
+DOMAIN_FIELD(__u32, idle_lb_count,
+ "load_balance() count on cpu idle", "%11u", true, v15);
+DOMAIN_FIELD(__u32, idle_lb_balanced,
+ "load_balance() found balanced on cpu idle", "%11u", true, v15);
+DOMAIN_FIELD(__u32, idle_lb_failed,
+ "load_balance() move task failed on cpu idle", "%11u", true, v15);
+DOMAIN_FIELD(__u32, idle_lb_imbalance,
+ "imbalance sum on cpu idle", "%11u", false, v15);
+DOMAIN_FIELD(__u32, idle_lb_gained,
+ "pull_task() count on cpu idle", "%11u", false, v15);
+DOMAIN_FIELD(__u32, idle_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu idle", "%11u", false, v15);
+DOMAIN_FIELD(__u32, idle_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu idle", "%11u", true, v15);
+DOMAIN_FIELD(__u32, idle_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu idle", "%11u", true, v15);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(idle_lb_success_count, "load_balance() success count on cpu idle", "%11u",
+ idle_lb_count, idle_lb_balanced, idle_lb_failed, v15);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(idle_lb_avg_pulled,
+ "avg task pulled per successful lb attempt (cpu idle)", "%11.2Lf",
+ idle_lb_count, idle_lb_balanced, idle_lb_failed, idle_lb_gained, v15);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category busy> ");
+#endif
+DOMAIN_FIELD(__u32, busy_lb_count,
+ "load_balance() count on cpu busy", "%11u", true, v15);
+DOMAIN_FIELD(__u32, busy_lb_balanced,
+ "load_balance() found balanced on cpu busy", "%11u", true, v15);
+DOMAIN_FIELD(__u32, busy_lb_failed,
+ "load_balance() move task failed on cpu busy", "%11u", true, v15);
+DOMAIN_FIELD(__u32, busy_lb_imbalance,
+ "imbalance sum on cpu busy", "%11u", false, v15);
+DOMAIN_FIELD(__u32, busy_lb_gained,
+ "pull_task() count on cpu busy", "%11u", false, v15);
+DOMAIN_FIELD(__u32, busy_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu busy", "%11u", false, v15);
+DOMAIN_FIELD(__u32, busy_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu busy", "%11u", true, v15);
+DOMAIN_FIELD(__u32, busy_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu busy", "%11u", true, v15);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(busy_lb_success_count, "load_balance() success count on cpu busy", "%11u",
+ busy_lb_count, busy_lb_balanced, busy_lb_failed, v15);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(busy_lb_avg_pulled,
+ "avg task pulled per successful lb attempt (cpu busy)", "%11.2Lf",
+ busy_lb_count, busy_lb_balanced, busy_lb_failed, busy_lb_gained, v15);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category newidle> ");
+#endif
+DOMAIN_FIELD(__u32, newidle_lb_count,
+ "load_balance() count on cpu newly idle", "%11u", true, v15);
+DOMAIN_FIELD(__u32, newidle_lb_balanced,
+ "load_balance() found balanced on cpu newly idle", "%11u", true, v15);
+DOMAIN_FIELD(__u32, newidle_lb_failed,
+ "load_balance() move task failed on cpu newly idle", "%11u", true, v15);
+DOMAIN_FIELD(__u32, newidle_lb_imbalance,
+ "imbalance sum on cpu newly idle", "%11u", false, v15);
+DOMAIN_FIELD(__u32, newidle_lb_gained,
+ "pull_task() count on cpu newly idle", "%11u", false, v15);
+DOMAIN_FIELD(__u32, newidle_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu newly idle", "%11u", false, v15);
+DOMAIN_FIELD(__u32, newidle_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu newly idle", "%11u", true, v15);
+DOMAIN_FIELD(__u32, newidle_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu newly idle", "%11u", true, v15);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(newidle_lb_success_count,
+ "load_balance() success count on cpu newly idle", "%11u",
+ newidle_lb_count, newidle_lb_balanced, newidle_lb_failed, v15);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(newidle_lb_avg_pulled,
+ "avg task pulled per successful lb attempt (cpu newly idle)", "%11.2Lf",
+ newidle_lb_count, newidle_lb_balanced, newidle_lb_failed, newidle_lb_gained, v15);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category active_load_balance()> ");
+#endif
+DOMAIN_FIELD(__u32, alb_count,
+ "active_load_balance() count", "%11u", false, v15);
+DOMAIN_FIELD(__u32, alb_failed,
+ "active_load_balance() move task failed", "%11u", false, v15);
+DOMAIN_FIELD(__u32, alb_pushed,
+ "active_load_balance() successfully moved a task", "%11u", false, v15);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category sched_balance_exec()> ");
+#endif
+DOMAIN_FIELD(__u32, sbe_count,
+ "sbe_count is not used", "%11u", false, v15);
+DOMAIN_FIELD(__u32, sbe_balanced,
+ "sbe_balanced is not used", "%11u", false, v15);
+DOMAIN_FIELD(__u32, sbe_pushed,
+ "sbe_pushed is not used", "%11u", false, v15);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category sched_balance_fork()> ");
+#endif
+DOMAIN_FIELD(__u32, sbf_count,
+ "sbf_count is not used", "%11u", false, v15);
+DOMAIN_FIELD(__u32, sbf_balanced,
+ "sbf_balanced is not used", "%11u", false, v15);
+DOMAIN_FIELD(__u32, sbf_pushed,
+ "sbf_pushed is not used", "%11u", false, v15);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Wakeup Info> ");
+#endif
+DOMAIN_FIELD(__u32, ttwu_wake_remote,
+ "try_to_wake_up() awoke a task that last ran on a diff cpu", "%11u", false, v15);
+DOMAIN_FIELD(__u32, ttwu_move_affine,
+ "try_to_wake_up() moved task because cache-cold on own cpu", "%11u", false, v15);
+DOMAIN_FIELD(__u32, ttwu_move_balance,
+ "try_to_wake_up() started passive balancing", "%11u", false, v15);
+#endif /* DOMAIN_FIELD */
diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c
index 587c180035b2..06735a7c495a 100644
--- a/tools/perf/builtin-inject.c
+++ b/tools/perf/builtin-inject.c
@@ -2528,6 +2528,8 @@ int cmd_inject(int argc, const char **argv)
inject.tool.compressed = perf_event__repipe_op4_synth;
inject.tool.auxtrace = perf_event__repipe_auxtrace;
inject.tool.bpf_metadata = perf_event__repipe_op2_synth;
+ inject.tool.schedstat_cpu = perf_event__repipe_op2_synth;
+ inject.tool.schedstat_domain = perf_event__repipe_op2_synth;
inject.tool.dont_split_sample_group = true;
inject.tool.merge_deferred_callchains = false;
inject.session = __perf_session__new(&data, &inject.tool,
diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c
index eca3b1c58c4b..ee3b4e42156e 100644
--- a/tools/perf/builtin-sched.c
+++ b/tools/perf/builtin-sched.c
@@ -28,6 +28,8 @@
#include "util/debug.h"
#include "util/event.h"
#include "util/util.h"
+#include "util/synthetic-events.h"
+#include "util/target.h"
#include <linux/kernel.h>
#include <linux/log2.h>
@@ -55,6 +57,7 @@
#define MAX_PRIO 140
static const char *cpu_list;
+static struct perf_cpu_map *user_requested_cpus;
static DECLARE_BITMAP(cpu_bitmap, MAX_NR_CPUS);
struct sched_atom;
@@ -236,6 +239,9 @@ struct perf_sched {
volatile bool thread_funcs_exit;
const char *prio_str;
DECLARE_BITMAP(prio_bitmap, MAX_PRIO);
+
+ struct perf_session *session;
+ struct perf_data *data;
};
/* per thread run time data */
@@ -3734,6 +3740,195 @@ static void setup_sorting(struct perf_sched *sched, const struct option *options
sort_dimension__add("pid", &sched->cmp_pid);
}
+static int process_synthesized_schedstat_event(const struct perf_tool *tool,
+ union perf_event *event,
+ struct perf_sample *sample __maybe_unused,
+ struct machine *machine __maybe_unused)
+{
+ struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
+
+ if (perf_data__write(sched->data, event, event->header.size) <= 0) {
+ pr_err("failed to write perf data, error: %m\n");
+ return -1;
+ }
+
+ sched->session->header.data_size += event->header.size;
+ return 0;
+}
+
+static void sighandler(int sig __maybe_unused)
+{
+}
+
+static int enable_sched_schedstats(int *reset)
+{
+ char path[PATH_MAX];
+ FILE *fp;
+ char ch;
+
+ snprintf(path, PATH_MAX, "%s/sys/kernel/sched_schedstats", procfs__mountpoint());
+ fp = fopen(path, "w+");
+ if (!fp) {
+ pr_err("Failed to open %s\n", path);
+ return -1;
+ }
+
+ ch = getc(fp);
+ if (ch == '0') {
+ *reset = 1;
+ rewind(fp);
+ putc('1', fp);
+ fclose(fp);
+ }
+ return 0;
+}
+
+static int disable_sched_schedstat(void)
+{
+ char path[PATH_MAX];
+ FILE *fp;
+
+ snprintf(path, PATH_MAX, "%s/sys/kernel/sched_schedstats", procfs__mountpoint());
+ fp = fopen(path, "w");
+ if (!fp) {
+ pr_err("Failed to open %s\n", path);
+ return -1;
+ }
+
+ putc('0', fp);
+ fclose(fp);
+ return 0;
+}
+
+/* perf.data or any other output file name used by stats subcommand (only). */
+const char *output_name;
+
+static int perf_sched__schedstat_record(struct perf_sched *sched,
+ int argc, const char **argv)
+{
+ struct perf_session *session;
+ struct target target = {};
+ struct evlist *evlist;
+ int reset = 0;
+ int err = 0;
+ int fd;
+ struct perf_data data = {
+ .path = output_name,
+ .mode = PERF_DATA_MODE_WRITE,
+ };
+
+ signal(SIGINT, sighandler);
+ signal(SIGCHLD, sighandler);
+ signal(SIGTERM, sighandler);
+
+ evlist = evlist__new();
+ if (!evlist)
+ return -ENOMEM;
+
+ session = perf_session__new(&data, &sched->tool);
+ if (IS_ERR(session)) {
+ pr_err("Perf session creation failed.\n");
+ evlist__delete(evlist);
+ return PTR_ERR(session);
+ }
+
+ session->evlist = evlist;
+
+ sched->session = session;
+ sched->data = &data;
+
+ fd = perf_data__fd(&data);
+
+ /*
+ * Capture all important metadata about the system. Although they are
+ * not used by `perf sched stats` tool directly, they provide useful
+ * information about profiled environment.
+ */
+ perf_header__set_feat(&session->header, HEADER_HOSTNAME);
+ perf_header__set_feat(&session->header, HEADER_OSRELEASE);
+ perf_header__set_feat(&session->header, HEADER_VERSION);
+ perf_header__set_feat(&session->header, HEADER_ARCH);
+ perf_header__set_feat(&session->header, HEADER_NRCPUS);
+ perf_header__set_feat(&session->header, HEADER_CPUDESC);
+ perf_header__set_feat(&session->header, HEADER_CPUID);
+ perf_header__set_feat(&session->header, HEADER_TOTAL_MEM);
+ perf_header__set_feat(&session->header, HEADER_CMDLINE);
+ perf_header__set_feat(&session->header, HEADER_CPU_TOPOLOGY);
+ perf_header__set_feat(&session->header, HEADER_NUMA_TOPOLOGY);
+ perf_header__set_feat(&session->header, HEADER_CACHE);
+ perf_header__set_feat(&session->header, HEADER_MEM_TOPOLOGY);
+ perf_header__set_feat(&session->header, HEADER_HYBRID_TOPOLOGY);
+ perf_header__set_feat(&session->header, HEADER_CPU_DOMAIN_INFO);
+
+ err = perf_session__write_header(session, evlist, fd, false);
+ if (err < 0)
+ goto out;
+
+ /*
+ * `perf sched stats` does not support workload profiling (-p pid)
+ * since /proc/schedstat file contains cpu specific data only. Hence, a
+ * profile target is either set of cpus or systemwide, never a process.
+ * Note that, although `-- <workload>` is supported, profile data are
+ * still cpu/systemwide.
+ */
+ if (cpu_list)
+ target.cpu_list = cpu_list;
+ else
+ target.system_wide = true;
+
+ if (argc) {
+ err = evlist__prepare_workload(evlist, &target, argv, false, NULL);
+ if (err)
+ goto out;
+ }
+
+ err = evlist__create_maps(evlist, &target);
+ if (err < 0)
+ goto out;
+
+ user_requested_cpus = evlist->core.user_requested_cpus;
+
+ err = perf_event__synthesize_schedstat(&(sched->tool),
+ process_synthesized_schedstat_event,
+ user_requested_cpus);
+ if (err < 0)
+ goto out;
+
+ err = enable_sched_schedstats(&reset);
+ if (err < 0)
+ goto out;
+
+ if (argc)
+ evlist__start_workload(evlist);
+
+ /* wait for signal */
+ pause();
+
+ if (reset) {
+ err = disable_sched_schedstat();
+ if (err < 0)
+ goto out;
+ }
+
+ err = perf_event__synthesize_schedstat(&(sched->tool),
+ process_synthesized_schedstat_event,
+ user_requested_cpus);
+ if (err < 0)
+ goto out;
+
+ err = perf_session__write_header(session, evlist, fd, true);
+
+out:
+ if (!err)
+ fprintf(stderr, "[ perf sched stats: Wrote samples to %s ]\n", data.path);
+ else
+ fprintf(stderr, "[ perf sched stats: Failed !! ]\n");
+
+ evlist__delete(evlist);
+ close(fd);
+ return err;
+}
+
static bool schedstat_events_exposed(void)
{
/*
@@ -3910,6 +4105,12 @@ int cmd_sched(int argc, const char **argv)
OPT_BOOLEAN('P', "pre-migrations", &sched.pre_migrations, "Show pre-migration wait time"),
OPT_PARENT(sched_options)
};
+ const struct option stats_options[] = {
+ OPT_STRING('o', "output", &output_name, "file",
+ "`stats record` with output filename"),
+ OPT_STRING('C', "cpu", &cpu_list, "cpu", "list of cpus to profile"),
+ OPT_END()
+ };
const char * const latency_usage[] = {
"perf sched latency [<options>]",
@@ -3927,9 +4128,13 @@ int cmd_sched(int argc, const char **argv)
"perf sched timehist [<options>]",
NULL
};
+ const char *stats_usage[] = {
+ "perf sched stats {record} [<options>]",
+ NULL
+ };
const char *const sched_subcommands[] = { "record", "latency", "map",
"replay", "script",
- "timehist", NULL };
+ "timehist", "stats", NULL };
const char *sched_usage[] = {
NULL,
NULL
@@ -4027,6 +4232,21 @@ int cmd_sched(int argc, const char **argv)
ret = symbol__validate_sym_arguments();
if (!ret)
ret = perf_sched__timehist(&sched);
+ } else if (!strcmp(argv[0], "stats")) {
+ const char *const stats_subcommands[] = {"record", NULL};
+
+ argc = parse_options_subcommand(argc, argv, stats_options,
+ stats_subcommands,
+ stats_usage,
+ PARSE_OPT_STOP_AT_NON_OPTION);
+
+ if (argv[0] && !strcmp(argv[0], "record")) {
+ if (argc)
+ argc = parse_options(argc, argv, stats_options,
+ stats_usage, 0);
+ return perf_sched__schedstat_record(&sched, argc, argv);
+ }
+ usage_with_options(stats_usage, stats_options);
} else {
usage_with_options(sched_usage, sched_options);
}
diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c
index 4c92cc1a952c..edacf13455d8 100644
--- a/tools/perf/util/event.c
+++ b/tools/perf/util/event.c
@@ -83,6 +83,8 @@ static const char *perf_event__names[] = {
[PERF_RECORD_FINISHED_INIT] = "FINISHED_INIT",
[PERF_RECORD_COMPRESSED2] = "COMPRESSED2",
[PERF_RECORD_BPF_METADATA] = "BPF_METADATA",
+ [PERF_RECORD_SCHEDSTAT_CPU] = "SCHEDSTAT_CPU",
+ [PERF_RECORD_SCHEDSTAT_DOMAIN] = "SCHEDSTAT_DOMAIN",
};
const char *perf_event__name(unsigned int id)
@@ -571,6 +573,44 @@ size_t perf_event__fprintf_text_poke(union perf_event *event, struct machine *ma
return ret;
}
+size_t perf_event__fprintf_schedstat_cpu(union perf_event *event, FILE *fp)
+{
+ struct perf_record_schedstat_cpu *cs = &event->schedstat_cpu;
+ size_t size = fprintf(fp, "\ncpu%u ", cs->cpu);
+ __u16 version = cs->version;
+
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \
+ size += fprintf(fp, "%" PRIu64 " ", (unsigned long)cs->_ver._name)
+
+ if (version == 15) {
+#include <perf/schedstat-v15.h>
+ return size;
+ }
+#undef CPU_FIELD
+
+ return fprintf(fp, "Unsupported /proc/schedstat version %d.\n",
+ event->schedstat_cpu.version);
+}
+
+size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp)
+{
+ struct perf_record_schedstat_domain *ds = &event->schedstat_domain;
+ __u16 version = ds->version;
+ size_t size = fprintf(fp, "\ndomain%u ", ds->domain);
+
+#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \
+ size += fprintf(fp, "%" PRIu64 " ", (unsigned long)ds->_ver._name)
+
+ if (version == 15) {
+#include <perf/schedstat-v15.h>
+ return size;
+ }
+#undef DOMAIN_FIELD
+
+ return fprintf(fp, "Unsupported /proc/schedstat version %d.\n",
+ event->schedstat_domain.version);
+}
+
size_t perf_event__fprintf(union perf_event *event, struct machine *machine, FILE *fp)
{
size_t ret = fprintf(fp, "PERF_RECORD_%s",
diff --git a/tools/perf/util/event.h b/tools/perf/util/event.h
index 64c63b59d617..2ea83fdf8a03 100644
--- a/tools/perf/util/event.h
+++ b/tools/perf/util/event.h
@@ -392,6 +392,8 @@ size_t perf_event__fprintf_ksymbol(union perf_event *event, FILE *fp);
size_t perf_event__fprintf_bpf(union perf_event *event, FILE *fp);
size_t perf_event__fprintf_bpf_metadata(union perf_event *event, FILE *fp);
size_t perf_event__fprintf_text_poke(union perf_event *event, struct machine *machine,FILE *fp);
+size_t perf_event__fprintf_schedstat_cpu(union perf_event *event, FILE *fp);
+size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp);
size_t perf_event__fprintf(union perf_event *event, struct machine *machine, FILE *fp);
int kallsyms__get_function_start(const char *kallsyms_filename,
diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
index 922ef6577bbb..475fe20d6c25 100644
--- a/tools/perf/util/session.c
+++ b/tools/perf/util/session.c
@@ -697,6 +697,20 @@ static void perf_event__time_conv_swap(union perf_event *event,
}
}
+static void
+perf_event__schedstat_cpu_swap(union perf_event *event __maybe_unused,
+ bool sample_id_all __maybe_unused)
+{
+ /* FIXME */
+}
+
+static void
+perf_event__schedstat_domain_swap(union perf_event *event __maybe_unused,
+ bool sample_id_all __maybe_unused)
+{
+ /* FIXME */
+}
+
typedef void (*perf_event__swap_op)(union perf_event *event,
bool sample_id_all);
@@ -736,6 +750,8 @@ static perf_event__swap_op perf_event__swap_ops[] = {
[PERF_RECORD_STAT_ROUND] = perf_event__stat_round_swap,
[PERF_RECORD_EVENT_UPDATE] = perf_event__event_update_swap,
[PERF_RECORD_TIME_CONV] = perf_event__time_conv_swap,
+ [PERF_RECORD_SCHEDSTAT_CPU] = perf_event__schedstat_cpu_swap,
+ [PERF_RECORD_SCHEDSTAT_DOMAIN] = perf_event__schedstat_domain_swap,
[PERF_RECORD_HEADER_MAX] = NULL,
};
@@ -1659,6 +1675,12 @@ static s64 perf_session__process_user_event(struct perf_session *session,
case PERF_RECORD_BPF_METADATA:
err = tool->bpf_metadata(tool, session, event);
break;
+ case PERF_RECORD_SCHEDSTAT_CPU:
+ err = tool->schedstat_cpu(tool, session, event);
+ break;
+ case PERF_RECORD_SCHEDSTAT_DOMAIN:
+ err = tool->schedstat_domain(tool, session, event);
+ break;
default:
err = -EINVAL;
break;
diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c
index 2ba9fa25e00a..5366ea921e70 100644
--- a/tools/perf/util/synthetic-events.c
+++ b/tools/perf/util/synthetic-events.c
@@ -2529,3 +2529,182 @@ int parse_synth_opt(char *synth)
return ret;
}
+
+static union perf_event *__synthesize_schedstat_cpu(struct io *io, __u16 version,
+ __u64 *cpu, __u64 timestamp)
+{
+ struct perf_record_schedstat_cpu *cs;
+ union perf_event *event;
+ size_t size;
+ char ch;
+
+ size = sizeof(*cs);
+ size = PERF_ALIGN(size, sizeof(u64));
+ event = zalloc(size);
+
+ if (!event)
+ return NULL;
+
+ cs = &event->schedstat_cpu;
+ cs->header.type = PERF_RECORD_SCHEDSTAT_CPU;
+ cs->header.size = size;
+ cs->timestamp = timestamp;
+
+ if (io__get_char(io) != 'p' || io__get_char(io) != 'u')
+ goto out_cpu;
+
+ if (io__get_dec(io, (__u64 *)cpu) != ' ')
+ goto out_cpu;
+
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \
+ do { \
+ __u64 _tmp; \
+ ch = io__get_dec(io, &_tmp); \
+ if (ch != ' ' && ch != '\n') \
+ goto out_cpu; \
+ cs->_ver._name = _tmp; \
+ } while (0)
+
+ if (version == 15) {
+#include <perf/schedstat-v15.h>
+ }
+#undef CPU_FIELD
+
+ cs->cpu = *cpu;
+ cs->version = version;
+
+ return event;
+out_cpu:
+ free(event);
+ return NULL;
+}
+
+static union perf_event *__synthesize_schedstat_domain(struct io *io, __u16 version,
+ __u64 cpu, __u64 timestamp)
+{
+ struct perf_record_schedstat_domain *ds;
+ union perf_event *event = NULL;
+ __u64 d_num;
+ size_t size;
+ char ch;
+
+ if (io__get_char(io) != 'o' || io__get_char(io) != 'm' || io__get_char(io) != 'a' ||
+ io__get_char(io) != 'i' || io__get_char(io) != 'n')
+ return NULL;
+
+ ch = io__get_dec(io, &d_num);
+
+ /* Skip cpumask as it can be extracted from perf header */
+ while (io__get_char(io) != ' ')
+ continue;
+
+ size = sizeof(*ds);
+ size = PERF_ALIGN(size, sizeof(u64));
+ event = zalloc(size);
+
+ ds = &event->schedstat_domain;
+ ds->header.type = PERF_RECORD_SCHEDSTAT_DOMAIN;
+ ds->header.size = size;
+ ds->version = version;
+ ds->timestamp = timestamp;
+ ds->domain = d_num;
+
+#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \
+ do { \
+ __u64 _tmp; \
+ ch = io__get_dec(io, &_tmp); \
+ if (ch != ' ' && ch != '\n') \
+ goto out_domain; \
+ ds->_ver._name = _tmp; \
+ } while (0)
+
+ if (version == 15) {
+#include <perf/schedstat-v15.h>
+ }
+#undef DOMAIN_FIELD
+
+ ds->cpu = cpu;
+ goto out;
+
+out_domain:
+ free(event);
+ event = NULL;
+out:
+ return event;
+}
+
+int perf_event__synthesize_schedstat(const struct perf_tool *tool,
+ perf_event__handler_t process,
+ struct perf_cpu_map *user_requested_cpus)
+{
+ char *line = NULL, path[PATH_MAX];
+ union perf_event *event = NULL;
+ size_t line_len = 0;
+ char bf[BUFSIZ];
+ __u64 timestamp;
+ __u64 cpu = -1;
+ __u16 version;
+ struct io io;
+ int ret = -1;
+ char ch;
+
+ snprintf(path, PATH_MAX, "%s/schedstat", procfs__mountpoint());
+ io.fd = open(path, O_RDONLY, 0);
+ if (io.fd < 0) {
+ pr_err("Failed to open %s. Possibly CONFIG_SCHEDSTAT is disabled.\n", path);
+ return -1;
+ }
+ io__init(&io, io.fd, bf, sizeof(bf));
+
+ if (io__getline(&io, &line, &line_len) < 0 || !line_len)
+ goto out;
+
+ if (!strcmp(line, "version 15\n")) {
+ version = 15;
+ } else {
+ pr_err("Unsupported %s version: %s", path, line + 8);
+ goto out_free_line;
+ }
+
+ if (io__getline(&io, &line, &line_len) < 0 || !line_len)
+ goto out_free_line;
+ timestamp = atol(line + 10);
+
+ /*
+ * FIXME: Can be optimized a bit by not synthesizing domain samples
+ * for filtered out cpus.
+ */
+ for (ch = io__get_char(&io); !io.eof; ch = io__get_char(&io)) {
+ struct perf_cpu this_cpu;
+
+ if (ch == 'c') {
+ event = __synthesize_schedstat_cpu(&io, version,
+ &cpu, timestamp);
+ } else if (ch == 'd') {
+ event = __synthesize_schedstat_domain(&io, version,
+ cpu, timestamp);
+ }
+ if (!event)
+ goto out_free_line;
+
+ this_cpu.cpu = cpu;
+
+ if (user_requested_cpus && !perf_cpu_map__has(user_requested_cpus, this_cpu))
+ continue;
+
+ if (process(tool, event, NULL, NULL) < 0) {
+ free(event);
+ goto out_free_line;
+ }
+
+ free(event);
+ }
+
+ ret = 0;
+
+out_free_line:
+ free(line);
+out:
+ close(io.fd);
+ return ret;
+}
diff --git a/tools/perf/util/synthetic-events.h b/tools/perf/util/synthetic-events.h
index f8588b6cf11a..b0edad0c3100 100644
--- a/tools/perf/util/synthetic-events.h
+++ b/tools/perf/util/synthetic-events.h
@@ -128,4 +128,7 @@ int perf_event__synthesize_for_pipe(const struct perf_tool *tool,
struct perf_data *data,
perf_event__handler_t process);
+int perf_event__synthesize_schedstat(const struct perf_tool *tool,
+ perf_event__handler_t process,
+ struct perf_cpu_map *user_requested_cpu);
#endif // __PERF_SYNTHETIC_EVENTS_H
diff --git a/tools/perf/util/tool.c b/tools/perf/util/tool.c
index 27ba5849c74a..013c7839e2cf 100644
--- a/tools/perf/util/tool.c
+++ b/tools/perf/util/tool.c
@@ -253,7 +253,25 @@ static int perf_event__process_bpf_metadata_stub(const struct perf_tool *tool __
{
if (dump_trace)
perf_event__fprintf_bpf_metadata(event, stdout);
+ dump_printf(": unhandled!\n");
+ return 0;
+}
+static int process_schedstat_cpu_stub(const struct perf_tool *tool __maybe_unused,
+ struct perf_session *perf_session __maybe_unused,
+ union perf_event *event)
+{
+ if (dump_trace)
+ perf_event__fprintf_schedstat_cpu(event, stdout);
+ dump_printf(": unhandled!\n");
+ return 0;
+}
+static int process_schedstat_domain_stub(const struct perf_tool *tool __maybe_unused,
+ struct perf_session *perf_session __maybe_unused,
+ union perf_event *event)
+{
+ if (dump_trace)
+ perf_event__fprintf_schedstat_domain(event, stdout);
dump_printf(": unhandled!\n");
return 0;
}
@@ -317,6 +335,8 @@ void perf_tool__init(struct perf_tool *tool, bool ordered_events)
#endif
tool->finished_init = process_event_op2_stub;
tool->bpf_metadata = perf_event__process_bpf_metadata_stub;
+ tool->schedstat_cpu = process_schedstat_cpu_stub;
+ tool->schedstat_domain = process_schedstat_domain_stub;
}
bool perf_tool__compressed_is_stub(const struct perf_tool *tool)
diff --git a/tools/perf/util/tool.h b/tools/perf/util/tool.h
index e96b69d25a5b..2d9a4b1ca9d0 100644
--- a/tools/perf/util/tool.h
+++ b/tools/perf/util/tool.h
@@ -81,7 +81,9 @@ struct perf_tool {
stat_round,
feature,
finished_init,
- bpf_metadata;
+ bpf_metadata,
+ schedstat_cpu,
+ schedstat_domain;
event_op4 compressed;
event_op3 auxtrace;
bool ordered_events;
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH v5 04/10] perf sched stats: Add schedstat v16 support
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (2 preceding siblings ...)
2026-01-19 17:58 ` [PATCH v5 03/10] perf sched stats: Add record and rawdump support Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 05/10] perf sched stats: Add schedstat v17 support Swapnil Sapkal
` (8 subsequent siblings)
12 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
/proc/schedstat file output is standardized with version number.
Add support to record and raw dump v16 version layout.
Version 16 of schedstats changed the order of definitions within
'enum cpu_idle_type', which changed the order of [CPU_MAX_IDLE_TYPES]
columns in show_schedstat(). In particular the position of CPU_IDLE
and __CPU_NOT_IDLE changed places.
Co-developed-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
Tested-by: James Clark <james.clark@linaro.org>
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/lib/perf/Makefile | 2 +-
tools/lib/perf/include/perf/event.h | 14 ++
tools/lib/perf/include/perf/schedstat-v16.h | 146 ++++++++++++++++++++
tools/perf/util/event.c | 6 +
tools/perf/util/synthetic-events.c | 6 +
5 files changed, 173 insertions(+), 1 deletion(-)
create mode 100644 tools/lib/perf/include/perf/schedstat-v16.h
diff --git a/tools/lib/perf/Makefile b/tools/lib/perf/Makefile
index 9fa28e512ca8..965e066fd780 100644
--- a/tools/lib/perf/Makefile
+++ b/tools/lib/perf/Makefile
@@ -179,7 +179,7 @@ install_lib: libs
cp -fpR $(LIBPERF_ALL) $(DESTDIR)$(libdir_SQ)
HDRS := bpf_perf.h core.h cpumap.h threadmap.h evlist.h evsel.h event.h mmap.h
-HDRS += schedstat-v15.h
+HDRS += schedstat-v15.h schedstat-v16.h
INTERNAL_HDRS := cpumap.h evlist.h evsel.h lib.h mmap.h rc_check.h threadmap.h xyarray.h
INSTALL_HDRS_PFX := $(DESTDIR)$(prefix)/include/perf
diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h
index ce04fed7cefc..bd4d507ea8ab 100644
--- a/tools/lib/perf/include/perf/event.h
+++ b/tools/lib/perf/include/perf/event.h
@@ -502,6 +502,12 @@ struct perf_record_schedstat_cpu_v15 {
#undef CPU_FIELD
};
+struct perf_record_schedstat_cpu_v16 {
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) _type _name
+#include "schedstat-v16.h"
+#undef CPU_FIELD
+};
+
struct perf_record_schedstat_cpu {
struct perf_event_header header;
__u64 timestamp;
@@ -511,6 +517,7 @@ struct perf_record_schedstat_cpu {
char __pad[2];
union {
struct perf_record_schedstat_cpu_v15 v15;
+ struct perf_record_schedstat_cpu_v16 v16;
};
};
@@ -520,6 +527,12 @@ struct perf_record_schedstat_domain_v15 {
#undef DOMAIN_FIELD
};
+struct perf_record_schedstat_domain_v16 {
+#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) _type _name
+#include "schedstat-v16.h"
+#undef DOMAIN_FIELD
+};
+
#define DOMAIN_NAME_LEN 16
struct perf_record_schedstat_domain {
@@ -530,6 +543,7 @@ struct perf_record_schedstat_domain {
__u16 domain;
union {
struct perf_record_schedstat_domain_v15 v15;
+ struct perf_record_schedstat_domain_v16 v16;
};
};
diff --git a/tools/lib/perf/include/perf/schedstat-v16.h b/tools/lib/perf/include/perf/schedstat-v16.h
new file mode 100644
index 000000000000..3462b79c29af
--- /dev/null
+++ b/tools/lib/perf/include/perf/schedstat-v16.h
@@ -0,0 +1,146 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifdef CPU_FIELD
+CPU_FIELD(__u32, yld_count, "sched_yield() count",
+ "%11u", false, yld_count, v16);
+CPU_FIELD(__u32, array_exp, "Legacy counter can be ignored",
+ "%11u", false, array_exp, v16);
+CPU_FIELD(__u32, sched_count, "schedule() called",
+ "%11u", false, sched_count, v16);
+CPU_FIELD(__u32, sched_goidle, "schedule() left the processor idle",
+ "%11u", true, sched_count, v16);
+CPU_FIELD(__u32, ttwu_count, "try_to_wake_up() was called",
+ "%11u", false, ttwu_count, v16);
+CPU_FIELD(__u32, ttwu_local, "try_to_wake_up() was called to wake up the local cpu",
+ "%11u", true, ttwu_count, v16);
+CPU_FIELD(__u64, rq_cpu_time, "total runtime by tasks on this processor (in jiffies)",
+ "%11llu", false, rq_cpu_time, v16);
+CPU_FIELD(__u64, run_delay, "total waittime by tasks on this processor (in jiffies)",
+ "%11llu", true, rq_cpu_time, v16);
+CPU_FIELD(__u64, pcount, "total timeslices run on this cpu",
+ "%11llu", false, pcount, v16);
+#endif /* CPU_FIELD */
+
+#ifdef DOMAIN_FIELD
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category busy> ");
+#endif
+DOMAIN_FIELD(__u32, busy_lb_count,
+ "load_balance() count on cpu busy", "%11u", true, v16);
+DOMAIN_FIELD(__u32, busy_lb_balanced,
+ "load_balance() found balanced on cpu busy", "%11u", true, v16);
+DOMAIN_FIELD(__u32, busy_lb_failed,
+ "load_balance() move task failed on cpu busy", "%11u", true, v16);
+DOMAIN_FIELD(__u32, busy_lb_imbalance,
+ "imbalance sum on cpu busy", "%11u", false, v16);
+DOMAIN_FIELD(__u32, busy_lb_gained,
+ "pull_task() count on cpu busy", "%11u", false, v16);
+DOMAIN_FIELD(__u32, busy_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu busy", "%11u", false, v16);
+DOMAIN_FIELD(__u32, busy_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu busy", "%11u", true, v16);
+DOMAIN_FIELD(__u32, busy_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu busy", "%11u", true, v16);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(busy_lb_success_count, "load_balance() success count on cpu busy", "%11u",
+ busy_lb_count, busy_lb_balanced, busy_lb_failed, v16);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(busy_lb_avg_pulled,
+ "avg task pulled per successful lb attempt (cpu busy)", "%11.2Lf",
+ busy_lb_count, busy_lb_balanced, busy_lb_failed, busy_lb_gained, v16);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category idle> ");
+#endif
+DOMAIN_FIELD(__u32, idle_lb_count,
+ "load_balance() count on cpu idle", "%11u", true, v16);
+DOMAIN_FIELD(__u32, idle_lb_balanced,
+ "load_balance() found balanced on cpu idle", "%11u", true, v16);
+DOMAIN_FIELD(__u32, idle_lb_failed,
+ "load_balance() move task failed on cpu idle", "%11u", true, v16);
+DOMAIN_FIELD(__u32, idle_lb_imbalance,
+ "imbalance sum on cpu idle", "%11u", false, v16);
+DOMAIN_FIELD(__u32, idle_lb_gained,
+ "pull_task() count on cpu idle", "%11u", false, v16);
+DOMAIN_FIELD(__u32, idle_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu idle", "%11u", false, v16);
+DOMAIN_FIELD(__u32, idle_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu idle", "%11u", true, v16);
+DOMAIN_FIELD(__u32, idle_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu idle", "%11u", true, v16);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(idle_lb_success_count, "load_balance() success count on cpu idle", "%11u",
+ idle_lb_count, idle_lb_balanced, idle_lb_failed, v16);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(idle_lb_avg_pulled,
+ "avg task pulled per successful lb attempt (cpu idle)", "%11.2Lf",
+ idle_lb_count, idle_lb_balanced, idle_lb_failed, idle_lb_gained, v16);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category newidle> ");
+#endif
+DOMAIN_FIELD(__u32, newidle_lb_count,
+ "load_balance() count on cpu newly idle", "%11u", true, v16);
+DOMAIN_FIELD(__u32, newidle_lb_balanced,
+ "load_balance() found balanced on cpu newly idle", "%11u", true, v16);
+DOMAIN_FIELD(__u32, newidle_lb_failed,
+ "load_balance() move task failed on cpu newly idle", "%11u", true, v16);
+DOMAIN_FIELD(__u32, newidle_lb_imbalance,
+ "imbalance sum on cpu newly idle", "%11u", false, v16);
+DOMAIN_FIELD(__u32, newidle_lb_gained,
+ "pull_task() count on cpu newly idle", "%11u", false, v16);
+DOMAIN_FIELD(__u32, newidle_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu newly idle", "%11u", false, v16);
+DOMAIN_FIELD(__u32, newidle_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu newly idle", "%11u", true, v16);
+DOMAIN_FIELD(__u32, newidle_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu newly idle", "%11u", true, v16);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(newidle_lb_success_count,
+ "load_balance() success count on cpu newly idle", "%11u",
+ newidle_lb_count, newidle_lb_balanced, newidle_lb_failed, v16);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(newidle_lb_avg_count,
+ "avg task pulled per successful lb attempt (cpu newly idle)", "%11.2Lf",
+ newidle_lb_count, newidle_lb_balanced, newidle_lb_failed, newidle_lb_gained, v16);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category active_load_balance()> ");
+#endif
+DOMAIN_FIELD(__u32, alb_count,
+ "active_load_balance() count", "%11u", false, v16);
+DOMAIN_FIELD(__u32, alb_failed,
+ "active_load_balance() move task failed", "%11u", false, v16);
+DOMAIN_FIELD(__u32, alb_pushed,
+ "active_load_balance() successfully moved a task", "%11u", false, v16);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category sched_balance_exec()> ");
+#endif
+DOMAIN_FIELD(__u32, sbe_count,
+ "sbe_count is not used", "%11u", false, v16);
+DOMAIN_FIELD(__u32, sbe_balanced,
+ "sbe_balanced is not used", "%11u", false, v16);
+DOMAIN_FIELD(__u32, sbe_pushed,
+ "sbe_pushed is not used", "%11u", false, v16);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category sched_balance_fork()> ");
+#endif
+DOMAIN_FIELD(__u32, sbf_count,
+ "sbf_count is not used", "%11u", false, v16);
+DOMAIN_FIELD(__u32, sbf_balanced,
+ "sbf_balanced is not used", "%11u", false, v16);
+DOMAIN_FIELD(__u32, sbf_pushed,
+ "sbf_pushed is not used", "%11u", false, v16);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Wakeup Info> ");
+#endif
+DOMAIN_FIELD(__u32, ttwu_wake_remote,
+ "try_to_wake_up() awoke a task that last ran on a diff cpu", "%11u", false, v16);
+DOMAIN_FIELD(__u32, ttwu_move_affine,
+ "try_to_wake_up() moved task because cache-cold on own cpu", "%11u", false, v16);
+DOMAIN_FIELD(__u32, ttwu_move_balance,
+ "try_to_wake_up() started passive balancing", "%11u", false, v16);
+#endif /* DOMAIN_FIELD */
diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c
index edacf13455d8..9a434e2e480f 100644
--- a/tools/perf/util/event.c
+++ b/tools/perf/util/event.c
@@ -585,6 +585,9 @@ size_t perf_event__fprintf_schedstat_cpu(union perf_event *event, FILE *fp)
if (version == 15) {
#include <perf/schedstat-v15.h>
return size;
+ } else if (version == 16) {
+#include <perf/schedstat-v16.h>
+ return size;
}
#undef CPU_FIELD
@@ -604,6 +607,9 @@ size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp)
if (version == 15) {
#include <perf/schedstat-v15.h>
return size;
+ } else if (version == 16) {
+#include <perf/schedstat-v16.h>
+ return size;
}
#undef DOMAIN_FIELD
diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c
index 5366ea921e70..4ce37357db05 100644
--- a/tools/perf/util/synthetic-events.c
+++ b/tools/perf/util/synthetic-events.c
@@ -2567,6 +2567,8 @@ static union perf_event *__synthesize_schedstat_cpu(struct io *io, __u16 version
if (version == 15) {
#include <perf/schedstat-v15.h>
+ } else if (version == 16) {
+#include <perf/schedstat-v16.h>
}
#undef CPU_FIELD
@@ -2620,6 +2622,8 @@ static union perf_event *__synthesize_schedstat_domain(struct io *io, __u16 vers
if (version == 15) {
#include <perf/schedstat-v15.h>
+ } else if (version == 16) {
+#include <perf/schedstat-v16.h>
}
#undef DOMAIN_FIELD
@@ -2661,6 +2665,8 @@ int perf_event__synthesize_schedstat(const struct perf_tool *tool,
if (!strcmp(line, "version 15\n")) {
version = 15;
+ } else if (!strcmp(line, "version 16\n")) {
+ version = 16;
} else {
pr_err("Unsupported %s version: %s", path, line + 8);
goto out_free_line;
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH v5 05/10] perf sched stats: Add schedstat v17 support
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (3 preceding siblings ...)
2026-01-19 17:58 ` [PATCH v5 04/10] perf sched stats: Add schedstat v16 support Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 06/10] perf sched stats: Add support for report subcommand Swapnil Sapkal
` (7 subsequent siblings)
12 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users
/proc/schedstat file output is standardized with version number.
Add support to record and raw dump v17 version layout.
Version 17 of schedstats removed 'lb_imbalance' field as it has no
significance anymore and instead added more relevant fields namely
'lb_imbalance_load', 'lb_imbalance_util', 'lb_imbalance_task' and
'lb_imbalance_misfit'. The domain field prints the name of the
corresponding sched domain from this version onwards.
Co-developed-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/lib/perf/Makefile | 2 +-
tools/lib/perf/include/perf/event.h | 14 ++
tools/lib/perf/include/perf/schedstat-v17.h | 164 ++++++++++++++++++++
tools/perf/util/event.c | 6 +
tools/perf/util/synthetic-events.c | 11 ++
5 files changed, 196 insertions(+), 1 deletion(-)
create mode 100644 tools/lib/perf/include/perf/schedstat-v17.h
diff --git a/tools/lib/perf/Makefile b/tools/lib/perf/Makefile
index 965e066fd780..27e6490f64dc 100644
--- a/tools/lib/perf/Makefile
+++ b/tools/lib/perf/Makefile
@@ -179,7 +179,7 @@ install_lib: libs
cp -fpR $(LIBPERF_ALL) $(DESTDIR)$(libdir_SQ)
HDRS := bpf_perf.h core.h cpumap.h threadmap.h evlist.h evsel.h event.h mmap.h
-HDRS += schedstat-v15.h schedstat-v16.h
+HDRS += schedstat-v15.h schedstat-v16.h schedstat-v17.h
INTERNAL_HDRS := cpumap.h evlist.h evsel.h lib.h mmap.h rc_check.h threadmap.h xyarray.h
INSTALL_HDRS_PFX := $(DESTDIR)$(prefix)/include/perf
diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h
index bd4d507ea8ab..9043dc72b5d6 100644
--- a/tools/lib/perf/include/perf/event.h
+++ b/tools/lib/perf/include/perf/event.h
@@ -508,6 +508,12 @@ struct perf_record_schedstat_cpu_v16 {
#undef CPU_FIELD
};
+struct perf_record_schedstat_cpu_v17 {
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) _type _name
+#include "schedstat-v17.h"
+#undef CPU_FIELD
+};
+
struct perf_record_schedstat_cpu {
struct perf_event_header header;
__u64 timestamp;
@@ -518,6 +524,7 @@ struct perf_record_schedstat_cpu {
union {
struct perf_record_schedstat_cpu_v15 v15;
struct perf_record_schedstat_cpu_v16 v16;
+ struct perf_record_schedstat_cpu_v17 v17;
};
};
@@ -533,6 +540,12 @@ struct perf_record_schedstat_domain_v16 {
#undef DOMAIN_FIELD
};
+struct perf_record_schedstat_domain_v17 {
+#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) _type _name
+#include "schedstat-v17.h"
+#undef DOMAIN_FIELD
+};
+
#define DOMAIN_NAME_LEN 16
struct perf_record_schedstat_domain {
@@ -544,6 +557,7 @@ struct perf_record_schedstat_domain {
union {
struct perf_record_schedstat_domain_v15 v15;
struct perf_record_schedstat_domain_v16 v16;
+ struct perf_record_schedstat_domain_v17 v17;
};
};
diff --git a/tools/lib/perf/include/perf/schedstat-v17.h b/tools/lib/perf/include/perf/schedstat-v17.h
new file mode 100644
index 000000000000..865dc7c1039c
--- /dev/null
+++ b/tools/lib/perf/include/perf/schedstat-v17.h
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifdef CPU_FIELD
+CPU_FIELD(__u32, yld_count, "sched_yield() count",
+ "%11u", false, yld_count, v17);
+CPU_FIELD(__u32, array_exp, "Legacy counter can be ignored",
+ "%11u", false, array_exp, v17);
+CPU_FIELD(__u32, sched_count, "schedule() called",
+ "%11u", false, sched_count, v17);
+CPU_FIELD(__u32, sched_goidle, "schedule() left the processor idle",
+ "%11u", true, sched_count, v17);
+CPU_FIELD(__u32, ttwu_count, "try_to_wake_up() was called",
+ "%11u", false, ttwu_count, v17);
+CPU_FIELD(__u32, ttwu_local, "try_to_wake_up() was called to wake up the local cpu",
+ "%11u", true, ttwu_count, v17);
+CPU_FIELD(__u64, rq_cpu_time, "total runtime by tasks on this processor (in jiffies)",
+ "%11llu", false, rq_cpu_time, v17);
+CPU_FIELD(__u64, run_delay, "total waittime by tasks on this processor (in jiffies)",
+ "%11llu", true, rq_cpu_time, v17);
+CPU_FIELD(__u64, pcount, "total timeslices run on this cpu",
+ "%11llu", false, pcount, v17);
+#endif /* CPU_FIELD */
+
+#ifdef DOMAIN_FIELD
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category busy> ");
+#endif
+DOMAIN_FIELD(__u32, busy_lb_count,
+ "load_balance() count on cpu busy", "%11u", true, v17);
+DOMAIN_FIELD(__u32, busy_lb_balanced,
+ "load_balance() found balanced on cpu busy", "%11u", true, v17);
+DOMAIN_FIELD(__u32, busy_lb_failed,
+ "load_balance() move task failed on cpu busy", "%11u", true, v17);
+DOMAIN_FIELD(__u32, busy_lb_imbalance_load,
+ "imbalance in load on cpu busy", "%11u", false, v17);
+DOMAIN_FIELD(__u32, busy_lb_imbalance_util,
+ "imbalance in utilization on cpu busy", "%11u", false, v17);
+DOMAIN_FIELD(__u32, busy_lb_imbalance_task,
+ "imbalance in number of tasks on cpu busy", "%11u", false, v17);
+DOMAIN_FIELD(__u32, busy_lb_imbalance_misfit,
+ "imbalance in misfit tasks on cpu busy", "%11u", false, v17);
+DOMAIN_FIELD(__u32, busy_lb_gained,
+ "pull_task() count on cpu busy", "%11u", false, v17);
+DOMAIN_FIELD(__u32, busy_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu busy", "%11u", false, v17);
+DOMAIN_FIELD(__u32, busy_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu busy", "%11u", true, v17);
+DOMAIN_FIELD(__u32, busy_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu busy", "%11u", true, v17);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(busy_lb_success_count, "load_balance() success count on cpu busy", "%11u",
+ busy_lb_count, busy_lb_balanced, busy_lb_failed, v17);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(busy_lb_avg_pulled,
+ "avg task pulled per successful lb attempt (cpu busy)", "%11.2Lf",
+ busy_lb_count, busy_lb_balanced, busy_lb_failed, busy_lb_gained, v17);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category idle> ");
+#endif
+DOMAIN_FIELD(__u32, idle_lb_count,
+ "load_balance() count on cpu idle", "%11u", true, v17);
+DOMAIN_FIELD(__u32, idle_lb_balanced,
+ "load_balance() found balanced on cpu idle", "%11u", true, v17);
+DOMAIN_FIELD(__u32, idle_lb_failed,
+ "load_balance() move task failed on cpu idle", "%11u", true, v17);
+DOMAIN_FIELD(__u32, idle_lb_imbalance_load,
+ "imbalance in load on cpu idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, idle_lb_imbalance_util,
+ "imbalance in utilization on cpu idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, idle_lb_imbalance_task,
+ "imbalance in number of tasks on cpu idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, idle_lb_imbalance_misfit,
+ "imbalance in misfit tasks on cpu idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, idle_lb_gained,
+ "pull_task() count on cpu idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, idle_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, idle_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu idle", "%11u", true, v17);
+DOMAIN_FIELD(__u32, idle_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu idle", "%11u", true, v17);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(idle_lb_success_count, "load_balance() success count on cpu idle", "%11u",
+ idle_lb_count, idle_lb_balanced, idle_lb_failed, v17);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(idle_lb_avg_pulled,
+ "avg task pulled per successful lb attempt (cpu idle)", "%11.2Lf",
+ idle_lb_count, idle_lb_balanced, idle_lb_failed, idle_lb_gained, v17);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category newidle> ");
+#endif
+DOMAIN_FIELD(__u32, newidle_lb_count,
+ "load_balance() count on cpu newly idle", "%11u", true, v17);
+DOMAIN_FIELD(__u32, newidle_lb_balanced,
+ "load_balance() found balanced on cpu newly idle", "%11u", true, v17);
+DOMAIN_FIELD(__u32, newidle_lb_failed,
+ "load_balance() move task failed on cpu newly idle", "%11u", true, v17);
+DOMAIN_FIELD(__u32, newidle_lb_imbalance_load,
+ "imbalance in load on cpu newly idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, newidle_lb_imbalance_util,
+ "imbalance in utilization on cpu newly idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, newidle_lb_imbalance_task,
+ "imbalance in number of tasks on cpu newly idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, newidle_lb_imbalance_misfit,
+ "imbalance in misfit tasks on cpu newly idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, newidle_lb_gained,
+ "pull_task() count on cpu newly idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, newidle_lb_hot_gained,
+ "pull_task() when target task was cache-hot on cpu newly idle", "%11u", false, v17);
+DOMAIN_FIELD(__u32, newidle_lb_nobusyq,
+ "load_balance() failed to find busier queue on cpu newly idle", "%11u", true, v17);
+DOMAIN_FIELD(__u32, newidle_lb_nobusyg,
+ "load_balance() failed to find busier group on cpu newly idle", "%11u", true, v17);
+#ifdef DERIVED_CNT_FIELD
+DERIVED_CNT_FIELD(newidle_lb_success_count,
+ "load_balance() success count on cpu newly idle", "%11u",
+ newidle_lb_count, newidle_lb_balanced, newidle_lb_failed, v17);
+#endif
+#ifdef DERIVED_AVG_FIELD
+DERIVED_AVG_FIELD(newidle_lb_avg_pulled,
+ "avg task pulled per successful lb attempt (cpu newly idle)", "%11.2Lf",
+ newidle_lb_count, newidle_lb_balanced, newidle_lb_failed, newidle_lb_gained, v17);
+#endif
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category active_load_balance()> ");
+#endif
+DOMAIN_FIELD(__u32, alb_count,
+ "active_load_balance() count", "%11u", false, v17);
+DOMAIN_FIELD(__u32, alb_failed,
+ "active_load_balance() move task failed", "%11u", false, v17);
+DOMAIN_FIELD(__u32, alb_pushed,
+ "active_load_balance() successfully moved a task", "%11u", false, v17);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category sched_balance_exec()> ");
+#endif
+DOMAIN_FIELD(__u32, sbe_count,
+ "sbe_count is not used", "%11u", false, v17);
+DOMAIN_FIELD(__u32, sbe_balanced,
+ "sbe_balanced is not used", "%11u", false, v17);
+DOMAIN_FIELD(__u32, sbe_pushed,
+ "sbe_pushed is not used", "%11u", false, v17);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Category sched_balance_fork()> ");
+#endif
+DOMAIN_FIELD(__u32, sbf_count,
+ "sbf_count is not used", "%11u", false, v17);
+DOMAIN_FIELD(__u32, sbf_balanced,
+ "sbf_balanced is not used", "%11u", false, v17);
+DOMAIN_FIELD(__u32, sbf_pushed,
+ "sbf_pushed is not used", "%11u", false, v17);
+#ifdef DOMAIN_CATEGORY
+DOMAIN_CATEGORY(" <Wakeup Info> ");
+#endif
+DOMAIN_FIELD(__u32, ttwu_wake_remote,
+ "try_to_wake_up() awoke a task that last ran on a diff cpu", "%11u", false, v17);
+DOMAIN_FIELD(__u32, ttwu_move_affine,
+ "try_to_wake_up() moved task because cache-cold on own cpu", "%11u", false, v17);
+DOMAIN_FIELD(__u32, ttwu_move_balance,
+ "try_to_wake_up() started passive balancing", "%11u", false, v17);
+#endif /* DOMAIN_FIELD */
diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c
index 9a434e2e480f..f40419d21034 100644
--- a/tools/perf/util/event.c
+++ b/tools/perf/util/event.c
@@ -588,6 +588,9 @@ size_t perf_event__fprintf_schedstat_cpu(union perf_event *event, FILE *fp)
} else if (version == 16) {
#include <perf/schedstat-v16.h>
return size;
+ } else if (version == 17) {
+#include <perf/schedstat-v17.h>
+ return size;
}
#undef CPU_FIELD
@@ -610,6 +613,9 @@ size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp)
} else if (version == 16) {
#include <perf/schedstat-v16.h>
return size;
+ } else if (version == 17) {
+#include <perf/schedstat-v17.h>
+ return size;
}
#undef DOMAIN_FIELD
diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c
index 4ce37357db05..ef79433ebc3a 100644
--- a/tools/perf/util/synthetic-events.c
+++ b/tools/perf/util/synthetic-events.c
@@ -2569,6 +2569,8 @@ static union perf_event *__synthesize_schedstat_cpu(struct io *io, __u16 version
#include <perf/schedstat-v15.h>
} else if (version == 16) {
#include <perf/schedstat-v16.h>
+ } else if (version == 17) {
+#include <perf/schedstat-v17.h>
}
#undef CPU_FIELD
@@ -2595,6 +2597,11 @@ static union perf_event *__synthesize_schedstat_domain(struct io *io, __u16 vers
return NULL;
ch = io__get_dec(io, &d_num);
+ if (version >= 17) {
+ /* Skip domain name as it can be extracted from perf header */
+ while (io__get_char(io) != ' ')
+ continue;
+ }
/* Skip cpumask as it can be extracted from perf header */
while (io__get_char(io) != ' ')
@@ -2624,6 +2631,8 @@ static union perf_event *__synthesize_schedstat_domain(struct io *io, __u16 vers
#include <perf/schedstat-v15.h>
} else if (version == 16) {
#include <perf/schedstat-v16.h>
+ } else if (version == 17) {
+#include <perf/schedstat-v17.h>
}
#undef DOMAIN_FIELD
@@ -2667,6 +2676,8 @@ int perf_event__synthesize_schedstat(const struct perf_tool *tool,
version = 15;
} else if (!strcmp(line, "version 16\n")) {
version = 16;
+ } else if (!strcmp(line, "version 17\n")) {
+ version = 17;
} else {
pr_err("Unsupported %s version: %s", path, line + 8);
goto out_free_line;
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH v5 06/10] perf sched stats: Add support for report subcommand
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (4 preceding siblings ...)
2026-01-19 17:58 ` [PATCH v5 05/10] perf sched stats: Add schedstat v17 support Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 07/10] perf sched stats: Add support for live mode Swapnil Sapkal
` (6 subsequent siblings)
12 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
`perf sched stats record` captures two sets of samples. For workload
profile, first set right before workload starts and second set after
workload finishes. For the systemwide profile, first set at the
beginning of profile and second set on receiving SIGINT signal.
Add `perf sched stats report` subcommand that will read both the set
of samples, get the diff and render a final report. Final report prints
scheduler stat at cpu granularity as well as sched domain granularity.
Example usage:
# ./perf sched stats record -- true
[ perf sched stats: Wrote samples to perf.data ]
# perf sched stats report
Description
----------------------------------------------------------------------------------------------------
DESC -> Description of the field
COUNT -> Value of the field
PCT_CHANGE -> Percent change with corresponding base value
AVG_JIFFIES -> Avg time in jiffies between two consecutive occurrence of event
----------------------------------------------------------------------------------------------------
Time elapsed (in jiffies) : 1
----------------------------------------------------------------------------------------------------
CPU: <ALL CPUS SUMMARY>
----------------------------------------------------------------------------------------------------
DESC COUNT PCT_CHANGE
----------------------------------------------------------------------------------------------------
yld_count : 0
array_exp : 0
sched_count : 0
sched_goidle : 0 ( 0.00% )
ttwu_count : 0
ttwu_local : 0 ( 0.00% )
rq_cpu_time : 33525
run_delay : 436 ( 1.30% )
pcount : 0
----------------------------------------------------------------------------------------------------
CPU: <ALL CPUS SUMMARY> | DOMAIN: SMT
----------------------------------------------------------------------------------------------------
DESC COUNT AVG_JIFFIES
----------------------------------------- <Category busy> ------------------------------------------
busy_lb_count : 0 $ 0.00 $
busy_lb_balanced : 0 $ 0.00 $
busy_lb_failed : 0 $ 0.00 $
busy_lb_imbalance_load : 0
busy_lb_imbalance_util : 0
busy_lb_imbalance_task : 0
busy_lb_imbalance_misfit : 0
busy_lb_gained : 0
busy_lb_hot_gained : 0
busy_lb_nobusyq : 0 $ 0.00 $
busy_lb_nobusyg : 0 $ 0.00 $
*busy_lb_success_count : 0
*busy_lb_avg_pulled : 0.00
... and so on. Output shows similar data for all the cpus in the
system.
Co-developed-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
Tested-by: James Clark <james.clark@linaro.org>
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/perf/builtin-sched.c | 509 ++++++++++++++++++++++++++++++++++++-
tools/perf/util/util.c | 6 +
tools/perf/util/util.h | 2 +
3 files changed, 515 insertions(+), 2 deletions(-)
diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c
index ee3b4e42156e..c6b054b9b12a 100644
--- a/tools/perf/builtin-sched.c
+++ b/tools/perf/builtin-sched.c
@@ -3929,6 +3929,503 @@ static int perf_sched__schedstat_record(struct perf_sched *sched,
return err;
}
+struct schedstat_domain {
+ struct list_head domain_list;
+ struct perf_record_schedstat_domain *domain_data;
+};
+
+struct schedstat_cpu {
+ struct list_head cpu_list;
+ struct list_head domain_head;
+ struct perf_record_schedstat_cpu *cpu_data;
+};
+
+static struct list_head cpu_head = LIST_HEAD_INIT(cpu_head);
+static struct schedstat_cpu *cpu_second_pass;
+static struct schedstat_domain *domain_second_pass;
+static bool after_workload_flag;
+static bool verbose_field;
+
+static void store_schedtstat_cpu_diff(struct schedstat_cpu *after_workload)
+{
+ struct perf_record_schedstat_cpu *before = cpu_second_pass->cpu_data;
+ struct perf_record_schedstat_cpu *after = after_workload->cpu_data;
+ __u16 version = after_workload->cpu_data->version;
+
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \
+ (before->_ver._name = after->_ver._name - before->_ver._name)
+
+ if (version == 15) {
+#include <perf/schedstat-v15.h>
+ } else if (version == 16) {
+#include <perf/schedstat-v16.h>
+ } else if (version == 17) {
+#include <perf/schedstat-v17.h>
+ }
+
+#undef CPU_FIELD
+}
+
+static void store_schedstat_domain_diff(struct schedstat_domain *after_workload)
+{
+ struct perf_record_schedstat_domain *before = domain_second_pass->domain_data;
+ struct perf_record_schedstat_domain *after = after_workload->domain_data;
+ __u16 version = after_workload->domain_data->version;
+
+#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \
+ (before->_ver._name = after->_ver._name - before->_ver._name)
+
+ if (version == 15) {
+#include <perf/schedstat-v15.h>
+ } else if (version == 16) {
+#include <perf/schedstat-v16.h>
+ } else if (version == 17) {
+#include <perf/schedstat-v17.h>
+ }
+#undef DOMAIN_FIELD
+}
+
+static inline void print_cpu_stats(struct perf_record_schedstat_cpu *cs)
+{
+ printf("%-65s %12s %12s\n", "DESC", "COUNT", "PCT_CHANGE");
+ printf("%.*s\n", 100, graph_dotted_line);
+
+#define CALC_PCT(_x, _y) ((_y) ? ((double)(_x) / (_y)) * 100 : 0.0)
+
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \
+ do { \
+ printf("%-65s: " _format, verbose_field ? _desc : #_name, \
+ cs->_ver._name); \
+ if (_is_pct) { \
+ printf(" ( %8.2lf%% )", \
+ CALC_PCT(cs->_ver._name, cs->_ver._pct_of)); \
+ } \
+ printf("\n"); \
+ } while (0)
+
+ if (cs->version == 15) {
+#include <perf/schedstat-v15.h>
+ } else if (cs->version == 16) {
+#include <perf/schedstat-v16.h>
+ } else if (cs->version == 17) {
+#include <perf/schedstat-v17.h>
+ }
+
+#undef CPU_FIELD
+#undef CALC_PCT
+}
+
+static inline void print_domain_stats(struct perf_record_schedstat_domain *ds,
+ __u64 jiffies)
+{
+ printf("%-65s %12s %14s\n", "DESC", "COUNT", "AVG_JIFFIES");
+
+#define DOMAIN_CATEGORY(_desc) \
+ do { \
+ size_t _len = strlen(_desc); \
+ size_t _pre_dash_cnt = (100 - _len) / 2; \
+ size_t _post_dash_cnt = 100 - _len - _pre_dash_cnt; \
+ print_separator2((int)_pre_dash_cnt, _desc, (int)_post_dash_cnt);\
+ } while (0)
+
+#define CALC_AVG(_x, _y) ((_y) ? (long double)(_x) / (_y) : 0.0)
+
+#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \
+ do { \
+ printf("%-65s: " _format, verbose_field ? _desc : #_name, \
+ ds->_ver._name); \
+ if (_is_jiffies) { \
+ printf(" $ %11.2Lf $", \
+ CALC_AVG(jiffies, ds->_ver._name)); \
+ } \
+ printf("\n"); \
+ } while (0)
+
+#define DERIVED_CNT_FIELD(_name, _desc, _format, _x, _y, _z, _ver) \
+ printf("*%-64s: " _format "\n", verbose_field ? _desc : #_name, \
+ (ds->_ver._x) - (ds->_ver._y) - (ds->_ver._z))
+
+#define DERIVED_AVG_FIELD(_name, _desc, _format, _x, _y, _z, _w, _ver) \
+ printf("*%-64s: " _format "\n", verbose_field ? _desc : #_name, \
+ CALC_AVG(ds->_ver._w, \
+ ((ds->_ver._x) - (ds->_ver._y) - (ds->_ver._z))))
+
+ if (ds->version == 15) {
+#include <perf/schedstat-v15.h>
+ } else if (ds->version == 16) {
+#include <perf/schedstat-v16.h>
+ } else if (ds->version == 17) {
+#include <perf/schedstat-v17.h>
+ }
+
+#undef DERIVED_AVG_FIELD
+#undef DERIVED_CNT_FIELD
+#undef DOMAIN_FIELD
+#undef CALC_AVG
+#undef DOMAIN_CATEGORY
+}
+
+static void summarize_schedstat_cpu(struct schedstat_cpu *summary_cpu,
+ struct schedstat_cpu *cptr,
+ int cnt, bool is_last)
+{
+ struct perf_record_schedstat_cpu *summary_cs = summary_cpu->cpu_data,
+ *temp_cs = cptr->cpu_data;
+
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \
+ do { \
+ summary_cs->_ver._name += temp_cs->_ver._name; \
+ if (is_last) \
+ summary_cs->_ver._name /= cnt; \
+ } while (0)
+
+ if (cptr->cpu_data->version == 15) {
+#include <perf/schedstat-v15.h>
+ } else if (cptr->cpu_data->version == 16) {
+#include <perf/schedstat-v16.h>
+ } else if (cptr->cpu_data->version == 17) {
+#include <perf/schedstat-v17.h>
+ }
+#undef CPU_FIELD
+}
+
+static void summarize_schedstat_domain(struct schedstat_domain *summary_domain,
+ struct schedstat_domain *dptr,
+ int cnt, bool is_last)
+{
+ struct perf_record_schedstat_domain *summary_ds = summary_domain->domain_data,
+ *temp_ds = dptr->domain_data;
+
+#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \
+ do { \
+ summary_ds->_ver._name += temp_ds->_ver._name; \
+ if (is_last) \
+ summary_ds->_ver._name /= cnt; \
+ } while (0)
+
+ if (dptr->domain_data->version == 15) {
+#include <perf/schedstat-v15.h>
+ } else if (dptr->domain_data->version == 16) {
+#include <perf/schedstat-v16.h>
+ } else if (dptr->domain_data->version == 17) {
+#include <perf/schedstat-v17.h>
+ }
+#undef DOMAIN_FIELD
+}
+
+/*
+ * get_all_cpu_stats() appends the summary to the head of the list.
+ */
+static int get_all_cpu_stats(struct list_head *head)
+{
+ struct schedstat_cpu *cptr = list_first_entry(head, struct schedstat_cpu, cpu_list);
+ struct schedstat_cpu *summary_head = NULL;
+ struct perf_record_schedstat_domain *ds;
+ struct perf_record_schedstat_cpu *cs;
+ struct schedstat_domain *dptr, *tdptr;
+ bool is_last = false;
+ int cnt = 1;
+ int ret = 0;
+
+ if (cptr) {
+ summary_head = zalloc(sizeof(*summary_head));
+ if (!summary_head)
+ return -ENOMEM;
+
+ summary_head->cpu_data = zalloc(sizeof(*cs));
+ memcpy(summary_head->cpu_data, cptr->cpu_data, sizeof(*cs));
+
+ INIT_LIST_HEAD(&summary_head->domain_head);
+
+ list_for_each_entry(dptr, &cptr->domain_head, domain_list) {
+ tdptr = zalloc(sizeof(*tdptr));
+ if (!tdptr)
+ return -ENOMEM;
+
+ tdptr->domain_data = zalloc(sizeof(*ds));
+ if (!tdptr->domain_data)
+ return -ENOMEM;
+
+ memcpy(tdptr->domain_data, dptr->domain_data, sizeof(*ds));
+ list_add_tail(&tdptr->domain_list, &summary_head->domain_head);
+ }
+ }
+
+ list_for_each_entry(cptr, head, cpu_list) {
+ if (list_is_first(&cptr->cpu_list, head))
+ continue;
+
+ if (list_is_last(&cptr->cpu_list, head))
+ is_last = true;
+
+ cnt++;
+ summarize_schedstat_cpu(summary_head, cptr, cnt, is_last);
+ tdptr = list_first_entry(&summary_head->domain_head, struct schedstat_domain,
+ domain_list);
+
+ list_for_each_entry(dptr, &cptr->domain_head, domain_list) {
+ summarize_schedstat_domain(tdptr, dptr, cnt, is_last);
+ tdptr = list_next_entry(tdptr, domain_list);
+ }
+ }
+
+ list_add(&summary_head->cpu_list, head);
+ return ret;
+}
+
+static int show_schedstat_data(struct list_head *head, struct cpu_domain_map **cd_map)
+{
+ struct schedstat_cpu *cptr = list_first_entry(head, struct schedstat_cpu, cpu_list);
+ __u64 jiffies = cptr->cpu_data->timestamp;
+ struct perf_record_schedstat_domain *ds;
+ struct perf_record_schedstat_cpu *cs;
+ struct schedstat_domain *dptr;
+ bool is_summary = true;
+ int ret = 0;
+
+ printf("Description\n");
+ print_separator2(100, "", 0);
+ printf("%-30s-> %s\n", "DESC", "Description of the field");
+ printf("%-30s-> %s\n", "COUNT", "Value of the field");
+ printf("%-30s-> %s\n", "PCT_CHANGE", "Percent change with corresponding base value");
+ printf("%-30s-> %s\n", "AVG_JIFFIES",
+ "Avg time in jiffies between two consecutive occurrence of event");
+
+ print_separator2(100, "", 0);
+ printf("\n");
+
+ printf("%-65s: %11llu\n", "Time elapsed (in jiffies)", jiffies);
+
+ ret = get_all_cpu_stats(head);
+
+ list_for_each_entry(cptr, head, cpu_list) {
+ cs = cptr->cpu_data;
+ print_separator2(100, "", 0);
+
+ if (is_summary)
+ printf("CPU: <ALL CPUS SUMMARY>\n");
+ else
+ printf("CPU: %d\n", cs->cpu);
+
+ print_separator2(100, "", 0);
+ print_cpu_stats(cs);
+ print_separator2(100, "", 0);
+
+ list_for_each_entry(dptr, &cptr->domain_head, domain_list) {
+ struct domain_info *dinfo;
+
+ ds = dptr->domain_data;
+ dinfo = cd_map[ds->cpu]->domains[ds->domain];
+ if (is_summary) {
+ if (dinfo->dname)
+ printf("CPU: <ALL CPUS SUMMARY> | DOMAIN: %s\n",
+ dinfo->dname);
+ else
+ printf("CPU: <ALL CPUS SUMMARY> | DOMAIN: %d\n",
+ dinfo->domain);
+ } else {
+ if (dinfo->dname)
+ printf("CPU: %d | DOMAIN: %s | DOMAIN_CPUS: ",
+ cs->cpu, dinfo->dname);
+ else
+ printf("CPU: %d | DOMAIN: %d | DOMAIN_CPUS: ",
+ cs->cpu, dinfo->domain);
+
+ printf("%s\n", dinfo->cpulist);
+ }
+ print_separator2(100, "", 0);
+ print_domain_stats(ds, jiffies);
+ print_separator2(100, "", 0);
+ }
+ is_summary = false;
+ }
+ return ret;
+}
+
+/*
+ * Creates a linked list of cpu_data and domain_data. Below represents the structure of the linked
+ * list where CPU0,CPU1,CPU2, ..., CPU(N-1) stores the cpu_data. Here N is the total number of cpus.
+ * Each of the CPU points to the list of domain_data. Here DOMAIN0, DOMAIN1, DOMAIN2, ... represents
+ * the domain_data. Here D0, D1, D2, ..., Dm are the number of domains in the respective cpus.
+ *
+ * +----------+
+ * | CPU_HEAD |
+ * +----------+
+ * |
+ * v
+ * +----------+ +---------+ +---------+ +---------+ +--------------+
+ * | CPU0 | -> | DOMAIN0 | -> | DOMAIN1 | -> | DOMAIN2 | -> ... -> | DOMAIN(D0-1) |
+ * +----------+ +---------+ +---------+ +---------+ +--------------+
+ * |
+ * v
+ * +----------+ +---------+ +---------+ +---------+ +--------------+
+ * | CPU1 | -> | DOMAIN0 | -> | DOMAIN1 | -> | DOMAIN2 | -> ... -> | DOMAIN(D1-1) |
+ * +----------+ +---------+ +---------+ +---------+ +--------------+
+ * |
+ * v
+ * +----------+ +---------+ +---------+ +---------+ +--------------+
+ * | CPU2 | -> | DOMAIN0 | -> | DOMAIN1 | -> | DOMAIN2 | -> ... -> | DOMAIN(D2-1) |
+ * +----------+ +---------+ +---------+ +---------+ +--------------+
+ * |
+ * v
+ * ...
+ * |
+ * v
+ * +----------+ +---------+ +---------+ +---------+ +--------------+
+ * | CPU(N-1) | -> | DOMAIN0 | -> | DOMAIN1 | -> | DOMAIN2 | -> ... -> | DOMAIN(Dm-1) |
+ * +----------+ +---------+ +---------+ +---------+ +--------------+
+ *
+ * Each cpu as well as domain has 2 enties in the event list one before the workload starts and
+ * other after completion of the workload. The above linked list stores the diff of the cpu and
+ * domain statistics.
+ */
+static int perf_sched__process_schedstat(const struct perf_tool *tool __maybe_unused,
+ struct perf_session *session __maybe_unused,
+ union perf_event *event)
+{
+ struct perf_cpu this_cpu;
+ static __u32 initial_cpu;
+
+ switch (event->header.type) {
+ case PERF_RECORD_SCHEDSTAT_CPU:
+ this_cpu.cpu = event->schedstat_cpu.cpu;
+ break;
+ case PERF_RECORD_SCHEDSTAT_DOMAIN:
+ this_cpu.cpu = event->schedstat_domain.cpu;
+ break;
+ default:
+ return 0;
+ }
+
+ if (user_requested_cpus && !perf_cpu_map__has(user_requested_cpus, this_cpu))
+ return 0;
+
+ if (event->header.type == PERF_RECORD_SCHEDSTAT_CPU) {
+ struct schedstat_cpu *temp = zalloc(sizeof(*temp));
+
+ if (!temp)
+ return -ENOMEM;
+
+ temp->cpu_data = zalloc(sizeof(*temp->cpu_data));
+ if (!temp->cpu_data)
+ return -ENOMEM;
+
+ memcpy(temp->cpu_data, &event->schedstat_cpu, sizeof(*temp->cpu_data));
+
+ if (!list_empty(&cpu_head) && temp->cpu_data->cpu == initial_cpu)
+ after_workload_flag = true;
+
+ if (!after_workload_flag) {
+ if (list_empty(&cpu_head))
+ initial_cpu = temp->cpu_data->cpu;
+
+ list_add_tail(&temp->cpu_list, &cpu_head);
+ INIT_LIST_HEAD(&temp->domain_head);
+ } else {
+ if (temp->cpu_data->cpu == initial_cpu) {
+ cpu_second_pass = list_first_entry(&cpu_head, struct schedstat_cpu,
+ cpu_list);
+ cpu_second_pass->cpu_data->timestamp =
+ temp->cpu_data->timestamp - cpu_second_pass->cpu_data->timestamp;
+ } else {
+ cpu_second_pass = list_next_entry(cpu_second_pass, cpu_list);
+ }
+ domain_second_pass = list_first_entry(&cpu_second_pass->domain_head,
+ struct schedstat_domain, domain_list);
+ store_schedtstat_cpu_diff(temp);
+ }
+ } else if (event->header.type == PERF_RECORD_SCHEDSTAT_DOMAIN) {
+ struct schedstat_cpu *cpu_tail;
+ struct schedstat_domain *temp = zalloc(sizeof(*temp));
+
+ if (!temp)
+ return -ENOMEM;
+
+ temp->domain_data = zalloc(sizeof(*temp->domain_data));
+ if (!temp->domain_data)
+ return -ENOMEM;
+
+ memcpy(temp->domain_data, &event->schedstat_domain, sizeof(*temp->domain_data));
+
+ if (!after_workload_flag) {
+ cpu_tail = list_last_entry(&cpu_head, struct schedstat_cpu, cpu_list);
+ list_add_tail(&temp->domain_list, &cpu_tail->domain_head);
+ } else {
+ store_schedstat_domain_diff(temp);
+ domain_second_pass = list_next_entry(domain_second_pass, domain_list);
+ }
+ }
+
+ return 0;
+}
+
+static void free_schedstat(struct list_head *head)
+{
+ struct schedstat_domain *dptr, *n1;
+ struct schedstat_cpu *cptr, *n2;
+
+ list_for_each_entry_safe(cptr, n2, head, cpu_list) {
+ list_for_each_entry_safe(dptr, n1, &cptr->domain_head, domain_list) {
+ list_del_init(&dptr->domain_list);
+ free(dptr);
+ }
+ list_del_init(&cptr->cpu_list);
+ free(cptr);
+ }
+}
+
+static int perf_sched__schedstat_report(struct perf_sched *sched)
+{
+ struct cpu_domain_map **cd_map;
+ struct perf_session *session;
+ struct target target = {};
+ struct perf_data data = {
+ .path = input_name,
+ .mode = PERF_DATA_MODE_READ,
+ };
+ int err = 0;
+
+ sched->tool.schedstat_cpu = perf_sched__process_schedstat;
+ sched->tool.schedstat_domain = perf_sched__process_schedstat;
+
+ session = perf_session__new(&data, &sched->tool);
+ if (IS_ERR(session)) {
+ pr_err("Perf session creation failed.\n");
+ return PTR_ERR(session);
+ }
+
+ if (cpu_list)
+ target.cpu_list = cpu_list;
+ else
+ target.system_wide = true;
+
+ err = evlist__create_maps(session->evlist, &target);
+ if (err < 0)
+ goto out;
+
+ user_requested_cpus = session->evlist->core.user_requested_cpus;
+
+ err = perf_session__process_events(session);
+
+ if (!err) {
+ setup_pager();
+
+ if (list_empty(&cpu_head)) {
+ pr_err("Data is not available\n");
+ err = -1;
+ goto out;
+ }
+
+ cd_map = session->header.env.cpu_domain;
+ err = show_schedstat_data(&cpu_head, cd_map);
+ }
+
+out:
+ free_schedstat(&cpu_head);
+ perf_session__delete(session);
+ return err;
+}
+
static bool schedstat_events_exposed(void)
{
/*
@@ -4106,9 +4603,12 @@ int cmd_sched(int argc, const char **argv)
OPT_PARENT(sched_options)
};
const struct option stats_options[] = {
+ OPT_STRING('i', "input", &input_name, "file",
+ "`stats report` with input filename"),
OPT_STRING('o', "output", &output_name, "file",
"`stats record` with output filename"),
OPT_STRING('C', "cpu", &cpu_list, "cpu", "list of cpus to profile"),
+ OPT_BOOLEAN('v', "verbose", &verbose_field, "Show explanation for fields in the report"),
OPT_END()
};
@@ -4129,7 +4629,7 @@ int cmd_sched(int argc, const char **argv)
NULL
};
const char *stats_usage[] = {
- "perf sched stats {record} [<options>]",
+ "perf sched stats {record|report} [<options>]",
NULL
};
const char *const sched_subcommands[] = { "record", "latency", "map",
@@ -4233,7 +4733,7 @@ int cmd_sched(int argc, const char **argv)
if (!ret)
ret = perf_sched__timehist(&sched);
} else if (!strcmp(argv[0], "stats")) {
- const char *const stats_subcommands[] = {"record", NULL};
+ const char *const stats_subcommands[] = {"record", "report", NULL};
argc = parse_options_subcommand(argc, argv, stats_options,
stats_subcommands,
@@ -4245,6 +4745,11 @@ int cmd_sched(int argc, const char **argv)
argc = parse_options(argc, argv, stats_options,
stats_usage, 0);
return perf_sched__schedstat_record(&sched, argc, argv);
+ } else if (argv[0] && !strcmp(argv[0], "report")) {
+ if (argc)
+ argc = parse_options(argc, argv, stats_options,
+ stats_usage, 0);
+ return perf_sched__schedstat_report(&sched);
}
usage_with_options(stats_usage, stats_options);
} else {
diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c
index b87ff96a9f45..03a603fbcd7d 100644
--- a/tools/perf/util/util.c
+++ b/tools/perf/util/util.c
@@ -299,6 +299,12 @@ void cpumask_to_cpulist(char *cpumask, char *cpulist)
free(bm);
}
+void print_separator2(int pre_dash_cnt, const char *s, int post_dash_cnt)
+{
+ printf("%.*s%s%.*s\n", pre_dash_cnt, graph_dotted_line, s, post_dash_cnt,
+ graph_dotted_line);
+}
+
int rm_rf_perf_data(const char *path)
{
const char *pat[] = {
diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h
index 1572c8cf04e5..394dbfa944ac 100644
--- a/tools/perf/util/util.h
+++ b/tools/perf/util/util.h
@@ -51,6 +51,8 @@ int perf_tip(char **strp, const char *dirpath);
void cpumask_to_cpulist(char *cpumask, char *cpulist);
+void print_separator2(int pre_dash_cnt, const char *s, int post_dash_cnt);
+
#ifndef HAVE_SCHED_GETCPU_SUPPORT
int sched_getcpu(void);
#endif
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH v5 07/10] perf sched stats: Add support for live mode
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (5 preceding siblings ...)
2026-01-19 17:58 ` [PATCH v5 06/10] perf sched stats: Add support for report subcommand Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-22 0:54 ` Arnaldo Carvalho de Melo
2026-03-03 18:47 ` Ian Rogers
2026-01-19 17:58 ` [PATCH v5 08/10] perf sched stats: Add support for diff subcommand Swapnil Sapkal
` (5 subsequent siblings)
12 siblings, 2 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
The live mode works similar to simple `perf stat` command, by profiling
the target and printing results on the terminal as soon as the target
finishes.
Example usage:
# perf sched stats -- true
Description
----------------------------------------------------------------------------------------------------
DESC -> Description of the field
COUNT -> Value of the field
PCT_CHANGE -> Percent change with corresponding base value
AVG_JIFFIES -> Avg time in jiffies between two consecutive occurrence of event
----------------------------------------------------------------------------------------------------
Time elapsed (in jiffies) : 1
----------------------------------------------------------------------------------------------------
CPU: <ALL CPUS SUMMARY>
----------------------------------------------------------------------------------------------------
DESC COUNT PCT_CHANGE
----------------------------------------------------------------------------------------------------
yld_count : 0
array_exp : 0
sched_count : 0
sched_goidle : 0 ( 0.00% )
ttwu_count : 0
ttwu_local : 0 ( 0.00% )
rq_cpu_time : 27875
run_delay : 0 ( 0.00% )
pcount : 0
----------------------------------------------------------------------------------------------------
CPU: <ALL CPUS SUMMARY> | DOMAIN: SMT
----------------------------------------------------------------------------------------------------
DESC COUNT AVG_JIFFIES
----------------------------------------- <Category busy> ------------------------------------------
busy_lb_count : 0 $ 0.00 $
busy_lb_balanced : 0 $ 0.00 $
busy_lb_failed : 0 $ 0.00 $
busy_lb_imbalance_load : 0
busy_lb_imbalance_util : 0
busy_lb_imbalance_task : 0
busy_lb_imbalance_misfit : 0
busy_lb_gained : 0
busy_lb_hot_gained : 0
busy_lb_nobusyq : 0 $ 0.00 $
busy_lb_nobusyg : 0 $ 0.00 $
*busy_lb_success_count : 0
*busy_lb_avg_pulled : 0.00
... and so on. Output will show similar data for all the cpus in the
system.
Co-developed-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
Tested-by: James Clark <james.clark@linaro.org>
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/perf/builtin-sched.c | 99 +++++++++++++++++++++++++++++++++++++-
| 3 +-
| 3 ++
3 files changed, 102 insertions(+), 3 deletions(-)
diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c
index c6b054b9b12a..8993308439bc 100644
--- a/tools/perf/builtin-sched.c
+++ b/tools/perf/builtin-sched.c
@@ -4426,6 +4426,103 @@ static int perf_sched__schedstat_report(struct perf_sched *sched)
return err;
}
+static int process_synthesized_event_live(const struct perf_tool *tool __maybe_unused,
+ union perf_event *event,
+ struct perf_sample *sample __maybe_unused,
+ struct machine *machine __maybe_unused)
+{
+ return perf_sched__process_schedstat(tool, NULL, event);
+}
+
+static int perf_sched__schedstat_live(struct perf_sched *sched,
+ int argc, const char **argv)
+{
+ struct cpu_domain_map **cd_map = NULL;
+ struct target target = {};
+ u32 __maybe_unused md;
+ struct evlist *evlist;
+ u32 nr = 0, sv;
+ int reset = 0;
+ int err = 0;
+
+ signal(SIGINT, sighandler);
+ signal(SIGCHLD, sighandler);
+ signal(SIGTERM, sighandler);
+
+ evlist = evlist__new();
+ if (!evlist)
+ return -ENOMEM;
+
+ /*
+ * `perf sched schedstat` does not support workload profiling (-p pid)
+ * since /proc/schedstat file contains cpu specific data only. Hence, a
+ * profile target is either set of cpus or systemwide, never a process.
+ * Note that, although `-- <workload>` is supported, profile data are
+ * still cpu/systemwide.
+ */
+ if (cpu_list)
+ target.cpu_list = cpu_list;
+ else
+ target.system_wide = true;
+
+ if (argc) {
+ err = evlist__prepare_workload(evlist, &target, argv, false, NULL);
+ if (err)
+ goto out;
+ }
+
+ err = evlist__create_maps(evlist, &target);
+ if (err < 0)
+ goto out;
+
+ user_requested_cpus = evlist->core.user_requested_cpus;
+
+ err = perf_event__synthesize_schedstat(&(sched->tool),
+ process_synthesized_event_live,
+ user_requested_cpus);
+ if (err < 0)
+ goto out;
+
+ err = enable_sched_schedstats(&reset);
+ if (err < 0)
+ goto out;
+
+ if (argc)
+ evlist__start_workload(evlist);
+
+ /* wait for signal */
+ pause();
+
+ if (reset) {
+ err = disable_sched_schedstat();
+ if (err < 0)
+ goto out;
+ }
+
+ err = perf_event__synthesize_schedstat(&(sched->tool),
+ process_synthesized_event_live,
+ user_requested_cpus);
+ if (err)
+ goto out;
+
+ setup_pager();
+
+ if (list_empty(&cpu_head)) {
+ pr_err("Data is not available\n");
+ err = -1;
+ goto out;
+ }
+
+ nr = cpu__max_present_cpu().cpu;
+ cd_map = build_cpu_domain_map(&sv, &md, nr);
+ show_schedstat_data(&cpu_head, cd_map);
+out:
+ free_cpu_domain_info(cd_map, sv, nr);
+ free_schedstat(&cpu_head);
+ evlist__delete(evlist);
+ return err;
+}
+
static bool schedstat_events_exposed(void)
{
/*
@@ -4751,7 +4848,7 @@ int cmd_sched(int argc, const char **argv)
stats_usage, 0);
return perf_sched__schedstat_report(&sched);
}
- usage_with_options(stats_usage, stats_options);
+ return perf_sched__schedstat_live(&sched, argc, argv);
} else {
usage_with_options(sched_usage, sched_options);
}
--git a/tools/perf/util/header.c b/tools/perf/util/header.c
index 673d53bb2a2c..9a15dd4b7640 100644
--- a/tools/perf/util/header.c
+++ b/tools/perf/util/header.c
@@ -1614,8 +1614,7 @@ static int write_pmu_caps(struct feat_fd *ff,
return 0;
}
-static struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains,
- u32 nr)
+struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains, u32 nr)
{
struct domain_info *domain_info;
struct cpu_domain_map **cd_map;
--git a/tools/perf/util/header.h b/tools/perf/util/header.h
index c62f3275a80f..36cc74e2d14d 100644
--- a/tools/perf/util/header.h
+++ b/tools/perf/util/header.h
@@ -211,4 +211,7 @@ char *get_cpuid_str(struct perf_cpu cpu);
char *get_cpuid_allow_env_override(struct perf_cpu cpu);
int strcmp_cpuid_str(const char *s1, const char *s2);
+
+struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains,
+ u32 nr);
#endif /* __PERF_HEADER_H */
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* Re: [PATCH v5 07/10] perf sched stats: Add support for live mode
2026-01-19 17:58 ` [PATCH v5 07/10] perf sched stats: Add support for live mode Swapnil Sapkal
@ 2026-01-22 0:54 ` Arnaldo Carvalho de Melo
2026-01-22 1:32 ` Arnaldo Carvalho de Melo
2026-01-22 15:42 ` Arnaldo Carvalho de Melo
2026-03-03 18:47 ` Ian Rogers
1 sibling, 2 replies; 36+ messages in thread
From: Arnaldo Carvalho de Melo @ 2026-01-22 0:54 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: peterz, mingo, namhyung, irogers, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
On Mon, Jan 19, 2026 at 05:58:29PM +0000, Swapnil Sapkal wrote:
> The live mode works similar to simple `perf stat` command, by profiling
> the target and printing results on the terminal as soon as the target
> finishes.
>
> Example usage:
>
> # perf sched stats -- true
> Description
> ----------------------------------------------------------------------------------------------------
> DESC -> Description of the field
> COUNT -> Value of the field
> PCT_CHANGE -> Percent change with corresponding base value
> AVG_JIFFIES -> Avg time in jiffies between two consecutive occurrence of event
> ----------------------------------------------------------------------------------------------------
>
> Time elapsed (in jiffies) : 1
> ----------------------------------------------------------------------------------------------------
> CPU: <ALL CPUS SUMMARY>
> ----------------------------------------------------------------------------------------------------
> DESC COUNT PCT_CHANGE
> ----------------------------------------------------------------------------------------------------
> yld_count : 0
> array_exp : 0
> sched_count : 0
> sched_goidle : 0 ( 0.00% )
> ttwu_count : 0
> ttwu_local : 0 ( 0.00% )
> rq_cpu_time : 27875
> run_delay : 0 ( 0.00% )
> pcount : 0
> ----------------------------------------------------------------------------------------------------
> CPU: <ALL CPUS SUMMARY> | DOMAIN: SMT
> ----------------------------------------------------------------------------------------------------
> DESC COUNT AVG_JIFFIES
> ----------------------------------------- <Category busy> ------------------------------------------
> busy_lb_count : 0 $ 0.00 $
> busy_lb_balanced : 0 $ 0.00 $
> busy_lb_failed : 0 $ 0.00 $
> busy_lb_imbalance_load : 0
> busy_lb_imbalance_util : 0
> busy_lb_imbalance_task : 0
> busy_lb_imbalance_misfit : 0
> busy_lb_gained : 0
> busy_lb_hot_gained : 0
> busy_lb_nobusyq : 0 $ 0.00 $
> busy_lb_nobusyg : 0 $ 0.00 $
> *busy_lb_success_count : 0
> *busy_lb_avg_pulled : 0.00
>
> ... and so on. Output will show similar data for all the cpus in the
> system.
>
> Co-developed-by: Ravi Bangoria <ravi.bangoria@amd.com>
> Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
> Tested-by: James Clark <james.clark@linaro.org>
> Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
> ---
> tools/perf/builtin-sched.c | 99 +++++++++++++++++++++++++++++++++++++-
> tools/perf/util/header.c | 3 +-
> tools/perf/util/header.h | 3 ++
> 3 files changed, 102 insertions(+), 3 deletions(-)
>
> diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c
> index c6b054b9b12a..8993308439bc 100644
> --- a/tools/perf/builtin-sched.c
> +++ b/tools/perf/builtin-sched.c
> @@ -4426,6 +4426,103 @@ static int perf_sched__schedstat_report(struct perf_sched *sched)
> return err;
> }
>
> +static int process_synthesized_event_live(const struct perf_tool *tool __maybe_unused,
> + union perf_event *event,
> + struct perf_sample *sample __maybe_unused,
> + struct machine *machine __maybe_unused)
> +{
> + return perf_sched__process_schedstat(tool, NULL, event);
> +}
> +
> +static int perf_sched__schedstat_live(struct perf_sched *sched,
> + int argc, const char **argv)
> +{
> + struct cpu_domain_map **cd_map = NULL;
> + struct target target = {};
> + u32 __maybe_unused md;
> + struct evlist *evlist;
> + u32 nr = 0, sv;
> + int reset = 0;
> + int err = 0;
> +
> + signal(SIGINT, sighandler);
> + signal(SIGCHLD, sighandler);
> + signal(SIGTERM, sighandler);
> +
> + evlist = evlist__new();
> + if (!evlist)
> + return -ENOMEM;
> +
> + /*
> + * `perf sched schedstat` does not support workload profiling (-p pid)
> + * since /proc/schedstat file contains cpu specific data only. Hence, a
> + * profile target is either set of cpus or systemwide, never a process.
> + * Note that, although `-- <workload>` is supported, profile data are
> + * still cpu/systemwide.
> + */
> + if (cpu_list)
> + target.cpu_list = cpu_list;
> + else
> + target.system_wide = true;
> +
> + if (argc) {
> + err = evlist__prepare_workload(evlist, &target, argv, false, NULL);
> + if (err)
> + goto out;
> + }
> +
> + err = evlist__create_maps(evlist, &target);
> + if (err < 0)
> + goto out;
> +
> + user_requested_cpus = evlist->core.user_requested_cpus;
> +
> + err = perf_event__synthesize_schedstat(&(sched->tool),
> + process_synthesized_event_live,
> + user_requested_cpus);
> + if (err < 0)
> + goto out;
> +
> + err = enable_sched_schedstats(&reset);
> + if (err < 0)
> + goto out;
> +
> + if (argc)
> + evlist__start_workload(evlist);
> +
> + /* wait for signal */
> + pause();
> +
> + if (reset) {
> + err = disable_sched_schedstat();
> + if (err < 0)
> + goto out;
> + }
> +
> + err = perf_event__synthesize_schedstat(&(sched->tool),
> + process_synthesized_event_live,
> + user_requested_cpus);
> + if (err)
> + goto out;
> +
> + setup_pager();
> +
> + if (list_empty(&cpu_head)) {
> + pr_err("Data is not available\n");
> + err = -1;
> + goto out;
> + }
> +
> + nr = cpu__max_present_cpu().cpu;
> + cd_map = build_cpu_domain_map(&sv, &md, nr);
> + show_schedstat_data(&cpu_head, cd_map);
> +out:
With clang on almalinux 10:
+ make ARCH= CROSS_COMPILE= EXTRA_CFLAGS= -C tools/perf O=/tmp/build/perf CC=clang
make: Entering directory '/git/perf-6.19.0-rc4/tools/perf'
GEN /tmp/build/perf/pmu-events/arch/powerpc/power8/memory.json
builtin-sched.c:4709:6: error: variable 'sv' is used uninitialized whenever 'if' condition is true [-Werror,-Wsometimes-uninitialized]
4709 | if (list_empty(&cpu_head)) {
| ^~~~~~~~~~~~~~~~~~~~~
GEN /tmp/build/perf/pmu-events/arch/powerpc/power8/metrics.json
GEN /tmp/build/perf/pmu-events/arch/powerpc/power8/other.json
CC /tmp/build/perf/tests/kmod-path.o
builtin-sched.c:4719:31: note: uninitialized use occurs here
GEN /tmp/build/perf/pmu-events/arch/powerpc/power8/pipeline.json
4719 | free_cpu_domain_info(cd_map, sv, nr);
| ^~
But then it doesn't build on 32-bit arches with:
20.64 almalinux:9-i386 : FAIL gcc version 11.4.1 20231218 (Red Hat 11.4.1-3) (GCC)
| ^~~
/tmp/build/perf/libperf/include/perf/schedstat-v15.h:4:1: note: in expansion of macro ‘CPU_FIELD’
4 | CPU_FIELD(__u32, yld_count, "sched_yield() count",
| ^~~~~~~~~
In file included from util/event.c:3:
/usr/include/inttypes.h:105:41: note: format string is defined here
105 | # define PRIu64 __PRI64_PREFIX "u"
util/event.c:583:29: error: format ‘%llu’ expects argument of type ‘long long unsigned int’, but argument 3 has type ‘long unsigned int’ [-Werror=format=]
583 | size += fprintf(fp, "%" PRIu64 " ", (unsigned long)cs->_ver._name)
| ^~~
/tmp/build/perf/libperf/include/perf/schedstat-v15.h:6:1: note: in expansion of macro ‘CPU_FIELD’
6 | CPU_FIELD(__u32, array_exp, "Legacy counter can be ignored",
| ^~~~~~~~~
In file included from util/event.c:3:
/usr/include/inttypes.h:105:41: note: format string is defined here
105 | # define PRIu64 __PRI64_PREFIX "u"
util/event.c:583:29: error: format ‘%llu’ expects argument of type ‘long long unsigned int’, but argument 3 has type ‘long unsigned int’ [-Werror=format=]
583 | size += fprintf(fp, "%" PRIu64 " ", (unsigned long)cs->_ver._name)
| ^~~
/tmp/build/perf/libperf/include/perf/schedstat-v15.h:8:1: note: in expansion of macro ‘CPU_FIELD’
8 | CPU_FIELD(__u32, sched_count, "schedule() called",
So continuing to test build up to:
⬢ [acme@toolbx perf-tools-next]$ git log --oneline -5
139b45df27c05531 (HEAD -> perf-tools-next) perf sched stats: Add schedstat v17 support
e092c5d4541da7f0 perf sched stats: Add schedstat v16 support
e659d5e11000b7ff perf sched stats: Add record and rawdump support
900884770020691c perf header: Support CPU DOMAIN relation info
a02829a0e6c65b12 tools/lib: Add list_is_first()
⬢ [acme@toolbx perf-tools-next]$
- Arnaldo
> + free_cpu_domain_info(cd_map, sv, nr);
> + free_schedstat(&cpu_head);
> + evlist__delete(evlist);
> + return err;
> +}
> +
> static bool schedstat_events_exposed(void)
> {
> /*
> @@ -4751,7 +4848,7 @@ int cmd_sched(int argc, const char **argv)
> stats_usage, 0);
> return perf_sched__schedstat_report(&sched);
> }
> - usage_with_options(stats_usage, stats_options);
> + return perf_sched__schedstat_live(&sched, argc, argv);
> } else {
> usage_with_options(sched_usage, sched_options);
> }
> diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c
> index 673d53bb2a2c..9a15dd4b7640 100644
> --- a/tools/perf/util/header.c
> +++ b/tools/perf/util/header.c
> @@ -1614,8 +1614,7 @@ static int write_pmu_caps(struct feat_fd *ff,
> return 0;
> }
>
> -static struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains,
> - u32 nr)
> +struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains, u32 nr)
> {
> struct domain_info *domain_info;
> struct cpu_domain_map **cd_map;
> diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h
> index c62f3275a80f..36cc74e2d14d 100644
> --- a/tools/perf/util/header.h
> +++ b/tools/perf/util/header.h
> @@ -211,4 +211,7 @@ char *get_cpuid_str(struct perf_cpu cpu);
> char *get_cpuid_allow_env_override(struct perf_cpu cpu);
>
> int strcmp_cpuid_str(const char *s1, const char *s2);
> +
> +struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains,
> + u32 nr);
> #endif /* __PERF_HEADER_H */
> --
> 2.43.0
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH v5 07/10] perf sched stats: Add support for live mode
2026-01-22 0:54 ` Arnaldo Carvalho de Melo
@ 2026-01-22 1:32 ` Arnaldo Carvalho de Melo
2026-01-22 15:34 ` Arnaldo Carvalho de Melo
2026-01-22 15:42 ` Arnaldo Carvalho de Melo
1 sibling, 1 reply; 36+ messages in thread
From: Arnaldo Carvalho de Melo @ 2026-01-22 1:32 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: peterz, mingo, namhyung, irogers, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
On Wed, Jan 21, 2026 at 09:54:16PM -0300, Arnaldo Carvalho de Melo wrote:
> So continuing to test build up to:
> ⬢ [acme@toolbx perf-tools-next]$ git log --oneline -5
> 139b45df27c05531 (HEAD -> perf-tools-next) perf sched stats: Add schedstat v17 support
> e092c5d4541da7f0 perf sched stats: Add schedstat v16 support
> e659d5e11000b7ff perf sched stats: Add record and rawdump support
> 900884770020691c perf header: Support CPU DOMAIN relation info
> a02829a0e6c65b12 tools/lib: Add list_is_first()
> ⬢ [acme@toolbx perf-tools-next]$
Just the first two patches build on 32-bit arches, I'll look at it
tomorrow after some coffee, for now at least the first two are merged,
to make some progress.
- Arnaldo
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 07/10] perf sched stats: Add support for live mode
2026-01-22 1:32 ` Arnaldo Carvalho de Melo
@ 2026-01-22 15:34 ` Arnaldo Carvalho de Melo
0 siblings, 0 replies; 36+ messages in thread
From: Arnaldo Carvalho de Melo @ 2026-01-22 15:34 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: peterz, mingo, namhyung, irogers, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
On Wed, Jan 21, 2026 at 10:32:54PM -0300, Arnaldo Carvalho de Melo wrote:
> On Wed, Jan 21, 2026 at 09:54:16PM -0300, Arnaldo Carvalho de Melo wrote:
> > So continuing to test build up to:
>
> > ⬢ [acme@toolbx perf-tools-next]$ git log --oneline -5
> > 139b45df27c05531 (HEAD -> perf-tools-next) perf sched stats: Add schedstat v17 support
> > e092c5d4541da7f0 perf sched stats: Add schedstat v16 support
> > e659d5e11000b7ff perf sched stats: Add record and rawdump support
> > 900884770020691c perf header: Support CPU DOMAIN relation info
> > a02829a0e6c65b12 tools/lib: Add list_is_first()
> > ⬢ [acme@toolbx perf-tools-next]$
> Just the first two patches build on 32-bit arches, I'll look at it
> tomorrow after some coffee, for now at least the first two are merged,
> to make some progress.
Its simple stuff like:
acme@five:~/git/perf-tools-next$ git diff
diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c
index f1051224124437c6..5a98c16e10923d57 100644
--- a/tools/perf/util/event.c
+++ b/tools/perf/util/event.c
@@ -599,7 +599,7 @@ size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp)
size_t size = fprintf(fp, "\ndomain%u ", ds->domain);
#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \
- size += fprintf(fp, "%" PRIu64 " ", (unsigned long)ds->_ver._name)
+ size += fprintf(fp, "%" PRIu64 " ", (uint64_t)ds->_ver._name)
if (version == 15) {
#include <perf/schedstat-v15.h>
acme@five:~/git/perf-tools-next$
I'm fixing it here,
- Arnaldo
^ permalink raw reply related [flat|nested] 36+ messages in thread
* Re: [PATCH v5 07/10] perf sched stats: Add support for live mode
2026-01-22 0:54 ` Arnaldo Carvalho de Melo
2026-01-22 1:32 ` Arnaldo Carvalho de Melo
@ 2026-01-22 15:42 ` Arnaldo Carvalho de Melo
1 sibling, 0 replies; 36+ messages in thread
From: Arnaldo Carvalho de Melo @ 2026-01-22 15:42 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: peterz, mingo, namhyung, irogers, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
On Wed, Jan 21, 2026 at 09:54:16PM -0300, Arnaldo Carvalho de Melo wrote:
> On Mon, Jan 19, 2026 at 05:58:29PM +0000, Swapnil Sapkal wrote:
> > The live mode works similar to simple `perf stat` command, by profiling
> > the target and printing results on the terminal as soon as the target
> > finishes.
> >
> > Example usage:
> >
> > # perf sched stats -- true
> > Description
> > ----------------------------------------------------------------------------------------------------
> > DESC -> Description of the field
> > COUNT -> Value of the field
> > PCT_CHANGE -> Percent change with corresponding base value
> > AVG_JIFFIES -> Avg time in jiffies between two consecutive occurrence of event
> > ----------------------------------------------------------------------------------------------------
> >
> > Time elapsed (in jiffies) : 1
> > ----------------------------------------------------------------------------------------------------
> > CPU: <ALL CPUS SUMMARY>
> > ----------------------------------------------------------------------------------------------------
> > DESC COUNT PCT_CHANGE
> > ----------------------------------------------------------------------------------------------------
> > yld_count : 0
> > array_exp : 0
> > sched_count : 0
> > sched_goidle : 0 ( 0.00% )
> > ttwu_count : 0
> > ttwu_local : 0 ( 0.00% )
> > rq_cpu_time : 27875
> > run_delay : 0 ( 0.00% )
> > pcount : 0
> > ----------------------------------------------------------------------------------------------------
> > CPU: <ALL CPUS SUMMARY> | DOMAIN: SMT
> > ----------------------------------------------------------------------------------------------------
> > DESC COUNT AVG_JIFFIES
> > ----------------------------------------- <Category busy> ------------------------------------------
> > busy_lb_count : 0 $ 0.00 $
> > busy_lb_balanced : 0 $ 0.00 $
> > busy_lb_failed : 0 $ 0.00 $
> > busy_lb_imbalance_load : 0
> > busy_lb_imbalance_util : 0
> > busy_lb_imbalance_task : 0
> > busy_lb_imbalance_misfit : 0
> > busy_lb_gained : 0
> > busy_lb_hot_gained : 0
> > busy_lb_nobusyq : 0 $ 0.00 $
> > busy_lb_nobusyg : 0 $ 0.00 $
> > *busy_lb_success_count : 0
> > *busy_lb_avg_pulled : 0.00
> >
> > ... and so on. Output will show similar data for all the cpus in the
> > system.
> >
> > Co-developed-by: Ravi Bangoria <ravi.bangoria@amd.com>
> > Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
> > Tested-by: James Clark <james.clark@linaro.org>
> > Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
> > ---
> > tools/perf/builtin-sched.c | 99 +++++++++++++++++++++++++++++++++++++-
> > tools/perf/util/header.c | 3 +-
> > tools/perf/util/header.h | 3 ++
> > 3 files changed, 102 insertions(+), 3 deletions(-)
> >
> > diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c
> > index c6b054b9b12a..8993308439bc 100644
> > --- a/tools/perf/builtin-sched.c
> > +++ b/tools/perf/builtin-sched.c
> > @@ -4426,6 +4426,103 @@ static int perf_sched__schedstat_report(struct perf_sched *sched)
> > return err;
> > }
> >
> > +static int process_synthesized_event_live(const struct perf_tool *tool __maybe_unused,
> > + union perf_event *event,
> > + struct perf_sample *sample __maybe_unused,
> > + struct machine *machine __maybe_unused)
> > +{
> > + return perf_sched__process_schedstat(tool, NULL, event);
> > +}
> > +
> > +static int perf_sched__schedstat_live(struct perf_sched *sched,
> > + int argc, const char **argv)
> > +{
> > + struct cpu_domain_map **cd_map = NULL;
> > + struct target target = {};
> > + u32 __maybe_unused md;
> > + struct evlist *evlist;
> > + u32 nr = 0, sv;
> > + int reset = 0;
> > + int err = 0;
> > +
> > + signal(SIGINT, sighandler);
> > + signal(SIGCHLD, sighandler);
> > + signal(SIGTERM, sighandler);
> > +
> > + evlist = evlist__new();
> > + if (!evlist)
> > + return -ENOMEM;
> > +
> > + /*
> > + * `perf sched schedstat` does not support workload profiling (-p pid)
> > + * since /proc/schedstat file contains cpu specific data only. Hence, a
> > + * profile target is either set of cpus or systemwide, never a process.
> > + * Note that, although `-- <workload>` is supported, profile data are
> > + * still cpu/systemwide.
> > + */
> > + if (cpu_list)
> > + target.cpu_list = cpu_list;
> > + else
> > + target.system_wide = true;
> > +
> > + if (argc) {
> > + err = evlist__prepare_workload(evlist, &target, argv, false, NULL);
> > + if (err)
> > + goto out;
> > + }
> > +
> > + err = evlist__create_maps(evlist, &target);
> > + if (err < 0)
> > + goto out;
> > +
> > + user_requested_cpus = evlist->core.user_requested_cpus;
> > +
> > + err = perf_event__synthesize_schedstat(&(sched->tool),
> > + process_synthesized_event_live,
> > + user_requested_cpus);
> > + if (err < 0)
> > + goto out;
> > +
> > + err = enable_sched_schedstats(&reset);
> > + if (err < 0)
> > + goto out;
> > +
> > + if (argc)
> > + evlist__start_workload(evlist);
> > +
> > + /* wait for signal */
> > + pause();
> > +
> > + if (reset) {
> > + err = disable_sched_schedstat();
> > + if (err < 0)
> > + goto out;
> > + }
> > +
> > + err = perf_event__synthesize_schedstat(&(sched->tool),
> > + process_synthesized_event_live,
> > + user_requested_cpus);
> > + if (err)
> > + goto out;
> > +
> > + setup_pager();
> > +
> > + if (list_empty(&cpu_head)) {
> > + pr_err("Data is not available\n");
> > + err = -1;
> > + goto out;
> > + }
> > +
> > + nr = cpu__max_present_cpu().cpu;
> > + cd_map = build_cpu_domain_map(&sv, &md, nr);
> > + show_schedstat_data(&cpu_head, cd_map);
> > +out:
>
> With clang on almalinux 10:
>
> + make ARCH= CROSS_COMPILE= EXTRA_CFLAGS= -C tools/perf O=/tmp/build/perf CC=clang
> make: Entering directory '/git/perf-6.19.0-rc4/tools/perf'
> GEN /tmp/build/perf/pmu-events/arch/powerpc/power8/memory.json
> builtin-sched.c:4709:6: error: variable 'sv' is used uninitialized whenever 'if' condition is true [-Werror,-Wsometimes-uninitialized]
> 4709 | if (list_empty(&cpu_head)) {
> | ^~~~~~~~~~~~~~~~~~~~~
> GEN /tmp/build/perf/pmu-events/arch/powerpc/power8/metrics.json
> GEN /tmp/build/perf/pmu-events/arch/powerpc/power8/other.json
> CC /tmp/build/perf/tests/kmod-path.o
> builtin-sched.c:4719:31: note: uninitialized use occurs here
> GEN /tmp/build/perf/pmu-events/arch/powerpc/power8/pipeline.json
> 4719 | free_cpu_domain_info(cd_map, sv, nr);
> | ^~
>
Moving the label does the trick as free_cpu_domain_info() only needs to
be called if build_cpu_domain_map() was called.
- Arnaldo
diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c
index 8993308439bc5998..ec9fa29196b24f5a 100644
--- a/tools/perf/builtin-sched.c
+++ b/tools/perf/builtin-sched.c
@@ -4516,8 +4516,8 @@ static int perf_sched__schedstat_live(struct perf_sched *sched,
nr = cpu__max_present_cpu().cpu;
cd_map = build_cpu_domain_map(&sv, &md, nr);
show_schedstat_data(&cpu_head, cd_map);
-out:
free_cpu_domain_info(cd_map, sv, nr);
+out:
free_schedstat(&cpu_head);
evlist__delete(evlist);
return err;
^ permalink raw reply related [flat|nested] 36+ messages in thread
* Re: [PATCH v5 07/10] perf sched stats: Add support for live mode
2026-01-19 17:58 ` [PATCH v5 07/10] perf sched stats: Add support for live mode Swapnil Sapkal
2026-01-22 0:54 ` Arnaldo Carvalho de Melo
@ 2026-03-03 18:47 ` Ian Rogers
2026-03-10 10:08 ` Swapnil Sapkal
1 sibling, 1 reply; 36+ messages in thread
From: Ian Rogers @ 2026-03-03 18:47 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: peterz, mingo, acme, namhyung, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
On Mon, Jan 19, 2026 at 10:02 AM Swapnil Sapkal <swapnil.sapkal@amd.com> wrote:
[ snip ]
> + /* wait for signal */
> + pause();
I'm seeing the perf sched stats test hang here, requiring me to kill
the perf process started by the test. It is unclear what signal is
being waited upon, but is there a race condition where the signal
could have occurred before the pause? Should the pause at least be
conditional on that happening? Given that a race condition would exist
even with a test, would it be better to use a condition variable?
Thanks,
Ian
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 07/10] perf sched stats: Add support for live mode
2026-03-03 18:47 ` Ian Rogers
@ 2026-03-10 10:08 ` Swapnil Sapkal
0 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-03-10 10:08 UTC (permalink / raw)
To: Ian Rogers
Cc: peterz, mingo, acme, namhyung, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, James Clark
Hi Ian,
Sorry for the delay.
On 04-03-2026 00:17, Ian Rogers wrote:
> On Mon, Jan 19, 2026 at 10:02 AM Swapnil Sapkal <swapnil.sapkal@amd.com> wrote:
> [ snip ]
>> + /* wait for signal */
>> + pause();
>
> I'm seeing the perf sched stats test hang here, requiring me to kill
> the perf process started by the test. It is unclear what signal is
> being waited upon, but is there a race condition where the signal
> could have occurred before the pause? Should the pause at least be
> conditional on that happening? Given that a race condition would exist
> even with a test, would it be better to use a condition variable?
>
I am able to reproduce the issue. There are other places also where the
same issue can be reproduced. (e.g. with `perf lock contention -b`)
I will propose a fix.
> Thanks,
> Ian
--
Thanks and Regards,
Swapnil
^ permalink raw reply [flat|nested] 36+ messages in thread
* [PATCH v5 08/10] perf sched stats: Add support for diff subcommand
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (6 preceding siblings ...)
2026-01-19 17:58 ` [PATCH v5 07/10] perf sched stats: Add support for live mode Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 09/10] perf sched stats: Add basic perf sched stats test Swapnil Sapkal
` (4 subsequent siblings)
12 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users
`perf sched stats diff` subcommand will take two perf.data files as an
input and it will print the diff between the two perf.data files. The
default input to this subcommnd is perf.data.old and perf.data.
Example usage:
# perf sched stats diff sample1.data sample2.data
Description
----------------------------------------------------------------------------------------------------
DESC -> Description of the field
COUNT -> Value of the field
PCT_CHANGE -> Percent change with corresponding base value
AVG_JIFFIES -> Avg time in jiffies between two consecutive occurrence of event
----------------------------------------------------------------------------------------------------
Time elapsed (in jiffies) : 1, 1
----------------------------------------------------------------------------------------------------
CPU: <ALL CPUS SUMMARY>
----------------------------------------------------------------------------------------------------
DESC COUNT1 COUNT2 PCT_CHANGE PCT_CHANGE1 PCT_CHANGE2
----------------------------------------------------------------------------------------------------
yld_count : 0, 0 | 0.00% |
array_exp : 0, 0 | 0.00% |
sched_count : 0, 0 | 0.00% |
sched_goidle : 0, 0 | 0.00% | ( 0.00%, 0.00% )
ttwu_count : 0, 0 | 0.00% |
ttwu_local : 0, 0 | 0.00% | ( 0.00%, 0.00% )
rq_cpu_time : 32565, 33525 | 2.95% |
run_delay : 0, 436 | 0.00% | ( 0.00%, 1.30% )
pcount : 0, 0 | 0.00% |
----------------------------------------------------------------------------------------------------
CPU: <ALL CPUS SUMMARY> | DOMAIN: SMT
----------------------------------------------------------------------------------------------------
DESC COUNT1 COUNT2 PCT_CHANGE AVG_JIFFIES1 AVG_JIFFIES2
----------------------------------------- <Category busy> ------------------------------------------
busy_lb_count : 0, 0 | 0.00% | $ 0.00, 0.00 $
busy_lb_balanced : 0, 0 | 0.00% | $ 0.00, 0.00 $
busy_lb_failed : 0, 0 | 0.00% | $ 0.00, 0.00 $
busy_lb_imbalance_load : 0, 0 | 0.00% |
busy_lb_imbalance_util : 0, 0 | 0.00% |
busy_lb_imbalance_task : 0, 0 | 0.00% |
busy_lb_imbalance_misfit : 0, 0 | 0.00% |
busy_lb_gained : 0, 0 | 0.00% |
busy_lb_hot_gained : 0, 0 | 0.00% |
busy_lb_nobusyq : 0, 0 | 0.00% | $ 0.00, 0.00 $
busy_lb_nobusyg : 0, 0 | 0.00% | $ 0.00, 0.00 $
*busy_lb_success_count : 0, 0 | 0.00% |
*busy_lb_avg_pulled : 0.00, 0.00 | 0.00% |
... and so on. Output contains the diff of aggregated data of all the
busy, idle and newidle categories for all the sched domains in the system.
Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/perf/builtin-sched.c | 316 ++++++++++++++++++++++++++++++-------
1 file changed, 260 insertions(+), 56 deletions(-)
diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c
index 8993308439bc..01e6cb6a2fbc 100644
--- a/tools/perf/builtin-sched.c
+++ b/tools/perf/builtin-sched.c
@@ -3985,29 +3985,46 @@ static void store_schedstat_domain_diff(struct schedstat_domain *after_workload)
#undef DOMAIN_FIELD
}
-static inline void print_cpu_stats(struct perf_record_schedstat_cpu *cs)
+#define PCT_CHNG(_x, _y) ((_x) ? ((double)((double)(_y) - (_x)) / (_x)) * 100 : 0.0)
+static inline void print_cpu_stats(struct perf_record_schedstat_cpu *cs1,
+ struct perf_record_schedstat_cpu *cs2)
{
- printf("%-65s %12s %12s\n", "DESC", "COUNT", "PCT_CHANGE");
- printf("%.*s\n", 100, graph_dotted_line);
+ printf("%-65s ", "DESC");
+ if (!cs2)
+ printf("%12s %12s", "COUNT", "PCT_CHANGE");
+ else
+ printf("%12s %11s %12s %14s %10s", "COUNT1", "COUNT2", "PCT_CHANGE",
+ "PCT_CHANGE1", "PCT_CHANGE2");
+
+ printf("\n");
+ print_separator2(100, "", 0);
#define CALC_PCT(_x, _y) ((_y) ? ((double)(_x) / (_y)) * 100 : 0.0)
-#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \
- do { \
- printf("%-65s: " _format, verbose_field ? _desc : #_name, \
- cs->_ver._name); \
- if (_is_pct) { \
- printf(" ( %8.2lf%% )", \
- CALC_PCT(cs->_ver._name, cs->_ver._pct_of)); \
- } \
- printf("\n"); \
+#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \
+ do { \
+ printf("%-65s: " _format, verbose_field ? _desc : #_name, \
+ cs1->_ver._name); \
+ if (!cs2) { \
+ if (_is_pct) \
+ printf(" ( %8.2lf%% )", \
+ CALC_PCT(cs1->_ver._name, cs1->_ver._pct_of)); \
+ } else { \
+ printf("," _format " | %8.2lf%% |", cs2->_ver._name, \
+ PCT_CHNG(cs1->_ver._name, cs2->_ver._name)); \
+ if (_is_pct) \
+ printf(" ( %8.2lf%%, %8.2lf%% )", \
+ CALC_PCT(cs1->_ver._name, cs1->_ver._pct_of), \
+ CALC_PCT(cs2->_ver._name, cs2->_ver._pct_of)); \
+ } \
+ printf("\n"); \
} while (0)
- if (cs->version == 15) {
+ if (cs1->version == 15) {
#include <perf/schedstat-v15.h>
- } else if (cs->version == 16) {
+ } else if (cs1->version == 16) {
#include <perf/schedstat-v16.h>
- } else if (cs->version == 17) {
+ } else if (cs1->version == 17) {
#include <perf/schedstat-v17.h>
}
@@ -4015,10 +4032,17 @@ static inline void print_cpu_stats(struct perf_record_schedstat_cpu *cs)
#undef CALC_PCT
}
-static inline void print_domain_stats(struct perf_record_schedstat_domain *ds,
- __u64 jiffies)
+static inline void print_domain_stats(struct perf_record_schedstat_domain *ds1,
+ struct perf_record_schedstat_domain *ds2,
+ __u64 jiffies1, __u64 jiffies2)
{
- printf("%-65s %12s %14s\n", "DESC", "COUNT", "AVG_JIFFIES");
+ printf("%-65s ", "DESC");
+ if (!ds2)
+ printf("%12s %14s", "COUNT", "AVG_JIFFIES");
+ else
+ printf("%12s %11s %12s %16s %12s", "COUNT1", "COUNT2", "PCT_CHANGE",
+ "AVG_JIFFIES1", "AVG_JIFFIES2");
+ printf("\n");
#define DOMAIN_CATEGORY(_desc) \
do { \
@@ -4033,28 +4057,54 @@ static inline void print_domain_stats(struct perf_record_schedstat_domain *ds,
#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \
do { \
printf("%-65s: " _format, verbose_field ? _desc : #_name, \
- ds->_ver._name); \
- if (_is_jiffies) { \
- printf(" $ %11.2Lf $", \
- CALC_AVG(jiffies, ds->_ver._name)); \
+ ds1->_ver._name); \
+ if (!ds2) { \
+ if (_is_jiffies) \
+ printf(" $ %11.2Lf $", \
+ CALC_AVG(jiffies1, ds1->_ver._name)); \
+ } else { \
+ printf("," _format " | %8.2lf%% |", ds2->_ver._name, \
+ PCT_CHNG(ds1->_ver._name, ds2->_ver._name)); \
+ if (_is_jiffies) \
+ printf(" $ %11.2Lf, %11.2Lf $", \
+ CALC_AVG(jiffies1, ds1->_ver._name), \
+ CALC_AVG(jiffies2, ds2->_ver._name)); \
} \
printf("\n"); \
} while (0)
#define DERIVED_CNT_FIELD(_name, _desc, _format, _x, _y, _z, _ver) \
- printf("*%-64s: " _format "\n", verbose_field ? _desc : #_name, \
- (ds->_ver._x) - (ds->_ver._y) - (ds->_ver._z))
+ do { \
+ __u32 t1 = ds1->_ver._x - ds1->_ver._y - ds1->_ver._z; \
+ printf("*%-64s: " _format, verbose_field ? _desc : #_name, t1); \
+ if (ds2) { \
+ __u32 t2 = ds2->_ver._x - ds2->_ver._y - ds2->_ver._z; \
+ printf("," _format " | %8.2lf%% |", t2, \
+ PCT_CHNG(t1, t2)); \
+ } \
+ printf("\n"); \
+ } while (0)
#define DERIVED_AVG_FIELD(_name, _desc, _format, _x, _y, _z, _w, _ver) \
- printf("*%-64s: " _format "\n", verbose_field ? _desc : #_name, \
- CALC_AVG(ds->_ver._w, \
- ((ds->_ver._x) - (ds->_ver._y) - (ds->_ver._z))))
+ do { \
+ __u32 t1 = ds1->_ver._x - ds1->_ver._y - ds1->_ver._z; \
+ printf("*%-64s: " _format, verbose_field ? _desc : #_name, \
+ CALC_AVG(ds1->_ver._w, t1)); \
+ if (ds2) { \
+ __u32 t2 = ds2->_ver._x - ds2->_ver._y - ds2->_ver._z; \
+ printf("," _format " | %8.2Lf%% |", \
+ CALC_AVG(ds2->_ver._w, t2), \
+ PCT_CHNG(CALC_AVG(ds1->_ver._w, t1), \
+ CALC_AVG(ds2->_ver._w, t2))); \
+ } \
+ printf("\n"); \
+ } while (0)
- if (ds->version == 15) {
+ if (ds1->version == 15) {
#include <perf/schedstat-v15.h>
- } else if (ds->version == 16) {
+ } else if (ds1->version == 16) {
#include <perf/schedstat-v16.h>
- } else if (ds->version == 17) {
+ } else if (ds1->version == 17) {
#include <perf/schedstat-v17.h>
}
@@ -4064,6 +4114,7 @@ static inline void print_domain_stats(struct perf_record_schedstat_domain *ds,
#undef CALC_AVG
#undef DOMAIN_CATEGORY
}
+#undef PCT_CHNG
static void summarize_schedstat_cpu(struct schedstat_cpu *summary_cpu,
struct schedstat_cpu *cptr,
@@ -4173,13 +4224,16 @@ static int get_all_cpu_stats(struct list_head *head)
return ret;
}
-static int show_schedstat_data(struct list_head *head, struct cpu_domain_map **cd_map)
+static int show_schedstat_data(struct list_head *head1, struct cpu_domain_map **cd_map1,
+ struct list_head *head2, struct cpu_domain_map **cd_map2,
+ bool summary_only)
{
- struct schedstat_cpu *cptr = list_first_entry(head, struct schedstat_cpu, cpu_list);
- __u64 jiffies = cptr->cpu_data->timestamp;
- struct perf_record_schedstat_domain *ds;
- struct perf_record_schedstat_cpu *cs;
- struct schedstat_domain *dptr;
+ struct schedstat_cpu *cptr1 = list_first_entry(head1, struct schedstat_cpu, cpu_list);
+ struct perf_record_schedstat_domain *ds1 = NULL, *ds2 = NULL;
+ struct perf_record_schedstat_cpu *cs1 = NULL, *cs2 = NULL;
+ struct schedstat_domain *dptr1 = NULL, *dptr2 = NULL;
+ struct schedstat_cpu *cptr2 = NULL;
+ __u64 jiffies1 = 0, jiffies2 = 0;
bool is_summary = true;
int ret = 0;
@@ -4194,49 +4248,100 @@ static int show_schedstat_data(struct list_head *head, struct cpu_domain_map **c
print_separator2(100, "", 0);
printf("\n");
- printf("%-65s: %11llu\n", "Time elapsed (in jiffies)", jiffies);
+ printf("%-65s: ", "Time elapsed (in jiffies)");
+ jiffies1 = cptr1->cpu_data->timestamp;
+ printf("%11llu", jiffies1);
+ if (head2) {
+ cptr2 = list_first_entry(head2, struct schedstat_cpu, cpu_list);
+ jiffies2 = cptr2->cpu_data->timestamp;
+ printf(",%11llu", jiffies2);
+ }
+ printf("\n");
+
+ ret = get_all_cpu_stats(head1);
+ if (cptr2) {
+ ret = get_all_cpu_stats(head2);
+ cptr2 = list_first_entry(head2, struct schedstat_cpu, cpu_list);
+ }
- ret = get_all_cpu_stats(head);
+ list_for_each_entry(cptr1, head1, cpu_list) {
+ struct cpu_domain_map *cd_info1 = NULL, *cd_info2 = NULL;
+
+ cs1 = cptr1->cpu_data;
+ cd_info1 = cd_map1[cs1->cpu];
+ if (cptr2) {
+ cs2 = cptr2->cpu_data;
+ cd_info2 = cd_map2[cs2->cpu];
+ dptr2 = list_first_entry(&cptr2->domain_head, struct schedstat_domain,
+ domain_list);
+ }
+
+ if (cs2 && cs1->cpu != cs2->cpu) {
+ pr_err("Failed because matching cpus not found for diff\n");
+ return -1;
+ }
+
+ if (cd_info2 && cd_info1->nr_domains != cd_info2->nr_domains) {
+ pr_err("Failed because nr_domains is not same for cpus\n");
+ return -1;
+ }
- list_for_each_entry(cptr, head, cpu_list) {
- cs = cptr->cpu_data;
print_separator2(100, "", 0);
if (is_summary)
printf("CPU: <ALL CPUS SUMMARY>\n");
else
- printf("CPU: %d\n", cs->cpu);
+ printf("CPU: %d\n", cs1->cpu);
print_separator2(100, "", 0);
- print_cpu_stats(cs);
+ print_cpu_stats(cs1, cs2);
print_separator2(100, "", 0);
- list_for_each_entry(dptr, &cptr->domain_head, domain_list) {
- struct domain_info *dinfo;
+ list_for_each_entry(dptr1, &cptr1->domain_head, domain_list) {
+ struct domain_info *dinfo1 = NULL, *dinfo2 = NULL;
+
+ ds1 = dptr1->domain_data;
+ dinfo1 = cd_info1->domains[ds1->domain];
+ if (dptr2) {
+ ds2 = dptr2->domain_data;
+ dinfo2 = cd_info2->domains[ds2->domain];
+ }
+
+ if (dinfo2 && dinfo1->domain != dinfo2->domain) {
+ pr_err("Failed because matching domain not found for diff\n");
+ return -1;
+ }
- ds = dptr->domain_data;
- dinfo = cd_map[ds->cpu]->domains[ds->domain];
if (is_summary) {
- if (dinfo->dname)
+ if (dinfo1->dname)
printf("CPU: <ALL CPUS SUMMARY> | DOMAIN: %s\n",
- dinfo->dname);
+ dinfo1->dname);
else
printf("CPU: <ALL CPUS SUMMARY> | DOMAIN: %d\n",
- dinfo->domain);
+ dinfo1->domain);
} else {
- if (dinfo->dname)
+ if (dinfo1->dname)
printf("CPU: %d | DOMAIN: %s | DOMAIN_CPUS: ",
- cs->cpu, dinfo->dname);
+ cs1->cpu, dinfo1->dname);
else
printf("CPU: %d | DOMAIN: %d | DOMAIN_CPUS: ",
- cs->cpu, dinfo->domain);
+ cs1->cpu, dinfo1->domain);
- printf("%s\n", dinfo->cpulist);
+ printf("%s\n", dinfo1->cpulist);
}
print_separator2(100, "", 0);
- print_domain_stats(ds, jiffies);
+ print_domain_stats(ds1, ds2, jiffies1, jiffies2);
print_separator2(100, "", 0);
+
+ if (dptr2)
+ dptr2 = list_next_entry(dptr2, domain_list);
}
+ if (summary_only)
+ break;
+
+ if (cptr2)
+ cptr2 = list_next_entry(cptr2, cpu_list);
+
is_summary = false;
}
return ret;
@@ -4417,7 +4522,7 @@ static int perf_sched__schedstat_report(struct perf_sched *sched)
}
cd_map = session->header.env.cpu_domain;
- err = show_schedstat_data(&cpu_head, cd_map);
+ err = show_schedstat_data(&cpu_head, cd_map, NULL, NULL, false);
}
out:
@@ -4426,6 +4531,100 @@ static int perf_sched__schedstat_report(struct perf_sched *sched)
return err;
}
+static int perf_sched__schedstat_diff(struct perf_sched *sched,
+ int argc, const char **argv)
+{
+ struct cpu_domain_map **cd_map0 = NULL, **cd_map1 = NULL;
+ struct list_head cpu_head_ses0, cpu_head_ses1;
+ struct perf_session *session[2];
+ struct perf_data data[2];
+ int ret = 0, err = 0;
+ static const char *defaults[] = {
+ "perf.data.old",
+ "perf.data",
+ };
+
+ if (argc) {
+ if (argc == 1)
+ defaults[1] = argv[0];
+ else if (argc == 2) {
+ defaults[0] = argv[0];
+ defaults[1] = argv[1];
+ } else {
+ pr_err("perf sched stats diff is not supported with more than 2 files.\n");
+ goto out_ret;
+ }
+ }
+
+ INIT_LIST_HEAD(&cpu_head_ses0);
+ INIT_LIST_HEAD(&cpu_head_ses1);
+
+ sched->tool.schedstat_cpu = perf_sched__process_schedstat;
+ sched->tool.schedstat_domain = perf_sched__process_schedstat;
+
+ data[0].path = defaults[0];
+ data[0].mode = PERF_DATA_MODE_READ;
+ session[0] = perf_session__new(&data[0], &sched->tool);
+ if (IS_ERR(session[0])) {
+ ret = PTR_ERR(session[0]);
+ pr_err("Failed to open %s\n", data[0].path);
+ goto out_delete_ses0;
+ }
+
+ err = perf_session__process_events(session[0]);
+ if (err)
+ goto out_delete_ses0;
+
+ cd_map0 = session[0]->header.env.cpu_domain;
+ list_replace_init(&cpu_head, &cpu_head_ses0);
+ after_workload_flag = false;
+
+ data[1].path = defaults[1];
+ data[1].mode = PERF_DATA_MODE_READ;
+ session[1] = perf_session__new(&data[1], &sched->tool);
+ if (IS_ERR(session[1])) {
+ ret = PTR_ERR(session[1]);
+ pr_err("Failed to open %s\n", data[1].path);
+ goto out_delete_ses1;
+ }
+
+ err = perf_session__process_events(session[1]);
+ if (err)
+ goto out_delete_ses1;
+
+ cd_map1 = session[1]->header.env.cpu_domain;
+ list_replace_init(&cpu_head, &cpu_head_ses1);
+ after_workload_flag = false;
+ setup_pager();
+
+ if (list_empty(&cpu_head_ses1)) {
+ pr_err("Data is not available\n");
+ ret = -1;
+ goto out_delete_ses1;
+ }
+
+ if (list_empty(&cpu_head_ses0)) {
+ pr_err("Data is not available\n");
+ ret = -1;
+ goto out_delete_ses0;
+ }
+
+ show_schedstat_data(&cpu_head_ses0, cd_map0, &cpu_head_ses1, cd_map1, true);
+
+out_delete_ses1:
+ free_schedstat(&cpu_head_ses1);
+ if (!IS_ERR(session[1]))
+ perf_session__delete(session[1]);
+
+out_delete_ses0:
+ free_schedstat(&cpu_head_ses0);
+ if (!IS_ERR(session[0]))
+ perf_session__delete(session[0]);
+
+out_ret:
+ return ret;
+}
+
static int process_synthesized_event_live(const struct perf_tool *tool __maybe_unused,
union perf_event *event,
struct perf_sample *sample __maybe_unused,
@@ -4515,7 +4714,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched,
nr = cpu__max_present_cpu().cpu;
cd_map = build_cpu_domain_map(&sv, &md, nr);
- show_schedstat_data(&cpu_head, cd_map);
+ show_schedstat_data(&cpu_head, cd_map, NULL, NULL, false);
out:
free_cpu_domain_info(cd_map, sv, nr);
free_schedstat(&cpu_head);
@@ -4847,6 +5046,11 @@ int cmd_sched(int argc, const char **argv)
argc = parse_options(argc, argv, stats_options,
stats_usage, 0);
return perf_sched__schedstat_report(&sched);
+ } else if (argv[0] && !strcmp(argv[0], "diff")) {
+ if (argc)
+ argc = parse_options(argc, argv, stats_options,
+ stats_usage, 0);
+ return perf_sched__schedstat_diff(&sched, argc, argv);
}
return perf_sched__schedstat_live(&sched, argc, argv);
} else {
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH v5 09/10] perf sched stats: Add basic perf sched stats test
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (7 preceding siblings ...)
2026-01-19 17:58 ` [PATCH v5 08/10] perf sched stats: Add support for diff subcommand Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-19 17:58 ` [PATCH v5 10/10] perf sched stats: Add details in man page Swapnil Sapkal
` (3 subsequent siblings)
12 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users
Add basic test for perf sched stats {record|report|diff} subcommand.
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/perf/tests/shell/perf_sched_stats.sh | 64 ++++++++++++++++++++++
1 file changed, 64 insertions(+)
create mode 100755 tools/perf/tests/shell/perf_sched_stats.sh
diff --git a/tools/perf/tests/shell/perf_sched_stats.sh b/tools/perf/tests/shell/perf_sched_stats.sh
new file mode 100755
index 000000000000..2b1410b050d0
--- /dev/null
+++ b/tools/perf/tests/shell/perf_sched_stats.sh
@@ -0,0 +1,64 @@
+#!/bin/sh
+# perf sched stats tests
+# SPDX-License-Identifier: GPL-2.0
+
+set -e
+
+err=0
+test_perf_sched_stats_record() {
+ echo "Basic perf sched stats record test"
+ if ! perf sched stats record true 2>&1 | \
+ grep -E -q "[ perf sched stats: Wrote samples to perf.data ]"
+ then
+ echo "Basic perf sched stats record test [Failed]"
+ err=1
+ return
+ fi
+ echo "Basic perf sched stats record test [Success]"
+}
+
+test_perf_sched_stats_report() {
+ echo "Basic perf sched stats report test"
+ perf sched stats record true > /dev/null
+ if ! perf sched stats report 2>&1 | grep -E -q "Description"
+ then
+ echo "Basic perf sched stats report test [Failed]"
+ err=1
+ rm perf.data
+ return
+ fi
+ rm perf.data
+ echo "Basic perf sched stats report test [Success]"
+}
+
+test_perf_sched_stats_live() {
+ echo "Basic perf sched stats live mode test"
+ if ! perf sched stats true 2>&1 | grep -E -q "Description"
+ then
+ echo "Basic perf sched stats live mode test [Failed]"
+ err=1
+ return
+ fi
+ echo "Basic perf sched stats live mode test [Success]"
+}
+
+test_perf_sched_stats_diff() {
+ echo "Basic perf sched stats diff test"
+ perf sched stats record true > /dev/null
+ perf sched stats record true > /dev/null
+ if ! perf sched stats diff > /dev/null
+ then
+ echo "Basic perf sched stats diff test [Failed]"
+ err=1
+ rm perf.data.old perf.data
+ return
+ fi
+ rm perf.data.old perf.data
+ echo "Basic perf sched stats diff test [Success]"
+}
+
+test_perf_sched_stats_record
+test_perf_sched_stats_report
+test_perf_sched_stats_live
+test_perf_sched_stats_diff
+exit $err
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* [PATCH v5 10/10] perf sched stats: Add details in man page
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (8 preceding siblings ...)
2026-01-19 17:58 ` [PATCH v5 09/10] perf sched stats: Add basic perf sched stats test Swapnil Sapkal
@ 2026-01-19 17:58 ` Swapnil Sapkal
2026-01-21 16:09 ` [PATCH v5 00/10] perf sched: Introduce stats tool Chen, Yu C
` (2 subsequent siblings)
12 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-19 17:58 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung, irogers, james.clark
Cc: ravi.bangoria, swapnil.sapkal, yu.c.chen, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users
Document perf sched stats purpose, usage examples and guide on
how to interpret the report data in the perf-sched man page.
Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---
tools/perf/Documentation/perf-sched.txt | 261 +++++++++++++++++++++++-
1 file changed, 260 insertions(+), 1 deletion(-)
diff --git a/tools/perf/Documentation/perf-sched.txt b/tools/perf/Documentation/perf-sched.txt
index 6dbbddb6464d..5bfb7bb6c633 100644
--- a/tools/perf/Documentation/perf-sched.txt
+++ b/tools/perf/Documentation/perf-sched.txt
@@ -8,7 +8,7 @@ perf-sched - Tool to trace/measure scheduler properties (latencies)
SYNOPSIS
--------
[verse]
-'perf sched' {record|latency|map|replay|script|timehist}
+'perf sched' {record|latency|map|replay|script|timehist|stats}
DESCRIPTION
-----------
@@ -80,8 +80,267 @@ There are several variants of 'perf sched':
Times are in msec.usec.
+ 'perf sched stats {record | report | diff} <command>' to capture, report the diff
+ in schedstat counters and show the difference between perf sched stats report
+ respectively. schedstat counters which are present in the linux kernel and are
+ exposed through the file ``/proc/schedstat``. These counters are enabled or disabled
+ via the sysctl governed by the file ``/proc/sys/kernel/sched_schedstats``. These
+ counters accounts for many scheduler events such as ``schedule()`` calls, load-balancing
+ events, ``try_to_wakeup()`` call among others. This is useful in understading the
+ scheduler behavior for the workload.
+
+ Note: The tool will not give correct results if there is topological reordering or
+ online/offline of cpus in between capturing snapshots of `/proc/schedstat`.
+
+ Example usage:
+ perf sched stats record -- sleep 1
+ perf sched stats report
+ perf sched stats diff
+
+ A detailed description of the schedstats can be found in the Kernel Documentation:
+ https://www.kernel.org/doc/html/latest/scheduler/sched-stats.html
+
+ The result can be interprested as follows:
+
+ The `perf sched stats report` starts with description of the columns present in
+ the report. These column names are given before cpu and domain stats to improve
+ the readability of the report.
+
+ ----------------------------------------------------------------------------------------------------
+ DESC -> Description of the field
+ COUNT -> Value of the field
+ PCT_CHANGE -> Percent change with corresponding base value
+ AVG_JIFFIES -> Avg time in jiffies between two consecutive occurrence of event
+ ----------------------------------------------------------------------------------------------------
+
+ Next is the total profiling time in terms of jiffies:
+
+ ----------------------------------------------------------------------------------------------------
+ Time elapsed (in jiffies) : 24537
+ ----------------------------------------------------------------------------------------------------
+
+ Next is CPU scheduling statistics. These are simple diffs of /proc/schedstat CPU lines
+ along with description. The report also prints % relative to base stat.
+
+ In the example below, schedule() left the CPU0 idle 36.58% of the time. 0.45% of total
+ try_to_wake_up() was to wakeup local CPU. And, the total waittime by tasks on CPU0 is
+ 48.70% of the total runtime by tasks on the same CPU.
+
+ ----------------------------------------------------------------------------------------------------
+ CPU 0
+ ----------------------------------------------------------------------------------------------------
+ DESC COUNT PCT_CHANGE
+ ----------------------------------------------------------------------------------------------------
+ yld_count : 0
+ array_exp : 0
+ sched_count : 402267
+ sched_goidle : 147161 ( 36.58% )
+ ttwu_count : 236309
+ ttwu_local : 1062 ( 0.45% )
+ rq_cpu_time : 7083791148
+ run_delay : 3449973971 ( 48.70% )
+ pcount : 255035
+ ----------------------------------------------------------------------------------------------------
+
+ Next is load balancing statistics. For each of the sched domains
+ (eg: `SMT`, `MC`, `DIE`...), the scheduler computes statistics under
+ the following three categories:
+
+ 1) Idle Load Balance: Load balancing performed on behalf of a long
+ idling CPU by some other CPU.
+ 2) Busy Load Balance: Load balancing performed when the CPU was busy.
+ 3) New Idle Balance : Load balancing performed when a CPU just became
+ idle.
+
+ Under each of these three categories, sched stats report provides
+ different load balancing statistics. Along with direct stats, the
+ report also contains derived metrics prefixed with *. Example:
+
+ ----------------------------------------------------------------------------------------------------
+ CPU 0, DOMAIN SMT CPUS 0,64
+ ----------------------------------------------------------------------------------------------------
+ DESC COUNT AVG_JIFFIES
+ ----------------------------------------- <Category busy> ------------------------------------------
+ busy_lb_count : 136 $ 17.08 $
+ busy_lb_balanced : 131 $ 17.73 $
+ busy_lb_failed : 0 $ 0.00 $
+ busy_lb_imbalance_load : 58
+ busy_lb_imbalance_util : 0
+ busy_lb_imbalance_task : 0
+ busy_lb_imbalance_misfit : 0
+ busy_lb_gained : 7
+ busy_lb_hot_gained : 0
+ busy_lb_nobusyq : 2 $ 1161.50 $
+ busy_lb_nobusyg : 129 $ 18.01 $
+ *busy_lb_success_count : 5
+ *busy_lb_avg_pulled : 1.40
+ ----------------------------------------- <Category idle> ------------------------------------------
+ idle_lb_count : 449 $ 5.17 $
+ idle_lb_balanced : 382 $ 6.08 $
+ idle_lb_failed : 3 $ 774.33 $
+ idle_lb_imbalance_load : 0
+ idle_lb_imbalance_util : 0
+ idle_lb_imbalance_task : 71
+ idle_lb_imbalance_misfit : 0
+ idle_lb_gained : 67
+ idle_lb_hot_gained : 0
+ idle_lb_nobusyq : 0 $ 0.00 $
+ idle_lb_nobusyg : 382 $ 6.08 $
+ *idle_lb_success_count : 64
+ *idle_lb_avg_pulled : 1.05
+ ---------------------------------------- <Category newidle> ----------------------------------------
+ newidle_lb_count : 30471 $ 0.08 $
+ newidle_lb_balanced : 28490 $ 0.08 $
+ newidle_lb_failed : 633 $ 3.67 $
+ newidle_lb_imbalance_load : 0
+ newidle_lb_imbalance_util : 0
+ newidle_lb_imbalance_task : 2040
+ newidle_lb_imbalance_misfit : 0
+ newidle_lb_gained : 1348
+ newidle_lb_hot_gained : 0
+ newidle_lb_nobusyq : 6 $ 387.17 $
+ newidle_lb_nobusyg : 26634 $ 0.09 $
+ *newidle_lb_success_count : 1348
+ *newidle_lb_avg_pulled : 1.00
+ ----------------------------------------------------------------------------------------------------
+
+ Consider following line:
+
+ newidle_lb_balanced : 28490 $ 0.08 $
+
+ While profiling was active, the load-balancer found 28490 times the load
+ needs to be balanced on a newly idle CPU 0. Following value encapsulated
+ inside $ is average jiffies between two events (28490 / 24537 = 0.08).
+
+ Next are active_load_balance() stats. alb did not trigger while the
+ profiling was active, hence it's all 0s.
+
+ --------------------------------- <Category active_load_balance()> ---------------------------------
+ alb_count : 0
+ alb_failed : 0
+ alb_pushed : 0
+ ----------------------------------------------------------------------------------------------------
+
+ Next are sched_balance_exec() and sched_balance_fork() stats. They are
+ not used but we kept it in RFC just for legacy purpose. Unless opposed,
+ we plan to remove them in next revision.
+
+ Next are wakeup statistics. For every domain, the report also shows
+ task-wakeup statistics. Example:
+
+ ------------------------------------------ <Wakeup Info> -------------------------------------------
+ ttwu_wake_remote : 1590
+ ttwu_move_affine : 84
+ ttwu_move_balance : 0
+ ----------------------------------------------------------------------------------------------------
+
+ Same set of stats are reported for each CPU and each domain level.
+
+ How to interpret the diff
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ The `perf sched stats diff` will also start with explaining the columns
+ present in the diff. Then it will show the diff in time in terms of
+ jiffies. The order of the values depends on the order of input data
+ files. It will take `perf.data.old` and `perf.data` respectively as the
+ defaults for comparison. Example:
+
+ ----------------------------------------------------------------------------------------------------
+ Time elapsed (in jiffies) : 2009, 2001
+ ----------------------------------------------------------------------------------------------------
+
+ Below is the sample representing the difference in cpu and domain stats of
+ two runs. Here third column or the values enclosed in `|...|` shows the
+ percent change between the two. Second and fourth columns shows the
+ side-by-side representions of the corresponding fields from `perf sched
+ stats report`.
+
+ ----------------------------------------------------------------------------------------------------
+ CPU <ALL CPUS SUMMARY>
+ ----------------------------------------------------------------------------------------------------
+ DESC COUNT1 COUNT2 PCT_CHANG>
+ ----------------------------------------------------------------------------------------------------
+ yld_count : 0, 0 | 0.00>
+ array_exp : 0, 0 | 0.00>
+ sched_count : 528533, 412573 | -21.94>
+ sched_goidle : 193426, 146082 | -24.48>
+ ttwu_count : 313134, 385975 | 23.26>
+ ttwu_local : 1126, 1282 | 13.85>
+ rq_cpu_time : 8257200244, 8301250047 | 0.53>
+ run_delay : 4728347053, 3997100703 | -15.47>
+ pcount : 335031, 266396 | -20.49>
+ ----------------------------------------------------------------------------------------------------
+
+ Below is the sample of domain stats diff:
+
+ ----------------------------------------------------------------------------------------------------
+ CPU <ALL CPUS SUMMARY>, DOMAIN SMT
+ ----------------------------------------------------------------------------------------------------
+ DESC COUNT1 COUNT2 PCT_CHANG>
+ ----------------------------------------- <Category busy> ------------------------------------------
+ busy_lb_count : 122, 80 | -34.43>
+ busy_lb_balanced : 115, 76 | -33.91>
+ busy_lb_failed : 1, 3 | 200.00>
+ busy_lb_imbalance_load : 35, 49 | 40.00>
+ busy_lb_imbalance_util : 0, 0 | 0.00>
+ busy_lb_imbalance_task : 0, 0 | 0.00>
+ busy_lb_imbalance_misfit : 0, 0 | 0.00>
+ busy_lb_gained : 7, 2 | -71.43>
+ busy_lb_hot_gained : 0, 0 | 0.00>
+ busy_lb_nobusyq : 0, 0 | 0.00>
+ busy_lb_nobusyg : 115, 76 | -33.91>
+ *busy_lb_success_count : 6, 1 | -83.33>
+ *busy_lb_avg_pulled : 1.17, 2.00 | 71.43>
+ ----------------------------------------- <Category idle> ------------------------------------------
+ idle_lb_count : 568, 620 | 9.15>
+ idle_lb_balanced : 462, 449 | -2.81>
+ idle_lb_failed : 11, 21 | 90.91>
+ idle_lb_imbalance_load : 0, 0 | 0.00>
+ idle_lb_imbalance_util : 0, 0 | 0.00>
+ idle_lb_imbalance_task : 115, 189 | 64.35>
+ idle_lb_imbalance_misfit : 0, 0 | 0.00>
+ idle_lb_gained : 103, 169 | 64.08>
+ idle_lb_hot_gained : 0, 0 | 0.00>
+ idle_lb_nobusyq : 0, 0 | 0.00>
+ idle_lb_nobusyg : 462, 449 | -2.81>
+ *idle_lb_success_count : 95, 150 | 57.89>
+ *idle_lb_avg_pulled : 1.08, 1.13 | 3.92>
+ ---------------------------------------- <Category newidle> ----------------------------------------
+ newidle_lb_count : 16961, 3155 | -81.40>
+ newidle_lb_balanced : 15646, 2556 | -83.66>
+ newidle_lb_failed : 397, 142 | -64.23>
+ newidle_lb_imbalance_load : 0, 0 | 0.00>
+ newidle_lb_imbalance_util : 0, 0 | 0.00>
+ newidle_lb_imbalance_task : 1376, 655 | -52.40>
+ newidle_lb_imbalance_misfit : 0, 0 | 0.00>
+ newidle_lb_gained : 917, 457 | -50.16>
+ newidle_lb_hot_gained : 0, 0 | 0.00>
+ newidle_lb_nobusyq : 3, 1 | -66.67>
+ newidle_lb_nobusyg : 14480, 2103 | -85.48>
+ *newidle_lb_success_count : 918, 457 | -50.22>
+ *newidle_lb_avg_pulled : 1.00, 1.00 | 0.11>
+ --------------------------------- <Category active_load_balance()> ---------------------------------
+ alb_count : 0, 1 | 0.00>
+ alb_failed : 0, 0 | 0.00>
+ alb_pushed : 0, 1 | 0.00>
+ --------------------------------- <Category sched_balance_exec()> ----------------------------------
+ sbe_count : 0, 0 | 0.00>
+ sbe_balanced : 0, 0 | 0.00>
+ sbe_pushed : 0, 0 | 0.00>
+ --------------------------------- <Category sched_balance_fork()> ----------------------------------
+ sbf_count : 0, 0 | 0.00>
+ sbf_balanced : 0, 0 | 0.00>
+ sbf_pushed : 0, 0 | 0.00>
+ ------------------------------------------ <Wakeup Info> -------------------------------------------
+ ttwu_wake_remote : 2031, 2914 | 43.48>
+ ttwu_move_affine : 73, 124 | 69.86>
+ ttwu_move_balance : 0, 0 | 0.00>
+ ----------------------------------------------------------------------------------------------------
+
OPTIONS
-------
+Applicable to {record|latency|map|replay|script}
+
-i::
--input=<file>::
Input file name. (default: perf.data unless stdin is a fifo)
--
2.43.0
^ permalink raw reply related [flat|nested] 36+ messages in thread* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (9 preceding siblings ...)
2026-01-19 17:58 ` [PATCH v5 10/10] perf sched stats: Add details in man page Swapnil Sapkal
@ 2026-01-21 16:09 ` Chen, Yu C
2026-01-21 16:33 ` Peter Zijlstra
2026-01-21 17:52 ` Shrikanth Hegde
2026-01-21 22:59 ` Namhyung Kim
12 siblings, 1 reply; 36+ messages in thread
From: Chen, Yu C @ 2026-01-21 16:09 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: ravi.bangoria, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, peterz, mingo, namhyung, irogers, james.clark,
acme
On 1/20/2026 1:58 AM, Swapnil Sapkal wrote:
> MOTIVATION
> ----------
>
> Existing `perf sched` is quite exhaustive and provides lot of insights
> into scheduler behavior but it quickly becomes impractical to use for
> long running or scheduler intensive workload. For ex, `perf sched record`
> has ~7.77% overhead on hackbench (with 25 groups each running 700K loops
> on a 2-socket 128 Cores 256 Threads 3rd Generation EPYC Server), and it
> generates huge 56G perf.data for which perf takes ~137 mins to prepare
> and write it to disk [1].
>
> Unlike `perf sched record`, which hooks onto set of scheduler tracepoints
> and generates samples on a tracepoint hit, `perf sched stats record` takes
> snapshot of the /proc/schedstat file before and after the workload, i.e.
> there is almost zero interference on workload run. Also, it takes very
> minimal time to parse /proc/schedstat, convert it into perf samples and
> save those samples into perf.data file. Result perf.data file is much
> smaller. So, overall `perf sched stats record` is much more light weight
> compare to `perf sched record`.
>
> We, internally at AMD, have been using this (a variant of this, known as
> "sched-scoreboard"[2]) and found it to be very useful to analyse impact
> of any scheduler code changes[3][4]. Prateek used v2[5] of this patch
> series to report the analysis[6][7].
>
> Please note that, this is not a replacement of perf sched record/report.
> The intended users of the new tool are scheduler developers, not regular
> users.
>
> USAGE
> -----
>
> # perf sched stats record
> # perf sched stats report
> # perf sched stats diff
>
> Note: Although `perf sched stats` tool supports workload profiling syntax
> (i.e. -- <workload> ), the recorded profile is still systemwide since the
> /proc/schedstat is a systemwide file.
>
I found this is useful for load balance analysis on my
384 CPUs system with 6.19.0-rc1, please feel free to add
Tested-by: Chen Yu <yu.c.chen@intel.com>
thanks,
Chenyu
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-21 16:09 ` [PATCH v5 00/10] perf sched: Introduce stats tool Chen, Yu C
@ 2026-01-21 16:33 ` Peter Zijlstra
2026-01-21 17:12 ` Ian Rogers
0 siblings, 1 reply; 36+ messages in thread
From: Peter Zijlstra @ 2026-01-21 16:33 UTC (permalink / raw)
To: Chen, Yu C
Cc: Swapnil Sapkal, ravi.bangoria, mark.rutland, alexander.shishkin,
jolsa, rostedt, vincent.guittot, adrian.hunter, kan.liang,
gautham.shenoy, kprateek.nayak, juri.lelli, yangjihong, void, tj,
sshegde, ctshao, quic_zhonhan, thomas.falcon, blakejones, ashelat,
leo.yan, dvyukov, ak, yujie.liu, graham.woodward, ben.gainey,
vineethr, tim.c.chen, linux, santosh.shukla, sandipan.das,
linux-kernel, linux-perf-users, mingo, namhyung, irogers,
james.clark, acme
On Thu, Jan 22, 2026 at 12:09:25AM +0800, Chen, Yu C wrote:
> On 1/20/2026 1:58 AM, Swapnil Sapkal wrote:
> > MOTIVATION
> > ----------
> >
> > Existing `perf sched` is quite exhaustive and provides lot of insights
> > into scheduler behavior but it quickly becomes impractical to use for
> > long running or scheduler intensive workload. For ex, `perf sched record`
> > has ~7.77% overhead on hackbench (with 25 groups each running 700K loops
> > on a 2-socket 128 Cores 256 Threads 3rd Generation EPYC Server), and it
> > generates huge 56G perf.data for which perf takes ~137 mins to prepare
> > and write it to disk [1].
> >
> > Unlike `perf sched record`, which hooks onto set of scheduler tracepoints
> > and generates samples on a tracepoint hit, `perf sched stats record` takes
> > snapshot of the /proc/schedstat file before and after the workload, i.e.
> > there is almost zero interference on workload run. Also, it takes very
> > minimal time to parse /proc/schedstat, convert it into perf samples and
> > save those samples into perf.data file. Result perf.data file is much
> > smaller. So, overall `perf sched stats record` is much more light weight
> > compare to `perf sched record`.
> >
> > We, internally at AMD, have been using this (a variant of this, known as
> > "sched-scoreboard"[2]) and found it to be very useful to analyse impact
> > of any scheduler code changes[3][4]. Prateek used v2[5] of this patch
> > series to report the analysis[6][7].
> >
> > Please note that, this is not a replacement of perf sched record/report.
> > The intended users of the new tool are scheduler developers, not regular
> > users.
> >
> > USAGE
> > -----
> >
> > # perf sched stats record
> > # perf sched stats report
> > # perf sched stats diff
> >
> > Note: Although `perf sched stats` tool supports workload profiling syntax
> > (i.e. -- <workload> ), the recorded profile is still systemwide since the
> > /proc/schedstat is a systemwide file.
> >
>
> I found this is useful for load balance analysis on my
> 384 CPUs system with 6.19.0-rc1, please feel free to add
>
> Tested-by: Chen Yu <yu.c.chen@intel.com>
Yeah, I've used a previous version for a while, was very nice.
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-21 16:33 ` Peter Zijlstra
@ 2026-01-21 17:12 ` Ian Rogers
2026-01-21 19:51 ` Peter Zijlstra
0 siblings, 1 reply; 36+ messages in thread
From: Ian Rogers @ 2026-01-21 17:12 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Chen, Yu C, Swapnil Sapkal, ravi.bangoria, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, mingo, namhyung, james.clark, acme
On Wed, Jan 21, 2026 at 8:33 AM Peter Zijlstra <peterz@infradead.org> wrote:
>
> On Thu, Jan 22, 2026 at 12:09:25AM +0800, Chen, Yu C wrote:
> > On 1/20/2026 1:58 AM, Swapnil Sapkal wrote:
> > > MOTIVATION
> > > ----------
> > >
> > > Existing `perf sched` is quite exhaustive and provides lot of insights
> > > into scheduler behavior but it quickly becomes impractical to use for
> > > long running or scheduler intensive workload. For ex, `perf sched record`
> > > has ~7.77% overhead on hackbench (with 25 groups each running 700K loops
> > > on a 2-socket 128 Cores 256 Threads 3rd Generation EPYC Server), and it
> > > generates huge 56G perf.data for which perf takes ~137 mins to prepare
> > > and write it to disk [1].
> > >
> > > Unlike `perf sched record`, which hooks onto set of scheduler tracepoints
> > > and generates samples on a tracepoint hit, `perf sched stats record` takes
> > > snapshot of the /proc/schedstat file before and after the workload, i.e.
> > > there is almost zero interference on workload run. Also, it takes very
> > > minimal time to parse /proc/schedstat, convert it into perf samples and
> > > save those samples into perf.data file. Result perf.data file is much
> > > smaller. So, overall `perf sched stats record` is much more light weight
> > > compare to `perf sched record`.
> > >
> > > We, internally at AMD, have been using this (a variant of this, known as
> > > "sched-scoreboard"[2]) and found it to be very useful to analyse impact
> > > of any scheduler code changes[3][4]. Prateek used v2[5] of this patch
> > > series to report the analysis[6][7].
> > >
> > > Please note that, this is not a replacement of perf sched record/report.
> > > The intended users of the new tool are scheduler developers, not regular
> > > users.
> > >
> > > USAGE
> > > -----
> > >
> > > # perf sched stats record
> > > # perf sched stats report
> > > # perf sched stats diff
> > >
> > > Note: Although `perf sched stats` tool supports workload profiling syntax
> > > (i.e. -- <workload> ), the recorded profile is still systemwide since the
> > > /proc/schedstat is a systemwide file.
> > >
> >
> > I found this is useful for load balance analysis on my
> > 384 CPUs system with 6.19.0-rc1, please feel free to add
> >
> > Tested-by: Chen Yu <yu.c.chen@intel.com>
>
> Yeah, I've used a previous version for a while, was very nice.
>
> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Ian Rogers <irogers@google.com>
I'm still wondering if we can make some of the /proc/schedstat data
appear as tool events similar to proposals for networking and memory
tool events in:
https://lore.kernel.org/lkml/20260104011738.475680-1-irogers@google.com/
Thanks,
Ian
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-21 17:12 ` Ian Rogers
@ 2026-01-21 19:51 ` Peter Zijlstra
2026-01-21 20:08 ` Arnaldo Carvalho de Melo
0 siblings, 1 reply; 36+ messages in thread
From: Peter Zijlstra @ 2026-01-21 19:51 UTC (permalink / raw)
To: Ian Rogers
Cc: Chen, Yu C, Swapnil Sapkal, ravi.bangoria, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, mingo, namhyung, james.clark, acme
On Wed, Jan 21, 2026 at 09:12:20AM -0800, Ian Rogers wrote:
> I'm still wondering if we can make some of the /proc/schedstat data
> appear as tool events similar to proposals for networking and memory
> tool events in:
> https://lore.kernel.org/lkml/20260104011738.475680-1-irogers@google.com/
Yeah, someone with a bit of spare time could make it happen:
https://lkml.kernel.org/r/20250703141800.GX1613200@noisy.programming.kicks-ass.net
It would be:
- convert all the accounting sites to tracepoints
- write modules that connect to relevant tracepoints to provide the
'legacy' interface
- convince world+dog to not auto-load above modules
- have beer and wait for people to complain about something :-)
I'm sure there's 'interesting' details glossed over, but I *think* it
should be doable.
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-21 19:51 ` Peter Zijlstra
@ 2026-01-21 20:08 ` Arnaldo Carvalho de Melo
2026-01-21 20:10 ` Peter Zijlstra
0 siblings, 1 reply; 36+ messages in thread
From: Arnaldo Carvalho de Melo @ 2026-01-21 20:08 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ian Rogers, Chen, Yu C, Swapnil Sapkal, ravi.bangoria,
mark.rutland, alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, mingo, namhyung, james.clark
On Wed, Jan 21, 2026 at 08:51:55PM +0100, Peter Zijlstra wrote:
> On Wed, Jan 21, 2026 at 09:12:20AM -0800, Ian Rogers wrote:
>
> > I'm still wondering if we can make some of the /proc/schedstat data
> > appear as tool events similar to proposals for networking and memory
> > tool events in:
> > https://lore.kernel.org/lkml/20260104011738.475680-1-irogers@google.com/
>
> Yeah, someone with a bit of spare time could make it happen:
>
> https://lkml.kernel.org/r/20250703141800.GX1613200@noisy.programming.kicks-ass.net
>
> It would be:
>
> - convert all the accounting sites to tracepoints
> - write modules that connect to relevant tracepoints to provide the
> 'legacy' interface
> - convince world+dog to not auto-load above modules
> - have beer and wait for people to complain about something :-)
>
> I'm sure there's 'interesting' details glossed over, but I *think* it
> should be doable.
Until then, since you and Ian acked this one, I'm going over it and
tentatively merging it,
Thanks,
- Arnaldo
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-21 20:08 ` Arnaldo Carvalho de Melo
@ 2026-01-21 20:10 ` Peter Zijlstra
2026-01-22 16:00 ` Arnaldo Carvalho de Melo
0 siblings, 1 reply; 36+ messages in thread
From: Peter Zijlstra @ 2026-01-21 20:10 UTC (permalink / raw)
To: Arnaldo Carvalho de Melo
Cc: Ian Rogers, Chen, Yu C, Swapnil Sapkal, ravi.bangoria,
mark.rutland, alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, mingo, namhyung, james.clark
On Wed, Jan 21, 2026 at 05:08:04PM -0300, Arnaldo Carvalho de Melo wrote:
> Until then, since you and Ian acked this one, I'm going over it and
> tentatively merging it,
Thanks!
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-21 20:10 ` Peter Zijlstra
@ 2026-01-22 16:00 ` Arnaldo Carvalho de Melo
2026-01-23 16:32 ` Swapnil Sapkal
0 siblings, 1 reply; 36+ messages in thread
From: Arnaldo Carvalho de Melo @ 2026-01-22 16:00 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ian Rogers, Chen, Yu C, Swapnil Sapkal, ravi.bangoria,
mark.rutland, alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, mingo, namhyung, james.clark
On Wed, Jan 21, 2026 at 09:10:34PM +0100, Peter Zijlstra wrote:
> On Wed, Jan 21, 2026 at 05:08:04PM -0300, Arnaldo Carvalho de Melo wrote:
> > Until then, since you and Ian acked this one, I'm going over it and
> > tentatively merging it,
> Thanks!
It is on
https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git tmp.perf-tools-next
Has the two minor fixes to overcome that 'sv' maybe used uninitialized
on the error path + 32-bit build, I will move to perf-tools-next after
it gets built on all the test containers, later today.
- Arnaldo
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-22 16:00 ` Arnaldo Carvalho de Melo
@ 2026-01-23 16:32 ` Swapnil Sapkal
2026-01-23 16:37 ` Arnaldo Carvalho de Melo
0 siblings, 1 reply; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-23 16:32 UTC (permalink / raw)
To: Arnaldo Carvalho de Melo, Peter Zijlstra
Cc: Ian Rogers, Chen, Yu C, ravi.bangoria, mark.rutland,
alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, mingo, namhyung, james.clark
Hello Arnaldo,
On 22-01-2026 21:30, Arnaldo Carvalho de Melo wrote:
> On Wed, Jan 21, 2026 at 09:10:34PM +0100, Peter Zijlstra wrote:
>> On Wed, Jan 21, 2026 at 05:08:04PM -0300, Arnaldo Carvalho de Melo wrote:
>>> Until then, since you and Ian acked this one, I'm going over it and
>>> tentatively merging it,
>
>> Thanks!
>
> It is on
>
> https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git tmp.perf-tools-next
>
> Has the two minor fixes to overcome that 'sv' maybe used uninitialized
> on the error path + 32-bit build, I will move to perf-tools-next after
> it gets built on all the test containers, later today.
>
> - Arnaldo
Thank you for the two fixes.
I have a fix for the man page, do you want me to send a separate fix
patch or will you please consider it with this series?
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-23 16:32 ` Swapnil Sapkal
@ 2026-01-23 16:37 ` Arnaldo Carvalho de Melo
2026-01-23 16:42 ` Swapnil Sapkal
0 siblings, 1 reply; 36+ messages in thread
From: Arnaldo Carvalho de Melo @ 2026-01-23 16:37 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: Peter Zijlstra, Ian Rogers, Chen, Yu C, ravi.bangoria,
mark.rutland, alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, mingo, namhyung, james.clark
On Fri, Jan 23, 2026 at 10:02:11PM +0530, Swapnil Sapkal wrote:
> Hello Arnaldo,
>
> On 22-01-2026 21:30, Arnaldo Carvalho de Melo wrote:
> > On Wed, Jan 21, 2026 at 09:10:34PM +0100, Peter Zijlstra wrote:
> > > On Wed, Jan 21, 2026 at 05:08:04PM -0300, Arnaldo Carvalho de Melo wrote:
> > > > Until then, since you and Ian acked this one, I'm going over it and
> > > > tentatively merging it,
> > > Thanks!
> >
> > It is on
> > https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git tmp.perf-tools-next
> > Has the two minor fixes to overcome that 'sv' maybe used uninitialized
> > on the error path + 32-bit build, I will move to perf-tools-next after
> > it gets built on all the test containers, later today.
> Thank you for the two fixes.
> I have a fix for the man page, do you want me to send a separate fix patch
> or will you please consider it with this series?
I pulled it already into perf-tools-next, so you can now go on working
from there, with patches fixing things like the NR_CPUS case, etc,
please add Fixes: tags pointing to the csets where the problems were
introduced.
- Arnaldo
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-23 16:37 ` Arnaldo Carvalho de Melo
@ 2026-01-23 16:42 ` Swapnil Sapkal
0 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-23 16:42 UTC (permalink / raw)
To: Arnaldo Carvalho de Melo
Cc: Peter Zijlstra, Ian Rogers, Chen, Yu C, ravi.bangoria,
mark.rutland, alexander.shishkin, jolsa, rostedt, vincent.guittot,
adrian.hunter, kan.liang, gautham.shenoy, kprateek.nayak,
juri.lelli, yangjihong, void, tj, sshegde, ctshao, quic_zhonhan,
thomas.falcon, blakejones, ashelat, leo.yan, dvyukov, ak,
yujie.liu, graham.woodward, ben.gainey, vineethr, tim.c.chen,
linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, mingo, namhyung, james.clark
Hello Arnaldo,
On 23-01-2026 22:07, Arnaldo Carvalho de Melo wrote:
> On Fri, Jan 23, 2026 at 10:02:11PM +0530, Swapnil Sapkal wrote:
>> Hello Arnaldo,
>>
>> On 22-01-2026 21:30, Arnaldo Carvalho de Melo wrote:
>>> On Wed, Jan 21, 2026 at 09:10:34PM +0100, Peter Zijlstra wrote:
>>>> On Wed, Jan 21, 2026 at 05:08:04PM -0300, Arnaldo Carvalho de Melo wrote:
>>>>> Until then, since you and Ian acked this one, I'm going over it and
>>>>> tentatively merging it,
>>>> Thanks!
>>>
>>> It is on
>
>>> https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git tmp.perf-tools-next
>
>>> Has the two minor fixes to overcome that 'sv' maybe used uninitialized
>>> on the error path + 32-bit build, I will move to perf-tools-next after
>>> it gets built on all the test containers, later today.
>
>> Thank you for the two fixes.
>
>> I have a fix for the man page, do you want me to send a separate fix patch
>> or will you please consider it with this series?
>
> I pulled it already into perf-tools-next, so you can now go on working
> from there, with patches fixing things like the NR_CPUS case, etc,
> please add Fixes: tags pointing to the csets where the problems were
> introduced.
>
> - Arnaldo
Thank you for pulling the changes. I will send the fixes separately.
- Swapnil
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (10 preceding siblings ...)
2026-01-21 16:09 ` [PATCH v5 00/10] perf sched: Introduce stats tool Chen, Yu C
@ 2026-01-21 17:52 ` Shrikanth Hegde
2026-01-23 16:19 ` Swapnil Sapkal
2026-01-21 22:59 ` Namhyung Kim
12 siblings, 1 reply; 36+ messages in thread
From: Shrikanth Hegde @ 2026-01-21 17:52 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: ravi.bangoria, yu.c.chen, mark.rutland, alexander.shishkin, jolsa,
rostedt, vincent.guittot, adrian.hunter, kan.liang,
gautham.shenoy, kprateek.nayak, juri.lelli, yangjihong, void, tj,
ctshao, quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, peterz, mingo, acme, namhyung, irogers,
james.clark
On 1/19/26 11:28 PM, Swapnil Sapkal wrote:
> MOTIVATION
> ----------
>
> Existing `perf sched` is quite exhaustive and provides lot of insights
> into scheduler behavior but it quickly becomes impractical to use for
> long running or scheduler intensive workload. For ex, `perf sched record`
> has ~7.77% overhead on hackbench (with 25 groups each running 700K loops
> on a 2-socket 128 Cores 256 Threads 3rd Generation EPYC Server), and it
> generates huge 56G perf.data for which perf takes ~137 mins to prepare
> and write it to disk [1].
>
> Unlike `perf sched record`, which hooks onto set of scheduler tracepoints
> and generates samples on a tracepoint hit, `perf sched stats record` takes
> snapshot of the /proc/schedstat file before and after the workload, i.e.
> there is almost zero interference on workload run. Also, it takes very
> minimal time to parse /proc/schedstat, convert it into perf samples and
> save those samples into perf.data file. Result perf.data file is much
> smaller. So, overall `perf sched stats record` is much more light weight
> compare to `perf sched record`.
>
> We, internally at AMD, have been using this (a variant of this, known as
> "sched-scoreboard"[2]) and found it to be very useful to analyse impact
> of any scheduler code changes[3][4]. Prateek used v2[5] of this patch
> series to report the analysis[6][7].
>
> Please note that, this is not a replacement of perf sched record/report.
> The intended users of the new tool are scheduler developers, not regular
> users.
>
> USAGE
> -----
>
> # perf sched stats record
> # perf sched stats report
> # perf sched stats diff
>
> Note: Although `perf sched stats` tool supports workload profiling syntax
> (i.e. -- <workload> ), the recorded profile is still systemwide since the
> /proc/schedstat is a systemwide file.
>
> HOW TO INTERPRET THE REPORT
> ---------------------------
>
> The `perf sched stats report` starts with description of the columns
> present in the report. These column names are given before cpu and
> domain stats to improve the readability of the report.
>
> ----------------------------------------------------------------------------------------------------
> DESC -> Description of the field
> COUNT -> Value of the field
> PCT_CHANGE -> Percent change with corresponding base value
> AVG_JIFFIES -> Avg time in jiffies between two consecutive occurrence of event
> ----------------------------------------------------------------------------------------------------
>
> Next is the total profiling time in terms of jiffies:
>
> ----------------------------------------------------------------------------------------------------
> Time elapsed (in jiffies) : 24537
> ----------------------------------------------------------------------------------------------------
>
nit:
Is there a way to export HZ value too here?
> Next is CPU scheduling statistics. These are simple diffs of
> /proc/schedstat CPU lines along with description. The report also
> prints % relative to base stat.
>
> In the example below, schedule() left the CPU0 idle 36.58% of the time.
> 0.45% of total try_to_wake_up() was to wakeup local CPU. And, the total
> waittime by tasks on CPU0 is 48.70% of the total runtime by tasks on the
> same CPU.
>
> ----------------------------------------------------------------------------------------------------
> CPU 0
> ----------------------------------------------------------------------------------------------------
> DESC COUNT PCT_CHANGE
> ----------------------------------------------------------------------------------------------------
> yld_count : 0
> array_exp : 0
> sched_count : 402267
> sched_goidle : 147161 ( 36.58% )
> ttwu_count : 236309
> ttwu_local : 1062 ( 0.45% )
> rq_cpu_time : 7083791148
> run_delay : 3449973971 ( 48.70% )
> pcount : 255035
> ----------------------------------------------------------------------------------------------------
>
> Next is load balancing statistics. For each of the sched domains
> (eg: `SMT`, `MC`, `DIE`...), the scheduler computes statistics under
> the following three categories:
>
> 1) Idle Load Balance: Load balancing performed on behalf of a long
> idling CPU by some other CPU.
> 2) Busy Load Balance: Load balancing performed when the CPU was busy.
> 3) New Idle Balance : Load balancing performed when a CPU just became
> idle.
>
> Under each of these three categories, sched stats report provides
> different load balancing statistics. Along with direct stats, the
> report also contains derived metrics prefixed with *. Example:
>
> ----------------------------------------------------------------------------------------------------
> CPU: 0 | DOMAIN: SMT | DOMAIN_CPUS: 0,64
> ----------------------------------------------------------------------------------------------------
> DESC COUNT AVG_JIFFIES
> ----------------------------------------- <Category busy> ------------------------------------------
> busy_lb_count : 136 $ 17.08 $
> busy_lb_balanced : 131 $ 17.73 $
> busy_lb_failed : 0 $ 0.00 $
> busy_lb_imbalance_load : 58
> busy_lb_imbalance_util : 0
> busy_lb_imbalance_task : 0
> busy_lb_imbalance_misfit : 0
> busy_lb_gained : 7
> busy_lb_hot_gained : 0
> busy_lb_nobusyq : 2 $ 1161.50 $
> busy_lb_nobusyg : 129 $ 18.01 $
> *busy_lb_success_count : 5
> *busy_lb_avg_pulled : 1.40
> ----------------------------------------- <Category idle> ------------------------------------------
> idle_lb_count : 449 $ 5.17 $
> idle_lb_balanced : 382 $ 6.08 $
> idle_lb_failed : 3 $ 774.33 $
> idle_lb_imbalance_load : 0
> idle_lb_imbalance_util : 0
> idle_lb_imbalance_task : 71
> idle_lb_imbalance_misfit : 0
> idle_lb_gained : 67
> idle_lb_hot_gained : 0
> idle_lb_nobusyq : 0 $ 0.00 $
> idle_lb_nobusyg : 382 $ 6.08 $
> *idle_lb_success_count : 64
> *idle_lb_avg_pulled : 1.05
> ---------------------------------------- <Category newidle> ----------------------------------------
> newidle_lb_count : 30471 $ 0.08 $
> newidle_lb_balanced : 28490 $ 0.08 $
> newidle_lb_failed : 633 $ 3.67 $
> newidle_lb_imbalance_load : 0
> newidle_lb_imbalance_util : 0
> newidle_lb_imbalance_task : 2040
> newidle_lb_imbalance_misfit : 0
> newidle_lb_gained : 1348
> newidle_lb_hot_gained : 0
> newidle_lb_nobusyq : 6 $ 387.17 $
> newidle_lb_nobusyg : 26634 $ 0.09 $
> *newidle_lb_success_count : 1348
> *newidle_lb_avg_pulled : 1.00
> ----------------------------------------------------------------------------------------------------
>
> Consider following line:
>
> newidle_lb_balanced : 28490 $ 0.08 $
>
> While profiling was active, the load-balancer found 28490 times the load
> needs to be balanced on a newly idle CPU 0. Following value encapsulated
> inside $ is average jiffies between two events (28490 / 24537 = 0.08).
>
Could you please explain this? I couldn't understand.
IIUC, you are parsing two instance of /proc/schedtstat,
once in the beginning and once in the end.
newidle_lb_balanced is a counter. In the beginning every iteration could
have decided domain is imbalanced and once load stabilized, it could have
decided now domain is balanced more often. i.e initially counter would add
quickly and then may stay more or less same value.
Also, what is this logic ? (28490 / 24537 = 0.08)?
> Next are active_load_balance() stats. alb did not trigger while the
> profiling was active, hence it's all 0s.
>
>
> --------------------------------- <Category active_load_balance()> ---------------------------------
> alb_count : 0
> alb_failed : 0
> alb_pushed : 0
> ----------------------------------------------------------------------------------------------------
>
> Next are sched_balance_exec() and sched_balance_fork() stats. They are
> not used but we kept it in RFC just for legacy purpose. Unless opposed,
> we plan to remove them in next revision.
>
> Next are wakeup statistics. For every domain, the report also shows
> task-wakeup statistics. Example:
>
> ------------------------------------------ <Wakeup Info> -------------------------------------------
> ttwu_wake_remote : 1590
> ttwu_move_affine : 84
> ttwu_move_balance : 0
> ----------------------------------------------------------------------------------------------------
>
> Same set of stats are reported for each CPU and each domain level.
>
> HOW TO INTERPRET THE DIFF
> -------------------------
>
> The `perf sched stats diff` will also start with explaining the columns
> present in the diff. Then it will show the diff in time in terms of
> jiffies. The order of the values depends on the order of input data
> files. Example:
>
> ----------------------------------------------------------------------------------------------------
> Time elapsed (in jiffies) : 2763, 2763
> ----------------------------------------------------------------------------------------------------
>
> Below is the sample representing the difference in cpu and domain stats of
> two runs. Here third column or the values enclosed in `|...|` shows the
> percent change between the two. Second and fourth columns shows the
> side-by-side representions of the corresponding fields from `perf sched
> stats report`.
>
> ----------------------------------------------------------------------------------------------------
> CPU: <ALL CPUS SUMMARY>
> ----------------------------------------------------------------------------------------------------
> DESC COUNT1 COUNT2 PCT_CHANG>
> ----------------------------------------------------------------------------------------------------
> yld_count : 0, 0 | 0.00>
> array_exp : 0, 0 | 0.00>
> sched_count : 528533, 412573 | -21.94>
> sched_goidle : 193426, 146082 | -24.48>
> ttwu_count : 313134, 385975 | 23.26>
> ttwu_local : 1126, 1282 | 13.85>
> rq_cpu_time : 8257200244, 8301250047 | 0.53>
> run_delay : 4728347053, 3997100703 | -15.47>
> pcount : 335031, 266396 | -20.49>
> ----------------------------------------------------------------------------------------------------
>
> Below is the sample of domain stats diff:
>
> ----------------------------------------------------------------------------------------------------
> CPU: <ALL CPUS SUMMARY> | DOMAIN: SMT
> ----------------------------------------------------------------------------------------------------
> DESC COUNT1 COUNT2 PCT_CHANG>
> ----------------------------------------- <Category busy> ------------------------------------------
> busy_lb_count : 122, 80 | -34.43>
> busy_lb_balanced : 115, 76 | -33.91>
> busy_lb_failed : 1, 3 | 200.00>
> busy_lb_imbalance_load : 35, 49 | 40.00>
> busy_lb_imbalance_util : 0, 0 | 0.00>
> busy_lb_imbalance_task : 0, 0 | 0.00>
> busy_lb_imbalance_misfit : 0, 0 | 0.00>
> busy_lb_gained : 7, 2 | -71.43>
> busy_lb_hot_gained : 0, 0 | 0.00>
> busy_lb_nobusyq : 0, 0 | 0.00>
> busy_lb_nobusyg : 115, 76 | -33.91>
> *busy_lb_success_count : 6, 1 | -83.33>
> *busy_lb_avg_pulled : 1.17, 2.00 | 71.43>
> ----------------------------------------- <Category idle> ------------------------------------------
> idle_lb_count : 568, 620 | 9.15>
> idle_lb_balanced : 462, 449 | -2.81>
> idle_lb_failed : 11, 21 | 90.91>
> idle_lb_imbalance_load : 0, 0 | 0.00>
> idle_lb_imbalance_util : 0, 0 | 0.00>
> idle_lb_imbalance_task : 115, 189 | 64.35>
> idle_lb_imbalance_misfit : 0, 0 | 0.00>
> idle_lb_gained : 103, 169 | 64.08>
> idle_lb_hot_gained : 0, 0 | 0.00>
> idle_lb_nobusyq : 0, 0 | 0.00>
> idle_lb_nobusyg : 462, 449 | -2.81>
> *idle_lb_success_count : 95, 150 | 57.89>
> *idle_lb_avg_pulled : 1.08, 1.13 | 3.92>
> ---------------------------------------- <Category newidle> ----------------------------------------
> newidle_lb_count : 16961, 3155 | -81.40>
> newidle_lb_balanced : 15646, 2556 | -83.66>
> newidle_lb_failed : 397, 142 | -64.23>
> newidle_lb_imbalance_load : 0, 0 | 0.00>
> newidle_lb_imbalance_util : 0, 0 | 0.00>
> newidle_lb_imbalance_task : 1376, 655 | -52.40>
> newidle_lb_imbalance_misfit : 0, 0 | 0.00>
> newidle_lb_gained : 917, 457 | -50.16>
> newidle_lb_hot_gained : 0, 0 | 0.00>
> newidle_lb_nobusyq : 3, 1 | -66.67>
> newidle_lb_nobusyg : 14480, 2103 | -85.48>
> *newidle_lb_success_count : 918, 457 | -50.22>
> *newidle_lb_avg_pulled : 1.00, 1.00 | 0.11>
> --------------------------------- <Category active_load_balance()> ---------------------------------
> alb_count : 0, 1 | 0.00>
> alb_failed : 0, 0 | 0.00>
> alb_pushed : 0, 1 | 0.00>
> --------------------------------- <Category sched_balance_exec()> ----------------------------------
> sbe_count : 0, 0 | 0.00>
> sbe_balanced : 0, 0 | 0.00>
> sbe_pushed : 0, 0 | 0.00>
> --------------------------------- <Category sched_balance_fork()> ----------------------------------
> sbf_count : 0, 0 | 0.00>
> sbf_balanced : 0, 0 | 0.00>
> sbf_pushed : 0, 0 | 0.00>
> ------------------------------------------ <Wakeup Info> -------------------------------------------
> ttwu_wake_remote : 2031, 2914 | 43.48>
> ttwu_move_affine : 73, 124 | 69.86>
> ttwu_move_balance : 0, 0 | 0.00>
> ----------------------------------------------------------------------------------------------------
>
> v4: https://lore.kernel.org/lkml/20250909114227.58802-1-swapnil.sapkal@amd.com/
> v4->v5:
> - Address review comments from v4 [Namhyung Kim]
> - Resolve the issue reported by kernel test rebot
> - Debug and resolve issue reported in the perf sched stats diff [Prateek]
> - Rebase on top of perf-tools-next(571d29baa07e)
>
> v3: https://lore.kernel.org/all/20250311120230.61774-1-swapnil.sapkal@amd.com/
> v3->v4:
> - All the review comments from v3 are addressed [Namhyung Kim].
> - Print short names instead of field descripion in the report [Peter Zijlstra]
> - Fix the double free issue [Cristian Prundeanu]
> - Documentation update related to `perf sched stats diff` [Chen yu]
> - Bail out `perf sched stats diff` if perf.data files have different schedstat
> versions [Peter Zijlstra]
>
> v2: https://lore.kernel.org/all/20241122084452.1064968-1-swapnil.sapkal@amd.com/
> v2->v3:
> - Add perf unit test for basic sched stats functionalities
> - Describe new tool, it's usage and interpretation of report data in the
> perf-sched man page.
> - Add /proc/schedstat version 17 support.
>
> v1: https://lore.kernel.org/lkml/20240916164722.1838-1-ravi.bangoria@amd.com
> v1->v2
> - Add the support for `perf sched stats diff`
> - Add column header in report for better readability. Use
> procfs__mountpoint for consistency. Add hint for enabling
> CONFIG_SCHEDSTAT if disabled. [James Clark]
> - Use a single header file for both cpu and domain fileds. Change
> the layout of structs to minimise the padding. I tried changing
> `v15` to `15` in the header files but it was not giving any
> benefits so drop the idea. [Namhyung Kim]
> - Add tested-by.
>
> RFC: https://lore.kernel.org/r/20240508060427.417-1-ravi.bangoria@amd.com
> RFC->v1:
> - [Kernel] Print domain name along with domain number in /proc/schedstat
> file.
> - s/schedstat/stats/ for the subcommand.
> - Record domain name and cpumask details, also show them in report.
> - Add CPU filtering capability at record and report time.
> - Add /proc/schedstat v16 support.
> - Live mode support. Similar to perf stat command, live mode prints the
> sched stats on the stdout.
> - Add pager support in `perf sched stats report` for better scrolling.
> - Some minor cosmetic changes in report output to improve readability.
> - Rebase to latest perf-tools-next/perf-tools-next (1de5b5dcb835).
>
> TODO:
> - perf sched stats records /proc/schedstat which is a CPU and domain
> level scheduler statistic. We are planning to add taskstat tool which
> reads task stats from procfs and generate scheduler statistic report
> at task granularity. this will probably a standalone tool, something
> like `perf sched taskstat record/report`.
> - Except pre-processor related checkpatch warnings, we have addressed
> most of the other possible warnings.
> - This version supports diff for two perf.data files captured for same
> schedstats version but the target is to show diff for multiple
> perf.data files. Plan is to support diff if perf.data files provided
> has different schedstat versions.
>
> Patches are prepared on top of perf-tools-next(571d29baa07e).
>
> [1] https://youtu.be/lg-9aG2ajA0?t=283
> [2] https://github.com/AMDESE/sched-scoreboard
> [3] https://lore.kernel.org/lkml/c50bdbfe-02ce-c1bc-c761-c95f8e216ca0@amd.com/
> [4] https://lore.kernel.org/lkml/3e32bec6-5e59-c66a-7676-7d15df2c961c@amd.com/
> [5] https://lore.kernel.org/all/20241122084452.1064968-1-swapnil.sapkal@amd.com/
> [6] https://lore.kernel.org/lkml/3170d16e-eb67-4db8-a327-eb8188397fdb@amd.com/
> [7] https://lore.kernel.org/lkml/feb31b6e-6457-454c-a4f3-ce8ad96bf8de@amd.com/
>
> Swapnil Sapkal (10):
> tools/lib: Add list_is_first()
> perf header: Support CPU DOMAIN relation info
> perf sched stats: Add record and rawdump support
> perf sched stats: Add schedstat v16 support
> perf sched stats: Add schedstat v17 support
> perf sched stats: Add support for report subcommand
> perf sched stats: Add support for live mode
> perf sched stats: Add support for diff subcommand
> perf sched stats: Add basic perf sched stats test
> perf sched stats: Add details in man page
>
> tools/include/linux/list.h | 10 +
> tools/lib/perf/Documentation/libperf.txt | 2 +
> tools/lib/perf/Makefile | 1 +
> tools/lib/perf/include/perf/event.h | 69 ++
> tools/lib/perf/include/perf/schedstat-v15.h | 146 +++
> tools/lib/perf/include/perf/schedstat-v16.h | 146 +++
> tools/lib/perf/include/perf/schedstat-v17.h | 164 +++
> tools/perf/Documentation/perf-sched.txt | 261 ++++-
> .../Documentation/perf.data-file-format.txt | 17 +
> tools/perf/builtin-inject.c | 3 +
> tools/perf/builtin-sched.c | 1028 ++++++++++++++++-
> tools/perf/tests/shell/perf_sched_stats.sh | 64 +
> tools/perf/util/env.c | 29 +
> tools/perf/util/env.h | 17 +
> tools/perf/util/event.c | 52 +
> tools/perf/util/event.h | 2 +
> tools/perf/util/header.c | 285 +++++
> tools/perf/util/header.h | 4 +
> tools/perf/util/session.c | 22 +
> tools/perf/util/synthetic-events.c | 196 ++++
> tools/perf/util/synthetic-events.h | 3 +
> tools/perf/util/tool.c | 20 +
> tools/perf/util/tool.h | 4 +-
> tools/perf/util/util.c | 48 +
> tools/perf/util/util.h | 5 +
> 25 files changed, 2595 insertions(+), 3 deletions(-)
> create mode 100644 tools/lib/perf/include/perf/schedstat-v15.h
> create mode 100644 tools/lib/perf/include/perf/schedstat-v16.h
> create mode 100644 tools/lib/perf/include/perf/schedstat-v17.h
> create mode 100755 tools/perf/tests/shell/perf_sched_stats.sh
>
^ permalink raw reply [flat|nested] 36+ messages in thread* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-21 17:52 ` Shrikanth Hegde
@ 2026-01-23 16:19 ` Swapnil Sapkal
0 siblings, 0 replies; 36+ messages in thread
From: Swapnil Sapkal @ 2026-01-23 16:19 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: ravi.bangoria, yu.c.chen, mark.rutland, alexander.shishkin, jolsa,
rostedt, vincent.guittot, adrian.hunter, kan.liang,
gautham.shenoy, kprateek.nayak, juri.lelli, yangjihong, void, tj,
ctshao, quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users, peterz, mingo, acme, namhyung, irogers,
james.clark
Hi Shrikanth,
On 21-01-2026 23:22, Shrikanth Hegde wrote:
>
>
> On 1/19/26 11:28 PM, Swapnil Sapkal wrote:
>> MOTIVATION
>> ----------
>>
>> Existing `perf sched` is quite exhaustive and provides lot of insights
>> into scheduler behavior but it quickly becomes impractical to use for
>> long running or scheduler intensive workload. For ex, `perf sched record`
>> has ~7.77% overhead on hackbench (with 25 groups each running 700K loops
>> on a 2-socket 128 Cores 256 Threads 3rd Generation EPYC Server), and it
>> generates huge 56G perf.data for which perf takes ~137 mins to prepare
>> and write it to disk [1].
>>
>> Unlike `perf sched record`, which hooks onto set of scheduler tracepoints
>> and generates samples on a tracepoint hit, `perf sched stats record`
>> takes
>> snapshot of the /proc/schedstat file before and after the workload, i.e.
>> there is almost zero interference on workload run. Also, it takes very
>> minimal time to parse /proc/schedstat, convert it into perf samples and
>> save those samples into perf.data file. Result perf.data file is much
>> smaller. So, overall `perf sched stats record` is much more light weight
>> compare to `perf sched record`.
>>
>> We, internally at AMD, have been using this (a variant of this, known as
>> "sched-scoreboard"[2]) and found it to be very useful to analyse impact
>> of any scheduler code changes[3][4]. Prateek used v2[5] of this patch
>> series to report the analysis[6][7].
>>
>> Please note that, this is not a replacement of perf sched record/report.
>> The intended users of the new tool are scheduler developers, not regular
>> users.
>>
>> USAGE
>> -----
>>
>> # perf sched stats record
>> # perf sched stats report
>> # perf sched stats diff
>>
>> Note: Although `perf sched stats` tool supports workload profiling syntax
>> (i.e. -- <workload> ), the recorded profile is still systemwide since the
>> /proc/schedstat is a systemwide file.
>>
>> HOW TO INTERPRET THE REPORT
>> ---------------------------
>>
>> The `perf sched stats report` starts with description of the columns
>> present in the report. These column names are given before cpu and
>> domain stats to improve the readability of the report.
>>
>>
>> ----------------------------------------------------------------------------------------------------
>> DESC -> Description of the field
>> COUNT -> Value of the field
>> PCT_CHANGE -> Percent change with corresponding base
>> value
>> AVG_JIFFIES -> Avg time in jiffies between two
>> consecutive occurrence of event
>>
>> ----------------------------------------------------------------------------------------------------
>>
>> Next is the total profiling time in terms of jiffies:
>>
>>
>> ----------------------------------------------------------------------------------------------------
>> Time elapsed (in jiffies) :
>> 24537
>>
>> ----------------------------------------------------------------------------------------------------
>>
>
> nit:
> Is there a way to export HZ value too here?
As per my knowledge, we can get this value from /proc/config.gz and this
depends on 'CONFIG_IKCONFIG_PROC' being enabled.
Peter, Is it okay to export the HZ value through a debugfs file, say
something like '/sys/kernel/debug/sched/hz_value'? Though I am not sure
if this is useful for anythyng else.
>
>> Next is CPU scheduling statistics. These are simple diffs of
>> /proc/schedstat CPU lines along with description. The report also
>> prints % relative to base stat.
>>
>> In the example below, schedule() left the CPU0 idle 36.58% of the time.
>> 0.45% of total try_to_wake_up() was to wakeup local CPU. And, the total
>> waittime by tasks on CPU0 is 48.70% of the total runtime by tasks on the
>> same CPU.
>>
>>
>> ----------------------------------------------------------------------------------------------------
>> CPU 0
>>
>> ----------------------------------------------------------------------------------------------------
>>
>> DESC COUNT PCT_CHANGE
>>
>> ----------------------------------------------------------------------------------------------------
>>
>> yld_count : 0
>>
>> array_exp : 0
>>
>> sched_count : 402267
>>
>> sched_goidle : 147161 ( 36.58% )
>>
>> ttwu_count : 236309
>>
>> ttwu_local : 1062 ( 0.45% )
>> rq_cpu_time :
>> 7083791148
>> run_delay :
>> 3449973971 ( 48.70% )
>>
>> pcount : 255035
>>
>> ----------------------------------------------------------------------------------------------------
>>
>> Next is load balancing statistics. For each of the sched domains
>> (eg: `SMT`, `MC`, `DIE`...), the scheduler computes statistics under
>> the following three categories:
>>
>> 1) Idle Load Balance: Load balancing performed on behalf of a long
>> idling CPU by some other CPU.
>> 2) Busy Load Balance: Load balancing performed when the CPU was busy.
>> 3) New Idle Balance : Load balancing performed when a CPU just became
>> idle.
>>
>> Under each of these three categories, sched stats report provides
>> different load balancing statistics. Along with direct stats, the
>> report also contains derived metrics prefixed with *. Example:
>>
>>
>> ----------------------------------------------------------------------------------------------------
>> CPU: 0 | DOMAIN: SMT | DOMAIN_CPUS: 0,64
>>
>> ----------------------------------------------------------------------------------------------------
>>
>> DESC COUNT AVG_JIFFIES
>> ----------------------------------------- <Category busy>
>> ------------------------------------------
>>
>> busy_lb_count : 136 $ 17.08 $
>>
>> busy_lb_balanced : 131 $ 17.73 $
>>
>> busy_lb_failed : 0 $ 0.00 $
>>
>> busy_lb_imbalance_load : 58
>>
>> busy_lb_imbalance_util : 0
>>
>> busy_lb_imbalance_task : 0
>>
>> busy_lb_imbalance_misfit : 0
>>
>> busy_lb_gained : 7
>>
>> busy_lb_hot_gained : 0
>>
>> busy_lb_nobusyq : 2 $ 1161.50 $
>>
>> busy_lb_nobusyg : 129 $ 18.01 $
>>
>> *busy_lb_success_count : 5
>>
>> *busy_lb_avg_pulled : 1.40
>> ----------------------------------------- <Category idle>
>> ------------------------------------------
>>
>> idle_lb_count : 449 $ 5.17 $
>>
>> idle_lb_balanced : 382 $ 6.08 $
>>
>> idle_lb_failed : 3 $ 774.33 $
>>
>> idle_lb_imbalance_load : 0
>>
>> idle_lb_imbalance_util : 0
>>
>> idle_lb_imbalance_task : 71
>>
>> idle_lb_imbalance_misfit : 0
>>
>> idle_lb_gained : 67
>>
>> idle_lb_hot_gained : 0
>>
>> idle_lb_nobusyq : 0 $ 0.00 $
>>
>> idle_lb_nobusyg : 382 $ 6.08 $
>>
>> *idle_lb_success_count : 64
>>
>> *idle_lb_avg_pulled : 1.05
>> ---------------------------------------- <Category newidle>
>> ----------------------------------------
>>
>> newidle_lb_count : 30471 $ 0.08 $
>>
>> newidle_lb_balanced : 28490 $ 0.08 $
>>
>> newidle_lb_failed : 633 $ 3.67 $
>>
>> newidle_lb_imbalance_load : 0
>>
>> newidle_lb_imbalance_util : 0
>>
>> newidle_lb_imbalance_task : 2040
>>
>> newidle_lb_imbalance_misfit : 0
>>
>> newidle_lb_gained : 1348
>>
>> newidle_lb_hot_gained : 0
>>
>> newidle_lb_nobusyq : 6 $ 387.17 $
>>
>> newidle_lb_nobusyg : 26634 $ 0.09 $
>>
>> *newidle_lb_success_count : 1348
>>
>> *newidle_lb_avg_pulled : 1.00
>>
>> ----------------------------------------------------------------------------------------------------
>>
>> Consider following line:
>>
>> newidle_lb_balanced : 28490 $ 0.08 $
>>
>> While profiling was active, the load-balancer found 28490 times the load
>> needs to be balanced on a newly idle CPU 0. Following value encapsulated
>> inside $ is average jiffies between two events (28490 / 24537 = 0.08).
>>
>
> Could you please explain this? I couldn't understand.
>
> IIUC, you are parsing two instance of /proc/schedtstat,
> once in the beginning and once in the end.
>
> newidle_lb_balanced is a counter. In the beginning every iteration could
> have decided domain is imbalanced and once load stabilized, it could have
> decided now domain is balanced more often. i.e initially counter would add
> quickly and then may stay more or less same value.
>
> Also, what is this logic ? (28490 / 24537 = 0.08)?
>
Thanks for catching this. This is a miss from my end while writing the
cover letter. Here the jiffies value and the counter values are from two
different runs. Total jiffies for the run was 2323.
The values inside $ .. $ represents per jiffie how many times this event
has occured. The calculation here is (total_jiffies / counter_value).
In the run it is (2323 / 24537 = 0.08)
I will fix this in man page also.
--
Thanks and Regards,
Swapnil
^ permalink raw reply [flat|nested] 36+ messages in thread
* Re: [PATCH v5 00/10] perf sched: Introduce stats tool
2026-01-19 17:58 [PATCH v5 00/10] perf sched: Introduce stats tool Swapnil Sapkal
` (11 preceding siblings ...)
2026-01-21 17:52 ` Shrikanth Hegde
@ 2026-01-21 22:59 ` Namhyung Kim
12 siblings, 0 replies; 36+ messages in thread
From: Namhyung Kim @ 2026-01-21 22:59 UTC (permalink / raw)
To: Swapnil Sapkal
Cc: peterz, mingo, acme, irogers, james.clark, ravi.bangoria,
yu.c.chen, mark.rutland, alexander.shishkin, jolsa, rostedt,
vincent.guittot, adrian.hunter, kan.liang, gautham.shenoy,
kprateek.nayak, juri.lelli, yangjihong, void, tj, sshegde, ctshao,
quic_zhonhan, thomas.falcon, blakejones, ashelat, leo.yan,
dvyukov, ak, yujie.liu, graham.woodward, ben.gainey, vineethr,
tim.c.chen, linux, santosh.shukla, sandipan.das, linux-kernel,
linux-perf-users
Hello,
On Mon, Jan 19, 2026 at 05:58:22PM +0000, Swapnil Sapkal wrote:
> MOTIVATION
> ----------
>
> Existing `perf sched` is quite exhaustive and provides lot of insights
> into scheduler behavior but it quickly becomes impractical to use for
> long running or scheduler intensive workload. For ex, `perf sched record`
> has ~7.77% overhead on hackbench (with 25 groups each running 700K loops
> on a 2-socket 128 Cores 256 Threads 3rd Generation EPYC Server), and it
> generates huge 56G perf.data for which perf takes ~137 mins to prepare
> and write it to disk [1].
>
> Unlike `perf sched record`, which hooks onto set of scheduler tracepoints
> and generates samples on a tracepoint hit, `perf sched stats record` takes
> snapshot of the /proc/schedstat file before and after the workload, i.e.
> there is almost zero interference on workload run. Also, it takes very
> minimal time to parse /proc/schedstat, convert it into perf samples and
> save those samples into perf.data file. Result perf.data file is much
> smaller. So, overall `perf sched stats record` is much more light weight
> compare to `perf sched record`.
>
> We, internally at AMD, have been using this (a variant of this, known as
> "sched-scoreboard"[2]) and found it to be very useful to analyse impact
> of any scheduler code changes[3][4]. Prateek used v2[5] of this patch
> series to report the analysis[6][7].
>
> Please note that, this is not a replacement of perf sched record/report.
> The intended users of the new tool are scheduler developers, not regular
> users.
>
> USAGE
> -----
>
> # perf sched stats record
> # perf sched stats report
> # perf sched stats diff
>
> Note: Although `perf sched stats` tool supports workload profiling syntax
> (i.e. -- <workload> ), the recorded profile is still systemwide since the
> /proc/schedstat is a systemwide file.
>
> HOW TO INTERPRET THE REPORT
> ---------------------------
>
> The `perf sched stats report` starts with description of the columns
> present in the report. These column names are given before cpu and
> domain stats to improve the readability of the report.
>
> ----------------------------------------------------------------------------------------------------
> DESC -> Description of the field
> COUNT -> Value of the field
> PCT_CHANGE -> Percent change with corresponding base value
> AVG_JIFFIES -> Avg time in jiffies between two consecutive occurrence of event
> ----------------------------------------------------------------------------------------------------
>
> Next is the total profiling time in terms of jiffies:
>
> ----------------------------------------------------------------------------------------------------
> Time elapsed (in jiffies) : 24537
> ----------------------------------------------------------------------------------------------------
>
> Next is CPU scheduling statistics. These are simple diffs of
> /proc/schedstat CPU lines along with description. The report also
> prints % relative to base stat.
>
> In the example below, schedule() left the CPU0 idle 36.58% of the time.
> 0.45% of total try_to_wake_up() was to wakeup local CPU. And, the total
> waittime by tasks on CPU0 is 48.70% of the total runtime by tasks on the
> same CPU.
>
> ----------------------------------------------------------------------------------------------------
> CPU 0
> ----------------------------------------------------------------------------------------------------
> DESC COUNT PCT_CHANGE
> ----------------------------------------------------------------------------------------------------
> yld_count : 0
> array_exp : 0
> sched_count : 402267
> sched_goidle : 147161 ( 36.58% )
> ttwu_count : 236309
> ttwu_local : 1062 ( 0.45% )
> rq_cpu_time : 7083791148
> run_delay : 3449973971 ( 48.70% )
> pcount : 255035
> ----------------------------------------------------------------------------------------------------
>
> Next is load balancing statistics. For each of the sched domains
> (eg: `SMT`, `MC`, `DIE`...), the scheduler computes statistics under
> the following three categories:
>
> 1) Idle Load Balance: Load balancing performed on behalf of a long
> idling CPU by some other CPU.
> 2) Busy Load Balance: Load balancing performed when the CPU was busy.
> 3) New Idle Balance : Load balancing performed when a CPU just became
> idle.
>
> Under each of these three categories, sched stats report provides
> different load balancing statistics. Along with direct stats, the
> report also contains derived metrics prefixed with *. Example:
>
> ----------------------------------------------------------------------------------------------------
> CPU: 0 | DOMAIN: SMT | DOMAIN_CPUS: 0,64
> ----------------------------------------------------------------------------------------------------
> DESC COUNT AVG_JIFFIES
> ----------------------------------------- <Category busy> ------------------------------------------
> busy_lb_count : 136 $ 17.08 $
> busy_lb_balanced : 131 $ 17.73 $
> busy_lb_failed : 0 $ 0.00 $
> busy_lb_imbalance_load : 58
> busy_lb_imbalance_util : 0
> busy_lb_imbalance_task : 0
> busy_lb_imbalance_misfit : 0
> busy_lb_gained : 7
> busy_lb_hot_gained : 0
> busy_lb_nobusyq : 2 $ 1161.50 $
> busy_lb_nobusyg : 129 $ 18.01 $
> *busy_lb_success_count : 5
> *busy_lb_avg_pulled : 1.40
> ----------------------------------------- <Category idle> ------------------------------------------
> idle_lb_count : 449 $ 5.17 $
> idle_lb_balanced : 382 $ 6.08 $
> idle_lb_failed : 3 $ 774.33 $
> idle_lb_imbalance_load : 0
> idle_lb_imbalance_util : 0
> idle_lb_imbalance_task : 71
> idle_lb_imbalance_misfit : 0
> idle_lb_gained : 67
> idle_lb_hot_gained : 0
> idle_lb_nobusyq : 0 $ 0.00 $
> idle_lb_nobusyg : 382 $ 6.08 $
> *idle_lb_success_count : 64
> *idle_lb_avg_pulled : 1.05
> ---------------------------------------- <Category newidle> ----------------------------------------
> newidle_lb_count : 30471 $ 0.08 $
> newidle_lb_balanced : 28490 $ 0.08 $
> newidle_lb_failed : 633 $ 3.67 $
> newidle_lb_imbalance_load : 0
> newidle_lb_imbalance_util : 0
> newidle_lb_imbalance_task : 2040
> newidle_lb_imbalance_misfit : 0
> newidle_lb_gained : 1348
> newidle_lb_hot_gained : 0
> newidle_lb_nobusyq : 6 $ 387.17 $
> newidle_lb_nobusyg : 26634 $ 0.09 $
> *newidle_lb_success_count : 1348
> *newidle_lb_avg_pulled : 1.00
> ----------------------------------------------------------------------------------------------------
>
> Consider following line:
>
> newidle_lb_balanced : 28490 $ 0.08 $
>
> While profiling was active, the load-balancer found 28490 times the load
> needs to be balanced on a newly idle CPU 0. Following value encapsulated
> inside $ is average jiffies between two events (28490 / 24537 = 0.08).
>
> Next are active_load_balance() stats. alb did not trigger while the
> profiling was active, hence it's all 0s.
>
>
> --------------------------------- <Category active_load_balance()> ---------------------------------
> alb_count : 0
> alb_failed : 0
> alb_pushed : 0
> ----------------------------------------------------------------------------------------------------
>
> Next are sched_balance_exec() and sched_balance_fork() stats. They are
> not used but we kept it in RFC just for legacy purpose. Unless opposed,
> we plan to remove them in next revision.
>
> Next are wakeup statistics. For every domain, the report also shows
> task-wakeup statistics. Example:
>
> ------------------------------------------ <Wakeup Info> -------------------------------------------
> ttwu_wake_remote : 1590
> ttwu_move_affine : 84
> ttwu_move_balance : 0
> ----------------------------------------------------------------------------------------------------
>
> Same set of stats are reported for each CPU and each domain level.
>
> HOW TO INTERPRET THE DIFF
> -------------------------
>
> The `perf sched stats diff` will also start with explaining the columns
> present in the diff. Then it will show the diff in time in terms of
> jiffies. The order of the values depends on the order of input data
> files. Example:
>
> ----------------------------------------------------------------------------------------------------
> Time elapsed (in jiffies) : 2763, 2763
> ----------------------------------------------------------------------------------------------------
>
> Below is the sample representing the difference in cpu and domain stats of
> two runs. Here third column or the values enclosed in `|...|` shows the
> percent change between the two. Second and fourth columns shows the
> side-by-side representions of the corresponding fields from `perf sched
> stats report`.
>
> ----------------------------------------------------------------------------------------------------
> CPU: <ALL CPUS SUMMARY>
> ----------------------------------------------------------------------------------------------------
> DESC COUNT1 COUNT2 PCT_CHANG>
> ----------------------------------------------------------------------------------------------------
> yld_count : 0, 0 | 0.00>
> array_exp : 0, 0 | 0.00>
> sched_count : 528533, 412573 | -21.94>
> sched_goidle : 193426, 146082 | -24.48>
> ttwu_count : 313134, 385975 | 23.26>
> ttwu_local : 1126, 1282 | 13.85>
> rq_cpu_time : 8257200244, 8301250047 | 0.53>
> run_delay : 4728347053, 3997100703 | -15.47>
> pcount : 335031, 266396 | -20.49>
> ----------------------------------------------------------------------------------------------------
>
> Below is the sample of domain stats diff:
>
> ----------------------------------------------------------------------------------------------------
> CPU: <ALL CPUS SUMMARY> | DOMAIN: SMT
> ----------------------------------------------------------------------------------------------------
> DESC COUNT1 COUNT2 PCT_CHANG>
> ----------------------------------------- <Category busy> ------------------------------------------
> busy_lb_count : 122, 80 | -34.43>
> busy_lb_balanced : 115, 76 | -33.91>
> busy_lb_failed : 1, 3 | 200.00>
> busy_lb_imbalance_load : 35, 49 | 40.00>
> busy_lb_imbalance_util : 0, 0 | 0.00>
> busy_lb_imbalance_task : 0, 0 | 0.00>
> busy_lb_imbalance_misfit : 0, 0 | 0.00>
> busy_lb_gained : 7, 2 | -71.43>
> busy_lb_hot_gained : 0, 0 | 0.00>
> busy_lb_nobusyq : 0, 0 | 0.00>
> busy_lb_nobusyg : 115, 76 | -33.91>
> *busy_lb_success_count : 6, 1 | -83.33>
> *busy_lb_avg_pulled : 1.17, 2.00 | 71.43>
> ----------------------------------------- <Category idle> ------------------------------------------
> idle_lb_count : 568, 620 | 9.15>
> idle_lb_balanced : 462, 449 | -2.81>
> idle_lb_failed : 11, 21 | 90.91>
> idle_lb_imbalance_load : 0, 0 | 0.00>
> idle_lb_imbalance_util : 0, 0 | 0.00>
> idle_lb_imbalance_task : 115, 189 | 64.35>
> idle_lb_imbalance_misfit : 0, 0 | 0.00>
> idle_lb_gained : 103, 169 | 64.08>
> idle_lb_hot_gained : 0, 0 | 0.00>
> idle_lb_nobusyq : 0, 0 | 0.00>
> idle_lb_nobusyg : 462, 449 | -2.81>
> *idle_lb_success_count : 95, 150 | 57.89>
> *idle_lb_avg_pulled : 1.08, 1.13 | 3.92>
> ---------------------------------------- <Category newidle> ----------------------------------------
> newidle_lb_count : 16961, 3155 | -81.40>
> newidle_lb_balanced : 15646, 2556 | -83.66>
> newidle_lb_failed : 397, 142 | -64.23>
> newidle_lb_imbalance_load : 0, 0 | 0.00>
> newidle_lb_imbalance_util : 0, 0 | 0.00>
> newidle_lb_imbalance_task : 1376, 655 | -52.40>
> newidle_lb_imbalance_misfit : 0, 0 | 0.00>
> newidle_lb_gained : 917, 457 | -50.16>
> newidle_lb_hot_gained : 0, 0 | 0.00>
> newidle_lb_nobusyq : 3, 1 | -66.67>
> newidle_lb_nobusyg : 14480, 2103 | -85.48>
> *newidle_lb_success_count : 918, 457 | -50.22>
> *newidle_lb_avg_pulled : 1.00, 1.00 | 0.11>
> --------------------------------- <Category active_load_balance()> ---------------------------------
> alb_count : 0, 1 | 0.00>
> alb_failed : 0, 0 | 0.00>
> alb_pushed : 0, 1 | 0.00>
> --------------------------------- <Category sched_balance_exec()> ----------------------------------
> sbe_count : 0, 0 | 0.00>
> sbe_balanced : 0, 0 | 0.00>
> sbe_pushed : 0, 0 | 0.00>
> --------------------------------- <Category sched_balance_fork()> ----------------------------------
> sbf_count : 0, 0 | 0.00>
> sbf_balanced : 0, 0 | 0.00>
> sbf_pushed : 0, 0 | 0.00>
> ------------------------------------------ <Wakeup Info> -------------------------------------------
> ttwu_wake_remote : 2031, 2914 | 43.48>
> ttwu_move_affine : 73, 124 | 69.86>
> ttwu_move_balance : 0, 0 | 0.00>
> ----------------------------------------------------------------------------------------------------
>
> v4: https://lore.kernel.org/lkml/20250909114227.58802-1-swapnil.sapkal@amd.com/
> v4->v5:
> - Address review comments from v4 [Namhyung Kim]
> - Resolve the issue reported by kernel test rebot
> - Debug and resolve issue reported in the perf sched stats diff [Prateek]
> - Rebase on top of perf-tools-next(571d29baa07e)
>
> v3: https://lore.kernel.org/all/20250311120230.61774-1-swapnil.sapkal@amd.com/
> v3->v4:
> - All the review comments from v3 are addressed [Namhyung Kim].
> - Print short names instead of field descripion in the report [Peter Zijlstra]
> - Fix the double free issue [Cristian Prundeanu]
> - Documentation update related to `perf sched stats diff` [Chen yu]
> - Bail out `perf sched stats diff` if perf.data files have different schedstat
> versions [Peter Zijlstra]
>
> v2: https://lore.kernel.org/all/20241122084452.1064968-1-swapnil.sapkal@amd.com/
> v2->v3:
> - Add perf unit test for basic sched stats functionalities
> - Describe new tool, it's usage and interpretation of report data in the
> perf-sched man page.
> - Add /proc/schedstat version 17 support.
>
> v1: https://lore.kernel.org/lkml/20240916164722.1838-1-ravi.bangoria@amd.com
> v1->v2
> - Add the support for `perf sched stats diff`
> - Add column header in report for better readability. Use
> procfs__mountpoint for consistency. Add hint for enabling
> CONFIG_SCHEDSTAT if disabled. [James Clark]
> - Use a single header file for both cpu and domain fileds. Change
> the layout of structs to minimise the padding. I tried changing
> `v15` to `15` in the header files but it was not giving any
> benefits so drop the idea. [Namhyung Kim]
> - Add tested-by.
>
> RFC: https://lore.kernel.org/r/20240508060427.417-1-ravi.bangoria@amd.com
> RFC->v1:
> - [Kernel] Print domain name along with domain number in /proc/schedstat
> file.
> - s/schedstat/stats/ for the subcommand.
> - Record domain name and cpumask details, also show them in report.
> - Add CPU filtering capability at record and report time.
> - Add /proc/schedstat v16 support.
> - Live mode support. Similar to perf stat command, live mode prints the
> sched stats on the stdout.
> - Add pager support in `perf sched stats report` for better scrolling.
> - Some minor cosmetic changes in report output to improve readability.
> - Rebase to latest perf-tools-next/perf-tools-next (1de5b5dcb835).
>
> TODO:
> - perf sched stats records /proc/schedstat which is a CPU and domain
> level scheduler statistic. We are planning to add taskstat tool which
> reads task stats from procfs and generate scheduler statistic report
> at task granularity. this will probably a standalone tool, something
> like `perf sched taskstat record/report`.
> - Except pre-processor related checkpatch warnings, we have addressed
> most of the other possible warnings.
> - This version supports diff for two perf.data files captured for same
> schedstats version but the target is to show diff for multiple
> perf.data files. Plan is to support diff if perf.data files provided
> has different schedstat versions.
>
> Patches are prepared on top of perf-tools-next(571d29baa07e).
>
> [1] https://youtu.be/lg-9aG2ajA0?t=283
> [2] https://github.com/AMDESE/sched-scoreboard
> [3] https://lore.kernel.org/lkml/c50bdbfe-02ce-c1bc-c761-c95f8e216ca0@amd.com/
> [4] https://lore.kernel.org/lkml/3e32bec6-5e59-c66a-7676-7d15df2c961c@amd.com/
> [5] https://lore.kernel.org/all/20241122084452.1064968-1-swapnil.sapkal@amd.com/
> [6] https://lore.kernel.org/lkml/3170d16e-eb67-4db8-a327-eb8188397fdb@amd.com/
> [7] https://lore.kernel.org/lkml/feb31b6e-6457-454c-a4f3-ce8ad96bf8de@amd.com/
>
> Swapnil Sapkal (10):
> tools/lib: Add list_is_first()
> perf header: Support CPU DOMAIN relation info
> perf sched stats: Add record and rawdump support
> perf sched stats: Add schedstat v16 support
> perf sched stats: Add schedstat v17 support
> perf sched stats: Add support for report subcommand
> perf sched stats: Add support for live mode
> perf sched stats: Add support for diff subcommand
> perf sched stats: Add basic perf sched stats test
> perf sched stats: Add details in man page
Nice work!
Acked-by: Namhyung Kim <namhyung@kernel.org>
Thanks,
Namhyung
>
> tools/include/linux/list.h | 10 +
> tools/lib/perf/Documentation/libperf.txt | 2 +
> tools/lib/perf/Makefile | 1 +
> tools/lib/perf/include/perf/event.h | 69 ++
> tools/lib/perf/include/perf/schedstat-v15.h | 146 +++
> tools/lib/perf/include/perf/schedstat-v16.h | 146 +++
> tools/lib/perf/include/perf/schedstat-v17.h | 164 +++
> tools/perf/Documentation/perf-sched.txt | 261 ++++-
> .../Documentation/perf.data-file-format.txt | 17 +
> tools/perf/builtin-inject.c | 3 +
> tools/perf/builtin-sched.c | 1028 ++++++++++++++++-
> tools/perf/tests/shell/perf_sched_stats.sh | 64 +
> tools/perf/util/env.c | 29 +
> tools/perf/util/env.h | 17 +
> tools/perf/util/event.c | 52 +
> tools/perf/util/event.h | 2 +
> tools/perf/util/header.c | 285 +++++
> tools/perf/util/header.h | 4 +
> tools/perf/util/session.c | 22 +
> tools/perf/util/synthetic-events.c | 196 ++++
> tools/perf/util/synthetic-events.h | 3 +
> tools/perf/util/tool.c | 20 +
> tools/perf/util/tool.h | 4 +-
> tools/perf/util/util.c | 48 +
> tools/perf/util/util.h | 5 +
> 25 files changed, 2595 insertions(+), 3 deletions(-)
> create mode 100644 tools/lib/perf/include/perf/schedstat-v15.h
> create mode 100644 tools/lib/perf/include/perf/schedstat-v16.h
> create mode 100644 tools/lib/perf/include/perf/schedstat-v17.h
> create mode 100755 tools/perf/tests/shell/perf_sched_stats.sh
>
> --
> 2.43.0
>
^ permalink raw reply [flat|nested] 36+ messages in thread