* + delaytop-add-delay-max-for-delaytop.patch added to mm-nonmm-unstable branch
@ 2026-07-02 23:02 Andrew Morton
0 siblings, 0 replies; 2+ messages in thread
From: Andrew Morton @ 2026-07-02 23:02 UTC (permalink / raw)
To: mm-commits, yang.yang29, xu.xin16, fan.yu9, corbet, wang.yaxin,
akpm
The patch titled
Subject: delaytop: add delay max for delaytop
has been added to the -mm mm-nonmm-unstable branch. Its filename is
delaytop-add-delay-max-for-delaytop.patch
This patch will shortly appear at
https://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new.git/tree/patches/delaytop-add-delay-max-for-delaytop.patch
This patch will later appear in the mm-nonmm-unstable branch at
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next via various
branches at git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
and is updated there most days
------------------------------------------------------
From: Wang Yaxin <wang.yaxin@zte.com.cn>
Subject: delaytop: add delay max for delaytop
Date: Thu, 2 Jul 2026 20:58:15 +0800 (CST)
Patch series "delaytop: add delay max, timestamp and sorting for top
latency analysis".
Previously delaytop only showed average delays. This patch adds:
1. delay_max fields to track the maximum delay value for each delay type
(cpu, blkio, irq, swapin, freepages, thrashing, compact, wpcopy)
per task.
2. The -t/--type option displays only the specified delay type with avg/max
values side by side, allowing focused analysis:
delaytop -t cpu # Show only CPU delay with avg/max
delaytop -t wpcopy # Show Copy-on-Write delay with avg/max
3. Wall-clock timestamp when each maximum delay occurred, displayed in the
MAX_TIMESTAMP column when using -t/--type option. This enables:
- Identifying the time when a process experienced an abnormal delay max
- Correlating delay max across multiple processes at the same timestamp
- Cross-referencing with logs, traces, or other metrics at that time
4. When using -t/--type option, tasks are sorted by maximum delay value in
descending order (largest delay first), enabling quick identification of
top N processes with highest delay spikes.
This patch (of 3):
Previously delaytop only showed average delays. Add delay_max fields to
track the maximum delay value for each delay type (cpu, blkio, irq,
swapin, freepages, thrashing, compact, wpcopy) per task.
This provides a global view of all tasks' delay spikes, which is essential
for identifying processes that experienced brief but significant latency
events that would be hidden by average-only metrics.
The -t/--type option displays only the specified delay type with avg/max
values side by side, allowing focused analysis:
delaytop -t cpu # Show only CPU delay with avg/max
delaytop -t wpcopy # Show Copy-on-Write delay with avg/max
Link: https://lore.kernel.org/20260702205704180NZ3cu_QF04KfBIL6vjTHL@zte.com.cn
Link: https://lore.kernel.org/202607022058152607Y25X-YgssuvncpVNHljz@zte.com.cn
Signed-off-by: Wang Yaxin <wang.yaxin@zte.com.cn>
Cc: Fan Yu <fan.yu9@zte.com.cn>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: xu xin <xu.xin16@zte.com.cn>
Cc: Yang Yang <yang.yang29@zte.com.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---
tools/accounting/delaytop.c | 165 ++++++++++++++++++++++++++--------
1 file changed, 131 insertions(+), 34 deletions(-)
--- a/tools/accounting/delaytop.c~delaytop-add-delay-max-for-delaytop
+++ a/tools/accounting/delaytop.c
@@ -75,13 +75,21 @@
{#name, #cmd, \
offsetof(struct task_info, name##_delay_total), \
offsetof(struct task_info, name##_count), \
+ offsetof(struct task_info, name##_delay_max), \
modes}
-#define END_FIELD {NULL, 0, 0}
+#define SORT_FIELD_NO_MAX(name, cmd, modes) \
+ {#name, #cmd, \
+ offsetof(struct task_info, name##_delay_total), \
+ offsetof(struct task_info, name##_count), \
+ 0, \
+ modes}
+#define END_FIELD {NULL, 0, 0, 0, 0, 0}
/* Display mode types */
#define MODE_TYPE_ALL (0xFFFFFFFF)
#define MODE_DEFAULT (1 << 0)
#define MODE_MEMVERBOSE (1 << 1)
+#define MODE_TYPE (1 << 2) /* Display specific type with avg/max */
/* PSI statistics structure */
struct psi_stats {
@@ -108,20 +116,28 @@ struct task_info {
char command[TASK_COMM_LEN];
unsigned long long cpu_count;
unsigned long long cpu_delay_total;
+ unsigned long long cpu_delay_max;
unsigned long long blkio_count;
unsigned long long blkio_delay_total;
+ unsigned long long blkio_delay_max;
unsigned long long swapin_count;
unsigned long long swapin_delay_total;
+ unsigned long long swapin_delay_max;
unsigned long long freepages_count;
unsigned long long freepages_delay_total;
+ unsigned long long freepages_delay_max;
unsigned long long thrashing_count;
unsigned long long thrashing_delay_total;
+ unsigned long long thrashing_delay_max;
unsigned long long compact_count;
unsigned long long compact_delay_total;
+ unsigned long long compact_delay_max;
unsigned long long wpcopy_count;
unsigned long long wpcopy_delay_total;
+ unsigned long long wpcopy_delay_max;
unsigned long long irq_count;
unsigned long long irq_delay_total;
+ unsigned long long irq_delay_max;
unsigned long long mem_count;
unsigned long long mem_delay_total;
};
@@ -141,6 +157,7 @@ struct field_desc {
const char *cmd_char; /* Interactive command */
unsigned long total_offset; /* Offset of total delay in task_info */
unsigned long count_offset; /* Offset of count in task_info */
+ unsigned long max_offset; /* Offset of max delay in task_info */
size_t supported_modes; /* Supported display modes */
};
@@ -153,6 +170,7 @@ struct config {
int monitor_pid; /* Monitor specific PID */
char *container_path; /* Path to container cgroup */
const struct field_desc *sort_field; /* Current sort field */
+ const struct field_desc *type_field; /* Type field for -t option */
size_t display_mode; /* Current display mode */
};
@@ -164,15 +182,15 @@ static int task_count;
static int running = 1;
static struct container_stats container_stats;
static const struct field_desc sort_fields[] = {
- SORT_FIELD(cpu, c, MODE_DEFAULT),
- SORT_FIELD(blkio, i, MODE_DEFAULT),
- SORT_FIELD(irq, q, MODE_DEFAULT),
- SORT_FIELD(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE),
- SORT_FIELD(swapin, s, MODE_MEMVERBOSE),
- SORT_FIELD(freepages, r, MODE_MEMVERBOSE),
- SORT_FIELD(thrashing, t, MODE_MEMVERBOSE),
- SORT_FIELD(compact, p, MODE_MEMVERBOSE),
- SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE),
+ SORT_FIELD(cpu, c, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD(blkio, i, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD(irq, q, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD_NO_MAX(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE),
+ SORT_FIELD(swapin, s, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(freepages, r, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(thrashing, t, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(compact, p, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE | MODE_TYPE),
END_FIELD
};
static int sort_selected;
@@ -265,6 +283,7 @@ static void usage(void)
" -p, --pid=PID Monitor only the specified PID\n"
" -C, --container=PATH Monitor the container at specified cgroup path\n"
" -s, --sort=FIELD Sort by delay field (default: cpu)\n"
+ " -t, --type=FIELD Display only specified delay type with avg/max\n"
" -M, --memverbose Display memory detailed information\n");
exit(0);
}
@@ -283,6 +302,7 @@ static void parse_args(int argc, char **
{"processes", required_argument, 0, 'P'},
{"sort", required_argument, 0, 's'},
{"container", required_argument, 0, 'C'},
+ {"type", required_argument, 0, 't'},
{"memverbose", no_argument, 0, 'M'},
{0, 0, 0, 0}
};
@@ -292,6 +312,7 @@ static void parse_args(int argc, char **
cfg.iterations = 0;
cfg.max_processes = 20;
cfg.sort_field = &sort_fields[0]; /* Default sorted by CPU delay */
+ cfg.type_field = NULL; /* No type field by default */
cfg.output_one_time = 0;
cfg.monitor_pid = 0; /* 0 means monitor all PIDs */
cfg.container_path = NULL;
@@ -300,7 +321,7 @@ static void parse_args(int argc, char **
while (1) {
int option_index = 0;
- c = getopt_long(argc, argv, "hd:n:p:oP:C:s:M", long_options, &option_index);
+ c = getopt_long(argc, argv, "hd:n:p:oP:C:s:t:M", long_options, &option_index);
if (c == -1)
break;
@@ -363,9 +384,32 @@ static void parse_args(int argc, char **
cfg.sort_field = field;
break;
+ case 't':
+ if (strlen(optarg) == 0) {
+ fprintf(stderr, "Error: empty type field\n");
+ exit(1);
+ }
+
+ field = get_field_by_name(optarg);
+ /* Show available fields if invalid option provided */
+ if (!field || !(field->supported_modes & MODE_TYPE)) {
+ fprintf(stderr, "Error: invalid type field '%s'\n", optarg);
+ display_available_fields(MODE_TYPE);
+ exit(1);
+ }
+
+ cfg.type_field = field;
+ cfg.display_mode = MODE_TYPE;
+ break;
case 'M':
cfg.display_mode = MODE_MEMVERBOSE;
- cfg.sort_field = get_field_by_name("mem");
+ /* Find first field supporting MODE_MEMVERBOSE for sorting */
+ for (field = sort_fields; field->name != NULL; field++) {
+ if (field->supported_modes & MODE_MEMVERBOSE) {
+ cfg.sort_field = field;
+ break;
+ }
+ }
break;
default:
fprintf(stderr, "Try 'delaytop --help' for more information.\n");
@@ -690,7 +734,12 @@ static void fetch_and_fill_task_info(int
nested_len = NLA_PAYLOAD(na->nla_len);
while (nested_len > 0) {
if (nested->nla_type == TASKSTATS_TYPE_STATS) {
- memcpy(&stats, NLA_DATA(nested), sizeof(stats));
+ size_t payload_len = NLA_PAYLOAD(nested->nla_len);
+
+ memset(&stats, 0, sizeof(stats));
+ if (payload_len > sizeof(stats))
+ payload_len = sizeof(stats);
+ memcpy(&stats, NLA_DATA(nested), payload_len);
if (task_count < MAX_TASKS) {
tasks[task_count].pid = pid;
tasks[task_count].tgid = pid;
@@ -699,20 +748,28 @@ static void fetch_and_fill_task_info(int
tasks[task_count].command[TASK_COMM_LEN - 1] = '\0';
SET_TASK_STAT(task_count, cpu_count);
SET_TASK_STAT(task_count, cpu_delay_total);
+ SET_TASK_STAT(task_count, cpu_delay_max);
SET_TASK_STAT(task_count, blkio_count);
SET_TASK_STAT(task_count, blkio_delay_total);
+ SET_TASK_STAT(task_count, blkio_delay_max);
SET_TASK_STAT(task_count, swapin_count);
SET_TASK_STAT(task_count, swapin_delay_total);
+ SET_TASK_STAT(task_count, swapin_delay_max);
SET_TASK_STAT(task_count, freepages_count);
SET_TASK_STAT(task_count, freepages_delay_total);
+ SET_TASK_STAT(task_count, freepages_delay_max);
SET_TASK_STAT(task_count, thrashing_count);
SET_TASK_STAT(task_count, thrashing_delay_total);
+ SET_TASK_STAT(task_count, thrashing_delay_max);
SET_TASK_STAT(task_count, compact_count);
SET_TASK_STAT(task_count, compact_delay_total);
+ SET_TASK_STAT(task_count, compact_delay_max);
SET_TASK_STAT(task_count, wpcopy_count);
SET_TASK_STAT(task_count, wpcopy_delay_total);
+ SET_TASK_STAT(task_count, wpcopy_delay_max);
SET_TASK_STAT(task_count, irq_count);
SET_TASK_STAT(task_count, irq_delay_total);
+ SET_TASK_STAT(task_count, irq_delay_max);
set_mem_count(&tasks[task_count]);
set_mem_delay_total(&tasks[task_count]);
task_count++;
@@ -777,14 +834,14 @@ static int compare_tasks(const void *a,
const struct task_info *t2 = (const struct task_info *)b;
unsigned long long total1;
unsigned long long total2;
- unsigned long count1;
- unsigned long count2;
+ unsigned long long count1;
+ unsigned long long count2;
double avg1, avg2;
total1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->total_offset);
total2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->total_offset);
- count1 = *(unsigned long *)((char *)t1 + cfg.sort_field->count_offset);
- count2 = *(unsigned long *)((char *)t2 + cfg.sort_field->count_offset);
+ count1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->count_offset);
+ count2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->count_offset);
avg1 = average_ms(total1, count1);
avg2 = average_ms(total2, count2);
@@ -794,6 +851,26 @@ static int compare_tasks(const void *a,
return 0;
}
+/* Get delay values for a specific field */
+static void get_field_delay_values(const struct task_info *task, const struct field_desc *field,
+ double *avg_ms, double *max_ms)
+{
+ unsigned long long total, count, max;
+
+ if (!field || !field->max_offset) {
+ *avg_ms = 0;
+ *max_ms = 0;
+ return;
+ }
+
+ total = *(unsigned long long *)((char *)task + field->total_offset);
+ count = *(unsigned long long *)((char *)task + field->count_offset);
+ *avg_ms = average_ms(total, count);
+
+ max = *(unsigned long long *)((char *)task + field->max_offset);
+ *max_ms = (double)max / 1000000.0; /* Convert nanoseconds to milliseconds */
+}
+
/* Sort tasks by selected field */
static void sort_tasks(void)
{
@@ -847,7 +924,12 @@ static void get_container_stats(void)
while (nl_len > 0) {
if (na->nla_type == CGROUPSTATS_TYPE_CGROUP_STATS) {
/* Get the cgroupstats structure */
- memcpy(&stats, NLA_DATA(na), sizeof(stats));
+ size_t payload_len = NLA_PAYLOAD(na->nla_len);
+
+ memset(&stats, 0, sizeof(stats));
+ if (payload_len > sizeof(stats))
+ payload_len = sizeof(stats);
+ memcpy(&stats, NLA_DATA(na), payload_len);
/* Fill container stats */
container_stats.nr_sleeping = stats.nr_sleeping;
@@ -950,29 +1032,44 @@ static void display_results(int psi_ret)
suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n",
cfg.max_processes, get_name_by_field(cfg.sort_field));
- suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND");
- if (cfg.display_mode == MODE_MEMVERBOSE) {
- suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
- "MEM(ms)", "SWAP(ms)", "RCL(ms)",
- "THR(ms)", "CMP(ms)", "WP(ms)");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "---------------------\n");
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+ /* Display mode for -t option: show only specified type with avg/max */
+ suc &= BOOL_FPRINT(out, "%8s %8s %-17s %12s %12s\n",
+ "PID", "TGID", "COMMAND",
+ "AVG(ms)", "MAX(ms)");
+ suc &= BOOL_FPRINT(out, "----------------------------------------------------\n");
} else {
- suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
- "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "--------------------------\n");
+ suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND");
+ if (cfg.display_mode == MODE_MEMVERBOSE) {
+ suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
+ "MEM(ms)", "SWAP(ms)", "RCL(ms)",
+ "THR(ms)", "CMP(ms)", "WP(ms)");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "---------------------\n");
+ } else {
+ suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
+ "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "--------------------------\n");
+ }
}
count = task_count < cfg.max_processes ? task_count : cfg.max_processes;
for (i = 0; i < count; i++) {
- suc &= BOOL_FPRINT(out, "%8d %8d %-15s",
+ suc &= BOOL_FPRINT(out, "%8d %8d %-17s",
tasks[i].pid, tasks[i].tgid, tasks[i].command);
- if (cfg.display_mode == MODE_MEMVERBOSE) {
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+ double avg_ms, max_ms;
+
+ get_field_delay_values(&tasks[i], cfg.type_field, &avg_ms, &max_ms);
+
+ suc &= BOOL_FPRINT(out, "%12.2f %12.2f\n",
+ avg_ms, max_ms);
+ } else if (cfg.display_mode == MODE_MEMVERBOSE) {
suc &= BOOL_FPRINT(out, DELAY_FMT_MEMVERBOSE,
TASK_AVG(tasks[i], mem),
TASK_AVG(tasks[i], swapin),
_
Patches currently in -mm which might be from wang.yaxin@zte.com.cn are
delaytop-add-delay-max-for-delaytop.patch
delaytop-add-timestamp-of-delay-max.patch
delaytop-sort-by-max-delay-to-highlight-top-latency-processes.patch
^ permalink raw reply [flat|nested] 2+ messages in thread
* + delaytop-add-delay-max-for-delaytop.patch added to mm-nonmm-unstable branch
@ 2026-10-31 1:06 Andrew Morton
0 siblings, 0 replies; 2+ messages in thread
From: Andrew Morton @ 2026-10-31 1:06 UTC (permalink / raw)
To: mm-commits, yang.yang29, xu.xin, thomas.weissschuh, fan.yu9,
corbet, wang.yaxin, akpm
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 18516 bytes --]
The patch titled
Subject: delaytop: add delay max for delaytop
has been added to the -mm mm-nonmm-unstable branch. Its filename is
delaytop-add-delay-max-for-delaytop.patch
This patch will shortly appear at
https://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new.git/tree/patches/delaytop-add-delay-max-for-delaytop.patch
This patch will later appear in the mm-nonmm-unstable branch at
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next via various
branches at git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
and is updated there most days
------------------------------------------------------
From: Wang Yaxin <wang.yaxin@zte.com.cn>
Subject: delaytop: add delay max for delaytop
Date: Wed, 23 Sep 2026 21:03:17 +0800
Patch series "delaytop: add delay max, timestamp and sorting for top
latency analysis", v5.
Add delay max, timestamp of delay max and max-delay sorting to delaytop,
which can help quickly detect potential abnormal delays from a global
perspective and locate the top-N processes with the highest latency
spikes.
Problem
=======
Delay accounting can track the average delay of processes to show
system workload. delaytop provides a global view of average delays
across all tasks, which differs from getdelays's single-task
perspective. However, when a process experiences a significant delay,
maybe a delay spike, which adversely affects performance, delaytop can
only display the average delay over a period of time. Yet, average
delay is unhelpful for diagnosing delay peak. A brief but severe
latency spike is averaged out and hidden by the average-only metric,
and the top-N list sorted by average cannot identify the processes that
actually contributed most to the delay peak, nor provide any time
context to correlate the spike with other system events.
Solution
========
Extend delaytop with the delay max, timestamp and sorting ability:
1. delay_max fields track the maximum delay value for each delay type
(cpu, blkio, irq, swapin, freepages, thrashing, compact, wpcopy)
per task, recording every delay peak since task start.
2. Record the timestamp when each maximum delay occurred, which is the
same capability as getdelays but provided from a global perspective.
3. The -t/--type option displays only the specified delay type with
avg/max/timestamp side by side, and sorts tasks by the maximum delay
in descending order. This highlights the top-N processes with the
highest delay spikes, so abnormal delays can be located from a
global perspective and root causes pinpointed by correlating the
timestamps with logs, traces or other metrics.
Use case
========
bash# ./delaytop -t cpu
System Pressure Information: (avg10/avg60/avg300/total)
CPU some: 0.4%/ 0.2%/ 0.1%/ 220(ms)
CPU full: 0.0%/ 0.0%/ 0.0%/ 0(ms)
Memory full: 0.0%/ 0.0%/ 0.0%/ 0(ms)
Memory some: 0.0%/ 0.0%/ 0.0%/ 0(ms)
IO full: 0.0%/ 0.0%/ 0.0%/ 12(ms)
IO some: 0.0%/ 0.0%/ 0.0%/ 13(ms)
IRQ full: 0.0%/ 0.0%/ 0.0%/ 0(ms)
[q]quit
Top 20 processes (sorted by cpu MAX delay, largest first):
PID TGID COMMAND AVG(ms) MAX(ms) MAX_TIMESTAMP
----------------------------------------------------------------
9 9 kworker/0:0-eve 0.59 16.87 2026-05-27T13:32:39
30 30 kworker/2:0H-kb 2.87 11.36 2026-05-27T13:32:36
27 27 migration/2 1.05 9.51 2026-05-27T13:32:37
50 50 kworker/2:1-eve 0.50 9.13 2026-05-27T13:32:37
15 15 rcu_preempt 0.11 8.98 2026-05-27T13:32:37
1 1 init 0.17 7.12 2026-05-27T13:32:38
This patch (of 3):
Previously delaytop only showed average delays. Add delay_max fields to
track the maximum delay value for each delay type (cpu, blkio, irq, swapin,
freepages, thrashing, compact, wpcopy) per task.
This provides a global view of all tasks' delay spikes, which is essential
for identifying processes that experienced brief but significant latency
events that would be hidden by average-only metrics.
The -t/--type option displays only the specified delay type with avg/max
values side by side, allowing focused analysis:
delaytop -t cpu # Show only CPU delay with avg/max
delaytop -t wpcopy # Show Copy-on-Write delay with avg/max
Link: https://lore.kernel.org/20260923210149015t-X8AvGrXfHE4PDn9gZOM@zte.com.cn
Link: https://lore.kernel.org/202609232103177521ISAibnqneYbXXYDwURYL@zte.com.cn
Signed-off-by: Wang Yaxin <wang.yaxin@zte.com.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Fan Yu <fan.yu9@zte.com.cn>
Cc: Yang Yang <yang.yang29@zte.com.cn>
Cc: Thomas Weißschuh <thomas.weissschuh@linutronix.de>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Xu Xin <xu.xin@linux.dev>
---
tools/accounting/delaytop.c | 165 ++++++++++++++++++++++++++--------
1 file changed, 131 insertions(+), 34 deletions(-)
--- a/tools/accounting/delaytop.c~delaytop-add-delay-max-for-delaytop
+++ a/tools/accounting/delaytop.c
@@ -75,13 +75,21 @@
{#name, #cmd, \
offsetof(struct task_info, name##_delay_total), \
offsetof(struct task_info, name##_count), \
+ offsetof(struct task_info, name##_delay_max), \
modes}
-#define END_FIELD {NULL, 0, 0}
+#define SORT_FIELD_NO_MAX(name, cmd, modes) \
+ {#name, #cmd, \
+ offsetof(struct task_info, name##_delay_total), \
+ offsetof(struct task_info, name##_count), \
+ 0, \
+ modes}
+#define END_FIELD {NULL, 0, 0, 0, 0, 0}
/* Display mode types */
#define MODE_TYPE_ALL (0xFFFFFFFF)
#define MODE_DEFAULT (1 << 0)
#define MODE_MEMVERBOSE (1 << 1)
+#define MODE_TYPE (1 << 2) /* Display specific type with avg/max */
/* PSI statistics structure */
struct psi_stats {
@@ -108,20 +116,28 @@ struct task_info {
char command[TASK_COMM_LEN];
unsigned long long cpu_count;
unsigned long long cpu_delay_total;
+ unsigned long long cpu_delay_max;
unsigned long long blkio_count;
unsigned long long blkio_delay_total;
+ unsigned long long blkio_delay_max;
unsigned long long swapin_count;
unsigned long long swapin_delay_total;
+ unsigned long long swapin_delay_max;
unsigned long long freepages_count;
unsigned long long freepages_delay_total;
+ unsigned long long freepages_delay_max;
unsigned long long thrashing_count;
unsigned long long thrashing_delay_total;
+ unsigned long long thrashing_delay_max;
unsigned long long compact_count;
unsigned long long compact_delay_total;
+ unsigned long long compact_delay_max;
unsigned long long wpcopy_count;
unsigned long long wpcopy_delay_total;
+ unsigned long long wpcopy_delay_max;
unsigned long long irq_count;
unsigned long long irq_delay_total;
+ unsigned long long irq_delay_max;
unsigned long long mem_count;
unsigned long long mem_delay_total;
};
@@ -141,6 +157,7 @@ struct field_desc {
const char *cmd_char; /* Interactive command */
unsigned long total_offset; /* Offset of total delay in task_info */
unsigned long count_offset; /* Offset of count in task_info */
+ unsigned long max_offset; /* Offset of max delay in task_info */
size_t supported_modes; /* Supported display modes */
};
@@ -153,6 +170,7 @@ struct config {
int monitor_pid; /* Monitor specific PID */
char *container_path; /* Path to container cgroup */
const struct field_desc *sort_field; /* Current sort field */
+ const struct field_desc *type_field; /* Type field for -t option */
size_t display_mode; /* Current display mode */
};
@@ -164,15 +182,15 @@ static int task_count;
static int running = 1;
static struct container_stats container_stats;
static const struct field_desc sort_fields[] = {
- SORT_FIELD(cpu, c, MODE_DEFAULT),
- SORT_FIELD(blkio, i, MODE_DEFAULT),
- SORT_FIELD(irq, q, MODE_DEFAULT),
- SORT_FIELD(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE),
- SORT_FIELD(swapin, s, MODE_MEMVERBOSE),
- SORT_FIELD(freepages, r, MODE_MEMVERBOSE),
- SORT_FIELD(thrashing, t, MODE_MEMVERBOSE),
- SORT_FIELD(compact, p, MODE_MEMVERBOSE),
- SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE),
+ SORT_FIELD(cpu, c, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD(blkio, i, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD(irq, q, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD_NO_MAX(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE),
+ SORT_FIELD(swapin, s, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(freepages, r, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(thrashing, t, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(compact, p, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE | MODE_TYPE),
END_FIELD
};
static int sort_selected;
@@ -265,6 +283,7 @@ static void usage(void)
" -p, --pid=PID Monitor only the specified PID\n"
" -C, --container=PATH Monitor the container at specified cgroup path\n"
" -s, --sort=FIELD Sort by delay field (default: cpu)\n"
+ " -t, --type=FIELD Display only specified delay type with avg/max\n"
" -M, --memverbose Display memory detailed information\n");
exit(0);
}
@@ -283,6 +302,7 @@ static void parse_args(int argc, char **
{"processes", required_argument, 0, 'P'},
{"sort", required_argument, 0, 's'},
{"container", required_argument, 0, 'C'},
+ {"type", required_argument, 0, 't'},
{"memverbose", no_argument, 0, 'M'},
{0, 0, 0, 0}
};
@@ -292,6 +312,7 @@ static void parse_args(int argc, char **
cfg.iterations = 0;
cfg.max_processes = 20;
cfg.sort_field = &sort_fields[0]; /* Default sorted by CPU delay */
+ cfg.type_field = NULL; /* No type field by default */
cfg.output_one_time = 0;
cfg.monitor_pid = 0; /* 0 means monitor all PIDs */
cfg.container_path = NULL;
@@ -300,7 +321,7 @@ static void parse_args(int argc, char **
while (1) {
int option_index = 0;
- c = getopt_long(argc, argv, "hd:n:p:oP:C:s:M", long_options, &option_index);
+ c = getopt_long(argc, argv, "hd:n:p:oP:C:s:t:M", long_options, &option_index);
if (c == -1)
break;
@@ -363,9 +384,32 @@ static void parse_args(int argc, char **
cfg.sort_field = field;
break;
+ case 't':
+ if (strlen(optarg) == 0) {
+ fprintf(stderr, "Error: empty type field\n");
+ exit(1);
+ }
+
+ field = get_field_by_name(optarg);
+ /* Show available fields if invalid option provided */
+ if (!field || !(field->supported_modes & MODE_TYPE)) {
+ fprintf(stderr, "Error: invalid type field '%s'\n", optarg);
+ display_available_fields(MODE_TYPE);
+ exit(1);
+ }
+
+ cfg.type_field = field;
+ cfg.display_mode = MODE_TYPE;
+ break;
case 'M':
cfg.display_mode = MODE_MEMVERBOSE;
- cfg.sort_field = get_field_by_name("mem");
+ /* Find first field supporting MODE_MEMVERBOSE for sorting */
+ for (field = sort_fields; field->name != NULL; field++) {
+ if (field->supported_modes & MODE_MEMVERBOSE) {
+ cfg.sort_field = field;
+ break;
+ }
+ }
break;
default:
fprintf(stderr, "Try 'delaytop --help' for more information.\n");
@@ -690,7 +734,12 @@ static void fetch_and_fill_task_info(int
nested_len = NLA_PAYLOAD(na->nla_len);
while (nested_len > 0) {
if (nested->nla_type == TASKSTATS_TYPE_STATS) {
- memcpy(&stats, NLA_DATA(nested), sizeof(stats));
+ size_t payload_len = NLA_PAYLOAD(nested->nla_len);
+
+ memset(&stats, 0, sizeof(stats));
+ if (payload_len > sizeof(stats))
+ payload_len = sizeof(stats);
+ memcpy(&stats, NLA_DATA(nested), payload_len);
if (task_count < MAX_TASKS) {
tasks[task_count].pid = pid;
tasks[task_count].tgid = pid;
@@ -699,20 +748,28 @@ static void fetch_and_fill_task_info(int
tasks[task_count].command[TASK_COMM_LEN - 1] = '\0';
SET_TASK_STAT(task_count, cpu_count);
SET_TASK_STAT(task_count, cpu_delay_total);
+ SET_TASK_STAT(task_count, cpu_delay_max);
SET_TASK_STAT(task_count, blkio_count);
SET_TASK_STAT(task_count, blkio_delay_total);
+ SET_TASK_STAT(task_count, blkio_delay_max);
SET_TASK_STAT(task_count, swapin_count);
SET_TASK_STAT(task_count, swapin_delay_total);
+ SET_TASK_STAT(task_count, swapin_delay_max);
SET_TASK_STAT(task_count, freepages_count);
SET_TASK_STAT(task_count, freepages_delay_total);
+ SET_TASK_STAT(task_count, freepages_delay_max);
SET_TASK_STAT(task_count, thrashing_count);
SET_TASK_STAT(task_count, thrashing_delay_total);
+ SET_TASK_STAT(task_count, thrashing_delay_max);
SET_TASK_STAT(task_count, compact_count);
SET_TASK_STAT(task_count, compact_delay_total);
+ SET_TASK_STAT(task_count, compact_delay_max);
SET_TASK_STAT(task_count, wpcopy_count);
SET_TASK_STAT(task_count, wpcopy_delay_total);
+ SET_TASK_STAT(task_count, wpcopy_delay_max);
SET_TASK_STAT(task_count, irq_count);
SET_TASK_STAT(task_count, irq_delay_total);
+ SET_TASK_STAT(task_count, irq_delay_max);
set_mem_count(&tasks[task_count]);
set_mem_delay_total(&tasks[task_count]);
task_count++;
@@ -777,14 +834,14 @@ static int compare_tasks(const void *a,
const struct task_info *t2 = (const struct task_info *)b;
unsigned long long total1;
unsigned long long total2;
- unsigned long count1;
- unsigned long count2;
+ unsigned long long count1;
+ unsigned long long count2;
double avg1, avg2;
total1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->total_offset);
total2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->total_offset);
- count1 = *(unsigned long *)((char *)t1 + cfg.sort_field->count_offset);
- count2 = *(unsigned long *)((char *)t2 + cfg.sort_field->count_offset);
+ count1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->count_offset);
+ count2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->count_offset);
avg1 = average_ms(total1, count1);
avg2 = average_ms(total2, count2);
@@ -794,6 +851,26 @@ static int compare_tasks(const void *a,
return 0;
}
+/* Get delay values for a specific field */
+static void get_field_delay_values(const struct task_info *task, const struct field_desc *field,
+ double *avg_ms, double *max_ms)
+{
+ unsigned long long total, count, max;
+
+ if (!field || !field->max_offset) {
+ *avg_ms = 0;
+ *max_ms = 0;
+ return;
+ }
+
+ total = *(unsigned long long *)((char *)task + field->total_offset);
+ count = *(unsigned long long *)((char *)task + field->count_offset);
+ *avg_ms = average_ms(total, count);
+
+ max = *(unsigned long long *)((char *)task + field->max_offset);
+ *max_ms = (double)max / 1000000.0; /* Convert nanoseconds to milliseconds */
+}
+
/* Sort tasks by selected field */
static void sort_tasks(void)
{
@@ -847,7 +924,12 @@ static void get_container_stats(void)
while (nl_len > 0) {
if (na->nla_type == CGROUPSTATS_TYPE_CGROUP_STATS) {
/* Get the cgroupstats structure */
- memcpy(&stats, NLA_DATA(na), sizeof(stats));
+ size_t payload_len = NLA_PAYLOAD(na->nla_len);
+
+ memset(&stats, 0, sizeof(stats));
+ if (payload_len > sizeof(stats))
+ payload_len = sizeof(stats);
+ memcpy(&stats, NLA_DATA(na), payload_len);
/* Fill container stats */
container_stats.nr_sleeping = stats.nr_sleeping;
@@ -950,29 +1032,44 @@ static void display_results(int psi_ret)
suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n",
cfg.max_processes, get_name_by_field(cfg.sort_field));
- suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND");
- if (cfg.display_mode == MODE_MEMVERBOSE) {
- suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
- "MEM(ms)", "SWAP(ms)", "RCL(ms)",
- "THR(ms)", "CMP(ms)", "WP(ms)");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "---------------------\n");
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+ /* Display mode for -t option: show only specified type with avg/max */
+ suc &= BOOL_FPRINT(out, "%8s %8s %-17s %12s %12s\n",
+ "PID", "TGID", "COMMAND",
+ "AVG(ms)", "MAX(ms)");
+ suc &= BOOL_FPRINT(out, "----------------------------------------------------\n");
} else {
- suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
- "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "--------------------------\n");
+ suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND");
+ if (cfg.display_mode == MODE_MEMVERBOSE) {
+ suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
+ "MEM(ms)", "SWAP(ms)", "RCL(ms)",
+ "THR(ms)", "CMP(ms)", "WP(ms)");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "---------------------\n");
+ } else {
+ suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
+ "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "--------------------------\n");
+ }
}
count = task_count < cfg.max_processes ? task_count : cfg.max_processes;
for (i = 0; i < count; i++) {
- suc &= BOOL_FPRINT(out, "%8d %8d %-15s",
+ suc &= BOOL_FPRINT(out, "%8d %8d %-17s",
tasks[i].pid, tasks[i].tgid, tasks[i].command);
- if (cfg.display_mode == MODE_MEMVERBOSE) {
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+ double avg_ms, max_ms;
+
+ get_field_delay_values(&tasks[i], cfg.type_field, &avg_ms, &max_ms);
+
+ suc &= BOOL_FPRINT(out, " %12.2f %12.2f\n",
+ avg_ms, max_ms);
+ } else if (cfg.display_mode == MODE_MEMVERBOSE) {
suc &= BOOL_FPRINT(out, DELAY_FMT_MEMVERBOSE,
TASK_AVG(tasks[i], mem),
TASK_AVG(tasks[i], swapin),
_
Patches currently in -mm which might be from wang.yaxin@zte.com.cn are
delaytop-add-delay-max-for-delaytop.patch
delaytop-add-timestamp-of-delay-max.patch
delaytop-sort-by-max-delay-to-highlight-top-latency-processes.patch
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2026-10-31 1:06 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-10-31 1:06 + delaytop-add-delay-max-for-delaytop.patch added to mm-nonmm-unstable branch Andrew Morton
-- strict thread matches above, loose matches on Subject: below --
2026-07-02 23:02 Andrew Morton
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.