* [PATCH 1/3 v4] delaytop: add delay max for delaytop
2026-09-08 14:50 [PATCH 0/3 v4] delaytop: add delay max, timestamp and sorting for top latency analysis wang.yaxin
@ 2026-09-08 14:52 ` wang.yaxin
2026-09-08 14:54 ` [PATCH 2/3 v4] delaytop: add timestamp of delay max wang.yaxin
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: wang.yaxin @ 2026-09-08 14:52 UTC (permalink / raw)
To: wang.yaxin
Cc: akpm, fan.yu9, yang.yang29, thomas.weissschuh, corbet,
linux-kernel, linux-doc, xu.xin16
From: Wang Yaxin <wang.yaxin@zte.com.cn>
Previously delaytop only showed average delays. Add delay_max fields to
track the maximum delay value for each delay type (cpu, blkio, irq, swapin,
freepages, thrashing, compact, wpcopy) per task.
This provides a global view of all tasks' delay spikes, which is essential
for identifying processes that experienced brief but significant latency
events that would be hidden by average-only metrics.
The -t/--type option displays only the specified delay type with avg/max
values side by side, allowing focused analysis:
delaytop -t cpu # Show only CPU delay with avg/max
delaytop -t wpcopy # Show Copy-on-Write delay with avg/max
Signed-off-by: Wang Yaxin <wang.yaxin@zte.com.cn>
---
tools/accounting/delaytop.c | 165 ++++++++++++++++++++++++++++--------
1 file changed, 131 insertions(+), 34 deletions(-)
diff --git a/tools/accounting/delaytop.c b/tools/accounting/delaytop.c
index 0a863dfeae8a..29b0fff1d4a0 100644
--- a/tools/accounting/delaytop.c
+++ b/tools/accounting/delaytop.c
@@ -75,13 +75,21 @@
{#name, #cmd, \
offsetof(struct task_info, name##_delay_total), \
offsetof(struct task_info, name##_count), \
+ offsetof(struct task_info, name##_delay_max), \
modes}
-#define END_FIELD {NULL, 0, 0}
+#define SORT_FIELD_NO_MAX(name, cmd, modes) \
+ {#name, #cmd, \
+ offsetof(struct task_info, name##_delay_total), \
+ offsetof(struct task_info, name##_count), \
+ 0, \
+ modes}
+#define END_FIELD {NULL, 0, 0, 0, 0, 0}
/* Display mode types */
#define MODE_TYPE_ALL (0xFFFFFFFF)
#define MODE_DEFAULT (1 << 0)
#define MODE_MEMVERBOSE (1 << 1)
+#define MODE_TYPE (1 << 2) /* Display specific type with avg/max */
/* PSI statistics structure */
struct psi_stats {
@@ -108,20 +116,28 @@ struct task_info {
char command[TASK_COMM_LEN];
unsigned long long cpu_count;
unsigned long long cpu_delay_total;
+ unsigned long long cpu_delay_max;
unsigned long long blkio_count;
unsigned long long blkio_delay_total;
+ unsigned long long blkio_delay_max;
unsigned long long swapin_count;
unsigned long long swapin_delay_total;
+ unsigned long long swapin_delay_max;
unsigned long long freepages_count;
unsigned long long freepages_delay_total;
+ unsigned long long freepages_delay_max;
unsigned long long thrashing_count;
unsigned long long thrashing_delay_total;
+ unsigned long long thrashing_delay_max;
unsigned long long compact_count;
unsigned long long compact_delay_total;
+ unsigned long long compact_delay_max;
unsigned long long wpcopy_count;
unsigned long long wpcopy_delay_total;
+ unsigned long long wpcopy_delay_max;
unsigned long long irq_count;
unsigned long long irq_delay_total;
+ unsigned long long irq_delay_max;
unsigned long long mem_count;
unsigned long long mem_delay_total;
};
@@ -141,6 +157,7 @@ struct field_desc {
const char *cmd_char; /* Interactive command */
unsigned long total_offset; /* Offset of total delay in task_info */
unsigned long count_offset; /* Offset of count in task_info */
+ unsigned long max_offset; /* Offset of max delay in task_info */
size_t supported_modes; /* Supported display modes */
};
@@ -153,6 +170,7 @@ struct config {
int monitor_pid; /* Monitor specific PID */
char *container_path; /* Path to container cgroup */
const struct field_desc *sort_field; /* Current sort field */
+ const struct field_desc *type_field; /* Type field for -t option */
size_t display_mode; /* Current display mode */
};
@@ -164,15 +182,15 @@ static int task_count;
static int running = 1;
static struct container_stats container_stats;
static const struct field_desc sort_fields[] = {
- SORT_FIELD(cpu, c, MODE_DEFAULT),
- SORT_FIELD(blkio, i, MODE_DEFAULT),
- SORT_FIELD(irq, q, MODE_DEFAULT),
- SORT_FIELD(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE),
- SORT_FIELD(swapin, s, MODE_MEMVERBOSE),
- SORT_FIELD(freepages, r, MODE_MEMVERBOSE),
- SORT_FIELD(thrashing, t, MODE_MEMVERBOSE),
- SORT_FIELD(compact, p, MODE_MEMVERBOSE),
- SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE),
+ SORT_FIELD(cpu, c, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD(blkio, i, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD(irq, q, MODE_DEFAULT | MODE_TYPE),
+ SORT_FIELD_NO_MAX(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE),
+ SORT_FIELD(swapin, s, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(freepages, r, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(thrashing, t, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(compact, p, MODE_MEMVERBOSE | MODE_TYPE),
+ SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE | MODE_TYPE),
END_FIELD
};
static int sort_selected;
@@ -265,6 +283,7 @@ static void usage(void)
" -p, --pid=PID Monitor only the specified PID\n"
" -C, --container=PATH Monitor the container at specified cgroup path\n"
" -s, --sort=FIELD Sort by delay field (default: cpu)\n"
+ " -t, --type=FIELD Display only specified delay type with avg/max\n"
" -M, --memverbose Display memory detailed information\n");
exit(0);
}
@@ -283,6 +302,7 @@ static void parse_args(int argc, char **argv)
{"processes", required_argument, 0, 'P'},
{"sort", required_argument, 0, 's'},
{"container", required_argument, 0, 'C'},
+ {"type", required_argument, 0, 't'},
{"memverbose", no_argument, 0, 'M'},
{0, 0, 0, 0}
};
@@ -292,6 +312,7 @@ static void parse_args(int argc, char **argv)
cfg.iterations = 0;
cfg.max_processes = 20;
cfg.sort_field = &sort_fields[0]; /* Default sorted by CPU delay */
+ cfg.type_field = NULL; /* No type field by default */
cfg.output_one_time = 0;
cfg.monitor_pid = 0; /* 0 means monitor all PIDs */
cfg.container_path = NULL;
@@ -300,7 +321,7 @@ static void parse_args(int argc, char **argv)
while (1) {
int option_index = 0;
- c = getopt_long(argc, argv, "hd:n:p:oP:C:s:M", long_options, &option_index);
+ c = getopt_long(argc, argv, "hd:n:p:oP:C:s:t:M", long_options, &option_index);
if (c == -1)
break;
@@ -363,9 +384,32 @@ static void parse_args(int argc, char **argv)
cfg.sort_field = field;
break;
+ case 't':
+ if (strlen(optarg) == 0) {
+ fprintf(stderr, "Error: empty type field\n");
+ exit(1);
+ }
+
+ field = get_field_by_name(optarg);
+ /* Show available fields if invalid option provided */
+ if (!field || !(field->supported_modes & MODE_TYPE)) {
+ fprintf(stderr, "Error: invalid type field '%s'\n", optarg);
+ display_available_fields(MODE_TYPE);
+ exit(1);
+ }
+
+ cfg.type_field = field;
+ cfg.display_mode = MODE_TYPE;
+ break;
case 'M':
cfg.display_mode = MODE_MEMVERBOSE;
- cfg.sort_field = get_field_by_name("mem");
+ /* Find first field supporting MODE_MEMVERBOSE for sorting */
+ for (field = sort_fields; field->name != NULL; field++) {
+ if (field->supported_modes & MODE_MEMVERBOSE) {
+ cfg.sort_field = field;
+ break;
+ }
+ }
break;
default:
fprintf(stderr, "Try 'delaytop --help' for more information.\n");
@@ -690,7 +734,12 @@ static void fetch_and_fill_task_info(int pid, const char *comm)
nested_len = NLA_PAYLOAD(na->nla_len);
while (nested_len > 0) {
if (nested->nla_type == TASKSTATS_TYPE_STATS) {
- memcpy(&stats, NLA_DATA(nested), sizeof(stats));
+ size_t payload_len = NLA_PAYLOAD(nested->nla_len);
+
+ memset(&stats, 0, sizeof(stats));
+ if (payload_len > sizeof(stats))
+ payload_len = sizeof(stats);
+ memcpy(&stats, NLA_DATA(nested), payload_len);
if (task_count < MAX_TASKS) {
tasks[task_count].pid = pid;
tasks[task_count].tgid = pid;
@@ -699,20 +748,28 @@ static void fetch_and_fill_task_info(int pid, const char *comm)
tasks[task_count].command[TASK_COMM_LEN - 1] = '\0';
SET_TASK_STAT(task_count, cpu_count);
SET_TASK_STAT(task_count, cpu_delay_total);
+ SET_TASK_STAT(task_count, cpu_delay_max);
SET_TASK_STAT(task_count, blkio_count);
SET_TASK_STAT(task_count, blkio_delay_total);
+ SET_TASK_STAT(task_count, blkio_delay_max);
SET_TASK_STAT(task_count, swapin_count);
SET_TASK_STAT(task_count, swapin_delay_total);
+ SET_TASK_STAT(task_count, swapin_delay_max);
SET_TASK_STAT(task_count, freepages_count);
SET_TASK_STAT(task_count, freepages_delay_total);
+ SET_TASK_STAT(task_count, freepages_delay_max);
SET_TASK_STAT(task_count, thrashing_count);
SET_TASK_STAT(task_count, thrashing_delay_total);
+ SET_TASK_STAT(task_count, thrashing_delay_max);
SET_TASK_STAT(task_count, compact_count);
SET_TASK_STAT(task_count, compact_delay_total);
+ SET_TASK_STAT(task_count, compact_delay_max);
SET_TASK_STAT(task_count, wpcopy_count);
SET_TASK_STAT(task_count, wpcopy_delay_total);
+ SET_TASK_STAT(task_count, wpcopy_delay_max);
SET_TASK_STAT(task_count, irq_count);
SET_TASK_STAT(task_count, irq_delay_total);
+ SET_TASK_STAT(task_count, irq_delay_max);
set_mem_count(&tasks[task_count]);
set_mem_delay_total(&tasks[task_count]);
task_count++;
@@ -777,14 +834,14 @@ static int compare_tasks(const void *a, const void *b)
const struct task_info *t2 = (const struct task_info *)b;
unsigned long long total1;
unsigned long long total2;
- unsigned long count1;
- unsigned long count2;
+ unsigned long long count1;
+ unsigned long long count2;
double avg1, avg2;
total1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->total_offset);
total2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->total_offset);
- count1 = *(unsigned long *)((char *)t1 + cfg.sort_field->count_offset);
- count2 = *(unsigned long *)((char *)t2 + cfg.sort_field->count_offset);
+ count1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->count_offset);
+ count2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->count_offset);
avg1 = average_ms(total1, count1);
avg2 = average_ms(total2, count2);
@@ -794,6 +851,26 @@ static int compare_tasks(const void *a, const void *b)
return 0;
}
+/* Get delay values for a specific field */
+static void get_field_delay_values(const struct task_info *task, const struct field_desc *field,
+ double *avg_ms, double *max_ms)
+{
+ unsigned long long total, count, max;
+
+ if (!field || !field->max_offset) {
+ *avg_ms = 0;
+ *max_ms = 0;
+ return;
+ }
+
+ total = *(unsigned long long *)((char *)task + field->total_offset);
+ count = *(unsigned long long *)((char *)task + field->count_offset);
+ *avg_ms = average_ms(total, count);
+
+ max = *(unsigned long long *)((char *)task + field->max_offset);
+ *max_ms = (double)max / 1000000.0; /* Convert nanoseconds to milliseconds */
+}
+
/* Sort tasks by selected field */
static void sort_tasks(void)
{
@@ -847,7 +924,12 @@ static void get_container_stats(void)
while (nl_len > 0) {
if (na->nla_type == CGROUPSTATS_TYPE_CGROUP_STATS) {
/* Get the cgroupstats structure */
- memcpy(&stats, NLA_DATA(na), sizeof(stats));
+ size_t payload_len = NLA_PAYLOAD(na->nla_len);
+
+ memset(&stats, 0, sizeof(stats));
+ if (payload_len > sizeof(stats))
+ payload_len = sizeof(stats);
+ memcpy(&stats, NLA_DATA(na), payload_len);
/* Fill container stats */
container_stats.nr_sleeping = stats.nr_sleeping;
@@ -950,29 +1032,44 @@ static void display_results(int psi_ret)
suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n",
cfg.max_processes, get_name_by_field(cfg.sort_field));
- suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND");
- if (cfg.display_mode == MODE_MEMVERBOSE) {
- suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
- "MEM(ms)", "SWAP(ms)", "RCL(ms)",
- "THR(ms)", "CMP(ms)", "WP(ms)");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "---------------------\n");
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+ /* Display mode for -t option: show only specified type with avg/max */
+ suc &= BOOL_FPRINT(out, "%8s %8s %-17s %12s %12s\n",
+ "PID", "TGID", "COMMAND",
+ "AVG(ms)", "MAX(ms)");
+ suc &= BOOL_FPRINT(out, "----------------------------------------------------\n");
} else {
- suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
- "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "-----------------------");
- suc &= BOOL_FPRINT(out, "--------------------------\n");
+ suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND");
+ if (cfg.display_mode == MODE_MEMVERBOSE) {
+ suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
+ "MEM(ms)", "SWAP(ms)", "RCL(ms)",
+ "THR(ms)", "CMP(ms)", "WP(ms)");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "---------------------\n");
+ } else {
+ suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
+ "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "-----------------------");
+ suc &= BOOL_FPRINT(out, "--------------------------\n");
+ }
}
count = task_count < cfg.max_processes ? task_count : cfg.max_processes;
for (i = 0; i < count; i++) {
- suc &= BOOL_FPRINT(out, "%8d %8d %-15s",
+ suc &= BOOL_FPRINT(out, "%8d %8d %-17s",
tasks[i].pid, tasks[i].tgid, tasks[i].command);
- if (cfg.display_mode == MODE_MEMVERBOSE) {
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+ double avg_ms, max_ms;
+
+ get_field_delay_values(&tasks[i], cfg.type_field, &avg_ms, &max_ms);
+
+ suc &= BOOL_FPRINT(out, "%12.2f %12.2f\n",
+ avg_ms, max_ms);
+ } else if (cfg.display_mode == MODE_MEMVERBOSE) {
suc &= BOOL_FPRINT(out, DELAY_FMT_MEMVERBOSE,
TASK_AVG(tasks[i], mem),
TASK_AVG(tasks[i], swapin),
--
2.47.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 2/3 v4] delaytop: add timestamp of delay max
2026-09-08 14:50 [PATCH 0/3 v4] delaytop: add delay max, timestamp and sorting for top latency analysis wang.yaxin
2026-09-08 14:52 ` [PATCH 1/3 v4] delaytop: add delay max for delaytop wang.yaxin
@ 2026-09-08 14:54 ` wang.yaxin
2026-09-08 14:55 ` [PATCH 3/3 v4] delaytop: sort by max delay to highlight top latency processes wang.yaxin
2026-09-08 18:23 ` [PATCH 0/3 v4] delaytop: add delay max, timestamp and sorting for top latency analysis Andrew Morton
3 siblings, 0 replies; 5+ messages in thread
From: wang.yaxin @ 2026-09-08 14:54 UTC (permalink / raw)
To: wang.yaxin
Cc: akpm, fan.yu9, yang.yang29, thomas.weissschuh, corbet,
linux-kernel, linux-doc, xu.xin16
From: Wang Yaxin <wang.yaxin@zte.com.cn>
Record the wall-clock timestamp when each maximum delay occurred for
all delay types. The timestamp is displayed in the MAX_TIMESTAMP column
when using -t/--type option.
This enables:
- Identifying the time when a process experienced an abnormal delay spike
- Correlating delay peaks across multiple processes at the same timestamp
- Cross-referencing with system logs, traces, or other metrics at that time
- Pinpointing the root cause of latency issues by finding concurrent events
Signed-off-by: Wang Yaxin <wang.yaxin@zte.com.cn>
---
tools/accounting/delaytop.c | 84 ++++++++++++++++++++++++++++++++-----
1 file changed, 74 insertions(+), 10 deletions(-)
diff --git a/tools/accounting/delaytop.c b/tools/accounting/delaytop.c
index 29b0fff1d4a0..7e80f3875a91 100644
--- a/tools/accounting/delaytop.c
+++ b/tools/accounting/delaytop.c
@@ -76,14 +76,16 @@
offsetof(struct task_info, name##_delay_total), \
offsetof(struct task_info, name##_count), \
offsetof(struct task_info, name##_delay_max), \
+ offsetof(struct task_info, name##_delay_max_ts), \
modes}
#define SORT_FIELD_NO_MAX(name, cmd, modes) \
{#name, #cmd, \
offsetof(struct task_info, name##_delay_total), \
offsetof(struct task_info, name##_count), \
0, \
+ 0, \
modes}
-#define END_FIELD {NULL, 0, 0, 0, 0, 0}
+#define END_FIELD {NULL, 0, 0, 0, 0, 0, 0}
/* Display mode types */
#define MODE_TYPE_ALL (0xFFFFFFFF)
@@ -117,27 +119,35 @@ struct task_info {
unsigned long long cpu_count;
unsigned long long cpu_delay_total;
unsigned long long cpu_delay_max;
+ struct __kernel_timespec cpu_delay_max_ts;
unsigned long long blkio_count;
unsigned long long blkio_delay_total;
unsigned long long blkio_delay_max;
+ struct __kernel_timespec blkio_delay_max_ts;
unsigned long long swapin_count;
unsigned long long swapin_delay_total;
unsigned long long swapin_delay_max;
+ struct __kernel_timespec swapin_delay_max_ts;
unsigned long long freepages_count;
unsigned long long freepages_delay_total;
unsigned long long freepages_delay_max;
+ struct __kernel_timespec freepages_delay_max_ts;
unsigned long long thrashing_count;
unsigned long long thrashing_delay_total;
unsigned long long thrashing_delay_max;
+ struct __kernel_timespec thrashing_delay_max_ts;
unsigned long long compact_count;
unsigned long long compact_delay_total;
unsigned long long compact_delay_max;
+ struct __kernel_timespec compact_delay_max_ts;
unsigned long long wpcopy_count;
unsigned long long wpcopy_delay_total;
unsigned long long wpcopy_delay_max;
+ struct __kernel_timespec wpcopy_delay_max_ts;
unsigned long long irq_count;
unsigned long long irq_delay_total;
unsigned long long irq_delay_max;
+ struct __kernel_timespec irq_delay_max_ts;
unsigned long long mem_count;
unsigned long long mem_delay_total;
};
@@ -158,6 +168,7 @@ struct field_desc {
unsigned long total_offset; /* Offset of total delay in task_info */
unsigned long count_offset; /* Offset of count in task_info */
unsigned long max_offset; /* Offset of max delay in task_info */
+ unsigned long max_ts_offset; /* Offset of max delay timestamp in task_info */
size_t supported_modes; /* Supported display modes */
};
@@ -283,7 +294,7 @@ static void usage(void)
" -p, --pid=PID Monitor only the specified PID\n"
" -C, --container=PATH Monitor the container at specified cgroup path\n"
" -s, --sort=FIELD Sort by delay field (default: cpu)\n"
- " -t, --type=FIELD Display only specified delay type with avg/max\n"
+ " -t, --type=FIELD Display only specified delay type with avg/max/timestamp\n"
" -M, --memverbose Display memory detailed information\n");
exit(0);
}
@@ -749,27 +760,35 @@ static void fetch_and_fill_task_info(int pid, const char *comm)
SET_TASK_STAT(task_count, cpu_count);
SET_TASK_STAT(task_count, cpu_delay_total);
SET_TASK_STAT(task_count, cpu_delay_max);
+ SET_TASK_STAT(task_count, cpu_delay_max_ts);
SET_TASK_STAT(task_count, blkio_count);
SET_TASK_STAT(task_count, blkio_delay_total);
SET_TASK_STAT(task_count, blkio_delay_max);
+ SET_TASK_STAT(task_count, blkio_delay_max_ts);
SET_TASK_STAT(task_count, swapin_count);
SET_TASK_STAT(task_count, swapin_delay_total);
SET_TASK_STAT(task_count, swapin_delay_max);
+ SET_TASK_STAT(task_count, swapin_delay_max_ts);
SET_TASK_STAT(task_count, freepages_count);
SET_TASK_STAT(task_count, freepages_delay_total);
SET_TASK_STAT(task_count, freepages_delay_max);
+ SET_TASK_STAT(task_count, freepages_delay_max_ts);
SET_TASK_STAT(task_count, thrashing_count);
SET_TASK_STAT(task_count, thrashing_delay_total);
SET_TASK_STAT(task_count, thrashing_delay_max);
+ SET_TASK_STAT(task_count, thrashing_delay_max_ts);
SET_TASK_STAT(task_count, compact_count);
SET_TASK_STAT(task_count, compact_delay_total);
SET_TASK_STAT(task_count, compact_delay_max);
+ SET_TASK_STAT(task_count, compact_delay_max_ts);
SET_TASK_STAT(task_count, wpcopy_count);
SET_TASK_STAT(task_count, wpcopy_delay_total);
SET_TASK_STAT(task_count, wpcopy_delay_max);
+ SET_TASK_STAT(task_count, wpcopy_delay_max_ts);
SET_TASK_STAT(task_count, irq_count);
SET_TASK_STAT(task_count, irq_delay_total);
SET_TASK_STAT(task_count, irq_delay_max);
+ SET_TASK_STAT(task_count, irq_delay_max_ts);
set_mem_count(&tasks[task_count]);
set_mem_delay_total(&tasks[task_count]);
task_count++;
@@ -827,6 +846,41 @@ static double average_ms(unsigned long long total, unsigned long long count)
return (double)total / 1000000.0 / count;
}
+/*
+ * Format __kernel_timespec to human readable string (YYYY-MM-DDTHH:MM:SS)
+ * Returns formatted string or "N/A" if timestamp is zero
+ */
+static const char *format_timespec64(struct __kernel_timespec *ts)
+{
+ static char buffer[32];
+ time_t time_sec;
+ struct tm tm_info;
+
+ /* Check if timestamp is zero (not set) */
+ if (ts->tv_sec == 0 && ts->tv_nsec == 0)
+ return "N/A";
+
+ /* Avoid Y2038 truncation: check if timestamp fits in time_t on 32-bit platforms */
+ if (sizeof(time_t) < sizeof(ts->tv_sec) &&
+ ts->tv_sec > (__u64)((1ULL << (sizeof(time_t) * 8 - 1)) - 1))
+ return "N/A";
+
+ time_sec = (time_t)ts->tv_sec;
+
+ if (localtime_r(&time_sec, &tm_info) == NULL)
+ return "N/A";
+
+ snprintf(buffer, sizeof(buffer), "%04d-%02d-%02dT%02d:%02d:%02d",
+ tm_info.tm_year + 1900,
+ tm_info.tm_mon + 1,
+ tm_info.tm_mday,
+ tm_info.tm_hour,
+ tm_info.tm_min,
+ tm_info.tm_sec);
+
+ return buffer;
+}
+
/* Comparison function for sorting tasks */
static int compare_tasks(const void *a, const void *b)
{
@@ -853,13 +907,15 @@ static int compare_tasks(const void *a, const void *b)
/* Get delay values for a specific field */
static void get_field_delay_values(const struct task_info *task, const struct field_desc *field,
- double *avg_ms, double *max_ms)
+ double *avg_ms, double *max_ms,
+ struct __kernel_timespec *max_ts)
{
unsigned long long total, count, max;
if (!field || !field->max_offset) {
*avg_ms = 0;
*max_ms = 0;
+ memset(max_ts, 0, sizeof(*max_ts));
return;
}
@@ -869,6 +925,11 @@ static void get_field_delay_values(const struct task_info *task, const struct fi
max = *(unsigned long long *)((char *)task + field->max_offset);
*max_ms = (double)max / 1000000.0; /* Convert nanoseconds to milliseconds */
+
+ if (field->max_ts_offset)
+ *max_ts = *(struct __kernel_timespec *)((char *)task + field->max_ts_offset);
+ else
+ memset(max_ts, 0, sizeof(*max_ts));
}
/* Sort tasks by selected field */
@@ -1033,11 +1094,12 @@ static void display_results(int psi_ret)
cfg.max_processes, get_name_by_field(cfg.sort_field));
if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
- /* Display mode for -t option: show only specified type with avg/max */
- suc &= BOOL_FPRINT(out, "%8s %8s %-17s %12s %12s\n",
+ /* Display mode for -t option: show only specified type with avg/max/timestamp */
+ suc &= BOOL_FPRINT(out, "%8s %8s %-17s %12s %12s %20s\n",
"PID", "TGID", "COMMAND",
- "AVG(ms)", "MAX(ms)");
- suc &= BOOL_FPRINT(out, "----------------------------------------------------\n");
+ "AVG(ms)", "MAX(ms)", "MAX_TIMESTAMP");
+ suc &= BOOL_FPRINT(out, "--------------------------------------------------------");
+ suc &= BOOL_FPRINT(out, "----------------------------------------\n");
} else {
suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND");
if (cfg.display_mode == MODE_MEMVERBOSE) {
@@ -1064,11 +1126,13 @@ static void display_results(int psi_ret)
tasks[i].pid, tasks[i].tgid, tasks[i].command);
if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
double avg_ms, max_ms;
+ struct __kernel_timespec max_ts;
- get_field_delay_values(&tasks[i], cfg.type_field, &avg_ms, &max_ms);
+ get_field_delay_values(&tasks[i], cfg.type_field, &avg_ms,
+ &max_ms, &max_ts);
- suc &= BOOL_FPRINT(out, "%12.2f %12.2f\n",
- avg_ms, max_ms);
+ suc &= BOOL_FPRINT(out, "%12.2f %12.2f %20s\n",
+ avg_ms, max_ms, format_timespec64(&max_ts));
} else if (cfg.display_mode == MODE_MEMVERBOSE) {
suc &= BOOL_FPRINT(out, DELAY_FMT_MEMVERBOSE,
TASK_AVG(tasks[i], mem),
--
2.47.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 3/3 v4] delaytop: sort by max delay to highlight top latency processes
2026-09-08 14:50 [PATCH 0/3 v4] delaytop: add delay max, timestamp and sorting for top latency analysis wang.yaxin
2026-09-08 14:52 ` [PATCH 1/3 v4] delaytop: add delay max for delaytop wang.yaxin
2026-09-08 14:54 ` [PATCH 2/3 v4] delaytop: add timestamp of delay max wang.yaxin
@ 2026-09-08 14:55 ` wang.yaxin
2026-09-08 18:23 ` [PATCH 0/3 v4] delaytop: add delay max, timestamp and sorting for top latency analysis Andrew Morton
3 siblings, 0 replies; 5+ messages in thread
From: wang.yaxin @ 2026-09-08 14:55 UTC (permalink / raw)
To: wang.yaxin
Cc: akpm, fan.yu9, yang.yang29, thomas.weissschuh, corbet,
linux-kernel, linux-doc, xu.xin16
From: Wang Yaxin <wang.yaxin@zte.com.cn>
When using -t/--type option, sort tasks by the maximum delay value of
the selected type in descending order (largest delay first).
This enables quickly identifying the top N processes with the highest
delay spikes, which is essential for diagnosing latency problems by
pinpointing which processes contributed most to system delays.
Signed-off-by: Wang Yaxin <wang.yaxin@zte.com.cn>
---
Documentation/accounting/delay-accounting.rst | 87 ++++++++++++++-----
Documentation/sphinx/maintainers_include.py | 2 +-
tools/accounting/delaytop.c | 71 ++++++++++++---
tools/accounting/getdelays.c | 5 ++
4 files changed, 131 insertions(+), 34 deletions(-)
diff --git a/Documentation/accounting/delay-accounting.rst b/Documentation/accounting/delay-accounting.rst
index e209c46241b0..d6cfe52f66f3 100644
--- a/Documentation/accounting/delay-accounting.rst
+++ b/Documentation/accounting/delay-accounting.rst
@@ -152,28 +152,28 @@ Basic usage with default settings (sorts by CPU delay, shows top 20 tasks, refre
IRQ full: 0.0%/ 0.0%/ 0.0%/ 0(ms)
[o]sort [M]memverbose [q]quit
Top 20 processes (sorted by cpu delay):
- PID TGID COMMAND CPU(ms) IO(ms) IRQ(ms) MEM(ms)
+ PID TGID COMMAND CPU(ms) IO(ms) IRQ(ms) MEM(ms)
------------------------------------------------------------------------
- 110 110 kworker/15:0H-s 27.91 0.00 0.00 0.00
- 57 57 cpuhp/7 3.18 0.00 0.00 0.00
- 99 99 cpuhp/14 2.97 0.00 0.00 0.00
- 51 51 cpuhp/6 0.90 0.00 0.00 0.00
- 44 44 kworker/4:0H-sy 0.80 0.00 0.00 0.00
- 60 60 ksoftirqd/7 0.74 0.00 0.00 0.00
- 76 76 idle_inject/10 0.31 0.00 0.00 0.00
- 100 100 idle_inject/14 0.30 0.00 0.00 0.00
- 1309 1309 systemsettings 0.29 0.00 0.00 0.00
- 45 45 cpuhp/5 0.22 0.00 0.00 0.00
- 63 63 cpuhp/8 0.20 0.00 0.00 0.00
- 87 87 cpuhp/12 0.18 0.00 0.00 0.00
- 93 93 cpuhp/13 0.17 0.00 0.00 0.00
- 1265 1265 acpid 0.17 0.00 0.00 0.00
- 1552 1552 sshd 0.17 0.00 0.00 0.00
- 2584 2584 sddm-helper 0.16 0.00 0.00 0.00
- 1284 1284 rtkit-daemon 0.15 0.00 0.00 0.00
- 1326 1326 nde-netfilter 0.14 0.00 0.00 0.00
- 27 27 cpuhp/2 0.13 0.00 0.00 0.00
- 631 631 kworker/11:2-rc 0.11 0.00 0.00 0.00
+ 110 110 kworker/15:0H-s 27.91 0.00 0.00 0.00
+ 57 57 cpuhp/7 3.18 0.00 0.00 0.00
+ 99 99 cpuhp/14 2.97 0.00 0.00 0.00
+ 51 51 cpuhp/6 0.90 0.00 0.00 0.00
+ 44 44 kworker/4:0H-sy 0.80 0.00 0.00 0.00
+ 60 60 ksoftirqd/7 0.74 0.00 0.00 0.00
+ 76 76 idle_inject/10 0.31 0.00 0.00 0.00
+ 100 100 idle_inject/14 0.30 0.00 0.00 0.00
+ 1309 1309 systemsettings 0.29 0.00 0.00 0.00
+ 45 45 cpuhp/5 0.22 0.00 0.00 0.00
+ 63 63 cpuhp/8 0.20 0.00 0.00 0.00
+ 87 87 cpuhp/12 0.18 0.00 0.00 0.00
+ 93 93 cpuhp/13 0.17 0.00 0.00 0.00
+ 1265 1265 acpid 0.17 0.00 0.00 0.00
+ 1552 1552 sshd 0.17 0.00 0.00 0.00
+ 2584 2584 sddm-helper 0.16 0.00 0.00 0.00
+ 1284 1284 rtkit-daemon 0.15 0.00 0.00 0.00
+ 1326 1326 nde-netfilter 0.14 0.00 0.00 0.00
+ 27 27 cpuhp/2 0.13 0.00 0.00 0.00
+ 631 631 kworker/11:2-rc 0.11 0.00 0.00 0.00
Interactive keyboard controls during runtime::
@@ -181,7 +181,7 @@ Interactive keyboard controls during runtime::
M - Toggle display mode (Default/Memory Verbose)
q - Quit
-Available sort fields(use -s/--sort or interactive command)::
+Available sort fields (use -s/--sort or interactive command)::
cpu(c) - CPU delay
blkio(i) - I/O delay
@@ -212,3 +212,46 @@ Advanced usage examples::
# ./delaytop -d secs
Specify refresh interval as secs
+
+ # ./delaytop -t type
+ Display only specified delay type with avg/max/timestamp
+ (rows sorted by MAX for that type, largest first)
+
+
+
+delaytop add delay_max fields to track the maximum delay value for each delay
+type (cpu, blkio, irq, swapin, freepages, thrashing, compact, wpcopy) per task::
+
+ bash# ./delaytop -t cpu
+ System Pressure Information: (avg10/avg60/avg300/total)
+ CPU some: 0.4%/ 0.2%/ 0.1%/ 220(ms)
+ CPU full: 0.0%/ 0.0%/ 0.0%/ 0(ms)
+ Memory full: 0.0%/ 0.0%/ 0.0%/ 0(ms)
+ Memory some: 0.0%/ 0.0%/ 0.0%/ 0(ms)
+ IO full: 0.0%/ 0.0%/ 0.0%/ 12(ms)
+ IO some: 0.0%/ 0.0%/ 0.0%/ 13(ms)
+ IRQ full: 0.0%/ 0.0%/ 0.0%/ 0(ms)
+ [q]quit
+ Top 20 processes (sorted by cpu MAX delay, largest first):
+ PID TGID COMMAND AVG(ms) MAX(ms) MAX_TIMESTAMP
+ ------------------------------------------------------------------------
+ 9 9 kworker/0:0-eve 0.59 16.87 2026-05-27T13:32:39
+ 30 30 kworker/2:0H-kb 2.87 11.36 2026-05-27T13:32:36
+ 27 27 migration/2 1.05 9.51 2026-05-27T13:32:37
+ 50 50 kworker/2:1-eve 0.50 9.13 2026-05-27T13:32:37
+ 15 15 rcu_preempt 0.11 8.98 2026-05-27T13:32:37
+ 1 1 init 0.17 7.12 2026-05-27T13:32:38
+ 67 67 scsi_eh_0 1.20 4.23 2026-05-27T13:32:37
+ 23 23 ksoftirqd/1 1.12 3.77 2026-05-27T13:32:36
+ 3 3 pool_workqueue 0.72 3.55 2026-05-27T13:32:38
+ 62 62 kworker/u20:2-a 0.49 3.03 2026-05-27T13:32:37
+ 2 2 kthreadd 0.18 2.82 2026-05-27T13:32:37
+ 11 11 kworker/0:1 1.42 2.76 2026-05-27T13:32:36
+ 39 39 kworker/u20:0-a 0.10 2.71 2026-05-27T13:32:38
+ 17 17 rcu_exp_gp_kthr 0.25 2.65 2026-05-27T13:32:36
+ 66 66 kworker/u20:3-e 0.38 2.55 2026-05-27T13:32:37
+ 20 20 cpuhp/0 0.53 2.51 2026-05-27T13:32:38
+ 28 28 ksoftirqd/2 0.59 2.48 2026-05-27T13:32:36
+ 55 55 kworker/u19:1 0.88 2.42 2026-05-27T13:32:36
+ 13 13 kworker/R-mm_pe 1.18 2.35 2026-05-27T13:32:36
+ 54 54 kworker/3:1-eve 0.14 2.20 2026-05-27T13:32:38
diff --git a/Documentation/sphinx/maintainers_include.py b/Documentation/sphinx/maintainers_include.py
index 7ffe19b5ed58..420c1b1808e7 100755
--- a/Documentation/sphinx/maintainers_include.py
+++ b/Documentation/sphinx/maintainers_include.py
@@ -155,7 +155,7 @@ class MaintainersParser:
ename = m.group(2)
entry = os.path.relpath(self.base_dir + fname, self.app_dir)
- entry = entry.removesuffix(".rst")
+ entry = entry[:-4] if entry.endswith(".rst") else entry
if entry.startswith("../"):
html = KERNELDOC_URL + ename + ".html"
diff --git a/tools/accounting/delaytop.c b/tools/accounting/delaytop.c
index 7e80f3875a91..af8a1e42a055 100644
--- a/tools/accounting/delaytop.c
+++ b/tools/accounting/delaytop.c
@@ -295,6 +295,7 @@ static void usage(void)
" -C, --container=PATH Monitor the container at specified cgroup path\n"
" -s, --sort=FIELD Sort by delay field (default: cpu)\n"
" -t, --type=FIELD Display only specified delay type with avg/max/timestamp\n"
+ " (rows sorted by MAX for that type, largest first)\n"
" -M, --memverbose Display memory detailed information\n");
exit(0);
}
@@ -838,6 +839,15 @@ static void get_task_delays(void)
closedir(dir);
}
+static void field_delay_max_and_ts(const struct task_info *task,
+ const struct field_desc *field,
+ unsigned long long *max_ns,
+ struct __kernel_timespec *max_ts);
+static void get_field_delay_values(const struct task_info *task,
+ const struct field_desc *field,
+ double *avg_ms, double *max_ms,
+ struct __kernel_timespec *max_ts);
+
/* Calculate average delay in milliseconds */
static double average_ms(unsigned long long total, unsigned long long count)
{
@@ -850,7 +860,7 @@ static double average_ms(unsigned long long total, unsigned long long count)
* Format __kernel_timespec to human readable string (YYYY-MM-DDTHH:MM:SS)
* Returns formatted string or "N/A" if timestamp is zero
*/
-static const char *format_timespec64(struct __kernel_timespec *ts)
+static const char *format_kernel_timespec(struct __kernel_timespec *ts)
{
static char buffer[32];
time_t time_sec;
@@ -891,6 +901,16 @@ static int compare_tasks(const void *a, const void *b)
unsigned long long count1;
unsigned long long count2;
double avg1, avg2;
+ unsigned long long max1, max2;
+
+ /* -t/--type: default sort by MAX column for the selected type (descending) */
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+ field_delay_max_and_ts(t1, cfg.type_field, &max1, NULL);
+ field_delay_max_and_ts(t2, cfg.type_field, &max2, NULL);
+ if (max1 != max2)
+ return max2 > max1 ? 1 : -1;
+ return 0;
+ }
total1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->total_offset);
total2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->total_offset);
@@ -905,6 +925,28 @@ static int compare_tasks(const void *a, const void *b)
return 0;
}
+/* Max delay (ns) and timestamp for field (shared by display and sort) */
+static void field_delay_max_and_ts(const struct task_info *task, const struct field_desc *field,
+ unsigned long long *max_ns, struct __kernel_timespec *max_ts)
+{
+ if (!field || !field->max_offset) {
+ *max_ns = 0;
+ if (max_ts)
+ memset(max_ts, 0, sizeof(*max_ts));
+ return;
+ }
+
+ *max_ns = *(unsigned long long *)((char *)task + field->max_offset);
+
+ if (max_ts) {
+ if (field->max_ts_offset)
+ *max_ts = *(struct __kernel_timespec *)((char *)task +
+ field->max_ts_offset);
+ else
+ memset(max_ts, 0, sizeof(*max_ts));
+ }
+}
+
/* Get delay values for a specific field */
static void get_field_delay_values(const struct task_info *task, const struct field_desc *field,
double *avg_ms, double *max_ms,
@@ -923,13 +965,8 @@ static void get_field_delay_values(const struct task_info *task, const struct fi
count = *(unsigned long long *)((char *)task + field->count_offset);
*avg_ms = average_ms(total, count);
- max = *(unsigned long long *)((char *)task + field->max_offset);
+ field_delay_max_and_ts(task, field, &max, max_ts);
*max_ms = (double)max / 1000000.0; /* Convert nanoseconds to milliseconds */
-
- if (field->max_ts_offset)
- *max_ts = *(struct __kernel_timespec *)((char *)task + field->max_ts_offset);
- else
- memset(max_ts, 0, sizeof(*max_ts));
}
/* Sort tasks by selected field */
@@ -1079,7 +1116,10 @@ static void display_results(int psi_ret)
}
/* Interacive command */
- suc &= BOOL_FPRINT(out, "[o]sort [M]memverbose [q]quit\n");
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field)
+ suc &= BOOL_FPRINT(out, "[q]quit\n");
+ else
+ suc &= BOOL_FPRINT(out, "[o]sort [M]memverbose [q]quit\n");
if (sort_selected) {
if (cfg.display_mode == MODE_MEMVERBOSE)
suc &= BOOL_FPRINT(out,
@@ -1090,8 +1130,13 @@ static void display_results(int psi_ret)
}
/* Task delay output */
- suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n",
- cfg.max_processes, get_name_by_field(cfg.sort_field));
+ if (cfg.display_mode == MODE_TYPE && cfg.type_field)
+ suc &= BOOL_FPRINT(out,
+ "Top %d processes (sorted by %s MAX delay, largest first):\n",
+ cfg.max_processes, get_name_by_field(cfg.type_field));
+ else
+ suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n",
+ cfg.max_processes, get_name_by_field(cfg.sort_field));
if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
/* Display mode for -t option: show only specified type with avg/max/timestamp */
@@ -1132,7 +1177,7 @@ static void display_results(int psi_ret)
&max_ms, &max_ts);
suc &= BOOL_FPRINT(out, "%12.2f %12.2f %20s\n",
- avg_ms, max_ms, format_timespec64(&max_ts));
+ avg_ms, max_ms, format_kernel_timespec(&max_ts));
} else if (cfg.display_mode == MODE_MEMVERBOSE) {
suc &= BOOL_FPRINT(out, DELAY_FMT_MEMVERBOSE,
TASK_AVG(tasks[i], mem),
@@ -1201,9 +1246,13 @@ static void handle_keypress(char ch, int *running)
} else {
switch (ch) {
case 'o':
+ if (cfg.display_mode == MODE_TYPE)
+ break;
sort_selected = 1;
break;
case 'M':
+ if (cfg.display_mode == MODE_TYPE)
+ break;
toggle_display_mode();
for (field = sort_fields; field->name != NULL; field++) {
if (field->supported_modes & cfg.display_mode) {
diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c
index 52930629ec3f..263ded4567fd 100644
--- a/tools/accounting/getdelays.c
+++ b/tools/accounting/getdelays.c
@@ -235,6 +235,11 @@ static const char *format_timespec(struct __kernel_timespec *ts)
if (ts->tv_sec == 0 && ts->tv_nsec == 0)
return "N/A";
+ /* Avoid Y2038 truncation on 32-bit platforms */
+ if (sizeof(time_sec) < sizeof(ts->tv_sec) &&
+ ts->tv_sec > (__u64)((1ULL << (sizeof(time_sec) * 8 - 1)) - 1))
+ return "N/A";
+
time_sec = ts->tv_sec;
/* Use thread-safe localtime_r */
--
2.47.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* Re: [PATCH 0/3 v4] delaytop: add delay max, timestamp and sorting for top latency analysis
2026-09-08 14:50 [PATCH 0/3 v4] delaytop: add delay max, timestamp and sorting for top latency analysis wang.yaxin
` (2 preceding siblings ...)
2026-09-08 14:55 ` [PATCH 3/3 v4] delaytop: sort by max delay to highlight top latency processes wang.yaxin
@ 2026-09-08 18:23 ` Andrew Morton
3 siblings, 0 replies; 5+ messages in thread
From: Andrew Morton @ 2026-09-08 18:23 UTC (permalink / raw)
To: wang.yaxin
Cc: fan.yu9, yang.yang29, thomas.weissschuh, corbet, linux-kernel,
linux-doc, xu.xin16
On Tue, 8 Sep 2026 22:50:56 +0800 (CST) <wang.yaxin@zte.com.cn> wrote:
> Previously delaytop only showed average delays. This patch adds:
>
> 1. delay_max fields to track the maximum delay value for each delay type
> (cpu, blkio, irq, swapin, freepages, thrashing, compact, wpcopy)
> per task.
>
> 2. The -t/--type option displays only the specified delay type with avg/max
> values side by side, allowing focused analysis:
> delaytop -t cpu # Show only CPU delay with avg/max
> delaytop -t wpcopy # Show Copy-on-Write delay with avg/max
>
> 3. Wall-clock timestamp when each maximum delay occurred, displayed in the
> MAX_TIMESTAMP column when using -t/--type option. This enables:
> - Identifying the time when a process experienced an abnormal delay max
> - Correlating delay max across multiple processes at the same timestamp
> - Cross-referencing with logs, traces, or other metrics at that time
>
> 4. When using -t/--type option, tasks are sorted by maximum delay value in
> descending order (largest delay first), enabling quick identification of
> top N processes with highest delay spikes.
>
We're missing the most important information: why make these changes?
Please update this [0/N] to explain the value of these changes. What
requirements are being satisfied? How will operators benefit from
these additions? How will you use them? Use cases. Things like that.
Also, AI review might have a found a few minor issues:
https://sashiko.dev/#/patchset/20260908225056266qjyBORsOReFvHLEQ3NKtR@zte.com.cn
Thanks.
^ permalink raw reply [flat|nested] 5+ messages in thread