Linux-mm Archive on lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] mm/vmstat: add per-order allocation slow path statistics
@ 2026-08-20 13:36 Daniil Tatianin
  2026-08-20 22:49 ` Andrew Morton
  2026-08-21  7:42 ` kernel test robot
  0 siblings, 2 replies; 5+ messages in thread
From: Daniil Tatianin @ 2026-08-20 13:36 UTC (permalink / raw)
  To: Andrew Morton, linux-mm
  Cc: Daniil Tatianin, Vlastimil Babka, Suren Baghdasaryan,
	Michal Hocko, Brendan Jackman, Johannes Weiner, Zi Yan,
	David Hildenbrand, Lorenzo Stoakes, Liam R. Howlett,
	Mike Rapoport, linux-kernel

Production incidents caused by bursts of high-order allocations all
entering direct compaction are currently hard to attribute from
/proc/vmstat: pgalloc_* has no order breakdown, and compact_stall does
not say which order stalled. Tracepoints can recover this on a single
machine, but they are impractical as an always-on fleet-wide monitoring
source, which is what is needed to correlate latency regressions with
allocation behavior after the fact.

Add per-order event counters to /proc/vmstat, covering only the
allocation slow path, so the page allocator fast path is not touched
at all:

 - pgalloc_slowpath_orderN: entries into __alloc_pages_slowpath(),
   counted once per allocation, before the restart loop
 - pgalloc_fail_orderN: allocations that returned NULL to the caller
   (including a successful allocation freed by memcg charge failure)
 - compact_stall_orderN / compact_success_orderN: per-order split of
   the existing direct compaction counters, order 0 is omitted since
   direct compaction is never entered for it

All new counters are purely additive: the existing keys are untouched
and compact_stall == sum of compact_stall_orderN.

alloc_pages_nolock() is deliberately not counted: it is opportunistic,
never enters the slow path, and its NULL returns are expected rather
than failures.

Counter names are generated for any MAX_PAGE_ORDER the arch Kconfig
ranges allow (10..13), a static_assert catches larger values.

A per-order split of PGALLOC itself was proposed in 2017 but stalled
over fast path overhead concerns, restricting the counters to the slow
path avoids that overhead entirely while still capturing the
allocations that cause latency.

Link: https://lore.kernel.org/all/1499346271-15653-1-git-send-email-guro@fb.com/

Signed-off-by: Daniil Tatianin <d-tatianin@yandex-team.ru>
---
 include/linux/vm_event_item.h | 19 +++++++++++++
 mm/page_alloc.c               |  7 +++++
 mm/vmstat.c                   | 52 +++++++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+)

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 03fe95f5a020..724561b7e800 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -175,6 +175,25 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		KSTACK_REST,
 #endif
 #endif /* CONFIG_DEBUG_STACK_USAGE */
+		/*
+		 * Per-order allocation statistics: each *_FIRST..*_LAST range
+		 * is indexed by allocation order.
+		 */
+		PGALLOC_SLOWPATH_ORDER_FIRST,
+		PGALLOC_SLOWPATH_ORDER_LAST =
+			PGALLOC_SLOWPATH_ORDER_FIRST + MAX_PAGE_ORDER,
+		PGALLOC_FAIL_ORDER_FIRST,
+		PGALLOC_FAIL_ORDER_LAST =
+			PGALLOC_FAIL_ORDER_FIRST + MAX_PAGE_ORDER,
+#ifdef CONFIG_COMPACTION
+		/* Direct compaction is never entered for order 0 */
+		COMPACTSTALL_ORDER_FIRST,
+		COMPACTSTALL_ORDER_LAST =
+			COMPACTSTALL_ORDER_FIRST + MAX_PAGE_ORDER - 1,
+		COMPACTSUCCESS_ORDER_FIRST,
+		COMPACTSUCCESS_ORDER_LAST =
+			COMPACTSUCCESS_ORDER_FIRST + MAX_PAGE_ORDER - 1,
+#endif
 		NR_VM_EVENT_ITEMS
 };
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index ee902a468c2f..2a2b14f3b516 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4169,6 +4169,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 	 * count a compaction stall
 	 */
 	count_vm_event(COMPACTSTALL);
+	count_vm_event(COMPACTSTALL_ORDER_FIRST + order - 1);
 
 	/* Prep a captured page if available */
 	if (page)
@@ -4184,6 +4185,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 		zone->compact_blockskip_flush = false;
 		compaction_defer_reset(zone, order, true);
 		count_vm_event(COMPACTSUCCESS);
+		count_vm_event(COMPACTSUCCESS_ORDER_FIRST + order - 1);
 		return page;
 	}
 
@@ -4757,6 +4759,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 		WARN_ON_ONCE(current->flags & PF_MEMALLOC);
 	}
 
+	count_vm_event(PGALLOC_SLOWPATH_ORDER_FIRST + order);
+
 restart:
 	compaction_retries = 0;
 	no_progress_loops = 0;
@@ -5323,6 +5327,9 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 		page = NULL;
 	}
 
+	if (unlikely(!page))
+		count_vm_event(PGALLOC_FAIL_ORDER_FIRST + order);
+
 	trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
 	kmsan_alloc_page(page, order, alloc_gfp);
 
diff --git a/mm/vmstat.c b/mm/vmstat.c
index f534972f517d..08596edc53ca 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1184,6 +1184,50 @@ int fragmentation_index(struct zone *zone, unsigned int order)
 	[xx##_MOVABLE] = yy "_movable",		\
 	TEXT_FOR_DEVICE(xx, yy)
 
+#if MAX_PAGE_ORDER >= 11
+#define TEXT_FOR_ORDER_11(xx, yy) [I((xx) + 11)] = yy "11",
+#else
+#define TEXT_FOR_ORDER_11(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 12
+#define TEXT_FOR_ORDER_12(xx, yy) [I((xx) + 12)] = yy "12",
+#else
+#define TEXT_FOR_ORDER_12(xx, yy)
+#endif
+
+#if MAX_PAGE_ORDER >= 13
+#define TEXT_FOR_ORDER_13(xx, yy) [I((xx) + 13)] = yy "13",
+#else
+#define TEXT_FOR_ORDER_13(xx, yy)
+#endif
+
+static_assert(MAX_PAGE_ORDER <= 13,
+	      "extend TEXT_FOR_ORDER_* for this MAX_PAGE_ORDER");
+
+/*
+ * (xx) + n must resolve to the vm_event_item for order n, so ranges that
+ * start at order 1 pass their *_ORDER_FIRST item minus one.
+ */
+#define TEXTS_FOR_NONZERO_ORDERS(xx, yy)	\
+	[I((xx) + 1)] = yy "1",			\
+	[I((xx) + 2)] = yy "2",			\
+	[I((xx) + 3)] = yy "3",			\
+	[I((xx) + 4)] = yy "4",			\
+	[I((xx) + 5)] = yy "5",			\
+	[I((xx) + 6)] = yy "6",			\
+	[I((xx) + 7)] = yy "7",			\
+	[I((xx) + 8)] = yy "8",			\
+	[I((xx) + 9)] = yy "9",			\
+	[I((xx) + 10)] = yy "10",		\
+	TEXT_FOR_ORDER_11(xx, yy)		\
+	TEXT_FOR_ORDER_12(xx, yy)		\
+	TEXT_FOR_ORDER_13(xx, yy)
+
+#define TEXTS_FOR_ORDERS(xx, yy)		\
+	[I(xx)] = yy "0",			\
+	TEXTS_FOR_NONZERO_ORDERS(xx, yy)
+
 const char * const vmstat_text[] = {
 	/* enum zone_stat_item counters */
 #define I(x) (x)
@@ -1488,6 +1532,14 @@ const char * const vmstat_text[] = {
 #if THREAD_SIZE > 65536
 	[I(KSTACK_REST)]			= "kstack_rest",
 #endif
+#endif
+	TEXTS_FOR_ORDERS(PGALLOC_SLOWPATH_ORDER_FIRST, "pgalloc_slowpath_order")
+	TEXTS_FOR_ORDERS(PGALLOC_FAIL_ORDER_FIRST, "pgalloc_fail_order")
+#ifdef CONFIG_COMPACTION
+	TEXTS_FOR_NONZERO_ORDERS(COMPACTSTALL_ORDER_FIRST - 1,
+				 "compact_stall_order")
+	TEXTS_FOR_NONZERO_ORDERS(COMPACTSUCCESS_ORDER_FIRST - 1,
+				 "compact_success_order")
 #endif
 #undef I
 #endif /* CONFIG_VM_EVENT_COUNTERS */


^ permalink raw reply related	[flat|nested] 5+ messages in thread
end of thread, other threads:[~2026-08-21  7:50 UTC | newest]

Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-08-20 13:36 [PATCH] mm/vmstat: add per-order allocation slow path statistics Daniil Tatianin
2026-08-20 22:49 ` Andrew Morton
2026-08-21  0:24   ` Daniil Tatianin
2026-08-21  7:50   ` Michal Hocko
2026-08-21  7:42 ` kernel test robot

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox