* [PATCH v4 6/7] tracing: move tracing declarations from kernel.h to a dedicated header
From: Yury Norov (NVIDIA) @ 2025-12-25 17:09 UTC (permalink / raw)
To: Steven Rostedt, Andrew Morton, Masami Hiramatsu,
Mathieu Desnoyers, Andy Shevchenko, Christophe Leroy,
Randy Dunlap, Ingo Molnar, Jani Nikula, Joonas Lahtinen,
David Laight, Petr Pavlu, Andi Shyti, Rodrigo Vivi,
Tvrtko Ursulin, Daniel Gomez, Greg Kroah-Hartman,
Rafael J. Wysocki, Danilo Krummrich, linux-kernel, intel-gfx,
dri-devel, linux-modules, linux-trace-kernel
Cc: Yury Norov (NVIDIA)
In-Reply-To: <20251225170930.1151781-1-yury.norov@gmail.com>
Tracing is a half of the kernel.h in terms of LOCs, although it's
a self-consistent part. It is intended for quick debugging purposes
and isn't used by the normal tracing utilities.
Move it to a separate header. If someone needs to just throw a
trace_printk() in their driver, they will not have to pull all
the heavy tracing machinery.
This is a pure move.
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
include/linux/kernel.h | 196 +--------------------------------
include/linux/trace_printk.h | 204 +++++++++++++++++++++++++++++++++++
2 files changed, 205 insertions(+), 195 deletions(-)
create mode 100644 include/linux/trace_printk.h
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 4ee48fb10dec..a377335e01da 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -32,7 +32,7 @@
#include <linux/build_bug.h>
#include <linux/sprintf.h>
#include <linux/static_call_types.h>
-#include <linux/instruction_pointer.h>
+#include <linux/trace_printk.h>
#include <linux/util_macros.h>
#include <linux/wordpart.h>
@@ -190,200 +190,6 @@ enum system_states {
};
extern enum system_states system_state;
-/*
- * General tracing related utility functions - trace_printk(),
- * tracing_on/tracing_off and tracing_start()/tracing_stop
- *
- * Use tracing_on/tracing_off when you want to quickly turn on or off
- * tracing. It simply enables or disables the recording of the trace events.
- * This also corresponds to the user space /sys/kernel/tracing/tracing_on
- * file, which gives a means for the kernel and userspace to interact.
- * Place a tracing_off() in the kernel where you want tracing to end.
- * From user space, examine the trace, and then echo 1 > tracing_on
- * to continue tracing.
- *
- * tracing_stop/tracing_start has slightly more overhead. It is used
- * by things like suspend to ram where disabling the recording of the
- * trace is not enough, but tracing must actually stop because things
- * like calling smp_processor_id() may crash the system.
- *
- * Most likely, you want to use tracing_on/tracing_off.
- */
-
-enum ftrace_dump_mode {
- DUMP_NONE,
- DUMP_ALL,
- DUMP_ORIG,
- DUMP_PARAM,
-};
-
-#ifdef CONFIG_TRACING
-void tracing_on(void);
-void tracing_off(void);
-int tracing_is_on(void);
-void tracing_snapshot(void);
-void tracing_snapshot_alloc(void);
-
-extern void tracing_start(void);
-extern void tracing_stop(void);
-
-static inline __printf(1, 2)
-void ____trace_printk_check_format(const char *fmt, ...)
-{
-}
-#define __trace_printk_check_format(fmt, args...) \
-do { \
- if (0) \
- ____trace_printk_check_format(fmt, ##args); \
-} while (0)
-
-/**
- * trace_printk - printf formatting in the ftrace buffer
- * @fmt: the printf format for printing
- *
- * Note: __trace_printk is an internal function for trace_printk() and
- * the @ip is passed in via the trace_printk() macro.
- *
- * This function allows a kernel developer to debug fast path sections
- * that printk is not appropriate for. By scattering in various
- * printk like tracing in the code, a developer can quickly see
- * where problems are occurring.
- *
- * This is intended as a debugging tool for the developer only.
- * Please refrain from leaving trace_printks scattered around in
- * your code. (Extra memory is used for special buffers that are
- * allocated when trace_printk() is used.)
- *
- * A little optimization trick is done here. If there's only one
- * argument, there's no need to scan the string for printf formats.
- * The trace_puts() will suffice. But how can we take advantage of
- * using trace_puts() when trace_printk() has only one argument?
- * By stringifying the args and checking the size we can tell
- * whether or not there are args. __stringify((__VA_ARGS__)) will
- * turn into "()\0" with a size of 3 when there are no args, anything
- * else will be bigger. All we need to do is define a string to this,
- * and then take its size and compare to 3. If it's bigger, use
- * do_trace_printk() otherwise, optimize it to trace_puts(). Then just
- * let gcc optimize the rest.
- */
-
-#define trace_printk(fmt, ...) \
-do { \
- char _______STR[] = __stringify((__VA_ARGS__)); \
- if (sizeof(_______STR) > 3) \
- do_trace_printk(fmt, ##__VA_ARGS__); \
- else \
- trace_puts(fmt); \
-} while (0)
-
-#define do_trace_printk(fmt, args...) \
-do { \
- static const char *trace_printk_fmt __used \
- __section("__trace_printk_fmt") = \
- __builtin_constant_p(fmt) ? fmt : NULL; \
- \
- __trace_printk_check_format(fmt, ##args); \
- \
- if (__builtin_constant_p(fmt)) \
- __trace_bprintk(_THIS_IP_, trace_printk_fmt, ##args); \
- else \
- __trace_printk(_THIS_IP_, fmt, ##args); \
-} while (0)
-
-extern __printf(2, 3)
-int __trace_bprintk(unsigned long ip, const char *fmt, ...);
-
-extern __printf(2, 3)
-int __trace_printk(unsigned long ip, const char *fmt, ...);
-
-/**
- * trace_puts - write a string into the ftrace buffer
- * @str: the string to record
- *
- * Note: __trace_bputs is an internal function for trace_puts and
- * the @ip is passed in via the trace_puts macro.
- *
- * This is similar to trace_printk() but is made for those really fast
- * paths that a developer wants the least amount of "Heisenbug" effects,
- * where the processing of the print format is still too much.
- *
- * This function allows a kernel developer to debug fast path sections
- * that printk is not appropriate for. By scattering in various
- * printk like tracing in the code, a developer can quickly see
- * where problems are occurring.
- *
- * This is intended as a debugging tool for the developer only.
- * Please refrain from leaving trace_puts scattered around in
- * your code. (Extra memory is used for special buffers that are
- * allocated when trace_puts() is used.)
- *
- * Returns: 0 if nothing was written, positive # if string was.
- * (1 when __trace_bputs is used, strlen(str) when __trace_puts is used)
- */
-
-#define trace_puts(str) ({ \
- static const char *trace_printk_fmt __used \
- __section("__trace_printk_fmt") = \
- __builtin_constant_p(str) ? str : NULL; \
- \
- if (__builtin_constant_p(str)) \
- __trace_bputs(_THIS_IP_, trace_printk_fmt); \
- else \
- __trace_puts(_THIS_IP_, str); \
-})
-extern int __trace_bputs(unsigned long ip, const char *str);
-extern int __trace_puts(unsigned long ip, const char *str);
-
-extern void trace_dump_stack(int skip);
-
-/*
- * The double __builtin_constant_p is because gcc will give us an error
- * if we try to allocate the static variable to fmt if it is not a
- * constant. Even with the outer if statement.
- */
-#define ftrace_vprintk(fmt, vargs) \
-do { \
- if (__builtin_constant_p(fmt)) { \
- static const char *trace_printk_fmt __used \
- __section("__trace_printk_fmt") = \
- __builtin_constant_p(fmt) ? fmt : NULL; \
- \
- __ftrace_vbprintk(_THIS_IP_, trace_printk_fmt, vargs); \
- } else \
- __ftrace_vprintk(_THIS_IP_, fmt, vargs); \
-} while (0)
-
-extern __printf(2, 0) int
-__ftrace_vbprintk(unsigned long ip, const char *fmt, va_list ap);
-
-extern __printf(2, 0) int
-__ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap);
-
-extern void ftrace_dump(enum ftrace_dump_mode oops_dump_mode);
-#else
-static inline void tracing_start(void) { }
-static inline void tracing_stop(void) { }
-static inline void trace_dump_stack(int skip) { }
-
-static inline void tracing_on(void) { }
-static inline void tracing_off(void) { }
-static inline int tracing_is_on(void) { return 0; }
-static inline void tracing_snapshot(void) { }
-static inline void tracing_snapshot_alloc(void) { }
-
-static inline __printf(1, 2)
-int trace_printk(const char *fmt, ...)
-{
- return 0;
-}
-static __printf(1, 0) inline int
-ftrace_vprintk(const char *fmt, va_list ap)
-{
- return 0;
-}
-static inline void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) { }
-#endif /* CONFIG_TRACING */
-
/* Rebuild everything on CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_DYNAMIC_FTRACE
# define REBUILD_DUE_TO_DYNAMIC_FTRACE
diff --git a/include/linux/trace_printk.h b/include/linux/trace_printk.h
new file mode 100644
index 000000000000..bb5874097f24
--- /dev/null
+++ b/include/linux/trace_printk.h
@@ -0,0 +1,204 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_TRACE_PRINTK_H
+#define _LINUX_TRACE_PRINTK_H
+
+#include <linux/compiler_attributes.h>
+#include <linux/instruction_pointer.h>
+#include <linux/stddef.h>
+#include <linux/stringify.h>
+
+/*
+ * General tracing related utility functions - trace_printk(),
+ * tracing_on/tracing_off and tracing_start()/tracing_stop
+ *
+ * Use tracing_on/tracing_off when you want to quickly turn on or off
+ * tracing. It simply enables or disables the recording of the trace events.
+ * This also corresponds to the user space /sys/kernel/tracing/tracing_on
+ * file, which gives a means for the kernel and userspace to interact.
+ * Place a tracing_off() in the kernel where you want tracing to end.
+ * From user space, examine the trace, and then echo 1 > tracing_on
+ * to continue tracing.
+ *
+ * tracing_stop/tracing_start has slightly more overhead. It is used
+ * by things like suspend to ram where disabling the recording of the
+ * trace is not enough, but tracing must actually stop because things
+ * like calling smp_processor_id() may crash the system.
+ *
+ * Most likely, you want to use tracing_on/tracing_off.
+ */
+
+enum ftrace_dump_mode {
+ DUMP_NONE,
+ DUMP_ALL,
+ DUMP_ORIG,
+ DUMP_PARAM,
+};
+
+#ifdef CONFIG_TRACING
+void tracing_on(void);
+void tracing_off(void);
+int tracing_is_on(void);
+void tracing_snapshot(void);
+void tracing_snapshot_alloc(void);
+
+extern void tracing_start(void);
+extern void tracing_stop(void);
+
+static inline __printf(1, 2)
+void ____trace_printk_check_format(const char *fmt, ...)
+{
+}
+#define __trace_printk_check_format(fmt, args...) \
+do { \
+ if (0) \
+ ____trace_printk_check_format(fmt, ##args); \
+} while (0)
+
+/**
+ * trace_printk - printf formatting in the ftrace buffer
+ * @fmt: the printf format for printing
+ *
+ * Note: __trace_printk is an internal function for trace_printk() and
+ * the @ip is passed in via the trace_printk() macro.
+ *
+ * This function allows a kernel developer to debug fast path sections
+ * that printk is not appropriate for. By scattering in various
+ * printk like tracing in the code, a developer can quickly see
+ * where problems are occurring.
+ *
+ * This is intended as a debugging tool for the developer only.
+ * Please refrain from leaving trace_printks scattered around in
+ * your code. (Extra memory is used for special buffers that are
+ * allocated when trace_printk() is used.)
+ *
+ * A little optimization trick is done here. If there's only one
+ * argument, there's no need to scan the string for printf formats.
+ * The trace_puts() will suffice. But how can we take advantage of
+ * using trace_puts() when trace_printk() has only one argument?
+ * By stringifying the args and checking the size we can tell
+ * whether or not there are args. __stringify((__VA_ARGS__)) will
+ * turn into "()\0" with a size of 3 when there are no args, anything
+ * else will be bigger. All we need to do is define a string to this,
+ * and then take its size and compare to 3. If it's bigger, use
+ * do_trace_printk() otherwise, optimize it to trace_puts(). Then just
+ * let gcc optimize the rest.
+ */
+
+#define trace_printk(fmt, ...) \
+do { \
+ char _______STR[] = __stringify((__VA_ARGS__)); \
+ if (sizeof(_______STR) > 3) \
+ do_trace_printk(fmt, ##__VA_ARGS__); \
+ else \
+ trace_puts(fmt); \
+} while (0)
+
+#define do_trace_printk(fmt, args...) \
+do { \
+ static const char *trace_printk_fmt __used \
+ __section("__trace_printk_fmt") = \
+ __builtin_constant_p(fmt) ? fmt : NULL; \
+ \
+ __trace_printk_check_format(fmt, ##args); \
+ \
+ if (__builtin_constant_p(fmt)) \
+ __trace_bprintk(_THIS_IP_, trace_printk_fmt, ##args); \
+ else \
+ __trace_printk(_THIS_IP_, fmt, ##args); \
+} while (0)
+
+extern __printf(2, 3)
+int __trace_bprintk(unsigned long ip, const char *fmt, ...);
+
+extern __printf(2, 3)
+int __trace_printk(unsigned long ip, const char *fmt, ...);
+
+/**
+ * trace_puts - write a string into the ftrace buffer
+ * @str: the string to record
+ *
+ * Note: __trace_bputs is an internal function for trace_puts and
+ * the @ip is passed in via the trace_puts macro.
+ *
+ * This is similar to trace_printk() but is made for those really fast
+ * paths that a developer wants the least amount of "Heisenbug" effects,
+ * where the processing of the print format is still too much.
+ *
+ * This function allows a kernel developer to debug fast path sections
+ * that printk is not appropriate for. By scattering in various
+ * printk like tracing in the code, a developer can quickly see
+ * where problems are occurring.
+ *
+ * This is intended as a debugging tool for the developer only.
+ * Please refrain from leaving trace_puts scattered around in
+ * your code. (Extra memory is used for special buffers that are
+ * allocated when trace_puts() is used.)
+ *
+ * Returns: 0 if nothing was written, positive # if string was.
+ * (1 when __trace_bputs is used, strlen(str) when __trace_puts is used)
+ */
+
+#define trace_puts(str) ({ \
+ static const char *trace_printk_fmt __used \
+ __section("__trace_printk_fmt") = \
+ __builtin_constant_p(str) ? str : NULL; \
+ \
+ if (__builtin_constant_p(str)) \
+ __trace_bputs(_THIS_IP_, trace_printk_fmt); \
+ else \
+ __trace_puts(_THIS_IP_, str); \
+})
+extern int __trace_bputs(unsigned long ip, const char *str);
+extern int __trace_puts(unsigned long ip, const char *str);
+
+extern void trace_dump_stack(int skip);
+
+/*
+ * The double __builtin_constant_p is because gcc will give us an error
+ * if we try to allocate the static variable to fmt if it is not a
+ * constant. Even with the outer if statement.
+ */
+#define ftrace_vprintk(fmt, vargs) \
+do { \
+ if (__builtin_constant_p(fmt)) { \
+ static const char *trace_printk_fmt __used \
+ __section("__trace_printk_fmt") = \
+ __builtin_constant_p(fmt) ? fmt : NULL; \
+ \
+ __ftrace_vbprintk(_THIS_IP_, trace_printk_fmt, vargs); \
+ } else \
+ __ftrace_vprintk(_THIS_IP_, fmt, vargs); \
+} while (0)
+
+extern __printf(2, 0) int
+__ftrace_vbprintk(unsigned long ip, const char *fmt, va_list ap);
+
+extern __printf(2, 0) int
+__ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap);
+
+extern void ftrace_dump(enum ftrace_dump_mode oops_dump_mode);
+#else
+static inline void tracing_start(void) { }
+static inline void tracing_stop(void) { }
+static inline void trace_dump_stack(int skip) { }
+
+static inline void tracing_on(void) { }
+static inline void tracing_off(void) { }
+static inline int tracing_is_on(void) { return 0; }
+static inline void tracing_snapshot(void) { }
+static inline void tracing_snapshot_alloc(void) { }
+
+static inline __printf(1, 2)
+int trace_printk(const char *fmt, ...)
+{
+ return 0;
+}
+static __printf(1, 0) inline int
+ftrace_vprintk(const char *fmt, va_list ap)
+{
+ return 0;
+}
+static inline void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) { }
+#endif /* CONFIG_TRACING */
+
+#endif
--
2.43.0
^ permalink raw reply related
* [PATCH v4 4/7] kernel.h: include linux/instruction_pointer.h explicitly
From: Yury Norov (NVIDIA) @ 2025-12-25 17:09 UTC (permalink / raw)
To: Steven Rostedt, Andrew Morton, Masami Hiramatsu,
Mathieu Desnoyers, Andy Shevchenko, Christophe Leroy,
Randy Dunlap, Ingo Molnar, Jani Nikula, Joonas Lahtinen,
David Laight, Petr Pavlu, Andi Shyti, Rodrigo Vivi,
Tvrtko Ursulin, Daniel Gomez, Greg Kroah-Hartman,
Rafael J. Wysocki, Danilo Krummrich, linux-kernel, intel-gfx,
dri-devel, linux-modules, linux-trace-kernel
Cc: Yury Norov (NVIDIA)
In-Reply-To: <20251225170930.1151781-1-yury.norov@gmail.com>
In preparation for decoupling linux/instruction_pointer.h and
linux/kernel.h, include instruction_pointer.h explicitly where needed.
Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
arch/s390/include/asm/processor.h | 1 +
include/linux/ww_mutex.h | 1 +
2 files changed, 2 insertions(+)
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 3affba95845b..cc187afa07b3 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -31,6 +31,7 @@
#include <linux/cpumask.h>
#include <linux/linkage.h>
#include <linux/irqflags.h>
+#include <linux/instruction_pointer.h>
#include <linux/bitops.h>
#include <asm/fpu-types.h>
#include <asm/cpu.h>
diff --git a/include/linux/ww_mutex.h b/include/linux/ww_mutex.h
index 45ff6f7a872b..9b30fa2ec508 100644
--- a/include/linux/ww_mutex.h
+++ b/include/linux/ww_mutex.h
@@ -17,6 +17,7 @@
#ifndef __LINUX_WW_MUTEX_H
#define __LINUX_WW_MUTEX_H
+#include <linux/instruction_pointer.h>
#include <linux/mutex.h>
#include <linux/rtmutex.h>
--
2.43.0
^ permalink raw reply related
* [PATCH v4 3/7] kernel.h: move VERIFY_OCTAL_PERMISSIONS() to sysfs.h
From: Yury Norov (NVIDIA) @ 2025-12-25 17:09 UTC (permalink / raw)
To: Steven Rostedt, Andrew Morton, Masami Hiramatsu,
Mathieu Desnoyers, Andy Shevchenko, Christophe Leroy,
Randy Dunlap, Ingo Molnar, Jani Nikula, Joonas Lahtinen,
David Laight, Petr Pavlu, Andi Shyti, Rodrigo Vivi,
Tvrtko Ursulin, Daniel Gomez, Greg Kroah-Hartman,
Rafael J. Wysocki, Danilo Krummrich, linux-kernel, intel-gfx,
dri-devel, linux-modules, linux-trace-kernel
Cc: Yury Norov (NVIDIA)
In-Reply-To: <20251225170930.1151781-1-yury.norov@gmail.com>
The macro is related to sysfs, but is defined in kernel.h. Move it to
the proper header, and unload the generic kernel.h.
Now that the macro is removed from kernel.h, linux/moduleparam.h is
decoupled, and kernel.h inclusion can be removed.
Acked-by: Randy Dunlap <rdunlap@infradead.org>
Tested-by: Randy Dunlap <rdunlap@infradead.org>
Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
Reviewed-by: Petr Pavlu <petr.pavlu@suse.com>
Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
Documentation/filesystems/sysfs.rst | 2 +-
include/linux/kernel.h | 12 ------------
include/linux/moduleparam.h | 2 +-
include/linux/sysfs.h | 13 +++++++++++++
4 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/Documentation/filesystems/sysfs.rst b/Documentation/filesystems/sysfs.rst
index 2703c04af7d0..ffcef4d6bc8d 100644
--- a/Documentation/filesystems/sysfs.rst
+++ b/Documentation/filesystems/sysfs.rst
@@ -120,7 +120,7 @@ is equivalent to doing::
.store = store_foo,
};
-Note as stated in include/linux/kernel.h "OTHER_WRITABLE? Generally
+Note as stated in include/linux/sysfs.h "OTHER_WRITABLE? Generally
considered a bad idea." so trying to set a sysfs file writable for
everyone will fail reverting to RO mode for "Others".
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 61d63c57bc2d..5b879bfea948 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -389,16 +389,4 @@ static inline void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) { }
# define REBUILD_DUE_TO_DYNAMIC_FTRACE
#endif
-/* Permissions on a sysfs file: you didn't miss the 0 prefix did you? */
-#define VERIFY_OCTAL_PERMISSIONS(perms) \
- (BUILD_BUG_ON_ZERO((perms) < 0) + \
- BUILD_BUG_ON_ZERO((perms) > 0777) + \
- /* USER_READABLE >= GROUP_READABLE >= OTHER_READABLE */ \
- BUILD_BUG_ON_ZERO((((perms) >> 6) & 4) < (((perms) >> 3) & 4)) + \
- BUILD_BUG_ON_ZERO((((perms) >> 3) & 4) < ((perms) & 4)) + \
- /* USER_WRITABLE >= GROUP_WRITABLE */ \
- BUILD_BUG_ON_ZERO((((perms) >> 6) & 2) < (((perms) >> 3) & 2)) + \
- /* OTHER_WRITABLE? Generally considered a bad idea. */ \
- BUILD_BUG_ON_ZERO((perms) & 2) + \
- (perms))
#endif
diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h
index 03a977168c52..281a006dc284 100644
--- a/include/linux/moduleparam.h
+++ b/include/linux/moduleparam.h
@@ -8,7 +8,7 @@
#include <linux/compiler.h>
#include <linux/init.h>
#include <linux/stringify.h>
-#include <linux/kernel.h>
+#include <linux/sysfs.h>
#include <linux/types.h>
/*
diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h
index c33a96b7391a..99b775f3ff46 100644
--- a/include/linux/sysfs.h
+++ b/include/linux/sysfs.h
@@ -808,4 +808,17 @@ static inline void sysfs_put(struct kernfs_node *kn)
kernfs_put(kn);
}
+/* Permissions on a sysfs file: you didn't miss the 0 prefix did you? */
+#define VERIFY_OCTAL_PERMISSIONS(perms) \
+ (BUILD_BUG_ON_ZERO((perms) < 0) + \
+ BUILD_BUG_ON_ZERO((perms) > 0777) + \
+ /* USER_READABLE >= GROUP_READABLE >= OTHER_READABLE */ \
+ BUILD_BUG_ON_ZERO((((perms) >> 6) & 4) < (((perms) >> 3) & 4)) + \
+ BUILD_BUG_ON_ZERO((((perms) >> 3) & 4) < ((perms) & 4)) + \
+ /* USER_WRITABLE >= GROUP_WRITABLE */ \
+ BUILD_BUG_ON_ZERO((((perms) >> 6) & 2) < (((perms) >> 3) & 2)) + \
+ /* OTHER_WRITABLE? Generally considered a bad idea. */ \
+ BUILD_BUG_ON_ZERO((perms) & 2) + \
+ (perms))
+
#endif /* _SYSFS_H_ */
--
2.43.0
^ permalink raw reply related
* [PATCH v4 2/7] moduleparam: include required headers explicitly
From: Yury Norov (NVIDIA) @ 2025-12-25 17:09 UTC (permalink / raw)
To: Steven Rostedt, Andrew Morton, Masami Hiramatsu,
Mathieu Desnoyers, Andy Shevchenko, Christophe Leroy,
Randy Dunlap, Ingo Molnar, Jani Nikula, Joonas Lahtinen,
David Laight, Petr Pavlu, Andi Shyti, Rodrigo Vivi,
Tvrtko Ursulin, Daniel Gomez, Greg Kroah-Hartman,
Rafael J. Wysocki, Danilo Krummrich, linux-kernel, intel-gfx,
dri-devel, linux-modules, linux-trace-kernel
Cc: Yury Norov (NVIDIA)
In-Reply-To: <20251225170930.1151781-1-yury.norov@gmail.com>
The following patch drops moduleparam.h dependency on kernel.h. In
preparation to it, list all the required headers explicitly.
Suggested-by: Petr Pavlu <petr.pavlu@suse.com>
Reviewed-by: Petr Pavlu <petr.pavlu@suse.com>
Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
include/linux/moduleparam.h | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h
index 915f32f7d888..03a977168c52 100644
--- a/include/linux/moduleparam.h
+++ b/include/linux/moduleparam.h
@@ -2,9 +2,14 @@
#ifndef _LINUX_MODULE_PARAMS_H
#define _LINUX_MODULE_PARAMS_H
/* (C) Copyright 2001, 2002 Rusty Russell IBM Corporation */
+
+#include <linux/array_size.h>
+#include <linux/build_bug.h>
+#include <linux/compiler.h>
#include <linux/init.h>
#include <linux/stringify.h>
#include <linux/kernel.h>
+#include <linux/types.h>
/*
* The maximum module name length, including the NUL byte.
--
2.43.0
^ permalink raw reply related
* [PATCH v4 1/7] kernel.h: drop STACK_MAGIC macro
From: Yury Norov (NVIDIA) @ 2025-12-25 17:09 UTC (permalink / raw)
To: Steven Rostedt, Andrew Morton, Masami Hiramatsu,
Mathieu Desnoyers, Andy Shevchenko, Christophe Leroy,
Randy Dunlap, Ingo Molnar, Jani Nikula, Joonas Lahtinen,
David Laight, Petr Pavlu, Andi Shyti, Rodrigo Vivi,
Tvrtko Ursulin, Daniel Gomez, Greg Kroah-Hartman,
Rafael J. Wysocki, Danilo Krummrich, linux-kernel, intel-gfx,
dri-devel, linux-modules, linux-trace-kernel
Cc: Yury Norov (NVIDIA), Jani Nikula, Aaron Tomlin, Andi Shyti
In-Reply-To: <20251225170930.1151781-1-yury.norov@gmail.com>
The macro was introduced in 1994, v1.0.4, for stacks protection. Since
that, people found better ways to protect stacks, and now the macro is
only used by i915 selftests. Move it to a local header and drop from
the kernel.h.
Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
Acked-by: Randy Dunlap <rdunlap@infradead.org>
Acked-by: Jani Nikula <jani.nikula@intel.com>
Reviewed-by: Christophe Leroy (CS GROUP) <chleroy@kernel.org>
Reviewed-by: Aaron Tomlin <atomlin@atomlin.com>
Reviewed-by: Andi Shyti <andi.shyti@linux.intel.com>
Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
drivers/gpu/drm/i915/gt/selftest_ring_submission.c | 1 +
drivers/gpu/drm/i915/i915_selftest.h | 2 ++
include/linux/kernel.h | 2 --
3 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
index 87ceb0f374b6..600333ae6c8c 100644
--- a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
@@ -3,6 +3,7 @@
* Copyright © 2020 Intel Corporation
*/
+#include "i915_selftest.h"
#include "intel_engine_pm.h"
#include "selftests/igt_flush_test.h"
diff --git a/drivers/gpu/drm/i915/i915_selftest.h b/drivers/gpu/drm/i915/i915_selftest.h
index bdf3e22c0a34..72922028f4ba 100644
--- a/drivers/gpu/drm/i915/i915_selftest.h
+++ b/drivers/gpu/drm/i915/i915_selftest.h
@@ -26,6 +26,8 @@
#include <linux/types.h>
+#define STACK_MAGIC 0xdeadbeef
+
struct pci_dev;
struct drm_i915_private;
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 5b46924fdff5..61d63c57bc2d 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -40,8 +40,6 @@
#include <uapi/linux/kernel.h>
-#define STACK_MAGIC 0xdeadbeef
-
struct completion;
struct user;
--
2.43.0
^ permalink raw reply related
* [PATCH v4 0/7] Unload linux/kernel.h
From: Yury Norov (NVIDIA) @ 2025-12-25 17:09 UTC (permalink / raw)
To: Steven Rostedt, Andrew Morton, Masami Hiramatsu,
Mathieu Desnoyers, Andy Shevchenko, Christophe Leroy,
Randy Dunlap, Ingo Molnar, Jani Nikula, Joonas Lahtinen,
David Laight, Petr Pavlu, Andi Shyti, Rodrigo Vivi,
Tvrtko Ursulin, Daniel Gomez, Greg Kroah-Hartman,
Rafael J. Wysocki, Danilo Krummrich, linux-kernel, intel-gfx,
dri-devel, linux-modules, linux-trace-kernel
Cc: Yury Norov (NVIDIA)
kernel.h hosts declarations that can be placed better. This series
decouples kernel.h with some explicit and implicit dependencies; also,
moves tracing functionality to a new independent header.
My local build testing shows ~2% performance improvement for GCC +
Ubuntu x86_64/localyesconfig.
v1: https://lore.kernel.org/all/20251129195304.204082-1-yury.norov@gmail.com/
v2: https://lore.kernel.org/all/20251203162329.280182-1-yury.norov@gmail.com/
v3: https://lore.kernel.org/all/20251205175237.242022-1-yury.norov@gmail.com/
v4:
- drop kernel.h dependency on linux/instruction_pointer.h (new patch #4);
- drop trace_printk.h dependency on string.h (new patch #5 - Steven);
- drop kernel.h dependency on trace_printk.h (new patch #7);
- explicitly tested CONFIG_FORTIFY x86_64 build with no issues.
0-DAY CI Kernel Test Service:
alpha allnoconfig gcc-15.1.0
alpha allyesconfig gcc-15.1.0
alpha defconfig gcc-15.1.0
arc allmodconfig clang-16
arc allmodconfig gcc-15.1.0
arc allnoconfig gcc-15.1.0
arc allyesconfig clang-22
arc allyesconfig gcc-15.1.0
arc defconfig gcc-15.1.0
arc randconfig-001-20251225 gcc-11.5.0
arc randconfig-002-20251225 gcc-11.5.0
arm allnoconfig clang-22
arm allnoconfig gcc-15.1.0
arm allyesconfig clang-16
arm allyesconfig gcc-15.1.0
arm defconfig gcc-15.1.0
arm exynos_defconfig gcc-15.1.0
arm randconfig-001-20251225 gcc-11.5.0
arm randconfig-002-20251225 gcc-11.5.0
arm randconfig-003-20251225 gcc-11.5.0
arm randconfig-004-20251225 gcc-11.5.0
arm spitz_defconfig gcc-15.1.0
arm64 allmodconfig clang-19
arm64 allmodconfig clang-22
arm64 allnoconfig gcc-15.1.0
arm64 defconfig gcc-15.1.0
arm64 randconfig-001-20251225 clang-18
arm64 randconfig-001-20251225 gcc-11.5.0
arm64 randconfig-002-20251225 gcc-11.5.0
arm64 randconfig-002-20251225 gcc-12.5.0
arm64 randconfig-003-20251225 clang-22
arm64 randconfig-003-20251225 gcc-11.5.0
arm64 randconfig-004-20251225 clang-22
arm64 randconfig-004-20251225 gcc-11.5.0
csky allmodconfig gcc-15.1.0
csky allnoconfig gcc-15.1.0
csky defconfig gcc-15.1.0
csky randconfig-001-20251225 gcc-11.5.0
csky randconfig-001-20251225 gcc-15.1.0
csky randconfig-002-20251225 gcc-11.5.0
hexagon allmodconfig clang-17
hexagon allmodconfig gcc-15.1.0
hexagon allnoconfig clang-22
hexagon allnoconfig gcc-15.1.0
hexagon defconfig gcc-15.1.0
hexagon randconfig-001-20251225 clang-22
hexagon randconfig-002-20251225 clang-22
i386 allmodconfig clang-20
i386 allmodconfig gcc-14
i386 allnoconfig gcc-14
i386 allnoconfig gcc-15.1.0
i386 allyesconfig clang-20
i386 allyesconfig gcc-14
i386 buildonly-randconfig-001-20251225 clang-20
i386 buildonly-randconfig-002-20251225 clang-20
i386 buildonly-randconfig-003-20251225 clang-20
i386 buildonly-randconfig-003-20251225 gcc-14
i386 buildonly-randconfig-004-20251225 clang-20
i386 buildonly-randconfig-005-20251225 clang-20
i386 buildonly-randconfig-006-20251225 clang-20
i386 randconfig-007-20251225 clang-20
i386 randconfig-011-20251225 clang-20
i386 randconfig-011-20251225 gcc-14
i386 randconfig-012-20251225 gcc-14
i386 randconfig-013-20251225 gcc-14
i386 randconfig-014-20251225 clang-20
i386 randconfig-014-20251225 gcc-14
i386 randconfig-015-20251225 gcc-14
i386 randconfig-016-20251225 clang-20
i386 randconfig-016-20251225 gcc-14
i386 randconfig-017-20251225 clang-20
i386 randconfig-017-20251225 gcc-14
loongarch allmodconfig clang-19
loongarch allmodconfig clang-22
loongarch allnoconfig clang-22
loongarch allnoconfig gcc-15.1.0
loongarch defconfig clang-19
loongarch randconfig-001-20251225 clang-22
loongarch randconfig-002-20251225 clang-22
loongarch randconfig-002-20251225 gcc-15.1.0
m68k allmodconfig gcc-15.1.0
m68k allnoconfig gcc-15.1.0
m68k allyesconfig clang-16
m68k allyesconfig gcc-15.1.0
m68k defconfig clang-19
microblaze allnoconfig gcc-15.1.0
microblaze allyesconfig gcc-15.1.0
microblaze defconfig clang-19
mips allmodconfig gcc-15.1.0
mips allnoconfig gcc-15.1.0
mips allyesconfig gcc-15.1.0
mips gcw0_defconfig gcc-15.1.0
nios2 allmodconfig clang-22
nios2 allmodconfig gcc-11.5.0
nios2 allnoconfig clang-22
nios2 allnoconfig gcc-11.5.0
nios2 defconfig clang-19
nios2 randconfig-001-20251225 clang-22
nios2 randconfig-001-20251225 gcc-9.5.0
nios2 randconfig-002-20251225 clang-22
nios2 randconfig-002-20251225 gcc-11.5.0
openrisc allmodconfig clang-22
openrisc allmodconfig gcc-15.1.0
openrisc allnoconfig clang-22
openrisc allnoconfig gcc-15.1.0
openrisc defconfig gcc-15.1.0
parisc allmodconfig gcc-15.1.0
parisc allnoconfig clang-22
parisc allnoconfig gcc-15.1.0
parisc allyesconfig clang-19
parisc allyesconfig gcc-15.1.0
parisc defconfig gcc-15.1.0
parisc randconfig-001-20251225 clang-22
parisc randconfig-002-20251225 clang-22
parisc64 defconfig clang-19
powerpc allmodconfig gcc-15.1.0
powerpc allnoconfig clang-22
powerpc allnoconfig gcc-15.1.0
powerpc cell_defconfig gcc-15.1.0
powerpc pmac32_defconfig gcc-15.1.0
powerpc randconfig-001-20251225 clang-22
powerpc randconfig-002-20251225 clang-22
powerpc64 randconfig-001-20251225 clang-22
powerpc64 randconfig-002-20251225 clang-22
riscv allmodconfig clang-22
riscv allnoconfig clang-22
riscv allnoconfig gcc-15.1.0
riscv allyesconfig clang-16
riscv defconfig clang-22
riscv defconfig gcc-15.1.0
riscv randconfig-001-20251225 clang-19
riscv randconfig-001-20251225 clang-22
riscv randconfig-002-20251225 clang-19
riscv randconfig-002-20251225 gcc-11.5.0
s390 allmodconfig clang-18
s390 allmodconfig clang-19
s390 allnoconfig clang-22
s390 allyesconfig gcc-15.1.0
s390 defconfig clang-22
s390 defconfig gcc-15.1.0
s390 randconfig-001-20251225 clang-19
s390 randconfig-001-20251225 gcc-14.3.0
s390 randconfig-002-20251225 clang-19
sh allmodconfig gcc-15.1.0
sh allnoconfig clang-22
sh allnoconfig gcc-15.1.0
sh allyesconfig clang-19
sh allyesconfig gcc-15.1.0
sh defconfig gcc-14
sh randconfig-001-20251225 clang-19
sh randconfig-001-20251225 gcc-15.1.0
sh randconfig-002-20251225 clang-19
sh randconfig-002-20251225 gcc-9.5.0
sh se7724_defconfig gcc-15.1.0
sparc allnoconfig clang-22
sparc allnoconfig gcc-15.1.0
sparc defconfig gcc-15.1.0
sparc randconfig-001-20251225 gcc-13
sparc randconfig-002-20251225 gcc-13
sparc64 allmodconfig clang-22
sparc64 defconfig gcc-14
sparc64 randconfig-001-20251225 gcc-13
sparc64 randconfig-002-20251225 gcc-13
um allmodconfig clang-19
um allnoconfig clang-22
um allyesconfig gcc-14
um allyesconfig gcc-15.1.0
um defconfig gcc-14
um i386_defconfig gcc-14
um randconfig-001-20251225 gcc-13
um randconfig-002-20251225 gcc-13
um x86_64_defconfig gcc-14
x86_64 allmodconfig clang-20
x86_64 allnoconfig clang-20
x86_64 allnoconfig clang-22
x86_64 allyesconfig clang-20
x86_64 buildonly-randconfig-001-20251225 clang-20
x86_64 buildonly-randconfig-001-20251225 gcc-14
x86_64 buildonly-randconfig-002-20251225 clang-20
x86_64 buildonly-randconfig-002-20251225 gcc-14
x86_64 buildonly-randconfig-003-20251225 gcc-14
x86_64 buildonly-randconfig-004-20251225 clang-20
x86_64 buildonly-randconfig-004-20251225 gcc-14
x86_64 buildonly-randconfig-005-20251225 gcc-14
x86_64 buildonly-randconfig-006-20251225 clang-20
x86_64 buildonly-randconfig-006-20251225 gcc-14
x86_64 defconfig gcc-14
x86_64 kexec clang-20
x86_64 randconfig-001-20251225 clang-20
x86_64 randconfig-002-20251225 clang-20
x86_64 randconfig-003-20251225 clang-20
x86_64 randconfig-004-20251225 clang-20
x86_64 randconfig-005-20251225 clang-20
x86_64 randconfig-006-20251225 clang-20
x86_64 randconfig-011-20251225 gcc-13
x86_64 randconfig-012-20251225 gcc-13
x86_64 randconfig-012-20251225 gcc-14
x86_64 randconfig-013-20251225 clang-20
x86_64 randconfig-013-20251225 gcc-13
x86_64 randconfig-014-20251225 clang-20
x86_64 randconfig-014-20251225 gcc-13
x86_64 randconfig-015-20251225 gcc-13
x86_64 randconfig-015-20251225 gcc-14
x86_64 randconfig-016-20251225 clang-20
x86_64 randconfig-016-20251225 gcc-13
x86_64 randconfig-071-20251225 clang-20
x86_64 randconfig-072-20251225 clang-20
x86_64 randconfig-073-20251225 clang-20
x86_64 randconfig-073-20251225 gcc-14
x86_64 randconfig-074-20251225 clang-20
x86_64 randconfig-075-20251225 clang-20
x86_64 randconfig-075-20251225 gcc-14
x86_64 randconfig-076-20251225 clang-20
x86_64 randconfig-076-20251225 gcc-14
x86_64 rhel-9.4 clang-20
x86_64 rhel-9.4-bpf gcc-14
x86_64 rhel-9.4-func clang-20
x86_64 rhel-9.4-kselftests clang-20
x86_64 rhel-9.4-kunit gcc-14
x86_64 rhel-9.4-ltp gcc-14
x86_64 rhel-9.4-rust clang-20
xtensa allnoconfig clang-22
xtensa allnoconfig gcc-15.1.0
xtensa allyesconfig clang-22
xtensa randconfig-001-20251225 gcc-13
xtensa randconfig-002-20251225 gcc-13
Merry Christmas everybody!
Steven Rostedt (1):
tracing: Remove size parameter in __trace_puts()
Yury Norov (NVIDIA) (6):
kernel.h: drop STACK_MAGIC macro
moduleparam: include required headers explicitly
kernel.h: move VERIFY_OCTAL_PERMISSIONS() to sysfs.h
kernel.h: include linux/instruction_pointer.h explicitly
tracing: move tracing declarations from kernel.h to a dedicated header
kernel.h: drop trace_printk.h
Documentation/filesystems/sysfs.rst | 2 +-
arch/powerpc/kvm/book3s_xics.c | 1 +
arch/powerpc/xmon/xmon.c | 1 +
arch/s390/include/asm/processor.h | 1 +
arch/s390/kernel/ipl.c | 1 +
arch/s390/kernel/machine_kexec.c | 1 +
drivers/gpu/drm/i915/gt/intel_gtt.h | 1 +
.../drm/i915/gt/selftest_ring_submission.c | 1 +
drivers/gpu/drm/i915/i915_gem.h | 1 +
drivers/gpu/drm/i915/i915_selftest.h | 2 +
drivers/hwtracing/stm/dummy_stm.c | 1 +
drivers/infiniband/hw/hfi1/trace_dbg.h | 1 +
drivers/tty/sysrq.c | 1 +
drivers/usb/early/xhci-dbc.c | 1 +
fs/ext4/inline.c | 1 +
include/linux/kernel.h | 209 ------------------
include/linux/moduleparam.h | 7 +-
include/linux/sunrpc/debug.h | 1 +
include/linux/sysfs.h | 13 ++
include/linux/trace_printk.h | 204 +++++++++++++++++
include/linux/ww_mutex.h | 1 +
kernel/debug/debug_core.c | 1 +
kernel/panic.c | 1 +
kernel/rcu/rcu.h | 1 +
kernel/rcu/rcutorture.c | 1 +
kernel/trace/error_report-traces.c | 1 +
kernel/trace/ring_buffer_benchmark.c | 1 +
kernel/trace/trace.c | 8 +-
kernel/trace/trace.h | 2 +-
kernel/trace/trace_benchmark.c | 1 +
kernel/trace/trace_events_trigger.c | 1 +
kernel/trace/trace_functions.c | 1 +
kernel/trace/trace_printk.c | 1 +
kernel/trace/trace_selftest.c | 1 +
lib/sys_info.c | 1 +
samples/fprobe/fprobe_example.c | 1 +
samples/ftrace/ftrace-direct-modify.c | 1 +
samples/ftrace/ftrace-direct-multi-modify.c | 1 +
samples/ftrace/ftrace-direct-multi.c | 1 +
samples/ftrace/ftrace-direct-too.c | 1 +
samples/ftrace/ftrace-direct.c | 1 +
samples/trace_printk/trace-printk.c | 1 +
sound/hda/common/sysfs.c | 1 +
43 files changed, 266 insertions(+), 216 deletions(-)
create mode 100644 include/linux/trace_printk.h
--
2.43.0
^ permalink raw reply
* [PATCH v11] dma-buf: add some tracepoints to debug.
From: Xiang Gao @ 2025-12-25 12:11 UTC (permalink / raw)
To: sumit.semwal, christian.koenig, rostedt, mhiramat
Cc: linux-media, dri-devel, linux-kernel, mathieu.desnoyers, dhowells,
kuba, brauner, akpm, linux-trace-kernel, gaoxiang17
From: gaoxiang17 <gaoxiang17@xiaomi.com>
Since we can only inspect dmabuf by iterating over process FDs or the
dmabuf_list, we need to add our own tracepoints to track its status in
real time in production.
For example:
binder:3016_1-3102 [006] ...1. 255.126521: dma_buf_export: exp_name=qcom,system size=12685312 ino=2738
binder:3016_1-3102 [006] ...1. 255.126528: dma_buf_fd: exp_name=qcom,system size=12685312 ino=2738 fd=8
binder:3016_1-3102 [006] ...1. 255.126642: dma_buf_mmap_internal: exp_name=qcom,system size=28672 ino=2739
kworker/6:1-86 [006] ...1. 255.127194: dma_buf_put: exp_name=qcom,system size=12685312 ino=2738
RenderThread-9293 [006] ...1. 316.618179: dma_buf_get: exp_name=qcom,system size=12771328 ino=2762 fd=176
RenderThread-9293 [006] ...1. 316.618195: dma_buf_dynamic_attach: exp_name=qcom,system size=12771328 ino=2762 attachment:ffffff880a18dd00 is_dynamic=0 dev_name=kgsl-3d0
RenderThread-9293 [006] ...1. 318.878220: dma_buf_detach: exp_name=qcom,system size=12771328 ino=2762 attachment:ffffff880a18dd00 is_dynamic=0 dev_name=kgsl-3d0
Signed-off-by: Xiang Gao <gaoxiang17@xiaomi.com>
---
Changes since v10: https://lore.kernel.org/all/20251224013455.1649879-1-gxxa03070307@gmail.com/
Changes since v9: https://lore.kernel.org/all/20251223032749.1371913-1-gxxa03070307@gmail.com/
Changes since v8: https://lore.kernel.org/all/20251218062853.819744-1-gxxa03070307@gmail.com/
Changes since v7: https://lore.kernel.org/all/20251217105132.643300-1-gxxa03070307@gmail.com/
Changes since v6: https://lore.kernel.org/all/20251216063952.516364-1-gxxa03070307@gmail.com/
Changes since v5: https://lore.kernel.org/all/20251201112148.843572-1-gxxa03070307@gmail.com/
Changes since v4: https://lore.kernel.org/all/20251128085215.550100-1-gxxa03070307@gmail.com/
Changes since v3: https://lore.kernel.org/all/20251127004352.376307-1-gxxa03070307@gmail.com/
Changes since v2: https://lore.kernel.org/all/20251125162949.220488-1-gxxa03070307@gmail.com/
Changes since v1: https://lore.kernel.org/all/20251124133648.72668-1-gxxa03070307@gmail.com/
drivers/dma-buf/dma-buf.c | 48 +++++++++-
include/trace/events/dma_buf.h | 159 +++++++++++++++++++++++++++++++++
2 files changed, 205 insertions(+), 2 deletions(-)
create mode 100644 include/trace/events/dma_buf.h
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index edaa9e4ee4ae..dee59d4c0b12 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -35,6 +35,26 @@
#include "dma-buf-sysfs-stats.h"
+#define CREATE_TRACE_POINTS
+#include <trace/events/dma_buf.h>
+
+/*
+ * dmabuf->name must be accessed with holding dmabuf->name_lock.
+ * we need to take the lock around the tracepoint call itself where
+ * it is called in the code.
+ *
+ * Note: FUNC##_enabled() is a static branch that will only
+ * be set when the trace event is enabled.
+ */
+#define DMA_BUF_TRACE(FUNC, ...) \
+ do { \
+ /* Always expose lock if lockdep is enabled */ \
+ if (IS_ENABLED(CONFIG_LOCKDEP) || FUNC##_enabled()) { \
+ guard(spinlock)(&dmabuf->name_lock); \
+ FUNC(__VA_ARGS__); \
+ } \
+ } while (0)
+
static inline int is_dma_buf_file(struct file *);
static DEFINE_MUTEX(dmabuf_list_mutex);
@@ -220,6 +240,8 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
dmabuf->size >> PAGE_SHIFT)
return -EINVAL;
+ DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf);
+
return dmabuf->ops->mmap(dmabuf, vma);
}
@@ -745,6 +767,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
__dma_buf_list_add(dmabuf);
+ DMA_BUF_TRACE(trace_dma_buf_export, dmabuf);
+
return dmabuf;
err_dmabuf:
@@ -768,10 +792,15 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF");
*/
int dma_buf_fd(struct dma_buf *dmabuf, int flags)
{
+ int fd;
+
if (!dmabuf || !dmabuf->file)
return -EINVAL;
- return FD_ADD(flags, dmabuf->file);
+ fd = FD_ADD(flags, dmabuf->file);
+ DMA_BUF_TRACE(trace_dma_buf_fd, dmabuf, fd);
+
+ return fd;
}
EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
@@ -786,6 +815,7 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
struct dma_buf *dma_buf_get(int fd)
{
struct file *file;
+ struct dma_buf *dmabuf;
file = fget(fd);
@@ -797,7 +827,11 @@ struct dma_buf *dma_buf_get(int fd)
return ERR_PTR(-EINVAL);
}
- return file->private_data;
+ dmabuf = file->private_data;
+
+ DMA_BUF_TRACE(trace_dma_buf_get, dmabuf, fd);
+
+ return dmabuf;
}
EXPORT_SYMBOL_NS_GPL(dma_buf_get, "DMA_BUF");
@@ -817,6 +851,8 @@ void dma_buf_put(struct dma_buf *dmabuf)
return;
fput(dmabuf->file);
+
+ DMA_BUF_TRACE(trace_dma_buf_put, dmabuf);
}
EXPORT_SYMBOL_NS_GPL(dma_buf_put, "DMA_BUF");
@@ -971,6 +1007,9 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
list_add(&attach->node, &dmabuf->attachments);
dma_resv_unlock(dmabuf->resv);
+ DMA_BUF_TRACE(trace_dma_buf_dynamic_attach, dmabuf, attach,
+ dma_buf_attachment_is_dynamic(attach), dev);
+
return attach;
err_attach:
@@ -1015,6 +1054,9 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
if (dmabuf->ops->detach)
dmabuf->ops->detach(dmabuf, attach);
+ DMA_BUF_TRACE(trace_dma_buf_detach, dmabuf, attach,
+ dma_buf_attachment_is_dynamic(attach), attach->dev);
+
kfree(attach);
}
EXPORT_SYMBOL_NS_GPL(dma_buf_detach, "DMA_BUF");
@@ -1480,6 +1522,8 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
vma_set_file(vma, dmabuf->file);
vma->vm_pgoff = pgoff;
+ DMA_BUF_TRACE(trace_dma_buf_mmap, dmabuf);
+
return dmabuf->ops->mmap(dmabuf, vma);
}
EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, "DMA_BUF");
diff --git a/include/trace/events/dma_buf.h b/include/trace/events/dma_buf.h
new file mode 100644
index 000000000000..3bb88d05bcc8
--- /dev/null
+++ b/include/trace/events/dma_buf.h
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM dma_buf
+
+#if !defined(_TRACE_DMA_BUF_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_DMA_BUF_H
+
+#include <linux/dma-buf.h>
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(dma_buf,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf),
+
+ TP_STRUCT__entry(
+ __string( exp_name, dmabuf->exp_name)
+ __field( size_t, size)
+ __field( ino_t, ino)
+ ),
+
+ TP_fast_assign(
+ __assign_str(exp_name);
+ __entry->size = dmabuf->size;
+ __entry->ino = dmabuf->file->f_inode->i_ino;
+ ),
+
+ TP_printk("exp_name=%s size=%zu ino=%lu",
+ __get_str(exp_name),
+ __entry->size,
+ __entry->ino)
+);
+
+DECLARE_EVENT_CLASS(dma_buf_attach_dev,
+
+ TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+ bool is_dynamic, struct device *dev),
+
+ TP_ARGS(dmabuf, attach, is_dynamic, dev),
+
+ TP_STRUCT__entry(
+ __string( dev_name, dev_name(dev))
+ __string( exp_name, dmabuf->exp_name)
+ __field( size_t, size)
+ __field( ino_t, ino)
+ __field( struct dma_buf_attachment *, attach)
+ __field( bool, is_dynamic)
+ ),
+
+ TP_fast_assign(
+ __assign_str(dev_name);
+ __assign_str(exp_name);
+ __entry->size = dmabuf->size;
+ __entry->ino = dmabuf->file->f_inode->i_ino;
+ __entry->is_dynamic = is_dynamic;
+ __entry->attach = attach;
+ ),
+
+ TP_printk("exp_name=%s size=%zu ino=%lu attachment:%p is_dynamic=%d dev_name=%s",
+ __get_str(exp_name),
+ __entry->size,
+ __entry->ino,
+ __entry->attach,
+ __entry->is_dynamic,
+ __get_str(dev_name))
+);
+
+DECLARE_EVENT_CLASS(dma_buf_fd,
+
+ TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+ TP_ARGS(dmabuf, fd),
+
+ TP_STRUCT__entry(
+ __string( exp_name, dmabuf->exp_name)
+ __field( size_t, size)
+ __field( ino_t, ino)
+ __field( int, fd)
+ ),
+
+ TP_fast_assign(
+ __assign_str(exp_name);
+ __entry->size = dmabuf->size;
+ __entry->ino = dmabuf->file->f_inode->i_ino;
+ __entry->fd = fd;
+ ),
+
+ TP_printk("exp_name=%s size=%zu ino=%lu fd=%d",
+ __get_str(exp_name),
+ __entry->size,
+ __entry->ino,
+ __entry->fd)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_export,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_mmap_internal,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_mmap,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_put,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf_attach_dev, dma_buf_dynamic_attach,
+
+ TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+ bool is_dynamic, struct device *dev),
+
+ TP_ARGS(dmabuf, attach, is_dynamic, dev)
+);
+
+DEFINE_EVENT(dma_buf_attach_dev, dma_buf_detach,
+
+ TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+ bool is_dynamic, struct device *dev),
+
+ TP_ARGS(dmabuf, attach, is_dynamic, dev)
+);
+
+DEFINE_EVENT_CONDITION(dma_buf_fd, dma_buf_fd,
+
+ TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+ TP_ARGS(dmabuf, fd),
+
+ TP_CONDITION(fd >= 0)
+);
+
+DEFINE_EVENT(dma_buf_fd, dma_buf_get,
+
+ TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+ TP_ARGS(dmabuf, fd)
+);
+
+#endif /* _TRACE_DMA_BUF_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
--
2.34.1
^ permalink raw reply related
* [v2 PATCH 1/1] tracing: Add bitmask-list option for human-readable bitmask display
From: Aaron Tomlin @ 2025-12-25 8:02 UTC (permalink / raw)
To: rostedt, mhiramat, mark.rutland, mathieu.desnoyers, corbet
Cc: sean, linux-kernel, linux-trace-kernel, linux-doc
In-Reply-To: <20251225080216.2196411-1-atomlin@atomlin.com>
Add support for displaying bitmasks in human-readable list format (e.g.,
0,2-5,7) in addition to the default hexadecimal bitmap representation.
This is particularly useful when tracing CPU masks and other large
bitmasks where individual bit positions are more meaningful than their
hexadecimal encoding.
When the "bitmask-list" option is enabled, the printk "%*pbl" format
specifier is used to render bitmasks as comma-separated ranges, making
trace output easier to interpret for complex CPU configurations and
large bitmask values.
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
Documentation/trace/ftrace.rst | 9 +++++++
include/linux/trace_events.h | 8 +++---
include/linux/trace_seq.h | 12 ++++++++-
include/trace/stages/stage3_trace_output.h | 4 +--
kernel/trace/trace.h | 1 +
kernel/trace/trace_output.c | 30 +++++++++++++++++++---
kernel/trace/trace_seq.c | 29 ++++++++++++++++++++-
7 files changed, 82 insertions(+), 11 deletions(-)
diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst
index d1f313a5f4ad..639f4d95732f 100644
--- a/Documentation/trace/ftrace.rst
+++ b/Documentation/trace/ftrace.rst
@@ -1290,6 +1290,15 @@ Here are the available options:
This will be useful if you want to find out which hashed
value is corresponding to the real value in trace log.
+ bitmask-list
+ When enabled, bitmasks are displayed as a human-readable list of
+ ranges (e.g., 0,2-5,7) using the printk "%*pbl" format specifier.
+ When disabled (the default), bitmasks are displayed in the
+ traditional hexadecimal bitmap representation. The list format is
+ particularly useful for tracing CPU masks and other large bitmasks
+ where individual bit positions are more meaningful than their
+ hexadecimal encoding.
+
record-cmd
When any event or tracer is enabled, a hook is enabled
in the sched_switch trace point to fill comm cache
diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
index 3690221ba3d8..0a2b8229b999 100644
--- a/include/linux/trace_events.h
+++ b/include/linux/trace_events.h
@@ -38,7 +38,10 @@ const char *trace_print_symbols_seq_u64(struct trace_seq *p,
*symbol_array);
#endif
-const char *trace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr,
+struct trace_iterator;
+struct trace_event;
+
+const char *trace_print_bitmask_seq(struct trace_iterator *iter, void *bitmask_ptr,
unsigned int bitmask_size);
const char *trace_print_hex_seq(struct trace_seq *p,
@@ -54,9 +57,6 @@ trace_print_hex_dump_seq(struct trace_seq *p, const char *prefix_str,
int prefix_type, int rowsize, int groupsize,
const void *buf, size_t len, bool ascii);
-struct trace_iterator;
-struct trace_event;
-
int trace_raw_output_prep(struct trace_iterator *iter,
struct trace_event *event);
extern __printf(2, 3)
diff --git a/include/linux/trace_seq.h b/include/linux/trace_seq.h
index 4a0b8c172d27..37772d8f97f8 100644
--- a/include/linux/trace_seq.h
+++ b/include/linux/trace_seq.h
@@ -114,7 +114,11 @@ extern void trace_seq_putmem_hex(struct trace_seq *s, const void *mem,
extern int trace_seq_path(struct trace_seq *s, const struct path *path);
extern void trace_seq_bitmask(struct trace_seq *s, const unsigned long *maskp,
- int nmaskbits);
+ int nmaskbits);
+
+extern void trace_seq_bitmask_list(struct trace_seq *s,
+ const unsigned long *maskp,
+ int nmaskbits);
extern int trace_seq_hex_dump(struct trace_seq *s, const char *prefix_str,
int prefix_type, int rowsize, int groupsize,
@@ -137,6 +141,12 @@ trace_seq_bitmask(struct trace_seq *s, const unsigned long *maskp,
{
}
+static inline void
+trace_seq_bitmask(struct trace_seq *s, const unsigned long *maskp,
+ int nmaskbits)
+{
+}
+
static inline int trace_print_seq(struct seq_file *m, struct trace_seq *s)
{
return 0;
diff --git a/include/trace/stages/stage3_trace_output.h b/include/trace/stages/stage3_trace_output.h
index 1e7b0bef95f5..fce85ea2df1c 100644
--- a/include/trace/stages/stage3_trace_output.h
+++ b/include/trace/stages/stage3_trace_output.h
@@ -39,7 +39,7 @@
void *__bitmask = __get_dynamic_array(field); \
unsigned int __bitmask_size; \
__bitmask_size = __get_dynamic_array_len(field); \
- trace_print_bitmask_seq(p, __bitmask, __bitmask_size); \
+ trace_print_bitmask_seq(iter, __bitmask, __bitmask_size); \
})
#undef __get_cpumask
@@ -51,7 +51,7 @@
void *__bitmask = __get_rel_dynamic_array(field); \
unsigned int __bitmask_size; \
__bitmask_size = __get_rel_dynamic_array_len(field); \
- trace_print_bitmask_seq(p, __bitmask, __bitmask_size); \
+ trace_print_bitmask_seq(iter, __bitmask, __bitmask_size); \
})
#undef __get_rel_cpumask
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index b6d42fe06115..8888fc9335b6 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -1411,6 +1411,7 @@ extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
C(COPY_MARKER, "copy_trace_marker"), \
C(PAUSE_ON_TRACE, "pause-on-trace"), \
C(HASH_PTR, "hash-ptr"), /* Print hashed pointer */ \
+ C(BITMASK_LIST, "bitmask-list"), \
FUNCTION_FLAGS \
FGRAPH_FLAGS \
STACK_FLAGS \
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index cc2d3306bb60..1996d7aba038 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -194,13 +194,37 @@ trace_print_symbols_seq_u64(struct trace_seq *p, unsigned long long val,
EXPORT_SYMBOL(trace_print_symbols_seq_u64);
#endif
+/**
+ * trace_print_bitmask_seq - print a bitmask to a sequence buffer
+ * @iter: The trace iterator for the current event instance
+ * @bitmask_ptr: The pointer to the bitmask data
+ * @bitmask_size: The size of the bitmask in bytes
+ *
+ * Prints a bitmask into a sequence buffer as either a hex string or a
+ * human-readable range list, depending on the instance's "bitmask-list"
+ * trace option. The bitmask is formatted into the iterator's temporary
+ * scratchpad rather than the primary sequence buffer. This avoids
+ * duplication and pointer-collision issues when the returned string is
+ * processed by a "%s" specifier in a TP_printk() macro.
+ *
+ * Returns a pointer to the formatted string within the temporary buffer.
+ */
const char *
-trace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr,
+trace_print_bitmask_seq(struct trace_iterator *iter, void *bitmask_ptr,
unsigned int bitmask_size)
{
- const char *ret = trace_seq_buffer_ptr(p);
+ struct trace_seq *p = &iter->tmp_seq;
+ const struct trace_array *tr = iter->tr;
+ const char *ret;
+
+ trace_seq_init(p);
+ ret = trace_seq_buffer_ptr(p);
+
+ if (tr->trace_flags & TRACE_ITER(BITMASK_LIST))
+ trace_seq_bitmask_list(p, bitmask_ptr, bitmask_size * 8);
+ else
+ trace_seq_bitmask(p, bitmask_ptr, bitmask_size * 8);
- trace_seq_bitmask(p, bitmask_ptr, bitmask_size * 8);
trace_seq_putc(p, 0);
return ret;
diff --git a/kernel/trace/trace_seq.c b/kernel/trace/trace_seq.c
index 32684ef4fb9d..85f6f10d107f 100644
--- a/kernel/trace/trace_seq.c
+++ b/kernel/trace/trace_seq.c
@@ -106,7 +106,7 @@ EXPORT_SYMBOL_GPL(trace_seq_printf);
* Writes a ASCII representation of a bitmask string into @s.
*/
void trace_seq_bitmask(struct trace_seq *s, const unsigned long *maskp,
- int nmaskbits)
+ int nmaskbits)
{
unsigned int save_len = s->seq.len;
@@ -124,6 +124,33 @@ void trace_seq_bitmask(struct trace_seq *s, const unsigned long *maskp,
}
EXPORT_SYMBOL_GPL(trace_seq_bitmask);
+/**
+ * trace_seq_bitmask_list - write a bitmask array in its list representation
+ * @s: trace sequence descriptor
+ * @maskp: points to an array of unsigned longs that represent a bitmask
+ * @nmaskbits: The number of bits that are valid in @maskp
+ *
+ * Writes a list representation (e.g., 0-3,5-7) of a bitmask string into @s.
+ */
+void trace_seq_bitmask_list(struct trace_seq *s, const unsigned long *maskp,
+ int nmaskbits)
+{
+ unsigned int save_len = s->seq.len;
+
+ if (s->full)
+ return;
+
+ __trace_seq_init(s);
+
+ seq_buf_printf(&s->seq, "%*pbl", nmaskbits, maskp);
+
+ if (unlikely(seq_buf_has_overflowed(&s->seq))) {
+ s->seq.len = save_len;
+ s->full = 1;
+ }
+}
+EXPORT_SYMBOL_GPL(trace_seq_bitmask_list);
+
/**
* trace_seq_vprintf - sequence printing of trace information
* @s: trace sequence descriptor
--
2.51.0
^ permalink raw reply related
* [v2 PATCH 0/1] tracing: Add bitmask-list option for human-readable bitmask display
From: Aaron Tomlin @ 2025-12-25 8:02 UTC (permalink / raw)
To: rostedt, mhiramat, mark.rutland, mathieu.desnoyers, corbet
Cc: sean, linux-kernel, linux-trace-kernel, linux-doc
Hi Steve,
Add support for displaying bitmasks in human-readable list format (e.g.,
0,2-5,7) in addition to the default hexadecimal bitmap representation.
This is particularly useful when tracing CPU masks and other large
bitmasks where individual bit positions are more meaningful than their
hexadecimal encoding.
When the "bitmask-list" option is enabled, the printk "%*pbl" format
specifier is used to render bitmasks as comma-separated ranges, making
trace output easier to interpret for complex CPU configurations and
large bitmask values.
This iteration incorporates the use of iter->tmp_seq to ensure the
implementation is robust, instance-aware, and free from buffer contention
or duplication issues.
Please let me know your thoughts.
Changes since v1 [1]:
- Introduce new helper trace_seq_bitmask_list() (Steven Rostedt)
- Use iter->tmp_seq as a scratchpad to prevent buffer collisions and
duplication
- Update trace_print_bitmask_seq() signature to accept trace_iterator
instead of trace_seq
- Add declaration for trace_seq_bitmask_list() and provide necessary stub
definitions
- Update __get_bitmask and __get_rel_bitmask macros to pass iter to the
bitmask helper
- Implement instance-aware bitmask formatting using iter->tmp_seq as a
scratchpad to prevent buffer collisions and duplication
[1]: https://lore.kernel.org/lkml/20251223035622.2084081-1-atomlin@atomlin.com/
Aaron Tomlin (1):
tracing: Add bitmask-list option for human-readable bitmask display
Documentation/trace/ftrace.rst | 9 +++++++
include/linux/trace_events.h | 8 +++---
include/linux/trace_seq.h | 12 ++++++++-
include/trace/stages/stage3_trace_output.h | 4 +--
kernel/trace/trace.h | 1 +
kernel/trace/trace_output.c | 30 +++++++++++++++++++---
kernel/trace/trace_seq.c | 29 ++++++++++++++++++++-
7 files changed, 82 insertions(+), 11 deletions(-)
--
2.51.0
^ permalink raw reply
* Re: [PATCH] tracing: Add bitmask-list option for human-readable bitmask display
From: Aaron Tomlin @ 2025-12-25 7:38 UTC (permalink / raw)
To: Steven Rostedt
Cc: mhiramat, mark.rutland, mathieu.desnoyers, corbet, sean,
linux-kernel, linux-trace-kernel, linux-doc
In-Reply-To: <20251224085848.26387f5d@gandalf.local.home>
[-- Attachment #1: Type: text/plain, Size: 1181 bytes --]
On Wed, Dec 24, 2025 at 08:58:48AM -0500, Steven Rostedt wrote:
> Should we just make all cpu bitmask range lists instead?
Hi Steve,
I am somewhat hesitant to adopt that suggestion as I would prefer to avoid
breaking any existing tooling that relies upon the default hexadecimal
bitmask format.
Whilst range lists are undoubtedly superior for human interpretation, the
hexadecimal output is a well-established standard throughout the kernel.
For instance, the hexadecimal format is still strictly adhered to for
"Cpus_allowed:" within /proc/[pid]/status. Introducing a global change to
ftrace defaults could disrupt parsers and scripts that expect this
consistency across the system.
By leveraging the existing bitmask-list trace option via
trace_print_bitmask_seq(), we offer users the requisite flexibility for
high-core-count systems whilst preserving backward compatibility for the
wider ecosystem.
I shall send a new version of the patch shortly. This version incorporates
the use of iter->tmp_seq to ensure the implementation is robust,
instance-aware, and free from buffer contention or duplication issues.
Kind regards,
--
Aaron Tomlin
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [PATCH v3 12/13] rv: Add deadline monitors
From: Nam Cao @ 2025-12-25 1:58 UTC (permalink / raw)
To: Gabriele Monaco, linux-kernel, Steven Rostedt, Gabriele Monaco,
Jonathan Corbet, Masami Hiramatsu, linux-trace-kernel, linux-doc
Cc: Tomas Glozar, Juri Lelli, Clark Williams, John Kacur
In-Reply-To: <874ipfbay7.fsf@yellow.woof>
Nam Cao <namcao@linutronix.de> writes:
> Gabriele Monaco <gmonaco@redhat.com> writes:
>> Add the deadline monitors collection to validate the deadline scheduler,
>> both for deadline tasks and servers.
>>
>> The currently implemented monitors are:
>> * throttle:
>> validate dl entities are throttled when they use up their runtime
>> * nomiss:
>> validate dl entities run to completion before their deadiline
>>
>> Signed-off-by: Gabriele Monaco <gmonaco@redhat.com>
>
> There are some helper functions that I am a bit uncomfortable with
> (e.g. pi_of, is_dl_boosted, dl_is_implicit, ...) because they are
> relying on implementation details of deadline scheduler which can be
> changed. So ideally this patch should have an Ack from the scheduler people.
Think about this again, perhaps we should move all these helpers to
include/linux/sched/deadline.h instead? The scheduler people must be
aware of these functions, and maintain them. I don't want a situation
where the scheduler people make changes, and something else (which they
do not know exist) breaks.
Ideally, non-scheduler code should not look at private fields of
scheduler's structs.
Nam
^ permalink raw reply
* Re: [PATCH v3 13/13] rv: Add dl_server specific monitors
From: Nam Cao @ 2025-12-25 1:51 UTC (permalink / raw)
To: Gabriele Monaco, linux-kernel, Steven Rostedt, Gabriele Monaco,
Jonathan Corbet, Masami Hiramatsu, linux-trace-kernel, linux-doc
Cc: Tomas Glozar, Juri Lelli, Clark Williams, John Kacur
In-Reply-To: <20251205131621.135513-14-gmonaco@redhat.com>
Gabriele Monaco <gmonaco@redhat.com> writes:
> Add monitors to validate the behaviour of the deadline server.
>
> The currently implemented monitors are:
> * boost
> fair tasks run either independently or boosted
> * laxity
> deferrable servers wait for zero-laxity and run
>
> Signed-off-by: Gabriele Monaco <gmonaco@redhat.com>
Same remarks in "[PATCH v3 12/13] rv: Add deadline monitors" apply.
Reviewed-by: Nam Cao <namcao@linutronix.de>
^ permalink raw reply
* Re: [PATCH v3 12/13] rv: Add deadline monitors
From: Nam Cao @ 2025-12-25 1:35 UTC (permalink / raw)
To: Gabriele Monaco, linux-kernel, Steven Rostedt, Gabriele Monaco,
Jonathan Corbet, Masami Hiramatsu, linux-trace-kernel, linux-doc
Cc: Tomas Glozar, Juri Lelli, Clark Williams, John Kacur
In-Reply-To: <20251205131621.135513-13-gmonaco@redhat.com>
Gabriele Monaco <gmonaco@redhat.com> writes:
> Add the deadline monitors collection to validate the deadline scheduler,
> both for deadline tasks and servers.
>
> The currently implemented monitors are:
> * throttle:
> validate dl entities are throttled when they use up their runtime
> * nomiss:
> validate dl entities run to completion before their deadiline
>
> Signed-off-by: Gabriele Monaco <gmonaco@redhat.com>
There are some helper functions that I am a bit uncomfortable with
(e.g. pi_of, is_dl_boosted, dl_is_implicit, ...) because they are
relying on implementation details of deadline scheduler which can be
changed. So ideally this patch should have an Ack from the scheduler people.
I can't comment on the model, since I don't know scheduler. But from RV
perspective:
Reviewed-by: Nam Cao <namcao@linutronix.de>
^ permalink raw reply
* Re: [PATCH v3 05/13] Documentation/rv: Add documentation about hybrid automata
From: Nam Cao @ 2025-12-25 1:16 UTC (permalink / raw)
To: Gabriele Monaco, linux-kernel, Steven Rostedt, Gabriele Monaco,
Jonathan Corbet, linux-trace-kernel, linux-doc
Cc: Tomas Glozar, Juri Lelli, Clark Williams, John Kacur
In-Reply-To: <20251205131621.135513-6-gmonaco@redhat.com>
Gabriele Monaco <gmonaco@redhat.com> writes:
> Describe theory and implementation of hybrid automata in the dedicated
> page hybrid_automata.rst
> Include a section on how to integrate a hybrid automaton in
> monitor_synthesis.rst
> Also remove a hanging $ in deterministic_automata.rst
>
> Signed-off-by: Gabriele Monaco <gmonaco@redhat.com>
Reviewed-by: Nam Cao <namcao@linutronix.de>
^ permalink raw reply
* Re: [PATCH v3 04/13] verification/rvgen: Add support for Hybrid Automata
From: Nam Cao @ 2025-12-25 1:05 UTC (permalink / raw)
To: Gabriele Monaco, linux-kernel, Steven Rostedt, Gabriele Monaco,
linux-trace-kernel
Cc: Tomas Glozar, Juri Lelli, Clark Williams, John Kacur
In-Reply-To: <20251205131621.135513-5-gmonaco@redhat.com>
Gabriele Monaco <gmonaco@redhat.com> writes:
> Add the possibility to parse dot files as hybrid automata and generate
> the necessary code from rvgen.
>
> Hybrid automata are very similar to deterministic ones and most
> functionality is shared, the dot files include also constraints together
> with event names (separated by ;) and state names (separated by \n).
>
> The tool can now generate the appropriate code to validate constraints
> at runtime according to the dot specification.
>
> Signed-off-by: Gabriele Monaco <gmonaco@redhat.com>
Reviewed-by: Nam Cao <namcao@linutronix.de>
^ permalink raw reply
* Re: [PATCH] tracing: Replace use of system_wq with system_percpu_wq
From: Steven Rostedt @ 2025-12-24 20:44 UTC (permalink / raw)
To: Marco Crivellari
Cc: linux-kernel, linux-trace-kernel, Tejun Heo, Lai Jiangshan,
Frederic Weisbecker, Sebastian Andrzej Siewior, Michal Hocko,
Masami Hiramatsu, Mathieu Desnoyers
In-Reply-To: <20251224162000.141731-1-marco.crivellari@suse.com>
On Wed, 24 Dec 2025 17:20:00 +0100
Marco Crivellari <marco.crivellari@suse.com> wrote:
> Before that to happen after a careful review and conversion of each individual
> case, workqueue users must be converted to the better named new workqueues with
> no intended behaviour changes:
>
> system_wq -> system_percpu_wq
> system_unbound_wq -> system_dfl_wq
>
> This way the old obsolete workqueues (system_wq, system_unbound_wq) can be
> removed in the future.
>
> Suggested-by: Tejun Heo <tj@kernel.org>
> Signed-off-by: Marco Crivellari <marco.crivellari@suse.com>
> ---
> kernel/trace/trace_events_filter.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
> index 385af8405392..bb6de1d38f78 100644
> --- a/kernel/trace/trace_events_filter.c
> +++ b/kernel/trace/trace_events_filter.c
> @@ -1375,7 +1375,7 @@ static void free_filter_list_tasks(struct rcu_head *rhp)
> struct filter_head *filter_list = container_of(rhp, struct filter_head, rcu);
>
> INIT_RCU_WORK(&filter_list->rwork, free_filter_list_work);
> - queue_rcu_work(system_wq, &filter_list->rwork);
> + queue_rcu_work(system_percpu_wq, &filter_list->rwork);
This is just garbage collection and has no need to be done on the CPU it
was queued on. I guess this can use system_dfl_wq?
-- Steve
> }
>
^ permalink raw reply
* [PATCH v11 3/3] mm: Implement precise OOM killer task selection
From: Mathieu Desnoyers @ 2025-12-24 17:46 UTC (permalink / raw)
To: Andrew Morton
Cc: linux-kernel, Mathieu Desnoyers, Paul E. McKenney, Steven Rostedt,
Masami Hiramatsu, Dennis Zhou, Tejun Heo, Christoph Lameter,
Martin Liu, David Rientjes, christian.koenig, Shakeel Butt,
SeongJae Park, Michal Hocko, Johannes Weiner, Sweet Tea Dorminy,
Lorenzo Stoakes, Liam R . Howlett, Mike Rapoport,
Suren Baghdasaryan, Vlastimil Babka, Christian Brauner, Wei Yang,
David Hildenbrand, Miaohe Lin, Al Viro, linux-mm,
linux-trace-kernel, Yu Zhao, Roman Gushchin, Mateusz Guzik,
Matthew Wilcox, Baolin Wang, Aboorva Devarajan
In-Reply-To: <20251224174638.650551-1-mathieu.desnoyers@efficios.com>
Use the hierarchical tree counter approximation to implement the OOM
killer task selection with a 2-pass algorithm. The first pass selects
the process that has the highest badness points approximation, and the
second pass compares each process using the current max badness points
approximation.
The second pass uses an approximate comparison to eliminate all processes
which are below the current max badness points approximation accuracy
range.
Summing the per-CPU counters to calculate the precise badness of tasks
is only required for tasks with an approximate badness within the
accuracy range of the current max points value.
Limit to 16 the maximum number of badness sums allowed for an OOM killer
task selection before falling back to the approximated comparison. This
ensures bounded execution time for scenarios where many tasks have
badness within the accuracy of the maximum badness approximation.
Tested with the following script:
#!/bin/sh
for a in $(seq 1 10); do (tail /dev/zero &); done
sleep 5
for a in $(seq 1 10); do (tail /dev/zero &); done
sleep 2
for a in $(seq 1 10); do (tail /dev/zero &); done
echo "Waiting for tasks to finish"
wait
Results: OOM kill order on a 128GB memory system
================================================
* systemd and sd-pam are chosen first due to their oom_score_ajd:100:
Out of memory: Killed process 3502 (systemd) total-vm:20096kB, anon-rss:0kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:72kB oom_score_adj:100
Out of memory: Killed process 3503 ((sd-pam)) total-vm:21432kB, anon-rss:0kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:76kB oom_score_adj:100
* The first batch of 10 processes are gradually killed, consecutively
picking the one that uses the most memory. The fact that we are
freeing memory from the previous processes increases the threshold
at which the remaining processes of that group are killed. Processes
from the second and third batches of 10 processes have time to start
before we complete killing the first 10 processes:
Out of memory: Killed process 3703 (tail) total-vm:6591280kB, anon-rss:6578176kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:12936kB oom_score_adj:0
Out of memory: Killed process 3705 (tail) total-vm:6731716kB, anon-rss:6709248kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:13212kB oom_score_adj:0
Out of memory: Killed process 3707 (tail) total-vm:6977216kB, anon-rss:6946816kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:13692kB oom_score_adj:0
Out of memory: Killed process 3699 (tail) total-vm:7205640kB, anon-rss:7184384kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:14136kB oom_score_adj:0
Out of memory: Killed process 3713 (tail) total-vm:7463204kB, anon-rss:7438336kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:14644kB oom_score_adj:0
Out of memory: Killed process 3701 (tail) total-vm:7739204kB, anon-rss:7716864kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:15180kB oom_score_adj:0
Out of memory: Killed process 3709 (tail) total-vm:8050176kB, anon-rss:8028160kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:15792kB oom_score_adj:0
Out of memory: Killed process 3711 (tail) total-vm:8362236kB, anon-rss:8339456kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:16404kB oom_score_adj:0
Out of memory: Killed process 3715 (tail) total-vm:8649360kB, anon-rss:8634368kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:16972kB oom_score_adj:0
Out of memory: Killed process 3697 (tail) total-vm:8951788kB, anon-rss:8929280kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:17560kB oom_score_adj:0
* Even though there is a 2 seconds delay between the 2nd and 3rd batches
those appear to execute in mixed order. Therefore, let's consider them
as a single batch of 20 processes. We are hitting oom at a lower
memory threshold because at this point the 20 remaining proceses are
running rather than the previous 10. The process with highest memory
usage is selected for oom, thus making room for the remaining
processes so they can use more memory before they fill the available
memory, thus explaining why the memory use for selected processes
gradually increases, until all system memory is used by the last one:
Out of memory: Killed process 3731 (tail) total-vm:7089868kB, anon-rss:7077888kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:13912kB oom_score_adj:0
Out of memory: Killed process 3721 (tail) total-vm:7417248kB, anon-rss:7405568kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:14556kB oom_score_adj:0
Out of memory: Killed process 3729 (tail) total-vm:7795864kB, anon-rss:7766016kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:15300kB oom_score_adj:0
Out of memory: Killed process 3723 (tail) total-vm:8259620kB, anon-rss:8224768kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:16208kB oom_score_adj:0
Out of memory: Killed process 3737 (tail) total-vm:8695984kB, anon-rss:8667136kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:17060kB oom_score_adj:0
Out of memory: Killed process 3735 (tail) total-vm:9295980kB, anon-rss:9265152kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:18240kB oom_score_adj:0
Out of memory: Killed process 3727 (tail) total-vm:9907900kB, anon-rss:9895936kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:19428kB oom_score_adj:0
Out of memory: Killed process 3719 (tail) total-vm:10631248kB, anon-rss:10600448kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:20844kB oom_score_adj:0
Out of memory: Killed process 3733 (tail) total-vm:11341720kB, anon-rss:11321344kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:22232kB oom_score_adj:0
Out of memory: Killed process 3725 (tail) total-vm:12348124kB, anon-rss:12320768kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:24204kB oom_score_adj:0
Out of memory: Killed process 3759 (tail) total-vm:12978888kB, anon-rss:12967936kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:25440kB oom_score_adj:0
Out of memory: Killed process 3751 (tail) total-vm:14386412kB, anon-rss:14352384kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:28196kB oom_score_adj:0
Out of memory: Killed process 3741 (tail) total-vm:16153168kB, anon-rss:16130048kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:31652kB oom_score_adj:0
Out of memory: Killed process 3753 (tail) total-vm:18414856kB, anon-rss:18391040kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:36076kB oom_score_adj:0
Out of memory: Killed process 3745 (tail) total-vm:21389456kB, anon-rss:21356544kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:41904kB oom_score_adj:0
Out of memory: Killed process 3747 (tail) total-vm:25659348kB, anon-rss:25632768kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:50260kB oom_score_adj:0
Out of memory: Killed process 3755 (tail) total-vm:32030820kB, anon-rss:32006144kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:62720kB oom_score_adj:0
Out of memory: Killed process 3743 (tail) total-vm:42648456kB, anon-rss:42614784kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:83504kB oom_score_adj:0
Out of memory: Killed process 3757 (tail) total-vm:63971028kB, anon-rss:63938560kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:125228kB oom_score_adj:0
Out of memory: Killed process 3749 (tail) total-vm:127799660kB, anon-rss:127778816kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:250140kB oom_score_adj:0
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Martin Liu <liumartin@google.com>
Cc: David Rientjes <rientjes@google.com>
Cc: christian.koenig@amd.com
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: SeongJae Park <sj@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Sweet Tea Dorminy <sweettea-kernel@dorminy.me>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: "Liam R . Howlett" <liam.howlett@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Wei Yang <richard.weiyang@gmail.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: linux-mm@kvack.org
Cc: linux-trace-kernel@vger.kernel.org
Cc: Yu Zhao <yuzhao@google.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Mateusz Guzik <mjguzik@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Aboorva Devarajan <aboorvad@linux.ibm.com>
---
fs/proc/base.c | 2 +-
include/linux/mm.h | 34 +++++++++++++++++----
include/linux/oom.h | 12 +++++++-
mm/oom_kill.c | 72 +++++++++++++++++++++++++++++++++++++--------
4 files changed, 101 insertions(+), 19 deletions(-)
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 4eec684baca9..d75d0ce97032 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -589,7 +589,7 @@ static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns,
unsigned long points = 0;
long badness;
- badness = oom_badness(task, totalpages);
+ badness = oom_badness(task, totalpages, false, NULL, NULL);
/*
* Special case OOM_SCORE_ADJ_MIN for all others scale the
* badness value into [0, 2000] range which we have been
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 79ac9895cb95..c8c0e054fb53 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2864,14 +2864,32 @@ static inline struct percpu_counter_tree_level_item *get_rss_stat_items(struct m
/*
* per-process(per-mm_struct) statistics.
*/
+static inline unsigned long __get_mm_counter(struct mm_struct *mm, int member, bool approximate,
+ unsigned int *accuracy_under, unsigned int *accuracy_over)
+{
+ if (approximate) {
+ if (accuracy_under && accuracy_over) {
+ unsigned int under, over;
+
+ percpu_counter_tree_approximate_accuracy_range(&mm->rss_stat[member], &under, &over);
+ *accuracy_under += under;
+ *accuracy_over += over;
+ }
+ return percpu_counter_tree_approximate_sum_positive(&mm->rss_stat[member]);
+ } else {
+ return percpu_counter_tree_precise_sum_positive(&mm->rss_stat[member]);
+ }
+}
+
static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
{
- return percpu_counter_tree_approximate_sum_positive(&mm->rss_stat[member]);
+ return __get_mm_counter(mm, member, true, NULL, NULL);
}
+
static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member)
{
- return percpu_counter_tree_precise_sum_positive(&mm->rss_stat[member]);
+ return get_mm_counter(mm, member);
}
void mm_trace_rss_stat(struct mm_struct *mm, int member);
@@ -2912,11 +2930,17 @@ static inline int mm_counter(struct folio *folio)
return mm_counter_file(folio);
}
+static inline unsigned long __get_mm_rss(struct mm_struct *mm, bool approximate,
+ unsigned int *accuracy_under, unsigned int *accuracy_over)
+{
+ return __get_mm_counter(mm, MM_FILEPAGES, approximate, accuracy_under, accuracy_over) +
+ __get_mm_counter(mm, MM_ANONPAGES, approximate, accuracy_under, accuracy_over) +
+ __get_mm_counter(mm, MM_SHMEMPAGES, approximate, accuracy_under, accuracy_over);
+}
+
static inline unsigned long get_mm_rss(struct mm_struct *mm)
{
- return get_mm_counter(mm, MM_FILEPAGES) +
- get_mm_counter(mm, MM_ANONPAGES) +
- get_mm_counter(mm, MM_SHMEMPAGES);
+ return __get_mm_rss(mm, true, NULL, NULL);
}
static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
diff --git a/include/linux/oom.h b/include/linux/oom.h
index 7b02bc1d0a7e..ef3919e4c7f2 100644
--- a/include/linux/oom.h
+++ b/include/linux/oom.h
@@ -48,6 +48,13 @@ struct oom_control {
unsigned long totalpages;
struct task_struct *chosen;
long chosen_points;
+ bool approximate;
+ /*
+ * Number of precise badness points sums performed by this task
+ * selection.
+ */
+ int nr_precise;
+ unsigned long accuracy_under;
/* Used to print the constraint info. */
enum oom_constraint constraint;
@@ -97,7 +104,10 @@ static inline vm_fault_t check_stable_address_space(struct mm_struct *mm)
}
long oom_badness(struct task_struct *p,
- unsigned long totalpages);
+ unsigned long totalpages,
+ bool approximate,
+ unsigned int *accuracy_under,
+ unsigned int *accuracy_over);
extern bool out_of_memory(struct oom_control *oc);
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 5eb11fbba704..91ecf2b90fe0 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -53,6 +53,14 @@
#define CREATE_TRACE_POINTS
#include <trace/events/oom.h>
+/*
+ * Maximum number of badness sums allowed before using an approximated
+ * comparison. This ensures bounded execution time for scenarios where
+ * many tasks have badness within the accuracy of the maximum badness
+ * approximation.
+ */
+static int max_precise_badness_sums = 16;
+
static int sysctl_panic_on_oom;
static int sysctl_oom_kill_allocating_task;
static int sysctl_oom_dump_tasks = 1;
@@ -194,12 +202,16 @@ static bool should_dump_unreclaim_slab(void)
* oom_badness - heuristic function to determine which candidate task to kill
* @p: task struct of which task we should calculate
* @totalpages: total present RAM allowed for page allocation
+ * @approximate: whether the value can be approximated
+ * @accuracy_under: accuracy of the badness value approximation (under value)
+ * @accuracy_over: accuracy of the badness value approximation (over value)
*
* The heuristic for determining which task to kill is made to be as simple and
* predictable as possible. The goal is to return the highest value for the
* task consuming the most memory to avoid subsequent oom failures.
*/
-long oom_badness(struct task_struct *p, unsigned long totalpages)
+long oom_badness(struct task_struct *p, unsigned long totalpages, bool approximate,
+ unsigned int *accuracy_under, unsigned int *accuracy_over)
{
long points;
long adj;
@@ -228,7 +240,8 @@ long oom_badness(struct task_struct *p, unsigned long totalpages)
* The baseline for the badness score is the proportion of RAM that each
* task's rss, pagetable and swap space use.
*/
- points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +
+ points = __get_mm_rss(p->mm, approximate, accuracy_under, accuracy_over) +
+ __get_mm_counter(p->mm, MM_SWAPENTS, approximate, accuracy_under, accuracy_over) +
mm_pgtables_bytes(p->mm) / PAGE_SIZE;
task_unlock(p);
@@ -309,6 +322,7 @@ static enum oom_constraint constrained_alloc(struct oom_control *oc)
static int oom_evaluate_task(struct task_struct *task, void *arg)
{
struct oom_control *oc = arg;
+ unsigned int accuracy_under = 0, accuracy_over = 0;
long points;
if (oom_unkillable_task(task))
@@ -339,9 +353,28 @@ static int oom_evaluate_task(struct task_struct *task, void *arg)
goto select;
}
- points = oom_badness(task, oc->totalpages);
- if (points == LONG_MIN || points < oc->chosen_points)
- goto next;
+ points = oom_badness(task, oc->totalpages, true, &accuracy_under, &accuracy_over);
+ if (oc->approximate) {
+ if (points == LONG_MIN || points < oc->chosen_points)
+ goto next;
+ } else {
+ /*
+ * Eliminate processes which are below the chosen
+ * points accuracy range with an approximation.
+ */
+ if (points == LONG_MIN || (long)(points + accuracy_over + oc->accuracy_under - oc->chosen_points) < 0)
+ goto next;
+
+ if (oc->nr_precise < max_precise_badness_sums) {
+ accuracy_under = 0;
+ accuracy_over = 0;
+ oc->nr_precise++;
+ /* Precise evaluation. */
+ points = oom_badness(task, oc->totalpages, false, NULL, NULL);
+ if (points == LONG_MIN || (long)(points + oc->accuracy_under - oc->chosen_points) < 0)
+ goto next;
+ }
+ }
select:
if (oc->chosen)
@@ -349,6 +382,7 @@ static int oom_evaluate_task(struct task_struct *task, void *arg)
get_task_struct(task);
oc->chosen = task;
oc->chosen_points = points;
+ oc->accuracy_under = accuracy_under;
next:
return 0;
abort:
@@ -358,14 +392,8 @@ static int oom_evaluate_task(struct task_struct *task, void *arg)
return 1;
}
-/*
- * Simple selection loop. We choose the process with the highest number of
- * 'points'. In case scan was aborted, oc->chosen is set to -1.
- */
-static void select_bad_process(struct oom_control *oc)
+static void select_bad_process_iter(struct oom_control *oc)
{
- oc->chosen_points = LONG_MIN;
-
if (is_memcg_oom(oc))
mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc);
else {
@@ -379,6 +407,26 @@ static void select_bad_process(struct oom_control *oc)
}
}
+/*
+ * Simple selection loop. We choose the process with the highest number of
+ * 'points'. In case scan was aborted, oc->chosen is set to -1.
+ */
+static void select_bad_process(struct oom_control *oc)
+{
+ oc->chosen_points = LONG_MIN;
+ oc->accuracy_under = 0;
+ oc->nr_precise = 0;
+
+ /* Approximate scan. */
+ oc->approximate = true;
+ select_bad_process_iter(oc);
+ if (oc->chosen == (void *)-1UL)
+ return;
+ /* Precise scan. */
+ oc->approximate = false;
+ select_bad_process_iter(oc);
+}
+
static int dump_task(struct task_struct *p, void *arg)
{
struct oom_control *oc = arg;
--
2.39.5
^ permalink raw reply related
* [PATCH v11 1/3] lib: Introduce hierarchical per-cpu counters
From: Mathieu Desnoyers @ 2025-12-24 17:46 UTC (permalink / raw)
To: Andrew Morton
Cc: linux-kernel, Mathieu Desnoyers, Paul E. McKenney, Steven Rostedt,
Masami Hiramatsu, Dennis Zhou, Tejun Heo, Christoph Lameter,
Martin Liu, David Rientjes, christian.koenig, Shakeel Butt,
SeongJae Park, Michal Hocko, Johannes Weiner, Sweet Tea Dorminy,
Lorenzo Stoakes, Liam R . Howlett, Mike Rapoport,
Suren Baghdasaryan, Vlastimil Babka, Christian Brauner, Wei Yang,
David Hildenbrand, Miaohe Lin, Al Viro, linux-mm,
linux-trace-kernel, Yu Zhao, Roman Gushchin, Mateusz Guzik,
Matthew Wilcox, Baolin Wang, Aboorva Devarajan
In-Reply-To: <20251224174638.650551-1-mathieu.desnoyers@efficios.com>
* Motivation
The purpose of this hierarchical split-counter scheme is to:
- Minimize contention when incrementing and decrementing counters,
- Provide fast access to a sum approximation,
- Provide a sum approximation with an acceptable accuracy level when
scaling to many-core systems.
- Provide approximate and precise comparison of two counters, and
between a counter and a value.
It aims at fixing the per-mm RSS tracking which has become too
inaccurate for OOM killer purposes on large many-core systems [1].
* Design
The hierarchical per-CPU counters propagate a sum approximation through
a N-way tree. When reaching the batch size, the carry is propagated
through a binary tree which consists of logN(nr_cpu_ids) levels. The
batch size for each level is twice the batch size of the prior level.
Example propagation diagram with 8 cpus through a binary tree:
Level 0: 0 1 2 3 4 5 6 7
| / | / | / | /
| / | / | / | /
| / | / | / | /
Level 1: 0 1 2 3
| / | /
| / | /
| / | /
Level 2: 0 1
| /
| /
| /
Level 3: 0
For a binary tree, the maximum inaccuracy is bound by:
batch_size * log2(nr_cpus) * nr_cpus
which evolves with O(n*log(n)) as the number of CPUs increases.
For a N-way tree, the maximum inaccuracy can be pre-calculated
based on the the N-arity of each level and the batch size.
Link: https://lore.kernel.org/lkml/20250331223516.7810-2-sweettea-kernel@dorminy.me/ # [1]
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Martin Liu <liumartin@google.com>
Cc: David Rientjes <rientjes@google.com>
Cc: christian.koenig@amd.com
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: SeongJae Park <sj@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Sweet Tea Dorminy <sweettea-kernel@dorminy.me>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: "Liam R . Howlett" <liam.howlett@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Wei Yang <richard.weiyang@gmail.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: linux-mm@kvack.org
Cc: linux-trace-kernel@vger.kernel.org
Cc: Yu Zhao <yuzhao@google.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Mateusz Guzik <mjguzik@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Aboorva Devarajan <aboorvad@linux.ibm.com>
---
Changes since v10:
- Fold "mm: Take into account hierarchical percpu tree items for static
mm_struct definitions".
Changes since v9:
- Introduce kerneldoc documentation.
- Document structure fields.
- Remove inline from percpu_counter_tree_add().
- Reject batch_size==1 which makes no sense.
- Fix copy-paste bug in percpu_counter_tree_precise_compare()
(wrong inequality comparison).
- Rename "inaccuracy" to "accuracy", which makes it easier to
document.
- Track accuracy limits more precisely. In two's complement
signed integers, the range before a n-bit underflow is one
unit larger than the range before a n-bit overflow. This sums
for all the counters within the tree. Therefore the "under"
vs "over" accuracy is not symmetrical.
Changes since v8:
- Remove migrate guard from the fast path. It does not
matter through which path the carry is propagated up
the tree.
- Rebase on top of v6.18-rc6.
- Introduce percpu_counter_tree_init_many and
percpu_counter_tree_destroy_many APIs.
- Move tree items allocation to the caller.
- Introduce percpu_counter_tree_items_size().
- Move percpu_counter_tree_subsystem_init() call before mm_core_init()
so percpu_counter_tree_items_size() is initialized before it is used.
Changes since v7:
- Explicitly initialize the subsystem from start_kernel() right
after mm_core_init() so it is up and running before the creation of
the first mm at boot.
- Remove the module.h include which is not needed with the explicit
initialization.
- Only consider levels>0 items for order={0,1} nr_items. No
functional change except to allocate only the amount of memory
which is strictly needed.
- Introduce positive precise sum API to handle a scenario where an
unlucky precise sum iteration would hit negative counter values
concurrently with counter updates.
Changes since v5:
- Introduce percpu_counter_tree_approximate_sum_positive.
- Introduce !CONFIG_SMP static inlines for UP build.
- Remove percpu_counter_tree_set_bias from the public API and make it
static.
Changes since v3:
- Add gfp flags to init function.
Changes since v2:
- Introduce N-way tree to reduce tree depth on larger systems.
Changes since v1:
- Remove percpu_counter_tree_precise_sum_unbiased from public header,
make this function static,
- Introduce precise and approximate comparisons between two counters,
- Reorder the struct percpu_counter_tree fields,
- Introduce approx_sum field, which points to the approximate sum
for the percpu_counter_tree_approximate_sum() fast path.
---
include/linux/mm_types.h | 6 +-
include/linux/percpu_counter_tree.h | 293 ++++++++++++
init/main.c | 2 +
lib/Makefile | 1 +
lib/percpu_counter_tree.c | 705 ++++++++++++++++++++++++++++
5 files changed, 1004 insertions(+), 3 deletions(-)
create mode 100644 include/linux/percpu_counter_tree.h
create mode 100644 lib/percpu_counter_tree.c
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index aa4639888f89..9f861ceabe61 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1366,9 +1366,9 @@ static inline void __mm_flags_set_mask_bits_word(struct mm_struct *mm,
MT_FLAGS_USE_RCU)
extern struct mm_struct init_mm;
-#define MM_STRUCT_FLEXIBLE_ARRAY_INIT \
-{ \
- [0 ... sizeof(cpumask_t) + MM_CID_STATIC_SIZE - 1] = 0 \
+#define MM_STRUCT_FLEXIBLE_ARRAY_INIT \
+{ \
+ [0 ... sizeof(cpumask_t) + MM_CID_STATIC_SIZE + PERCPU_COUNTER_TREE_ITEMS_STATIC_SIZE - 1] = 0 \
}
/* Pointer magic because the dynamic array size confuses some compilers. */
diff --git a/include/linux/percpu_counter_tree.h b/include/linux/percpu_counter_tree.h
new file mode 100644
index 000000000000..2e8b1ce5cd13
--- /dev/null
+++ b/include/linux/percpu_counter_tree.h
@@ -0,0 +1,293 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR MIT */
+/* SPDX-FileCopyrightText: 2025 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
+
+#ifndef _PERCPU_COUNTER_TREE_H
+#define _PERCPU_COUNTER_TREE_H
+
+#include <linux/preempt.h>
+#include <linux/atomic.h>
+#include <linux/percpu.h>
+
+#ifdef CONFIG_SMP
+
+#if NR_CPUS == (1U << 0)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 0
+#elif NR_CPUS <= (1U << 1)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 1
+#elif NR_CPUS <= (1U << 2)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 3
+#elif NR_CPUS <= (1U << 3)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 7
+#elif NR_CPUS <= (1U << 4)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 7
+#elif NR_CPUS <= (1U << 5)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 11
+#elif NR_CPUS <= (1U << 6)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 21
+#elif NR_CPUS <= (1U << 7)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 21
+#elif NR_CPUS <= (1U << 8)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 37
+#elif NR_CPUS <= (1U << 9)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 73
+#elif NR_CPUS <= (1U << 10)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 149
+#elif NR_CPUS <= (1U << 11)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 293
+#elif NR_CPUS <= (1U << 12)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 585
+#elif NR_CPUS <= (1U << 13)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 1173
+#elif NR_CPUS <= (1U << 14)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 2341
+#elif NR_CPUS <= (1U << 15)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 4681
+#elif NR_CPUS <= (1U << 16)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 4681
+#elif NR_CPUS <= (1U << 17)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 8777
+#elif NR_CPUS <= (1U << 18)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 17481
+#elif NR_CPUS <= (1U << 19)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 34953
+#elif NR_CPUS <= (1U << 20)
+# define PERCPU_COUNTER_TREE_STATIC_NR_ITEMS 69905
+#else
+# error "Unsupported number of CPUs."
+#endif
+
+struct percpu_counter_tree_level_item {
+ atomic_t count; /*
+ * Count the number of carry fort this tree item.
+ * The carry counter is kept at the order of the
+ * carry accounted for at this tree level.
+ */
+} ____cacheline_aligned_in_smp;
+
+#define PERCPU_COUNTER_TREE_ITEMS_STATIC_SIZE \
+ (PERCPU_COUNTER_TREE_STATIC_NR_ITEMS * sizeof(struct percpu_counter_tree_level_item))
+
+struct percpu_counter_tree {
+ /* Fast-path fields. */
+ unsigned int __percpu *level0; /* Pointer to per-CPU split counters (tree level 0). */
+ unsigned int level0_bit_mask; /* Bit mask to apply to detect carry propagation from tree level 0. */
+ union {
+ unsigned int *i; /* Approximate sum for single-CPU topology. */
+ atomic_t *a; /* Approximate sum for SMP topology. */
+ } approx_sum;
+ int bias; /* Bias to apply to counter precise and approximate values. */
+
+ /* Slow-path fields. */
+ struct percpu_counter_tree_level_item *items; /* Array of tree items for levels 1 to N. */
+ unsigned int batch_size; /*
+ * The batch size is the increment step at level 0 which
+ * triggers a carry propagation. The batch size is required
+ * to be greater than 1, and a power of 2.
+ */
+ /*
+ * The tree approximate sum is guaranteed to be within this accuracy range:
+ * (precise_sum - approx_accuracy_range.under) <= approx_sum <= (precise_sum + approx_accuracy_range.over).
+ * This accuracy is derived from the hardware topology and the tree batch_size.
+ * The "under" accuracy is larger than the "over" accuracy because the negative range of a
+ * two's complement signed integer is one unit larger than the positive range. This delta
+ * is summed for each tree item, which leads to a significantly larger "under" accuracy range
+ * compared to the "over" accuracy range.
+ */
+ struct {
+ unsigned int under;
+ unsigned int over;
+ } approx_accuracy_range;
+};
+
+size_t percpu_counter_tree_items_size(void);
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+ unsigned int nr_counters, unsigned int batch_size, gfp_t gfp_flags);
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+ unsigned int batch_size, gfp_t gfp_flags);
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counter, unsigned int nr_counters);
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter);
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, int inc);
+int percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter);
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b);
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, int v);
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b);
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, int v);
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, int v);
+void percpu_counter_tree_approximate_accuracy_range(struct percpu_counter_tree *counter,
+ unsigned int *under, unsigned int *over);
+int percpu_counter_tree_subsystem_init(void);
+
+/**
+ * percpu_counter_tree_approximate_sum() - Return approximate counter sum.
+ * @counter: The counter to sum.
+ *
+ * Querying the approximate sum is fast, but it is only accurate within
+ * the bounds delimited by percpu_counter_tree_approximate_accuracy_range().
+ * This is meant to be used when speed is preferred over accuracy.
+ *
+ * Return: The current approximate counter sum.
+ */
+static inline
+int percpu_counter_tree_approximate_sum(struct percpu_counter_tree *counter)
+{
+ unsigned int v;
+
+ if (!counter->level0_bit_mask)
+ v = READ_ONCE(*counter->approx_sum.i);
+ else
+ v = atomic_read(counter->approx_sum.a);
+ return (int) (v + (unsigned int)READ_ONCE(counter->bias));
+}
+
+#else /* !CONFIG_SMP */
+
+#define PERCPU_COUNTER_TREE_ITEMS_STATIC_SIZE 0
+
+struct percpu_counter_tree_level_item;
+
+struct percpu_counter_tree {
+ atomic_t count;
+};
+
+static inline
+size_t percpu_counter_tree_items_size(void)
+{
+ return 0;
+}
+
+static inline
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+ unsigned int nr_counters, unsigned int batch_size, gfp_t gfp_flags)
+{
+ for (unsigned int i = 0; i < nr_counters; i++)
+ atomic_set(&counters[i].count, 0);
+ return 0;
+}
+
+static inline
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+ unsigned int batch_size, gfp_t gfp_flags)
+{
+ return percpu_counter_tree_init_many(counter, items, 1, batch_size, gfp_flags);
+}
+
+static inline
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counter, unsigned int nr_counters)
+{
+}
+
+static inline
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter)
+{
+}
+
+static inline
+int percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter)
+{
+ return atomic_read(&counter->count);
+}
+
+static inline
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+ int count_a = percpu_counter_tree_precise_sum(a),
+ count_b = percpu_counter_tree_precise_sum(b);
+
+ if (count_a == count_b)
+ return 0;
+ if (count_a < count_b)
+ return -1;
+ return 1;
+}
+
+static inline
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, int v)
+{
+ int count = percpu_counter_tree_precise_sum(counter);
+
+ if (count == v)
+ return 0;
+ if (count < v)
+ return -1;
+ return 1;
+}
+
+static inline
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+ return percpu_counter_tree_precise_compare(a, b);
+}
+
+static inline
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, int v)
+{
+ return percpu_counter_tree_precise_compare_value(counter, v);
+}
+
+static inline
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, int v)
+{
+ atomic_set(&counter->count, v);
+}
+
+static inline
+void percpu_counter_tree_approximate_accuracy_range(struct percpu_counter_tree *counter,
+ unsigned int *under, unsigned int *over)
+{
+ *under = 0;
+ *over = 0;
+}
+
+static inline
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, int inc)
+{
+ atomic_add(inc, &counter->count);
+}
+
+static inline
+int percpu_counter_tree_approximate_sum(struct percpu_counter_tree *counter)
+{
+ return percpu_counter_tree_precise_sum(counter);
+}
+
+static inline
+int percpu_counter_tree_subsystem_init(void)
+{
+ return 0;
+}
+
+#endif /* CONFIG_SMP */
+
+/**
+ * percpu_counter_tree_approximate_sum_positive() - Return a positive approximate counter sum.
+ * @counter: The counter to sum.
+ *
+ * Return an approximate counter sum which is guaranteed to be greater
+ * or equal to 0.
+ *
+ * Return: The current positive approximate counter sum.
+ */
+static inline
+int percpu_counter_tree_approximate_sum_positive(struct percpu_counter_tree *counter)
+{
+ int v = percpu_counter_tree_approximate_sum(counter);
+ return v > 0 ? v : 0;
+}
+
+/**
+ * percpu_counter_tree_precise_sum_positive() - Return a positive precise counter sum.
+ * @counter: The counter to sum.
+ *
+ * Return a precise counter sum which is guaranteed to be greater
+ * or equal to 0.
+ *
+ * Return: The current positive precise counter sum.
+ */
+static inline
+int percpu_counter_tree_precise_sum_positive(struct percpu_counter_tree *counter)
+{
+ int v = percpu_counter_tree_precise_sum(counter);
+ return v > 0 ? v : 0;
+}
+
+#endif /* _PERCPU_COUNTER_TREE_H */
diff --git a/init/main.c b/init/main.c
index b84818ad9685..85bcc7c1ccd0 100644
--- a/init/main.c
+++ b/init/main.c
@@ -104,6 +104,7 @@
#include <linux/pidfs.h>
#include <linux/ptdump.h>
#include <linux/time_namespace.h>
+#include <linux/percpu_counter_tree.h>
#include <net/net_namespace.h>
#include <asm/io.h>
@@ -1063,6 +1064,7 @@ void start_kernel(void)
vfs_caches_init_early();
sort_main_extable();
trap_init();
+ percpu_counter_tree_subsystem_init();
mm_core_init();
maple_tree_init();
poking_init();
diff --git a/lib/Makefile b/lib/Makefile
index aaf677cf4527..bff117556424 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -181,6 +181,7 @@ obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
obj-$(CONFIG_TEXTSEARCH_BM) += ts_bm.o
obj-$(CONFIG_TEXTSEARCH_FSM) += ts_fsm.o
obj-$(CONFIG_SMP) += percpu_counter.o
+obj-$(CONFIG_SMP) += percpu_counter_tree.o
obj-$(CONFIG_AUDIT_GENERIC) += audit.o
obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o
diff --git a/lib/percpu_counter_tree.c b/lib/percpu_counter_tree.c
new file mode 100644
index 000000000000..9b7ae7df5530
--- /dev/null
+++ b/lib/percpu_counter_tree.c
@@ -0,0 +1,705 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+// SPDX-FileCopyrightText: 2025 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+
+/*
+ * Split Counters With Tree Approximation Propagation
+ *
+ * * Propagation diagram when reaching batch size thresholds (± batch size):
+ *
+ * Example diagram for 8 CPUs:
+ *
+ * log2(8) = 3 levels
+ *
+ * At each level, each pair propagates its values to the next level when
+ * reaching the batch size thresholds.
+ *
+ * Counters at levels 0, 1, 2 can be kept on a single byte ([-128 .. +127] range),
+ * although it may be relevant to keep them on 32-bit counters for
+ * simplicity. (complexity vs memory footprint tradeoff)
+ *
+ * Counter at level 3 can be kept on a 32-bit counter.
+ *
+ * Level 0: 0 1 2 3 4 5 6 7
+ * | / | / | / | /
+ * | / | / | / | /
+ * | / | / | / | /
+ * Level 1: 0 1 2 3
+ * | / | /
+ * | / | /
+ * | / | /
+ * Level 2: 0 1
+ * | /
+ * | /
+ * | /
+ * Level 3: 0
+ *
+ * * Approximation accuracy:
+ *
+ * BATCH(level N): Level N batch size.
+ *
+ * Example for BATCH(level 0) = 32.
+ *
+ * BATCH(level 0) = 32
+ * BATCH(level 1) = 64
+ * BATCH(level 2) = 128
+ * BATCH(level N) = BATCH(level 0) * 2^N
+ *
+ * per-counter global
+ * accuracy accuracy
+ * Level 0: [ -32 .. +31] ±256 (8 * 32)
+ * Level 1: [ -64 .. +63] ±256 (4 * 64)
+ * Level 2: [-128 .. +127] ±256 (2 * 128)
+ * Total: ------ ±768 (log2(nr_cpu_ids) * BATCH(level 0) * nr_cpu_ids)
+ *
+ * Note that the global accuracy can be calculated more precisely
+ * by taking into account that the positive accuracy range is
+ * 31 rather than 32.
+ *
+ * -----
+ *
+ * Approximate Sum Carry Propagation
+ *
+ * Let's define a number of counter bits for each level, e.g.:
+ *
+ * log2(BATCH(level 0)) = log2(32) = 5
+ *
+ * nr_bit value_mask range
+ * Level 0: 5 bits v 0 .. +31
+ * Level 1: 1 bit (v & ~((1UL << 5) - 1)) 0 .. +63
+ * Level 2: 1 bit (v & ~((1UL << 6) - 1)) 0 .. +127
+ * Level 3: 25 bits (v & ~((1UL << 7) - 1)) 0 .. 2^32-1
+ *
+ * Note: Use a full 32-bit per-cpu counter at level 0 to allow precise sum.
+ *
+ * Note: Use cacheline aligned counters at levels above 0 to prevent false sharing.
+ * If memory footprint is an issue, a specialized allocator could be used
+ * to eliminate padding.
+ *
+ * Example with expanded values:
+ *
+ * counter_add(counter, inc):
+ *
+ * if (!inc)
+ * return;
+ *
+ * res = percpu_add_return(counter @ Level 0, inc);
+ * orig = res - inc;
+ * if (inc < 0) {
+ * inc = -(-inc & ~0b00011111); // Clear used bits
+ * // xor bit 5: underflow
+ * if ((inc ^ orig ^ res) & 0b00100000)
+ * inc -= 0b00100000;
+ * } else {
+ * inc &= ~0b00011111; // Clear used bits
+ * // xor bit 5: overflow
+ * if ((inc ^ orig ^ res) & 0b00100000)
+ * inc += 0b00100000;
+ * }
+ * if (!inc)
+ * return;
+ *
+ * res = atomic_add_return(counter @ Level 1, inc);
+ * orig = res - inc;
+ * if (inc < 0) {
+ * inc = -(-inc & ~0b00111111); // Clear used bits
+ * // xor bit 6: underflow
+ * if ((inc ^ orig ^ res) & 0b01000000)
+ * inc -= 0b01000000;
+ * } else {
+ * inc &= ~0b00111111; // Clear used bits
+ * // xor bit 6: overflow
+ * if ((inc ^ orig ^ res) & 0b01000000)
+ * inc += 0b01000000;
+ * }
+ * if (!inc)
+ * return;
+ *
+ * res = atomic_add_return(counter @ Level 2, inc);
+ * orig = res - inc;
+ * if (inc < 0) {
+ * inc = -(-inc & ~0b01111111); // Clear used bits
+ * // xor bit 7: underflow
+ * if ((inc ^ orig ^ res) & 0b10000000)
+ * inc -= 0b10000000;
+ * } else {
+ * inc &= ~0b01111111; // Clear used bits
+ * // xor bit 7: overflow
+ * if ((inc ^ orig ^ res) & 0b10000000)
+ * inc += 0b10000000;
+ * }
+ * if (!inc)
+ * return;
+ *
+ * atomic_add(counter @ Level 3, inc);
+ */
+
+#include <linux/percpu_counter_tree.h>
+#include <linux/cpumask.h>
+#include <linux/percpu.h>
+#include <linux/atomic.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/math.h>
+
+#define MAX_NR_LEVELS 5
+
+/*
+ * The counter configuration is selected at boot time based on the
+ * hardware topology.
+ */
+struct counter_config {
+ unsigned int nr_items; /*
+ * nr_items is the number of items in the tree for levels 1
+ * up to and including the final level (approximate sum).
+ * It excludes the level 0 per-CPU counters.
+ */
+ unsigned char nr_levels; /*
+ * nr_levels is the number of hierarchical counter tree levels.
+ * It excludes the final level (approximate sum).
+ */
+ unsigned char n_arity_order[MAX_NR_LEVELS]; /*
+ * n-arity of tree nodes for each level from
+ * 0 to (nr_levels - 1).
+ */
+};
+
+static const struct counter_config per_nr_cpu_order_config[] = {
+ [0] = { .nr_items = 0, .nr_levels = 0, .n_arity_order = { 0 } },
+ [1] = { .nr_items = 1, .nr_levels = 1, .n_arity_order = { 1 } },
+ [2] = { .nr_items = 3, .nr_levels = 2, .n_arity_order = { 1, 1 } },
+ [3] = { .nr_items = 7, .nr_levels = 3, .n_arity_order = { 1, 1, 1 } },
+ [4] = { .nr_items = 7, .nr_levels = 3, .n_arity_order = { 2, 1, 1 } },
+ [5] = { .nr_items = 11, .nr_levels = 3, .n_arity_order = { 2, 2, 1 } },
+ [6] = { .nr_items = 21, .nr_levels = 3, .n_arity_order = { 2, 2, 2 } },
+ [7] = { .nr_items = 21, .nr_levels = 3, .n_arity_order = { 3, 2, 2 } },
+ [8] = { .nr_items = 37, .nr_levels = 3, .n_arity_order = { 3, 3, 2 } },
+ [9] = { .nr_items = 73, .nr_levels = 3, .n_arity_order = { 3, 3, 3 } },
+ [10] = { .nr_items = 149, .nr_levels = 4, .n_arity_order = { 3, 3, 2, 2 } },
+ [11] = { .nr_items = 293, .nr_levels = 4, .n_arity_order = { 3, 3, 3, 2 } },
+ [12] = { .nr_items = 585, .nr_levels = 4, .n_arity_order = { 3, 3, 3, 3 } },
+ [13] = { .nr_items = 1173, .nr_levels = 5, .n_arity_order = { 3, 3, 3, 2, 2 } },
+ [14] = { .nr_items = 2341, .nr_levels = 5, .n_arity_order = { 3, 3, 3, 3, 2 } },
+ [15] = { .nr_items = 4681, .nr_levels = 5, .n_arity_order = { 3, 3, 3, 3, 3 } },
+ [16] = { .nr_items = 4681, .nr_levels = 5, .n_arity_order = { 4, 3, 3, 3, 3 } },
+ [17] = { .nr_items = 8777, .nr_levels = 5, .n_arity_order = { 4, 4, 3, 3, 3 } },
+ [18] = { .nr_items = 17481, .nr_levels = 5, .n_arity_order = { 4, 4, 4, 3, 3 } },
+ [19] = { .nr_items = 34953, .nr_levels = 5, .n_arity_order = { 4, 4, 4, 4, 3 } },
+ [20] = { .nr_items = 69905, .nr_levels = 5, .n_arity_order = { 4, 4, 4, 4, 4 } },
+};
+
+static const struct counter_config *counter_config; /* Hierarchical counter configuration for the hardware topology. */
+static unsigned int nr_cpus_order, /* Order of nr_cpu_ids. */
+ accuracy_multiplier; /* Calculate accuracy for a given batch size (multiplication factor). */
+
+static
+int __percpu_counter_tree_init(struct percpu_counter_tree *counter,
+ unsigned int batch_size, gfp_t gfp_flags,
+ unsigned int __percpu *level0,
+ struct percpu_counter_tree_level_item *items)
+{
+ /* Batch size must be greater than 1, and a power of 2. */
+ if (WARN_ON(batch_size <= 1 || (batch_size & (batch_size - 1))))
+ return -EINVAL;
+ counter->batch_size = batch_size;
+ counter->bias = 0;
+ counter->level0 = level0;
+ counter->items = items;
+ if (!nr_cpus_order) {
+ counter->approx_sum.i = per_cpu_ptr(counter->level0, 0);
+ counter->level0_bit_mask = 0;
+ } else {
+ counter->approx_sum.a = &counter->items[counter_config->nr_items - 1].count;
+ counter->level0_bit_mask = 1UL << get_count_order(batch_size);
+ }
+ /*
+ * Each tree item signed integer has a negative range which is
+ * one unit greater than the positive range.
+ */
+ counter->approx_accuracy_range.under = batch_size * accuracy_multiplier;
+ counter->approx_accuracy_range.over = (batch_size - 1) * accuracy_multiplier;
+ return 0;
+}
+
+/**
+ * percpu_counter_tree_init_many() - Initialize many per-CPU counter trees.
+ * @counters: An array of @nr_counters counters to initialize.
+ * Their memory is provided by the caller.
+ * @items: Pointer to memory area where to store tree items.
+ * This memory is provided by the caller.
+ * Its size needs to be at least @nr_counters * percpu_counter_tree_items_size().
+ * @nr_counters: The number of counter trees to initialize
+ * @batch_size: The batch size is the increment step at level 0 which triggers a
+ * carry propagation.
+ * The batch size is required to be greater than 1, and a power of 2.
+ * @gfp_flags: gfp flags to pass to the per-CPU allocator.
+ *
+ * Initialize many per-CPU counter trees using a single per-CPU
+ * allocator invocation for @nr_counters counters.
+ *
+ * Return:
+ * * %0: Success
+ * * %-EINVAL: - Invalid @batch_size argument
+ * * %-ENOMEM: - Out of memory
+ */
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+ unsigned int nr_counters, unsigned int batch_size, gfp_t gfp_flags)
+{
+ void __percpu *level0, *level0_iter;
+ size_t counter_size, items_size = 0;
+ void *items_iter;
+ unsigned int i;
+ int ret;
+
+ counter_size = ALIGN(sizeof(*counters), __alignof__(*counters));
+ level0 = __alloc_percpu_gfp(nr_counters * counter_size,
+ __alignof__(*counters), gfp_flags);
+ if (!level0)
+ return -ENOMEM;
+ if (nr_cpus_order) {
+ items_size = percpu_counter_tree_items_size();
+ memset(items, 0, items_size * nr_counters);
+ }
+ level0_iter = level0;
+ items_iter = items;
+ for (i = 0; i < nr_counters; i++) {
+ ret = __percpu_counter_tree_init(&counters[i], batch_size, gfp_flags, level0_iter, items_iter);
+ if (ret)
+ goto free_level0;
+ level0_iter += counter_size;
+ if (nr_cpus_order)
+ items_iter += items_size;
+ }
+ return 0;
+
+free_level0:
+ free_percpu(level0);
+ return ret;
+}
+
+/**
+ * percpu_counter_tree_init() - Initialize one per-CPU counter tree.
+ * @counter: Counter to initialize.
+ * Its memory is provided by the caller.
+ * @items: Pointer to memory area where to store tree items.
+ * This memory is provided by the caller.
+ * Its size needs to be at least percpu_counter_tree_items_size().
+ * @batch_size: The batch size is the increment step at level 0 which triggers a
+ * carry propagation.
+ * The batch size is required to be greater than 1, and a power of 2.
+ * @gfp_flags: gfp flags to pass to the per-CPU allocator.
+ *
+ * Initialize one per-CPU counter tree.
+ *
+ * Return:
+ * * %0: Success
+ * * %-EINVAL: - Invalid @batch_size argument
+ * * %-ENOMEM: - Out of memory
+ */
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+ unsigned int batch_size, gfp_t gfp_flags)
+{
+ return percpu_counter_tree_init_many(counter, items, 1, batch_size, gfp_flags);
+}
+
+/**
+ * percpu_counter_tree_destroy_many() - Destroy many per-CPU counter trees.
+ * @counters: Array of counters trees to destroy.
+ * @nr_counters: The number of counter trees to destroy.
+ *
+ * Release internal resources allocated for @nr_counters per-CPU counter trees.
+ */
+
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counters, unsigned int nr_counters)
+{
+ free_percpu(counters->level0);
+}
+
+/**
+ * percpu_counter_tree_destroy() - Destroy one per-CPU counter tree.
+ * @counter: Counter to destroy.
+ *
+ * Release internal resources allocated for one per-CPU counter tree.
+ */
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter)
+{
+ return percpu_counter_tree_destroy_many(counter, 1);
+}
+
+static
+int percpu_counter_tree_carry(int orig, int res, int inc, unsigned int bit_mask)
+{
+ if (inc < 0) {
+ inc = -(-inc & ~(bit_mask - 1));
+ /*
+ * xor bit_mask: underflow.
+ *
+ * If inc has bit set, decrement an additional bit if
+ * there is _no_ bit transition between orig and res.
+ * Else, inc has bit cleared, decrement an additional
+ * bit if there is a bit transition between orig and
+ * res.
+ */
+ if ((inc ^ orig ^ res) & bit_mask)
+ inc -= bit_mask;
+ } else {
+ inc &= ~(bit_mask - 1);
+ /*
+ * xor bit_mask: overflow.
+ *
+ * If inc has bit set, increment an additional bit if
+ * there is _no_ bit transition between orig and res.
+ * Else, inc has bit cleared, increment an additional
+ * bit if there is a bit transition between orig and
+ * res.
+ */
+ if ((inc ^ orig ^ res) & bit_mask)
+ inc += bit_mask;
+ }
+ return inc;
+}
+
+/*
+ * It does not matter through which path the carry propagates up the
+ * tree, therefore there is no need to disable preemption because the
+ * cpu number is only used to favor cache locality.
+ */
+static
+void percpu_counter_tree_add_slowpath(struct percpu_counter_tree *counter, int inc)
+{
+ unsigned int level_items, nr_levels = counter_config->nr_levels,
+ level, n_arity_order, bit_mask;
+ struct percpu_counter_tree_level_item *item = counter->items;
+ unsigned int cpu = raw_smp_processor_id();
+
+ WARN_ON_ONCE(!nr_cpus_order); /* Should never be called for 1 cpu. */
+
+ n_arity_order = counter_config->n_arity_order[0];
+ bit_mask = counter->level0_bit_mask << n_arity_order;
+ level_items = 1U << (nr_cpus_order - n_arity_order);
+
+ for (level = 1; level < nr_levels; level++) {
+ atomic_t *count = &item[cpu & (level_items - 1)].count;
+ unsigned int orig, res;
+
+ res = atomic_add_return_relaxed(inc, count);
+ orig = res - inc;
+ inc = percpu_counter_tree_carry(orig, res, inc, bit_mask);
+ if (likely(!inc))
+ return;
+ item += level_items;
+ n_arity_order = counter_config->n_arity_order[level];
+ level_items >>= n_arity_order;
+ bit_mask <<= n_arity_order;
+ }
+ atomic_add(inc, counter->approx_sum.a);
+}
+
+/**
+ * percpu_counter_tree_add() - Add to a per-CPU counter tree.
+ * @counter: Counter added to.
+ * @inc: Increment value (either positive or negative).
+ *
+ * Add @inc to a per-CPU counter tree. This is a fast-path which will
+ * typically increment per-CPU counters as long as there is no carry
+ * greater or equal to the counter tree batch size.
+ */
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, int inc)
+{
+ unsigned int bit_mask = counter->level0_bit_mask, orig, res;
+
+ res = this_cpu_add_return(*counter->level0, inc);
+ orig = res - inc;
+ inc = percpu_counter_tree_carry(orig, res, inc, bit_mask);
+ if (likely(!inc))
+ return;
+ percpu_counter_tree_add_slowpath(counter, inc);
+}
+
+
+static
+int percpu_counter_tree_precise_sum_unbiased(struct percpu_counter_tree *counter)
+{
+ unsigned int sum = 0;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ sum += *per_cpu_ptr(counter->level0, cpu);
+ return (int) sum;
+}
+
+/**
+ * percpu_counter_tree_precise_sum() - Return precise counter sum.
+ * @counter: The counter to sum.
+ *
+ * Querying the precise sum is relatively expensive because it needs to
+ * iterate over all CPUs.
+ * This is meant to be used when accuracy is preferred over speed.
+ *
+ * Return: The current precise counter sum.
+ */
+int percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter)
+{
+ return percpu_counter_tree_precise_sum_unbiased(counter) + READ_ONCE(counter->bias);
+}
+
+static
+int compare_delta(int delta, unsigned int accuracy_neg, unsigned int accuracy_pos)
+{
+ if (delta >= 0) {
+ if (delta <= accuracy_pos)
+ return 0;
+ else
+ return 1;
+ } else {
+ if (-delta <= accuracy_neg)
+ return 0;
+ else
+ return -1;
+ }
+}
+
+/**
+ * percpu_counter_tree_approximate_compare - Approximated comparison of two counter trees.
+ * @a: First counter to compare.
+ * @b: Second counter to compare.
+ *
+ * Evaluate an approximate comparison of two counter trees.
+ * This approximation comparison is fast, and provides an accurate
+ * answer if the counters are found to be either less than or greater
+ * than the other. However, if the approximated comparison returns
+ * 0, the counters respective sums are found to be within the two
+ * counters accuracy range.
+ *
+ * Return:
+ * * %0 - Counters @a and @b do not differ by more than the sum of their respective
+ * accuracy ranges.
+ * * %-1 - Counter @a less than counter @b.
+ * * %1 - Counter @a is greater than counter @b.
+ */
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+ return compare_delta(percpu_counter_tree_approximate_sum(a) - percpu_counter_tree_approximate_sum(b),
+ a->approx_accuracy_range.over + b->approx_accuracy_range.under,
+ a->approx_accuracy_range.under + b->approx_accuracy_range.over);
+}
+
+/**
+ * percpu_counter_tree_approximate_compare_value - Approximated comparison of a counter tree against a given value.
+ * @counter: Counter to compare.
+ * @v: Value to compare.
+ *
+ * Evaluate an approximate comparison of a counter tree against a given value.
+ * This approximation comparison is fast, and provides an accurate
+ * answer if the counter is found to be either less than or greater
+ * than the value. However, if the approximated comparison returns
+ * 0, the value is within the counter accuracy range.
+ *
+ * Return:
+ * * %0 - The value @v is within the accuracy range of the counter.
+ * * %-1 - The value @v is less than the counter.
+ * * %1 - The value @v is greater than the counter.
+ */
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, int v)
+{
+ return compare_delta(v - percpu_counter_tree_approximate_sum(counter),
+ counter->approx_accuracy_range.under,
+ counter->approx_accuracy_range.over);
+}
+
+/**
+ * percpu_counter_tree_precise_compare - Precise comparison of two counter trees.
+ * @a: First counter to compare.
+ * @b: Second counter to compare.
+ *
+ * Evaluate a precise comparison of two counter trees.
+ * As an optimization, it uses the approximate counter comparison
+ * to quickly compare counters which are far apart. Only cases where
+ * counter sums are within the accuracy range require precise counter
+ * sums.
+ *
+ * Return:
+ * * %0 - Counters are equal.
+ * * %-1 - Counter @a less than counter @b.
+ * * %1 - Counter @a is greater than counter @b.
+ */
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+ int count_a = percpu_counter_tree_approximate_sum(a),
+ count_b = percpu_counter_tree_approximate_sum(b);
+ unsigned int accuracy_a, accuracy_b;
+ int delta = count_a - count_b;
+ int res;
+
+ res = compare_delta(delta,
+ a->approx_accuracy_range.over + b->approx_accuracy_range.under,
+ a->approx_accuracy_range.under + b->approx_accuracy_range.over);
+ /* The values are distanced enough for an accurate approximated comparison. */
+ if (res)
+ return res;
+
+ /*
+ * The approximated comparison is within the accuracy range, therefore at least one
+ * precise sum is needed. Sum the counter which has the largest accuracy first.
+ */
+ if (delta >= 0) {
+ accuracy_a = a->approx_accuracy_range.under;
+ accuracy_b = b->approx_accuracy_range.over;
+ } else {
+ accuracy_a = a->approx_accuracy_range.over;
+ accuracy_b = b->approx_accuracy_range.under;
+ }
+ if (accuracy_b < accuracy_a) {
+ count_a = percpu_counter_tree_precise_sum(a);
+ res = compare_delta(count_a - count_b,
+ b->approx_accuracy_range.under,
+ b->approx_accuracy_range.over);
+ if (res)
+ return res;
+ /* Precise sum of second counter is required. */
+ count_b = percpu_counter_tree_precise_sum(b);
+ } else {
+ count_b = percpu_counter_tree_precise_sum(b);
+ res = compare_delta(count_a - count_b,
+ a->approx_accuracy_range.over,
+ a->approx_accuracy_range.under);
+ if (res)
+ return res;
+ /* Precise sum of second counter is required. */
+ count_a = percpu_counter_tree_precise_sum(a);
+ }
+ if (count_a - count_b < 0)
+ return -1;
+ if (count_a - count_b > 0)
+ return 1;
+ return 0;
+}
+
+/**
+ * percpu_counter_tree_precise_compare_value - Precise comparison of a counter tree against a given value.
+ * @counter: Counter to compare.
+ * @v: Value to compare.
+ *
+ * Evaluate a precise comparison of a counter tree against a given value.
+ * As an optimization, it uses the approximate counter comparison
+ * to quickly identify whether the counter and value are far apart.
+ * Only cases where the value is within the counter accuracy range
+ * require a precise counter sum.
+ *
+ * Return:
+ * * %0 - The value @v is equal to the counter.
+ * * %-1 - The value @v is less than the counter.
+ * * %1 - The value @v is greater than the counter.
+ */
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, int v)
+{
+ int count = percpu_counter_tree_approximate_sum(counter);
+ int res;
+
+ res = compare_delta(v - count,
+ counter->approx_accuracy_range.under,
+ counter->approx_accuracy_range.over);
+ /* The values are distanced enough for an accurate approximated comparison. */
+ if (res)
+ return res;
+
+ /* Precise sum is required. */
+ count = percpu_counter_tree_precise_sum(counter);
+ if (v - count < 0)
+ return -1;
+ if (v - count > 0)
+ return 1;
+ return 0;
+}
+
+static
+void percpu_counter_tree_set_bias(struct percpu_counter_tree *counter, int bias)
+{
+ WRITE_ONCE(counter->bias, bias);
+}
+
+/**
+ * percpu_counter_tree_set - Set the counter tree sum to a given value.
+ * @counter: Counter to set.
+ * @v: Value to set.
+ *
+ * Set the counter sum to a given value. It can be useful for instance
+ * to reset the counter sum to 0. Note that even after setting the
+ * counter sum to a given value, the counter sum approximation can
+ * return any value within the accuracy range around that value.
+ */
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, int v)
+{
+ percpu_counter_tree_set_bias(counter,
+ v - percpu_counter_tree_precise_sum_unbiased(counter));
+}
+
+/**
+ * percpu_counter_tree_approximate_accuracy_range - Query the accuracy range for a counter tree.
+ * @counter: Counter to query.
+ * @under: Pointer where to store the to the accuracy range below the approximation.
+ * @over: Pointer where to store the to the accuracy range above the approximation.
+ *
+ * Query the accuracy range limits for the counter.
+ * Because of two's complement binary representation, the "under" range is typically
+ * slightly larger than the "over" range.
+ * Those values are derived from the hardware topology and the counter tree batch size.
+ * They are invariant for a given counter tree.
+ * Using this function should not be typically required, see the following functions instead:
+ * * percpu_counter_tree_approximate_compare(),
+ * * percpu_counter_tree_approximate_compare_value(),
+ * * percpu_counter_tree_precise_compare(),
+ * * percpu_counter_tree_precise_compare_value().
+ */
+void percpu_counter_tree_approximate_accuracy_range(struct percpu_counter_tree *counter,
+ unsigned int *under, unsigned int *over)
+{
+ *under = counter->approx_accuracy_range.under;
+ *over = counter->approx_accuracy_range.over;
+}
+
+/*
+ * percpu_counter_tree_items_size - Query the size required for counter tree items.
+ *
+ * Query the size of the memory area required to hold the counter tree
+ * items. This depends on the hardware topology and is invariant after
+ * boot.
+ *
+ * Return: Size required to hold tree items.
+ */
+size_t percpu_counter_tree_items_size(void)
+{
+ if (!nr_cpus_order)
+ return 0;
+ return counter_config->nr_items * sizeof(struct percpu_counter_tree_level_item);
+}
+
+static void __init calculate_accuracy_topology(void)
+{
+ unsigned int nr_levels = counter_config->nr_levels, level;
+ unsigned int level_items = 1U << nr_cpus_order;
+ unsigned int batch_size = 1;
+
+ for (level = 0; level < nr_levels; level++) {
+ unsigned int n_arity_order = counter_config->n_arity_order[level];
+
+ /*
+ * The accuracy multiplier is derived from a batch size of 1
+ * to speed up calculating the accuracy at tree initialization.
+ */
+ accuracy_multiplier += batch_size * level_items;
+ batch_size <<= n_arity_order;
+ level_items >>= n_arity_order;
+ }
+}
+
+int __init percpu_counter_tree_subsystem_init(void)
+{
+ nr_cpus_order = get_count_order(nr_cpu_ids);
+ if (WARN_ON_ONCE(nr_cpus_order >= ARRAY_SIZE(per_nr_cpu_order_config))) {
+ printk(KERN_ERR "Unsupported number of CPUs (%u)\n", nr_cpu_ids);
+ return -1;
+ }
+ counter_config = &per_nr_cpu_order_config[nr_cpus_order];
+ calculate_accuracy_topology();
+ return 0;
+}
--
2.39.5
^ permalink raw reply related
* [PATCH v11 2/3] mm: Fix OOM killer inaccuracy on large many-core systems
From: Mathieu Desnoyers @ 2025-12-24 17:46 UTC (permalink / raw)
To: Andrew Morton
Cc: linux-kernel, Mathieu Desnoyers, Paul E. McKenney, Steven Rostedt,
Masami Hiramatsu, Dennis Zhou, Tejun Heo, Christoph Lameter,
Martin Liu, David Rientjes, christian.koenig, Shakeel Butt,
SeongJae Park, Michal Hocko, Johannes Weiner, Sweet Tea Dorminy,
Lorenzo Stoakes, Liam R . Howlett, Mike Rapoport,
Suren Baghdasaryan, Vlastimil Babka, Christian Brauner, Wei Yang,
David Hildenbrand, Miaohe Lin, Al Viro, linux-mm,
linux-trace-kernel, Yu Zhao, Roman Gushchin, Mateusz Guzik,
Matthew Wilcox, Baolin Wang, Aboorva Devarajan
In-Reply-To: <20251224174638.650551-1-mathieu.desnoyers@efficios.com>
Use hierarchical per-cpu counters for rss tracking to fix the per-mm RSS
tracking which has become too inaccurate for OOM killer purposes on
large many-core systems.
The following rss tracking issues were noted by Sweet Tea Dorminy [1],
which lead to picking wrong tasks as OOM kill target:
Recently, several internal services had an RSS usage regression as part of a
kernel upgrade. Previously, they were on a pre-6.2 kernel and were able to
read RSS statistics in a backup watchdog process to monitor and decide if
they'd overrun their memory budget. Now, however, a representative service
with five threads, expected to use about a hundred MB of memory, on a 250-cpu
machine had memory usage tens of megabytes different from the expected amount
-- this constituted a significant percentage of inaccuracy, causing the
watchdog to act.
This was a result of commit f1a7941243c1 ("mm: convert mm's rss stats
into percpu_counter") [1]. Previously, the memory error was bounded by
64*nr_threads pages, a very livable megabyte. Now, however, as a result of
scheduler decisions moving the threads around the CPUs, the memory error could
be as large as a gigabyte.
This is a really tremendous inaccuracy for any few-threaded program on a
large machine and impedes monitoring significantly. These stat counters are
also used to make OOM killing decisions, so this additional inaccuracy could
make a big difference in OOM situations -- either resulting in the wrong
process being killed, or in less memory being returned from an OOM-kill than
expected.
Here is a (possibly incomplete) list of the prior approaches that were
used or proposed, along with their downside:
1) Per-thread rss tracking: large error on many-thread processes.
2) Per-CPU counters: up to 12% slower for short-lived processes and 9%
increased system time in make test workloads [1]. Moreover, the
inaccuracy increases with O(n^2) with the number of CPUs.
3) Per-NUMA-node counters: requires atomics on fast-path (overhead),
error is high with systems that have lots of NUMA nodes (32 times
the number of NUMA nodes).
The approach proposed here is to replace this by the hierarchical
per-cpu counters, which bounds the inaccuracy based on the system
topology with O(N*logN).
commit 82241a83cd15 ("mm: fix the inaccurate memory statistics issue for
users") introduced get_mm_counter_sum() for precise /proc memory status
queries. Implement it with percpu_counter_tree_precise_sum() since it
is not a fast path and precision is preferred over speed.
* Testing results:
Test hardware: 2 sockets AMD EPYC 9654 96-Core Processor (384 logical CPUs total)
Methodology:
Comparing the current upstream implementation with the hierarchical
counters is done by keeping both implementations wired up in parallel,
and running a single-process, single-threaded program which hops
randomly across CPUs in the system, calling mmap(2) and munmap(2) on
random CPUs, keeping track of an array of allocated mappings, randomly
choosing entries to either map or unmap.
get_mm_counter() is instrumented to compare the upstream counter
approximation to the precise value, and print the delta when going over
a given threshold. The delta of the hierarchical counter approximation
to the precise value is also printed for comparison.
After a few minutes running this test, the upstream implementation
counter approximation reaches a 1GB delta from the
precise value, compared to 80MB delta with the hierarchical counter.
The hierarchical counter provides a guaranteed maximum approximation
inaccuracy of 192MB on that hardware topology.
* Fast path implementation comparison
The new inline percpu_counter_tree_add() uses a this_cpu_add_return()
for the fast path (under a certain allocation size threshold). Above
that, it calls a slow path which "trickles up" the carry to upper level
counters with atomic_add_return.
In comparison, the upstream counters implementation calls
percpu_counter_add_batch which uses this_cpu_try_cmpxchg() on the fast
path, and does a raw_spin_lock_irqsave above a certain threshold.
The hierarchical implementation is therefore expected to have less
contention on mid-sized allocations than the upstream counters because
the atomic counters tracking those bits are only shared across nearby
CPUs. In comparison, the upstream counters immediately use a global
spinlock when reaching the threshold.
* Benchmarks
Using will-it-scale page_fault1 benchmarks to compare the upstream
counters to the hierarchical counters. This is done with hyperthreading
disabled. The speedup is within the standard deviation of the upstream
runs, so the overhead is not significant.
upstream hierarchical speedup
page_fault1_processes -s 100 -t 1 614783 615558 +0.1%
page_fault1_threads -s 100 -t 1 612788 612447 -0.1%
page_fault1_processes -s 100 -t 96 37994977 37932035 -0.2%
page_fault1_threads -s 100 -t 96 2484130 2504860 +0.8%
page_fault1_processes -s 100 -t 192 71262917 71118830 -0.2%
page_fault1_threads -s 100 -t 192 2446437 2469296 +0.1%
Link: https://lore.kernel.org/lkml/20250331223516.7810-2-sweettea-kernel@dorminy.me/ # [1]
Link: https://lore.kernel.org/lkml/20250704150226.47980-1-mathieu.desnoyers@efficios.com/
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Martin Liu <liumartin@google.com>
Cc: David Rientjes <rientjes@google.com>
Cc: christian.koenig@amd.com
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: SeongJae Park <sj@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Sweet Tea Dorminy <sweettea-kernel@dorminy.me>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: "Liam R . Howlett" <liam.howlett@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Wei Yang <richard.weiyang@gmail.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: linux-mm@kvack.org
Cc: linux-trace-kernel@vger.kernel.org
Cc: Yu Zhao <yuzhao@google.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Mateusz Guzik <mjguzik@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Aboorva Devarajan <aboorvad@linux.ibm.com>
---
Changes since v10:
- Rebase on top of mm_struct static init fixes.
Changes since v8:
- Use percpu_counter_tree_init_many and
percpu_counter_tree_destroy_many APIs.
- Remove percpu tree items allocation. Extend mm_struct size to include
rss items. Those are handled through the new helpers
get_rss_stat_items() and get_rss_stat_items_size() and passed
as parameter to percpu_counter_tree_init_many().
Changes since v7:
- Use precise sum positive API to handle a scenario where an unlucky
precise sum iteration would observe negative counter values due to
concurrent updates.
Changes since v6:
- Rebased on v6.18-rc3.
- Implement get_mm_counter_sum as percpu_counter_tree_precise_sum for
/proc virtual files memory state queries.
Changes since v5:
- Use percpu_counter_tree_approximate_sum_positive.
Change since v4:
- get_mm_counter needs to return 0 or a positive value.
get_mm_counter_sum -> precise sum positive
---
include/linux/mm.h | 28 +++++++++++++++++++++++-----
include/linux/mm_types.h | 4 ++--
include/trace/events/kmem.h | 2 +-
kernel/fork.c | 24 ++++++++++++++----------
4 files changed, 40 insertions(+), 18 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 15076261d0c2..79ac9895cb95 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2843,38 +2843,56 @@ static inline bool get_user_page_fast_only(unsigned long addr,
{
return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1;
}
+
+static inline size_t get_rss_stat_items_size(void)
+{
+ return percpu_counter_tree_items_size() * NR_MM_COUNTERS;
+}
+
+static inline struct percpu_counter_tree_level_item *get_rss_stat_items(struct mm_struct *mm)
+{
+ unsigned long ptr = (unsigned long)mm;
+
+ ptr += offsetof(struct mm_struct, flexible_array);
+ /* Skip cpu_bitmap */
+ ptr += cpumask_size();
+ /* Skip mm_cidmask */
+ ptr += mm_cid_size();
+ return (struct percpu_counter_tree_level_item *)ptr;
+}
+
/*
* per-process(per-mm_struct) statistics.
*/
static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
{
- return percpu_counter_read_positive(&mm->rss_stat[member]);
+ return percpu_counter_tree_approximate_sum_positive(&mm->rss_stat[member]);
}
static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member)
{
- return percpu_counter_sum_positive(&mm->rss_stat[member]);
+ return percpu_counter_tree_precise_sum_positive(&mm->rss_stat[member]);
}
void mm_trace_rss_stat(struct mm_struct *mm, int member);
static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
{
- percpu_counter_add(&mm->rss_stat[member], value);
+ percpu_counter_tree_add(&mm->rss_stat[member], value);
mm_trace_rss_stat(mm, member);
}
static inline void inc_mm_counter(struct mm_struct *mm, int member)
{
- percpu_counter_inc(&mm->rss_stat[member]);
+ percpu_counter_tree_add(&mm->rss_stat[member], 1);
mm_trace_rss_stat(mm, member);
}
static inline void dec_mm_counter(struct mm_struct *mm, int member)
{
- percpu_counter_dec(&mm->rss_stat[member]);
+ percpu_counter_tree_add(&mm->rss_stat[member], -1);
mm_trace_rss_stat(mm, member);
}
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 9f861ceabe61..38495715de89 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -18,7 +18,7 @@
#include <linux/page-flags-layout.h>
#include <linux/workqueue.h>
#include <linux/seqlock.h>
-#include <linux/percpu_counter.h>
+#include <linux/percpu_counter_tree.h>
#include <linux/types.h>
#include <linux/rseq_types.h>
#include <linux/bitmap.h>
@@ -1215,7 +1215,7 @@ struct mm_struct {
unsigned long saved_e_flags;
#endif
- struct percpu_counter rss_stat[NR_MM_COUNTERS];
+ struct percpu_counter_tree rss_stat[NR_MM_COUNTERS];
struct linux_binfmt *binfmt;
diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h
index 7f93e754da5c..91c81c44f884 100644
--- a/include/trace/events/kmem.h
+++ b/include/trace/events/kmem.h
@@ -442,7 +442,7 @@ TRACE_EVENT(rss_stat,
__entry->mm_id = mm_ptr_to_hash(mm);
__entry->curr = !!(current->mm == mm);
__entry->member = member;
- __entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member])
+ __entry->size = (percpu_counter_tree_approximate_sum_positive(&mm->rss_stat[member])
<< PAGE_SHIFT);
),
diff --git a/kernel/fork.c b/kernel/fork.c
index b1f3915d5f8e..949ac019a7b1 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -133,6 +133,11 @@
*/
#define MAX_THREADS FUTEX_TID_MASK
+/*
+ * Batch size of rss stat approximation
+ */
+#define RSS_STAT_BATCH_SIZE 32
+
/*
* Protected counters by write_lock_irq(&tasklist_lock)
*/
@@ -626,14 +631,12 @@ static void check_mm(struct mm_struct *mm)
"Please make sure 'struct resident_page_types[]' is updated as well");
for (i = 0; i < NR_MM_COUNTERS; i++) {
- long x = percpu_counter_sum(&mm->rss_stat[i]);
-
- if (unlikely(x)) {
- pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n",
- mm, resident_page_types[i], x,
+ if (unlikely(percpu_counter_tree_precise_compare_value(&mm->rss_stat[i], 0) != 0))
+ pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%d Comm:%s Pid:%d\n",
+ mm, resident_page_types[i],
+ percpu_counter_tree_precise_sum(&mm->rss_stat[i]),
current->comm,
task_pid_nr(current));
- }
}
if (mm_pgtables_bytes(mm))
@@ -731,7 +734,7 @@ void __mmdrop(struct mm_struct *mm)
put_user_ns(mm->user_ns);
mm_pasid_drop(mm);
mm_destroy_cid(mm);
- percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
+ percpu_counter_tree_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
free_mm(mm);
}
@@ -1123,8 +1126,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
if (mm_alloc_cid(mm, p))
goto fail_cid;
- if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
- NR_MM_COUNTERS))
+ if (percpu_counter_tree_init_many(mm->rss_stat, get_rss_stat_items(mm),
+ NR_MM_COUNTERS, RSS_STAT_BATCH_SIZE,
+ GFP_KERNEL_ACCOUNT))
goto fail_pcpu;
mm->user_ns = get_user_ns(user_ns);
@@ -3006,7 +3010,7 @@ void __init mm_cache_init(void)
* dynamically sized based on the maximum CPU number this system
* can have, taking hotplug into account (nr_cpu_ids).
*/
- mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size();
+ mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size() + get_rss_stat_items_size();
mm_cachep = kmem_cache_create_usercopy("mm_struct",
mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
--
2.39.5
^ permalink raw reply related
* [PATCH v11 0/3] mm: Fix OOM killer inaccuracy on large many-core systems
From: Mathieu Desnoyers @ 2025-12-24 17:46 UTC (permalink / raw)
To: Andrew Morton
Cc: linux-kernel, Mathieu Desnoyers, Paul E. McKenney, Steven Rostedt,
Masami Hiramatsu, Dennis Zhou, Tejun Heo, Christoph Lameter,
Martin Liu, David Rientjes, christian.koenig, Shakeel Butt,
SeongJae Park, Michal Hocko, Johannes Weiner, Sweet Tea Dorminy,
Lorenzo Stoakes, Liam R . Howlett, Mike Rapoport,
Suren Baghdasaryan, Vlastimil Babka, Christian Brauner, Wei Yang,
David Hildenbrand, Miaohe Lin, Al Viro, linux-mm,
linux-trace-kernel, Yu Zhao, Roman Gushchin, Mateusz Guzik,
Matthew Wilcox, Baolin Wang, Aboorva Devarajan
Introduce hierarchical per-cpu counters and use them for RSS tracking to
fix the per-mm RSS tracking which has become too inaccurate for OOM
killer purposes on large many-core systems.
The following rss tracking issues were noted by Sweet Tea Dorminy [1],
which lead to picking wrong tasks as OOM kill target:
Recently, several internal services had an RSS usage regression as part of a
kernel upgrade. Previously, they were on a pre-6.2 kernel and were able to
read RSS statistics in a backup watchdog process to monitor and decide if
they'd overrun their memory budget. Now, however, a representative service
with five threads, expected to use about a hundred MB of memory, on a 250-cpu
machine had memory usage tens of megabytes different from the expected amount
-- this constituted a significant percentage of inaccuracy, causing the
watchdog to act.
This was a result of commit f1a7941243c1 ("mm: convert mm's rss stats
into percpu_counter") [1]. Previously, the memory error was bounded by
64*nr_threads pages, a very livable megabyte. Now, however, as a result of
scheduler decisions moving the threads around the CPUs, the memory error could
be as large as a gigabyte.
This is a really tremendous inaccuracy for any few-threaded program on a
large machine and impedes monitoring significantly. These stat counters are
also used to make OOM killing decisions, so this additional inaccuracy could
make a big difference in OOM situations -- either resulting in the wrong
process being killed, or in less memory being returned from an OOM-kill than
expected.
The approach proposed here is to replace this by the hierarchical
per-cpu counters, which bounds the inaccuracy based on the system
topology with O(N*logN).
Notable change for v11: Rebased on preparation patches fixing mm_struct
static init for init_mm and efi_mm.
I've done moderate testing of this series on a 256-core VM with 128GB
RAM. Figuring out whether this indeed helps solve issues with real-life
workloads will require broader feedback from the community.
This series is based on v6.19-rc2, on top of the following two
preparation series:
https://lore.kernel.org/linux-mm/20251224173358.647691-1-mathieu.desnoyers@efficios.com/T/#t
https://lore.kernel.org/linux-mm/20251224173810.648699-1-mathieu.desnoyers@efficios.com/T/#t
Andrew, this series replaces v10, for testing in mm-new if you're still
up for it.
Thanks!
Mathieu
Link: https://lore.kernel.org/lkml/20250331223516.7810-2-sweettea-kernel@dorminy.me/ # [1]
To: Andrew Morton <akpm@linux-foundation.org>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Martin Liu <liumartin@google.com>
Cc: David Rientjes <rientjes@google.com>
Cc: christian.koenig@amd.com
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: SeongJae Park <sj@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Sweet Tea Dorminy <sweettea-kernel@dorminy.me>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: "Liam R . Howlett" <liam.howlett@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Christian Brauner <brauner@kernel.org>
Cc: Wei Yang <richard.weiyang@gmail.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: linux-mm@kvack.org
Cc: linux-trace-kernel@vger.kernel.org
Cc: Yu Zhao <yuzhao@google.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Mateusz Guzik <mjguzik@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Aboorva Devarajan <aboorvad@linux.ibm.com>
Mathieu Desnoyers (3):
lib: Introduce hierarchical per-cpu counters
mm: Fix OOM killer inaccuracy on large many-core systems
mm: Implement precise OOM killer task selection
fs/proc/base.c | 2 +-
include/linux/mm.h | 58 ++-
include/linux/mm_types.h | 10 +-
include/linux/oom.h | 12 +-
include/linux/percpu_counter_tree.h | 293 ++++++++++++
include/trace/events/kmem.h | 2 +-
init/main.c | 2 +
kernel/fork.c | 24 +-
lib/Makefile | 1 +
lib/percpu_counter_tree.c | 705 ++++++++++++++++++++++++++++
mm/oom_kill.c | 72 ++-
11 files changed, 1143 insertions(+), 38 deletions(-)
create mode 100644 include/linux/percpu_counter_tree.h
create mode 100644 lib/percpu_counter_tree.c
--
2.39.5
^ permalink raw reply
* [PATCH] tracing: Replace use of system_wq with system_percpu_wq
From: Marco Crivellari @ 2025-12-24 16:20 UTC (permalink / raw)
To: linux-kernel, linux-trace-kernel
Cc: Tejun Heo, Lai Jiangshan, Frederic Weisbecker,
Sebastian Andrzej Siewior, Marco Crivellari, Michal Hocko,
Steven Rostedt, Masami Hiramatsu, Mathieu Desnoyers
This patch continues the effort to refactor workqueue APIs, which has begun
with the changes introducing new workqueues and a new alloc_workqueue flag:
commit 128ea9f6ccfb ("workqueue: Add system_percpu_wq and system_dfl_wq")
commit 930c2ea566af ("workqueue: Add new WQ_PERCPU flag")
The point of the refactoring is to eventually alter the default behavior of
workqueues to become unbound by default so that their workload placement is
optimized by the scheduler.
Before that to happen after a careful review and conversion of each individual
case, workqueue users must be converted to the better named new workqueues with
no intended behaviour changes:
system_wq -> system_percpu_wq
system_unbound_wq -> system_dfl_wq
This way the old obsolete workqueues (system_wq, system_unbound_wq) can be
removed in the future.
Suggested-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Marco Crivellari <marco.crivellari@suse.com>
---
kernel/trace/trace_events_filter.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index 385af8405392..bb6de1d38f78 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -1375,7 +1375,7 @@ static void free_filter_list_tasks(struct rcu_head *rhp)
struct filter_head *filter_list = container_of(rhp, struct filter_head, rcu);
INIT_RCU_WORK(&filter_list->rwork, free_filter_list_work);
- queue_rcu_work(system_wq, &filter_list->rwork);
+ queue_rcu_work(system_percpu_wq, &filter_list->rwork);
}
/*
--
2.52.0
^ permalink raw reply related
* Re: [PATCH v10] dma-buf: add some tracepoints to debug.
From: Steven Rostedt @ 2025-12-24 14:31 UTC (permalink / raw)
To: Xiang Gao
Cc: sumit.semwal, christian.koenig, mhiramat, linux-media, dri-devel,
linux-kernel, mathieu.desnoyers, dhowells, kuba, brauner, akpm,
linux-trace-kernel, gaoxiang17
In-Reply-To: <20251224013455.1649879-1-gxxa03070307@gmail.com>
On Wed, 24 Dec 2025 09:34:55 +0800
Xiang Gao <gxxa03070307@gmail.com> wrote:
> From: gaoxiang17 <gaoxiang17@xiaomi.com>
>
> Since we can only inspect dmabuf by iterating over process FDs or the
> dmabuf_list, we need to add our own tracepoints to track its status in
> real time in production.
>
> For example:
> binder:3016_1-3102 [006] ...1. 255.126521: dma_buf_export: exp_name=qcom,system size=12685312 ino=2738
> binder:3016_1-3102 [006] ...1. 255.126528: dma_buf_fd: exp_name=qcom,system size=12685312 ino=2738 fd=8
> binder:3016_1-3102 [006] ...1. 255.126642: dma_buf_mmap_internal: exp_name=qcom,system size=28672 ino=2739
> kworker/6:1-86 [006] ...1. 255.127194: dma_buf_put: exp_name=qcom,system size=12685312 ino=2738
> RenderThread-9293 [006] ...1. 316.618179: dma_buf_get: exp_name=qcom,system size=12771328 ino=2762 fd=176
> RenderThread-9293 [006] ...1. 316.618195: dma_buf_dynamic_attach: exp_name=qcom,system size=12771328 ino=2762 attachment:ffffff880a18dd00 is_dynamic=0 dev_name=kgsl-3d0
> RenderThread-9293 [006] ...1. 318.878220: dma_buf_detach: exp_name=qcom,system size=12771328 ino=2762 attachment:ffffff880a18dd00 is_dynamic=0 dev_name=kgsl-3d0
>
> Signed-off-by: Xiang Gao <gaoxiang17@xiaomi.com>
> ---
BTW, it's sometimes nice to have in new versions of a patch with a list of
changes below the above "---" (here):
Changes since v9: https://lore.kernel.org/all/20251223032749.1371913-1-gxxa03070307@gmail.com/
- <list changes here>
That way it keeps a nice history of all the versions of the patch.
No need to resend. Just giving you some advice for future patches.
> drivers/dma-buf/dma-buf.c | 49 +++++++++-
> include/trace/events/dma_buf.h | 157 +++++++++++++++++++++++++++++++++
> 2 files changed, 204 insertions(+), 2 deletions(-)
> create mode 100644 include/trace/events/dma_buf.h
>
> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> index edaa9e4ee4ae..5e6f65cd0306 100644
> --- a/drivers/dma-buf/dma-buf.c
> +++ b/drivers/dma-buf/dma-buf.c
> @@ -35,6 +35,26 @@
>
> #include "dma-buf-sysfs-stats.h"
>
> +#define CREATE_TRACE_POINTS
> +#include <trace/events/dma_buf.h>
> +
> +/*
> + * dmabuf->name must be accessed with holding dmabuf->name_lock.
> + * we need to take the lock around the tracepoint call itself where
> + * it is called in the code.
> + *
> + * Note: FUNC##_enabled() is a static branch that will only
> + * be set when the trace event is enabled.
> + */
> +#define DMA_BUF_TRACE(FUNC, ...) \
> + do { \
> + /* Always expose lock if lockdep is enabled */ \
> + if (IS_ENABLED(CONFIG_LOCKDEP) || FUNC##_enabled()) { \
> + guard(spinlock)(&dmabuf->name_lock); \
> + FUNC(__VA_ARGS__); \
> + } \
> + } while (0)
I'm curious. Are the above backslashes lined up nicely in the code?
> +
> static inline int is_dma_buf_file(struct file *);
>
> static DEFINE_MUTEX(dmabuf_list_mutex);
> @@ -220,6 +240,8 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
> dmabuf->size >> PAGE_SHIFT)
> return -EINVAL;
>
> + DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf);
> +
> return dmabuf->ops->mmap(dmabuf, vma);
> }
>
> @@ -745,6 +767,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
>
> __dma_buf_list_add(dmabuf);
>
> + DMA_BUF_TRACE(trace_dma_buf_export, dmabuf);
> +
> return dmabuf;
>
> err_dmabuf:
> @@ -768,10 +792,16 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF");
> */
> int dma_buf_fd(struct dma_buf *dmabuf, int flags)
> {
> + int fd;
> +
> if (!dmabuf || !dmabuf->file)
> return -EINVAL;
>
> - return FD_ADD(flags, dmabuf->file);
> + fd = FD_ADD(flags, dmabuf->file);
> + if (fd >= 0)
> + DMA_BUF_TRACE(trace_dma_buf_fd, dmabuf, fd);
Instead of adding the above if statement in the code, you can make the
event conditional (See below). Then this could just be:
- return FD_ADD(flags, dmabuf->file);
+ fd = FD_ADD(flags, dmabuf->file);
+ DMA_BUF_TRACE(trace_dma_buf_fd, dmabuf, fd);
And not have the condition hit when tracing isn't enabled.
> +
> + return fd;
> }
> EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
>
[..]
> diff --git a/include/trace/events/dma_buf.h b/include/trace/events/dma_buf.h
> new file mode 100644
> index 000000000000..2c9ba8533467
> --- /dev/null
> +++ b/include/trace/events/dma_buf.h
> @@ -0,0 +1,157 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#undef TRACE_SYSTEM
> +#define TRACE_SYSTEM dma_buf
> +
> +#if !defined(_TRACE_DMA_BUF_H) || defined(TRACE_HEADER_MULTI_READ)
> +#define _TRACE_DMA_BUF_H
> +
> +#include <linux/dma-buf.h>
> +#include <linux/tracepoint.h>
> +
> +DECLARE_EVENT_CLASS(dma_buf,
> +
> + TP_PROTO(struct dma_buf *dmabuf),
> +
> + TP_ARGS(dmabuf),
> +
> + TP_STRUCT__entry(
> + __string( exp_name, dmabuf->exp_name)
> + __field( size_t, size)
> + __field( ino_t, ino)
> + ),
> +
> + TP_fast_assign(
> + __assign_str(exp_name);
> + __entry->size = dmabuf->size;
> + __entry->ino = dmabuf->file->f_inode->i_ino;
> + ),
> +
> + TP_printk("exp_name=%s size=%zu ino=%lu",
> + __get_str(exp_name),
> + __entry->size,
> + __entry->ino)
> +);
> +
> +DECLARE_EVENT_CLASS(dma_buf_attach_dev,
> +
> + TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
> + bool is_dynamic, struct device *dev),
> +
> + TP_ARGS(dmabuf, attach, is_dynamic, dev),
> +
> + TP_STRUCT__entry(
> + __string( dev_name, dev_name(dev))
> + __string( exp_name, dmabuf->exp_name)
> + __field( size_t, size)
> + __field( ino_t, ino)
> + __field( struct dma_buf_attachment *, attach)
> + __field( bool, is_dynamic)
> + ),
> +
> + TP_fast_assign(
> + __assign_str(dev_name);
> + __assign_str(exp_name);
> + __entry->size = dmabuf->size;
> + __entry->ino = dmabuf->file->f_inode->i_ino;
> + __entry->is_dynamic = is_dynamic;
> + __entry->attach = attach;
> + ),
> +
> + TP_printk("exp_name=%s size=%zu ino=%lu attachment:%p is_dynamic=%d dev_name=%s",
> + __get_str(exp_name),
> + __entry->size,
> + __entry->ino,
> + __entry->attach,
> + __entry->is_dynamic,
> + __get_str(dev_name))
> +);
> +
> +DECLARE_EVENT_CLASS(dma_buf_fd,
> +
> + TP_PROTO(struct dma_buf *dmabuf, int fd),
> +
> + TP_ARGS(dmabuf, fd),
> +
> + TP_STRUCT__entry(
> + __string( exp_name, dmabuf->exp_name)
> + __field( size_t, size)
> + __field( ino_t, ino)
> + __field( int, fd)
> + ),
> +
> + TP_fast_assign(
> + __assign_str(exp_name);
> + __entry->size = dmabuf->size;
> + __entry->ino = dmabuf->file->f_inode->i_ino;
> + __entry->fd = fd;
> + ),
> +
> + TP_printk("exp_name=%s size=%zu ino=%lu fd=%d",
> + __get_str(exp_name),
> + __entry->size,
> + __entry->ino,
> + __entry->fd)
> +);
> +
[..]
> +DEFINE_EVENT(dma_buf_fd, dma_buf_fd,
> +
> + TP_PROTO(struct dma_buf *dmabuf, int fd),
> +
> + TP_ARGS(dmabuf, fd)
> +);
If fd needs to be greater or equal to zero, you can make the above:
DEFINE_EVENT_CONDITION(dma_buf_fd, dma_buf_fd,
TP_PROTO(struct dma_buf *dmabuf, int fd),
TP_ARGS(dmabuf, fd),
TP_CONDITION(fd >= 0)
);
This places the "fd >= 0" into the code that is called when the tracepoint
is enabled. If the condition isn't met, then the tracepoint doesn't get
recorded.
-- Steve
> +
> +DEFINE_EVENT(dma_buf_fd, dma_buf_get,
> +
> + TP_PROTO(struct dma_buf *dmabuf, int fd),
> +
> + TP_ARGS(dmabuf, fd)
> +);
> +
> +#endif /* _TRACE_DMA_BUF_H */
> +
> +/* This part must be outside protection */
> +#include <trace/define_trace.h>
^ permalink raw reply
* Re: [PATCH] tracing: Add bitmask-list option for human-readable bitmask display
From: Steven Rostedt @ 2025-12-24 13:58 UTC (permalink / raw)
To: Aaron Tomlin
Cc: mhiramat, mark.rutland, mathieu.desnoyers, corbet, sean,
linux-kernel, linux-trace-kernel, linux-doc
In-Reply-To: <wgkjg2bpsyonl5qkgxgdrpmzzaduuyiti7vtifxbtdnmlrhptl@jchrtoaltfv3>
On Tue, 23 Dec 2025 17:14:35 -0500
Aaron Tomlin <atomlin@atomlin.com> wrote:
> When dealing with 128+ logical cores, interpreting a raw hexadecimal bitmap
> to identify targeted CPUs is mentally taxing and prone to error. For
> example, if I am investigating why CPU 6 is being interrupted, I might use
> a filter such as "cpumask & CPU{6}". Seeing the resulting output as a range
> list (e.g., 0-7) rather than a hexadecimal bitmask allows one to deduce
> almost instantly which cluster of CPUs is involved in the IPI broadcast.
Should we just make all cpu bitmask range lists instead?
-- Steve
^ permalink raw reply
* [PATCH v1] tools/rtla: Deduplicate cgroup path opening code
From: Costa Shulyupin @ 2025-12-24 12:50 UTC (permalink / raw)
To: Steven Rostedt, Tomas Glozar, Costa Shulyupin, Tiezhu Yang,
Ivan Pravdin, linux-trace-kernel, linux-kernel
Both set_pid_cgroup() and set_comm_cgroup() functions contain
identical code for opening the cgroup.procs file.
Extract this common code into a new helper function open_cgroup_procs()
to reduce code duplication and improve maintainability.
Signed-off-by: Costa Shulyupin <costa.shul@redhat.com>
---
tools/tracing/rtla/src/utils.c | 65 +++++++++++++++++-----------------
1 file changed, 32 insertions(+), 33 deletions(-)
diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c
index 9cf5a0098e9a..0b84e02b13df 100644
--- a/tools/tracing/rtla/src/utils.c
+++ b/tools/tracing/rtla/src/utils.c
@@ -784,27 +784,27 @@ static int get_self_cgroup(char *self_cg, int sizeof_self_cg)
}
/*
- * set_comm_cgroup - Set cgroup to pid_t pid
+ * open_cgroup_procs - Open the cgroup.procs file for the given cgroup
*
- * If cgroup argument is not NULL, the threads will move to the given cgroup.
- * Otherwise, the cgroup of the calling, i.e., rtla, thread will be used.
+ * If cgroup argument is not NULL, the cgroup.procs file for that cgroup
+ * will be opened. Otherwise, the cgroup of the calling, i.e., rtla, thread
+ * will be used.
*
* Supports cgroup v2.
*
- * Returns 1 on success, 0 otherwise.
+ * Returns the file descriptor on success, -1 otherwise.
*/
-int set_pid_cgroup(pid_t pid, const char *cgroup)
+static int open_cgroup_procs(const char *cgroup)
{
char cgroup_path[MAX_PATH - strlen("/cgroup.procs")];
char cgroup_procs[MAX_PATH];
- char pid_str[24];
int retval;
int cg_fd;
retval = find_mount("cgroup2", cgroup_path, sizeof(cgroup_path));
if (!retval) {
err_msg("Did not find cgroupv2 mount point\n");
- return 0;
+ return -1;
}
if (!cgroup) {
@@ -812,7 +812,7 @@ int set_pid_cgroup(pid_t pid, const char *cgroup)
sizeof(cgroup_path) - strlen(cgroup_path));
if (!retval) {
err_msg("Did not find self cgroup\n");
- return 0;
+ return -1;
}
} else {
snprintf(&cgroup_path[strlen(cgroup_path)],
@@ -824,6 +824,29 @@ int set_pid_cgroup(pid_t pid, const char *cgroup)
debug_msg("Using cgroup path at: %s\n", cgroup_procs);
cg_fd = open(cgroup_procs, O_RDWR);
+ if (cg_fd < 0)
+ return -1;
+
+ return cg_fd;
+}
+
+/*
+ * set_pid_cgroup - Set cgroup to pid_t pid
+ *
+ * If cgroup argument is not NULL, the threads will move to the given cgroup.
+ * Otherwise, the cgroup of the calling, i.e., rtla, thread will be used.
+ *
+ * Supports cgroup v2.
+ *
+ * Returns 1 on success, 0 otherwise.
+ */
+int set_pid_cgroup(pid_t pid, const char *cgroup)
+{
+ char pid_str[24];
+ int retval;
+ int cg_fd;
+
+ cg_fd = open_cgroup_procs(cgroup);
if (cg_fd < 0)
return 0;
@@ -853,8 +876,6 @@ int set_pid_cgroup(pid_t pid, const char *cgroup)
*/
int set_comm_cgroup(const char *comm_prefix, const char *cgroup)
{
- char cgroup_path[MAX_PATH - strlen("/cgroup.procs")];
- char cgroup_procs[MAX_PATH];
struct dirent *proc_entry;
DIR *procfs;
int retval;
@@ -866,29 +887,7 @@ int set_comm_cgroup(const char *comm_prefix, const char *cgroup)
return 0;
}
- retval = find_mount("cgroup2", cgroup_path, sizeof(cgroup_path));
- if (!retval) {
- err_msg("Did not find cgroupv2 mount point\n");
- return 0;
- }
-
- if (!cgroup) {
- retval = get_self_cgroup(&cgroup_path[strlen(cgroup_path)],
- sizeof(cgroup_path) - strlen(cgroup_path));
- if (!retval) {
- err_msg("Did not find self cgroup\n");
- return 0;
- }
- } else {
- snprintf(&cgroup_path[strlen(cgroup_path)],
- sizeof(cgroup_path) - strlen(cgroup_path), "%s/", cgroup);
- }
-
- snprintf(cgroup_procs, MAX_PATH, "%s/cgroup.procs", cgroup_path);
-
- debug_msg("Using cgroup path at: %s\n", cgroup_procs);
-
- cg_fd = open(cgroup_procs, O_RDWR);
+ cg_fd = open_cgroup_procs(cgroup);
if (cg_fd < 0)
return 0;
--
2.52.0
^ permalink raw reply related
* [PATCH v10] dma-buf: add some tracepoints to debug.
From: Xiang Gao @ 2025-12-24 1:34 UTC (permalink / raw)
To: sumit.semwal, christian.koenig, rostedt, mhiramat
Cc: linux-media, dri-devel, linux-kernel, mathieu.desnoyers, dhowells,
kuba, brauner, akpm, linux-trace-kernel, gaoxiang17
From: gaoxiang17 <gaoxiang17@xiaomi.com>
Since we can only inspect dmabuf by iterating over process FDs or the
dmabuf_list, we need to add our own tracepoints to track its status in
real time in production.
For example:
binder:3016_1-3102 [006] ...1. 255.126521: dma_buf_export: exp_name=qcom,system size=12685312 ino=2738
binder:3016_1-3102 [006] ...1. 255.126528: dma_buf_fd: exp_name=qcom,system size=12685312 ino=2738 fd=8
binder:3016_1-3102 [006] ...1. 255.126642: dma_buf_mmap_internal: exp_name=qcom,system size=28672 ino=2739
kworker/6:1-86 [006] ...1. 255.127194: dma_buf_put: exp_name=qcom,system size=12685312 ino=2738
RenderThread-9293 [006] ...1. 316.618179: dma_buf_get: exp_name=qcom,system size=12771328 ino=2762 fd=176
RenderThread-9293 [006] ...1. 316.618195: dma_buf_dynamic_attach: exp_name=qcom,system size=12771328 ino=2762 attachment:ffffff880a18dd00 is_dynamic=0 dev_name=kgsl-3d0
RenderThread-9293 [006] ...1. 318.878220: dma_buf_detach: exp_name=qcom,system size=12771328 ino=2762 attachment:ffffff880a18dd00 is_dynamic=0 dev_name=kgsl-3d0
Signed-off-by: Xiang Gao <gaoxiang17@xiaomi.com>
---
drivers/dma-buf/dma-buf.c | 49 +++++++++-
include/trace/events/dma_buf.h | 157 +++++++++++++++++++++++++++++++++
2 files changed, 204 insertions(+), 2 deletions(-)
create mode 100644 include/trace/events/dma_buf.h
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index edaa9e4ee4ae..5e6f65cd0306 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -35,6 +35,26 @@
#include "dma-buf-sysfs-stats.h"
+#define CREATE_TRACE_POINTS
+#include <trace/events/dma_buf.h>
+
+/*
+ * dmabuf->name must be accessed with holding dmabuf->name_lock.
+ * we need to take the lock around the tracepoint call itself where
+ * it is called in the code.
+ *
+ * Note: FUNC##_enabled() is a static branch that will only
+ * be set when the trace event is enabled.
+ */
+#define DMA_BUF_TRACE(FUNC, ...) \
+ do { \
+ /* Always expose lock if lockdep is enabled */ \
+ if (IS_ENABLED(CONFIG_LOCKDEP) || FUNC##_enabled()) { \
+ guard(spinlock)(&dmabuf->name_lock); \
+ FUNC(__VA_ARGS__); \
+ } \
+ } while (0)
+
static inline int is_dma_buf_file(struct file *);
static DEFINE_MUTEX(dmabuf_list_mutex);
@@ -220,6 +240,8 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
dmabuf->size >> PAGE_SHIFT)
return -EINVAL;
+ DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf);
+
return dmabuf->ops->mmap(dmabuf, vma);
}
@@ -745,6 +767,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
__dma_buf_list_add(dmabuf);
+ DMA_BUF_TRACE(trace_dma_buf_export, dmabuf);
+
return dmabuf;
err_dmabuf:
@@ -768,10 +792,16 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF");
*/
int dma_buf_fd(struct dma_buf *dmabuf, int flags)
{
+ int fd;
+
if (!dmabuf || !dmabuf->file)
return -EINVAL;
- return FD_ADD(flags, dmabuf->file);
+ fd = FD_ADD(flags, dmabuf->file);
+ if (fd >= 0)
+ DMA_BUF_TRACE(trace_dma_buf_fd, dmabuf, fd);
+
+ return fd;
}
EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
@@ -786,6 +816,7 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
struct dma_buf *dma_buf_get(int fd)
{
struct file *file;
+ struct dma_buf *dmabuf;
file = fget(fd);
@@ -797,7 +828,11 @@ struct dma_buf *dma_buf_get(int fd)
return ERR_PTR(-EINVAL);
}
- return file->private_data;
+ dmabuf = file->private_data;
+
+ DMA_BUF_TRACE(trace_dma_buf_get, dmabuf, fd);
+
+ return dmabuf;
}
EXPORT_SYMBOL_NS_GPL(dma_buf_get, "DMA_BUF");
@@ -817,6 +852,8 @@ void dma_buf_put(struct dma_buf *dmabuf)
return;
fput(dmabuf->file);
+
+ DMA_BUF_TRACE(trace_dma_buf_put, dmabuf);
}
EXPORT_SYMBOL_NS_GPL(dma_buf_put, "DMA_BUF");
@@ -971,6 +1008,9 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
list_add(&attach->node, &dmabuf->attachments);
dma_resv_unlock(dmabuf->resv);
+ DMA_BUF_TRACE(trace_dma_buf_dynamic_attach, dmabuf, attach,
+ dma_buf_attachment_is_dynamic(attach), dev);
+
return attach;
err_attach:
@@ -1015,6 +1055,9 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
if (dmabuf->ops->detach)
dmabuf->ops->detach(dmabuf, attach);
+ DMA_BUF_TRACE(trace_dma_buf_detach, dmabuf, attach,
+ dma_buf_attachment_is_dynamic(attach), attach->dev);
+
kfree(attach);
}
EXPORT_SYMBOL_NS_GPL(dma_buf_detach, "DMA_BUF");
@@ -1480,6 +1523,8 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
vma_set_file(vma, dmabuf->file);
vma->vm_pgoff = pgoff;
+ DMA_BUF_TRACE(trace_dma_buf_mmap, dmabuf);
+
return dmabuf->ops->mmap(dmabuf, vma);
}
EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, "DMA_BUF");
diff --git a/include/trace/events/dma_buf.h b/include/trace/events/dma_buf.h
new file mode 100644
index 000000000000..2c9ba8533467
--- /dev/null
+++ b/include/trace/events/dma_buf.h
@@ -0,0 +1,157 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM dma_buf
+
+#if !defined(_TRACE_DMA_BUF_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_DMA_BUF_H
+
+#include <linux/dma-buf.h>
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(dma_buf,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf),
+
+ TP_STRUCT__entry(
+ __string( exp_name, dmabuf->exp_name)
+ __field( size_t, size)
+ __field( ino_t, ino)
+ ),
+
+ TP_fast_assign(
+ __assign_str(exp_name);
+ __entry->size = dmabuf->size;
+ __entry->ino = dmabuf->file->f_inode->i_ino;
+ ),
+
+ TP_printk("exp_name=%s size=%zu ino=%lu",
+ __get_str(exp_name),
+ __entry->size,
+ __entry->ino)
+);
+
+DECLARE_EVENT_CLASS(dma_buf_attach_dev,
+
+ TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+ bool is_dynamic, struct device *dev),
+
+ TP_ARGS(dmabuf, attach, is_dynamic, dev),
+
+ TP_STRUCT__entry(
+ __string( dev_name, dev_name(dev))
+ __string( exp_name, dmabuf->exp_name)
+ __field( size_t, size)
+ __field( ino_t, ino)
+ __field( struct dma_buf_attachment *, attach)
+ __field( bool, is_dynamic)
+ ),
+
+ TP_fast_assign(
+ __assign_str(dev_name);
+ __assign_str(exp_name);
+ __entry->size = dmabuf->size;
+ __entry->ino = dmabuf->file->f_inode->i_ino;
+ __entry->is_dynamic = is_dynamic;
+ __entry->attach = attach;
+ ),
+
+ TP_printk("exp_name=%s size=%zu ino=%lu attachment:%p is_dynamic=%d dev_name=%s",
+ __get_str(exp_name),
+ __entry->size,
+ __entry->ino,
+ __entry->attach,
+ __entry->is_dynamic,
+ __get_str(dev_name))
+);
+
+DECLARE_EVENT_CLASS(dma_buf_fd,
+
+ TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+ TP_ARGS(dmabuf, fd),
+
+ TP_STRUCT__entry(
+ __string( exp_name, dmabuf->exp_name)
+ __field( size_t, size)
+ __field( ino_t, ino)
+ __field( int, fd)
+ ),
+
+ TP_fast_assign(
+ __assign_str(exp_name);
+ __entry->size = dmabuf->size;
+ __entry->ino = dmabuf->file->f_inode->i_ino;
+ __entry->fd = fd;
+ ),
+
+ TP_printk("exp_name=%s size=%zu ino=%lu fd=%d",
+ __get_str(exp_name),
+ __entry->size,
+ __entry->ino,
+ __entry->fd)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_export,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_mmap_internal,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_mmap,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_put,
+
+ TP_PROTO(struct dma_buf *dmabuf),
+
+ TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf_attach_dev, dma_buf_dynamic_attach,
+
+ TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+ bool is_dynamic, struct device *dev),
+
+ TP_ARGS(dmabuf, attach, is_dynamic, dev)
+);
+
+DEFINE_EVENT(dma_buf_attach_dev, dma_buf_detach,
+
+ TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+ bool is_dynamic, struct device *dev),
+
+ TP_ARGS(dmabuf, attach, is_dynamic, dev)
+);
+
+DEFINE_EVENT(dma_buf_fd, dma_buf_fd,
+
+ TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+ TP_ARGS(dmabuf, fd)
+);
+
+DEFINE_EVENT(dma_buf_fd, dma_buf_get,
+
+ TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+ TP_ARGS(dmabuf, fd)
+);
+
+#endif /* _TRACE_DMA_BUF_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
--
2.34.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox