* Re: [PATCH v6 13/37] lib: add allocation tagging support for memory allocation profiling
@ 2024-04-05 13:54 Klara Modin
2024-04-06 21:47 ` Kent Overstreet
0 siblings, 1 reply; 5+ messages in thread
From: Klara Modin @ 2024-04-05 13:54 UTC (permalink / raw)
To: Suren Baghdasaryan, akpm
Cc: kent.overstreet, mhocko, vbabka, hannes, roman.gushchin, mgorman,
dave, willy, liam.howlett, penguin-kernel, corbet, void,
Peter Zijlstra (Intel), juri.lelli, catalin.marinas, will, arnd,
tglx, mingo, dave.hansen, x86, peterx, david, axboe, mcgrof,
masahiroy, Nathan Chancellor, dennis, jhubbard, tj, muchun.song,
rppt, paulmck, pasha.tatashin, yosryahmed, yuzhao, David Howells,
hughd, andreyknvl, keescook, ndesaulniers, vvvvvv, gregkh,
ebiggers, ytcoode, vincent.guittot, dietmar.eggemann, rostedt,
bsegall, bristot, vschneid, cl, penberg, iamjoonsoo.kim,
42.hyeyoo, glider, elver, dvyukov, songmuchun, jbaron, aliceryhl,
rientjes, minchan, kaleshsingh, kernel-team, linux-doc,
linux-kernel, iommu, linux-arch, linux-fsdevel, linux-mm,
linux-modules, kasan-dev, cgroups
[-- Attachment #1: Type: text/plain, Size: 6607 bytes --]
Hi,
On 2024-03-21 17:36, Suren Baghdasaryan wrote:
> Introduce CONFIG_MEM_ALLOC_PROFILING which provides definitions to easily
> instrument memory allocators. It registers an "alloc_tags" codetag type
> with /proc/allocinfo interface to output allocation tag information when
> the feature is enabled.
> CONFIG_MEM_ALLOC_PROFILING_DEBUG is provided for debugging the memory
> allocation profiling instrumentation.
> Memory allocation profiling can be enabled or disabled at runtime using
> /proc/sys/vm/mem_profiling sysctl when CONFIG_MEM_ALLOC_PROFILING_DEBUG=n.
> CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT enables memory allocation
> profiling by default.
>
> Signed-off-by: Suren Baghdasaryan <surenb@google.com>
> Co-developed-by: Kent Overstreet <kent.overstreet@linux.dev>
> Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
With this commit (9e2dcefa791e9d14006b360fba3455510fd3325d in
next-20240404), randconfig with KCONFIG_SEED=0xE6264236 fails to build
with the attached error. The following patch fixes the build error for
me, but I don't know if it's correct.
Kind regards,
Klara Modin
diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
index 100ddf66eb8e..1c765d80298b 100644
--- a/include/linux/alloc_tag.h
+++ b/include/linux/alloc_tag.h
@@ -12,6 +12,7 @@
#include <asm/percpu.h>
#include <linux/cpumask.h>
#include <linux/static_key.h>
+#include <linux/irqflags.h>
struct alloc_tag_counters {
u64 bytes;
> diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
> new file mode 100644
> index 000000000000..b970ff1c80dc
> --- /dev/null
> +++ b/include/linux/alloc_tag.h
> @@ -0,0 +1,145 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * allocation tagging
> + */
> +#ifndef _LINUX_ALLOC_TAG_H
> +#define _LINUX_ALLOC_TAG_H
> +
> +#include <linux/bug.h>
> +#include <linux/codetag.h>
> +#include <linux/container_of.h>
> +#include <linux/preempt.h>
> +#include <asm/percpu.h>
> +#include <linux/cpumask.h>
> +#include <linux/static_key.h>
> +
> +struct alloc_tag_counters {
> + u64 bytes;
> + u64 calls;
> +};
> +
> +/*
> + * An instance of this structure is created in a special ELF section at every
> + * allocation callsite. At runtime, the special section is treated as
> + * an array of these. Embedded codetag utilizes codetag framework.
> + */
> +struct alloc_tag {
> + struct codetag ct;
> + struct alloc_tag_counters __percpu *counters;
> +} __aligned(8);
> +
> +#ifdef CONFIG_MEM_ALLOC_PROFILING
> +
> +static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
> +{
> + return container_of(ct, struct alloc_tag, ct);
> +}
> +
> +#ifdef ARCH_NEEDS_WEAK_PER_CPU
> +/*
> + * When percpu variables are required to be defined as weak, static percpu
> + * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).
> + */
> +#error "Memory allocation profiling is incompatible with ARCH_NEEDS_WEAK_PER_CPU"
> +#endif
> +
> +#define DEFINE_ALLOC_TAG(_alloc_tag) \
> + static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr); \
> + static struct alloc_tag _alloc_tag __used __aligned(8) \
> + __section("alloc_tags") = { \
> + .ct = CODE_TAG_INIT, \
> + .counters = &_alloc_tag_cntr };
> +
> +DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
> + mem_alloc_profiling_key);
> +
> +static inline bool mem_alloc_profiling_enabled(void)
> +{
> + return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
> + &mem_alloc_profiling_key);
> +}
> +
> +static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
> +{
> + struct alloc_tag_counters v = { 0, 0 };
> + struct alloc_tag_counters *counter;
> + int cpu;
> +
> + for_each_possible_cpu(cpu) {
> + counter = per_cpu_ptr(tag->counters, cpu);
> + v.bytes += counter->bytes;
> + v.calls += counter->calls;
> + }
> +
> + return v;
> +}
> +
> +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
> +static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag)
> +{
> + WARN_ONCE(ref && ref->ct,
> + "alloc_tag was not cleared (got tag for %s:%u)\n",
> + ref->ct->filename, ref->ct->lineno);
> +
> + WARN_ONCE(!tag, "current->alloc_tag not set");
> +}
> +
> +static inline void alloc_tag_sub_check(union codetag_ref *ref)
> +{
> + WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
> +}
> +#else
> +static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
> +static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
> +#endif
> +
> +/* Caller should verify both ref and tag to be valid */
> +static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
> +{
> + ref->ct = &tag->ct;
> + /*
> + * We need in increment the call counter every time we have a new
> + * allocation or when we split a large allocation into smaller ones.
> + * Each new reference for every sub-allocation needs to increment call
> + * counter because when we free each part the counter will be decremented.
> + */
> + this_cpu_inc(tag->counters->calls);
> +}
> +
> +static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)
> +{
> + alloc_tag_add_check(ref, tag);
> + if (!ref || !tag)
> + return;
> +
> + __alloc_tag_ref_set(ref, tag);
> + this_cpu_add(tag->counters->bytes, bytes);
> +}
> +
> +static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)
> +{
> + struct alloc_tag *tag;
> +
> + alloc_tag_sub_check(ref);
> + if (!ref || !ref->ct)
> + return;
> +
> + tag = ct_to_alloc_tag(ref->ct);
> +
> + this_cpu_sub(tag->counters->bytes, bytes);
> + this_cpu_dec(tag->counters->calls);
> +
> + ref->ct = NULL;
> +}
> +
> +#else /* CONFIG_MEM_ALLOC_PROFILING */
> +
> +#define DEFINE_ALLOC_TAG(_alloc_tag)
> +static inline bool mem_alloc_profiling_enabled(void) { return false; }
> +static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
> + size_t bytes) {}
> +static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
> +
> +#endif /* CONFIG_MEM_ALLOC_PROFILING */
> +
> +#endif /* _LINUX_ALLOC_TAG_H */
[-- Attachment #2: error-in-alloc_tag --]
[-- Type: text/plain, Size: 3290 bytes --]
In file included from ./arch/x86/include/asm/percpu.h:615,
from ./arch/x86/include/asm/preempt.h:6,
from ./include/linux/preempt.h:79,
from ./include/linux/alloc_tag.h:11,
from lib/alloc_tag.c:2:
./include/linux/alloc_tag.h: In function ‘__alloc_tag_ref_set’:
./include/asm-generic/percpu.h:155:9: error: implicit declaration of function ‘raw_local_irq_save’ [-Werror=implicit-function-declaration]
155 | raw_local_irq_save(__flags); \
| ^~~~~~~~~~~~~~~~~~
./include/asm-generic/percpu.h:410:41: note: in expansion of macro ‘this_cpu_generic_to_op’
410 | #define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
| ^~~~~~~~~~~~~~~~~~~~~~
./include/linux/percpu-defs.h:368:25: note: in expansion of macro ‘this_cpu_add_8’
368 | case 8: stem##8(variable, __VA_ARGS__);break; \
| ^~~~
./include/linux/percpu-defs.h:491:41: note: in expansion of macro ‘__pcpu_size_call’
491 | #define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
| ^~~~~~~~~~~~~~~~
./include/linux/percpu-defs.h:501:41: note: in expansion of macro ‘this_cpu_add’
501 | #define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
| ^~~~~~~~~~~~
./include/linux/alloc_tag.h:106:9: note: in expansion of macro ‘this_cpu_inc’
106 | this_cpu_inc(tag->counters->calls);
| ^~~~~~~~~~~~
./include/asm-generic/percpu.h:157:9: error: implicit declaration of function ‘raw_local_irq_restore’ [-Werror=implicit-function-declaration]
157 | raw_local_irq_restore(__flags); \
| ^~~~~~~~~~~~~~~~~~~~~
./include/asm-generic/percpu.h:410:41: note: in expansion of macro ‘this_cpu_generic_to_op’
410 | #define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
| ^~~~~~~~~~~~~~~~~~~~~~
./include/linux/percpu-defs.h:368:25: note: in expansion of macro ‘this_cpu_add_8’
368 | case 8: stem##8(variable, __VA_ARGS__);break; \
| ^~~~
./include/linux/percpu-defs.h:491:41: note: in expansion of macro ‘__pcpu_size_call’
491 | #define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
| ^~~~~~~~~~~~~~~~
./include/linux/percpu-defs.h:501:41: note: in expansion of macro ‘this_cpu_add’
501 | #define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
| ^~~~~~~~~~~~
./include/linux/alloc_tag.h:106:9: note: in expansion of macro ‘this_cpu_inc’
106 | this_cpu_inc(tag->counters->calls);
| ^~~~~~~~~~~~
cc1: some warnings being treated as errors
make[3]: *** [scripts/Makefile.build:244: lib/alloc_tag.o] Error 1
make[2]: *** [scripts/Makefile.build:485: lib] Error 2
make[1]: *** [/home/klara/git/linux/Makefile:1919: .] Error 2
make: *** [Makefile:240: __sub-make] Error 2
[-- Attachment #3: randconfig.gz --]
[-- Type: application/gzip, Size: 43697 bytes --]
[-- Attachment #4: bisect-alloc_tag --]
[-- Type: text/plain, Size: 2535 bytes --]
# bad: [2b3d5988ae2cb5cd945ddbc653f0a71706231fdd] Add linux-next specific files for 20240404
git bisect start 'next/master'
# status: waiting for good commit(s), bad commit known
# good: [39cd87c4eb2b893354f3b850f916353f2658ae6f] Linux 6.9-rc2
git bisect good 39cd87c4eb2b893354f3b850f916353f2658ae6f
# bad: [cc7b62666779616ff52d389a344ffe2c041e36e2] Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git
git bisect bad cc7b62666779616ff52d389a344ffe2c041e36e2
# bad: [d6b7dd0f8d84f9fdf2af65fceb608e3206276e81] Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git
git bisect bad d6b7dd0f8d84f9fdf2af65fceb608e3206276e81
# bad: [ad6a31687713a8f12165e730e0eb6e0de3beae56] Merge branch 'mm-everything' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
git bisect bad ad6a31687713a8f12165e730e0eb6e0de3beae56
# good: [59266d9886adb5c9e240129ccc606727fd3a881d] Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git
git bisect good 59266d9886adb5c9e240129ccc606727fd3a881d
# bad: [085e5fe7388cf36ab5c02d91022229e5fade5b30] mm: merge folio_is_secretmem() and folio_fast_pin_allowed() into gup_fast_folio_allowed()
git bisect bad 085e5fe7388cf36ab5c02d91022229e5fade5b30
# bad: [f6a61baa9139d174170acdae8667b3246ce44db6] lib: add memory allocations report in show_mem()
git bisect bad f6a61baa9139d174170acdae8667b3246ce44db6
# good: [302519d9e80a7fbf2cf8d0b8961d491af648759f] asm-generic/io.h: kill vmalloc.h dependency
git bisect good 302519d9e80a7fbf2cf8d0b8961d491af648759f
# bad: [e6942003e682e3883847459c3d07e23c796a2782] mm: create new codetag references during page splitting
git bisect bad e6942003e682e3883847459c3d07e23c796a2782
# good: [ed97151dec736c1541bfac2b801108d54ebee5bc] lib: code tagging module support
git bisect good ed97151dec736c1541bfac2b801108d54ebee5bc
# bad: [95767bde5020afefef4205b60e71f4ebf96da74e] lib: introduce early boot parameter to avoid page_ext memory overhead
git bisect bad 95767bde5020afefef4205b60e71f4ebf96da74e
# bad: [9e2dcefa791e9d14006b360fba3455510fd3325d] lib: add allocation tagging support for memory allocation profiling
git bisect bad 9e2dcefa791e9d14006b360fba3455510fd3325d
# good: [0eccd42fbf9d7c4ae0cbec48cce637da89813c2c] lib: prevent module unloading if memory is not freed
git bisect good 0eccd42fbf9d7c4ae0cbec48cce637da89813c2c
# first bad commit: [9e2dcefa791e9d14006b360fba3455510fd3325d] lib: add allocation tagging support for memory allocation profiling
^ permalink raw reply related [flat|nested] 5+ messages in thread* Re: [PATCH v6 13/37] lib: add allocation tagging support for memory allocation profiling
2024-04-05 13:54 [PATCH v6 13/37] lib: add allocation tagging support for memory allocation profiling Klara Modin
@ 2024-04-06 21:47 ` Kent Overstreet
2024-04-07 13:44 ` Klara Modin
0 siblings, 1 reply; 5+ messages in thread
From: Kent Overstreet @ 2024-04-06 21:47 UTC (permalink / raw)
To: Klara Modin
Cc: Suren Baghdasaryan, akpm, mhocko, vbabka, hannes, roman.gushchin,
mgorman, dave, willy, liam.howlett, penguin-kernel, corbet, void,
Peter Zijlstra (Intel), juri.lelli, catalin.marinas, will, arnd,
tglx, mingo, dave.hansen, x86, peterx, david, axboe, mcgrof,
masahiroy, Nathan Chancellor, dennis, jhubbard, tj, muchun.song,
rppt, paulmck, pasha.tatashin, yosryahmed, yuzhao, David Howells,
hughd, andreyknvl, keescook, ndesaulniers, vvvvvv, gregkh,
ebiggers, ytcoode, vincent.guittot, dietmar.eggemann, rostedt,
bsegall, bristot, vschneid, cl, penberg, iamjoonsoo.kim,
42.hyeyoo, glider, elver, dvyukov, songmuchun, jbaron, aliceryhl,
rientjes, minchan, kaleshsingh, kernel-team, linux-doc,
linux-kernel, iommu, linux-arch, linux-fsdevel, linux-mm,
linux-modules, kasan-dev, cgroups
On Fri, Apr 05, 2024 at 03:54:45PM +0200, Klara Modin wrote:
> Hi,
>
> On 2024-03-21 17:36, Suren Baghdasaryan wrote:
> > Introduce CONFIG_MEM_ALLOC_PROFILING which provides definitions to easily
> > instrument memory allocators. It registers an "alloc_tags" codetag type
> > with /proc/allocinfo interface to output allocation tag information when
> > the feature is enabled.
> > CONFIG_MEM_ALLOC_PROFILING_DEBUG is provided for debugging the memory
> > allocation profiling instrumentation.
> > Memory allocation profiling can be enabled or disabled at runtime using
> > /proc/sys/vm/mem_profiling sysctl when CONFIG_MEM_ALLOC_PROFILING_DEBUG=n.
> > CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT enables memory allocation
> > profiling by default.
> >
> > Signed-off-by: Suren Baghdasaryan <surenb@google.com>
> > Co-developed-by: Kent Overstreet <kent.overstreet@linux.dev>
> > Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
>
> With this commit (9e2dcefa791e9d14006b360fba3455510fd3325d in
> next-20240404), randconfig with KCONFIG_SEED=0xE6264236 fails to build
> with the attached error. The following patch fixes the build error for me,
> but I don't know if it's correct.
Looks good - if you sound out an official patch I'll ack it.
>
> Kind regards,
> Klara Modin
>
> diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
> index 100ddf66eb8e..1c765d80298b 100644
> --- a/include/linux/alloc_tag.h
> +++ b/include/linux/alloc_tag.h
> @@ -12,6 +12,7 @@
> #include <asm/percpu.h>
> #include <linux/cpumask.h>
> #include <linux/static_key.h>
> +#include <linux/irqflags.h>
>
> struct alloc_tag_counters {
> u64 bytes;
>
> > diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
> > new file mode 100644
> > index 000000000000..b970ff1c80dc
> > --- /dev/null
> > +++ b/include/linux/alloc_tag.h
> > @@ -0,0 +1,145 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * allocation tagging
> > + */
> > +#ifndef _LINUX_ALLOC_TAG_H
> > +#define _LINUX_ALLOC_TAG_H
> > +
> > +#include <linux/bug.h>
> > +#include <linux/codetag.h>
> > +#include <linux/container_of.h>
> > +#include <linux/preempt.h>
> > +#include <asm/percpu.h>
> > +#include <linux/cpumask.h>
> > +#include <linux/static_key.h>
> > +
> > +struct alloc_tag_counters {
> > + u64 bytes;
> > + u64 calls;
> > +};
> > +
> > +/*
> > + * An instance of this structure is created in a special ELF section at every
> > + * allocation callsite. At runtime, the special section is treated as
> > + * an array of these. Embedded codetag utilizes codetag framework.
> > + */
> > +struct alloc_tag {
> > + struct codetag ct;
> > + struct alloc_tag_counters __percpu *counters;
> > +} __aligned(8);
> > +
> > +#ifdef CONFIG_MEM_ALLOC_PROFILING
> > +
> > +static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
> > +{
> > + return container_of(ct, struct alloc_tag, ct);
> > +}
> > +
> > +#ifdef ARCH_NEEDS_WEAK_PER_CPU
> > +/*
> > + * When percpu variables are required to be defined as weak, static percpu
> > + * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).
> > + */
> > +#error "Memory allocation profiling is incompatible with ARCH_NEEDS_WEAK_PER_CPU"
> > +#endif
> > +
> > +#define DEFINE_ALLOC_TAG(_alloc_tag) \
> > + static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr); \
> > + static struct alloc_tag _alloc_tag __used __aligned(8) \
> > + __section("alloc_tags") = { \
> > + .ct = CODE_TAG_INIT, \
> > + .counters = &_alloc_tag_cntr };
> > +
> > +DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
> > + mem_alloc_profiling_key);
> > +
> > +static inline bool mem_alloc_profiling_enabled(void)
> > +{
> > + return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
> > + &mem_alloc_profiling_key);
> > +}
> > +
> > +static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
> > +{
> > + struct alloc_tag_counters v = { 0, 0 };
> > + struct alloc_tag_counters *counter;
> > + int cpu;
> > +
> > + for_each_possible_cpu(cpu) {
> > + counter = per_cpu_ptr(tag->counters, cpu);
> > + v.bytes += counter->bytes;
> > + v.calls += counter->calls;
> > + }
> > +
> > + return v;
> > +}
> > +
> > +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
> > +static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag)
> > +{
> > + WARN_ONCE(ref && ref->ct,
> > + "alloc_tag was not cleared (got tag for %s:%u)\n",
> > + ref->ct->filename, ref->ct->lineno);
> > +
> > + WARN_ONCE(!tag, "current->alloc_tag not set");
> > +}
> > +
> > +static inline void alloc_tag_sub_check(union codetag_ref *ref)
> > +{
> > + WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
> > +}
> > +#else
> > +static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
> > +static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
> > +#endif
> > +
> > +/* Caller should verify both ref and tag to be valid */
> > +static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
> > +{
> > + ref->ct = &tag->ct;
> > + /*
> > + * We need in increment the call counter every time we have a new
> > + * allocation or when we split a large allocation into smaller ones.
> > + * Each new reference for every sub-allocation needs to increment call
> > + * counter because when we free each part the counter will be decremented.
> > + */
> > + this_cpu_inc(tag->counters->calls);
> > +}
> > +
> > +static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)
> > +{
> > + alloc_tag_add_check(ref, tag);
> > + if (!ref || !tag)
> > + return;
> > +
> > + __alloc_tag_ref_set(ref, tag);
> > + this_cpu_add(tag->counters->bytes, bytes);
> > +}
> > +
> > +static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)
> > +{
> > + struct alloc_tag *tag;
> > +
> > + alloc_tag_sub_check(ref);
> > + if (!ref || !ref->ct)
> > + return;
> > +
> > + tag = ct_to_alloc_tag(ref->ct);
> > +
> > + this_cpu_sub(tag->counters->bytes, bytes);
> > + this_cpu_dec(tag->counters->calls);
> > +
> > + ref->ct = NULL;
> > +}
> > +
> > +#else /* CONFIG_MEM_ALLOC_PROFILING */
> > +
> > +#define DEFINE_ALLOC_TAG(_alloc_tag)
> > +static inline bool mem_alloc_profiling_enabled(void) { return false; }
> > +static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
> > + size_t bytes) {}
> > +static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
> > +
> > +#endif /* CONFIG_MEM_ALLOC_PROFILING */
> > +
> > +#endif /* _LINUX_ALLOC_TAG_H */
> In file included from ./arch/x86/include/asm/percpu.h:615,
> from ./arch/x86/include/asm/preempt.h:6,
> from ./include/linux/preempt.h:79,
> from ./include/linux/alloc_tag.h:11,
> from lib/alloc_tag.c:2:
> ./include/linux/alloc_tag.h: In function ‘__alloc_tag_ref_set’:
> ./include/asm-generic/percpu.h:155:9: error: implicit declaration of function ‘raw_local_irq_save’ [-Werror=implicit-function-declaration]
> 155 | raw_local_irq_save(__flags); \
> | ^~~~~~~~~~~~~~~~~~
> ./include/asm-generic/percpu.h:410:41: note: in expansion of macro ‘this_cpu_generic_to_op’
> 410 | #define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
> | ^~~~~~~~~~~~~~~~~~~~~~
> ./include/linux/percpu-defs.h:368:25: note: in expansion of macro ‘this_cpu_add_8’
> 368 | case 8: stem##8(variable, __VA_ARGS__);break; \
> | ^~~~
> ./include/linux/percpu-defs.h:491:41: note: in expansion of macro ‘__pcpu_size_call’
> 491 | #define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
> | ^~~~~~~~~~~~~~~~
> ./include/linux/percpu-defs.h:501:41: note: in expansion of macro ‘this_cpu_add’
> 501 | #define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
> | ^~~~~~~~~~~~
> ./include/linux/alloc_tag.h:106:9: note: in expansion of macro ‘this_cpu_inc’
> 106 | this_cpu_inc(tag->counters->calls);
> | ^~~~~~~~~~~~
> ./include/asm-generic/percpu.h:157:9: error: implicit declaration of function ‘raw_local_irq_restore’ [-Werror=implicit-function-declaration]
> 157 | raw_local_irq_restore(__flags); \
> | ^~~~~~~~~~~~~~~~~~~~~
> ./include/asm-generic/percpu.h:410:41: note: in expansion of macro ‘this_cpu_generic_to_op’
> 410 | #define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
> | ^~~~~~~~~~~~~~~~~~~~~~
> ./include/linux/percpu-defs.h:368:25: note: in expansion of macro ‘this_cpu_add_8’
> 368 | case 8: stem##8(variable, __VA_ARGS__);break; \
> | ^~~~
> ./include/linux/percpu-defs.h:491:41: note: in expansion of macro ‘__pcpu_size_call’
> 491 | #define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
> | ^~~~~~~~~~~~~~~~
> ./include/linux/percpu-defs.h:501:41: note: in expansion of macro ‘this_cpu_add’
> 501 | #define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
> | ^~~~~~~~~~~~
> ./include/linux/alloc_tag.h:106:9: note: in expansion of macro ‘this_cpu_inc’
> 106 | this_cpu_inc(tag->counters->calls);
> | ^~~~~~~~~~~~
> cc1: some warnings being treated as errors
> make[3]: *** [scripts/Makefile.build:244: lib/alloc_tag.o] Error 1
> make[2]: *** [scripts/Makefile.build:485: lib] Error 2
> make[1]: *** [/home/klara/git/linux/Makefile:1919: .] Error 2
> make: *** [Makefile:240: __sub-make] Error 2
> # bad: [2b3d5988ae2cb5cd945ddbc653f0a71706231fdd] Add linux-next specific files for 20240404
> git bisect start 'next/master'
> # status: waiting for good commit(s), bad commit known
> # good: [39cd87c4eb2b893354f3b850f916353f2658ae6f] Linux 6.9-rc2
> git bisect good 39cd87c4eb2b893354f3b850f916353f2658ae6f
> # bad: [cc7b62666779616ff52d389a344ffe2c041e36e2] Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git
> git bisect bad cc7b62666779616ff52d389a344ffe2c041e36e2
> # bad: [d6b7dd0f8d84f9fdf2af65fceb608e3206276e81] Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git
> git bisect bad d6b7dd0f8d84f9fdf2af65fceb608e3206276e81
> # bad: [ad6a31687713a8f12165e730e0eb6e0de3beae56] Merge branch 'mm-everything' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
> git bisect bad ad6a31687713a8f12165e730e0eb6e0de3beae56
> # good: [59266d9886adb5c9e240129ccc606727fd3a881d] Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git
> git bisect good 59266d9886adb5c9e240129ccc606727fd3a881d
> # bad: [085e5fe7388cf36ab5c02d91022229e5fade5b30] mm: merge folio_is_secretmem() and folio_fast_pin_allowed() into gup_fast_folio_allowed()
> git bisect bad 085e5fe7388cf36ab5c02d91022229e5fade5b30
> # bad: [f6a61baa9139d174170acdae8667b3246ce44db6] lib: add memory allocations report in show_mem()
> git bisect bad f6a61baa9139d174170acdae8667b3246ce44db6
> # good: [302519d9e80a7fbf2cf8d0b8961d491af648759f] asm-generic/io.h: kill vmalloc.h dependency
> git bisect good 302519d9e80a7fbf2cf8d0b8961d491af648759f
> # bad: [e6942003e682e3883847459c3d07e23c796a2782] mm: create new codetag references during page splitting
> git bisect bad e6942003e682e3883847459c3d07e23c796a2782
> # good: [ed97151dec736c1541bfac2b801108d54ebee5bc] lib: code tagging module support
> git bisect good ed97151dec736c1541bfac2b801108d54ebee5bc
> # bad: [95767bde5020afefef4205b60e71f4ebf96da74e] lib: introduce early boot parameter to avoid page_ext memory overhead
> git bisect bad 95767bde5020afefef4205b60e71f4ebf96da74e
> # bad: [9e2dcefa791e9d14006b360fba3455510fd3325d] lib: add allocation tagging support for memory allocation profiling
> git bisect bad 9e2dcefa791e9d14006b360fba3455510fd3325d
> # good: [0eccd42fbf9d7c4ae0cbec48cce637da89813c2c] lib: prevent module unloading if memory is not freed
> git bisect good 0eccd42fbf9d7c4ae0cbec48cce637da89813c2c
> # first bad commit: [9e2dcefa791e9d14006b360fba3455510fd3325d] lib: add allocation tagging support for memory allocation profiling
^ permalink raw reply [flat|nested] 5+ messages in thread* Re: [PATCH v6 13/37] lib: add allocation tagging support for memory allocation profiling
2024-04-06 21:47 ` Kent Overstreet
@ 2024-04-07 13:44 ` Klara Modin
2024-04-07 16:50 ` Klara Modin
0 siblings, 1 reply; 5+ messages in thread
From: Klara Modin @ 2024-04-07 13:44 UTC (permalink / raw)
To: Kent Overstreet
Cc: Suren Baghdasaryan, akpm, mhocko, vbabka, hannes, roman.gushchin,
mgorman, dave, willy, liam.howlett, penguin-kernel, corbet, void,
Peter Zijlstra (Intel), juri.lelli, catalin.marinas, will, arnd,
tglx, mingo, dave.hansen, x86, peterx, david, axboe, mcgrof,
masahiroy, Nathan Chancellor, dennis, jhubbard, tj, muchun.song,
rppt, paulmck, pasha.tatashin, yosryahmed, yuzhao, David Howells,
hughd, andreyknvl, keescook, ndesaulniers, vvvvvv, gregkh,
ebiggers, ytcoode, vincent.guittot, dietmar.eggemann, rostedt,
bsegall, bristot, vschneid, cl, penberg, iamjoonsoo.kim,
42.hyeyoo, glider, elver, dvyukov, songmuchun, jbaron, aliceryhl,
rientjes, minchan, kaleshsingh, kernel-team, linux-doc,
linux-kernel, iommu, linux-arch, linux-fsdevel, linux-mm,
linux-modules, kasan-dev, cgroups
On 2024-04-06 23:47, Kent Overstreet wrote:
> On Fri, Apr 05, 2024 at 03:54:45PM +0200, Klara Modin wrote:
>> Hi,
>>
>> On 2024-03-21 17:36, Suren Baghdasaryan wrote:
>>> Introduce CONFIG_MEM_ALLOC_PROFILING which provides definitions to easily
>>> instrument memory allocators. It registers an "alloc_tags" codetag type
>>> with /proc/allocinfo interface to output allocation tag information when
>>> the feature is enabled.
>>> CONFIG_MEM_ALLOC_PROFILING_DEBUG is provided for debugging the memory
>>> allocation profiling instrumentation.
>>> Memory allocation profiling can be enabled or disabled at runtime using
>>> /proc/sys/vm/mem_profiling sysctl when CONFIG_MEM_ALLOC_PROFILING_DEBUG=n.
>>> CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT enables memory allocation
>>> profiling by default.
>>>
>>> Signed-off-by: Suren Baghdasaryan <surenb@google.com>
>>> Co-developed-by: Kent Overstreet <kent.overstreet@linux.dev>
>>> Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
>>
>> With this commit (9e2dcefa791e9d14006b360fba3455510fd3325d in
>> next-20240404), randconfig with KCONFIG_SEED=0xE6264236 fails to build
>> with the attached error. The following patch fixes the build error for me,
>> but I don't know if it's correct.
>
> Looks good - if you sound out an official patch I'll ack it.
>
I gave it a try and sent out a patch [1]. This is my first time doing
that and it's likely not without mistakes.
1.
https://lore.kernel.org/lkml/20240407133252.173636-1-klarasmodin@gmail.com/T/#u
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH v6 13/37] lib: add allocation tagging support for memory allocation profiling
2024-04-07 13:44 ` Klara Modin
@ 2024-04-07 16:50 ` Klara Modin
0 siblings, 0 replies; 5+ messages in thread
From: Klara Modin @ 2024-04-07 16:50 UTC (permalink / raw)
To: Kent Overstreet
Cc: Suren Baghdasaryan, akpm, mhocko, vbabka, hannes, roman.gushchin,
mgorman, dave, willy, liam.howlett, penguin-kernel, corbet, void,
Peter Zijlstra (Intel), juri.lelli, catalin.marinas, will, arnd,
tglx, mingo, dave.hansen, x86, peterx, david, axboe, mcgrof,
masahiroy, Nathan Chancellor, dennis, jhubbard, tj, muchun.song,
rppt, paulmck, pasha.tatashin, yosryahmed, yuzhao, David Howells,
hughd, andreyknvl, keescook, ndesaulniers, vvvvvv, gregkh,
ebiggers, ytcoode, vincent.guittot, dietmar.eggemann, rostedt,
bsegall, bristot, vschneid, cl, penberg, iamjoonsoo.kim,
42.hyeyoo, glider, elver, dvyukov, songmuchun, jbaron, aliceryhl,
rientjes, minchan, kaleshsingh, kernel-team, linux-doc,
linux-kernel, iommu, linux-arch, linux-fsdevel, linux-mm,
linux-modules, kasan-dev, cgroups
[-- Attachment #1: Type: text/plain, Size: 2215 bytes --]
On 2024-04-07 15:44, Klara Modin wrote:
> On 2024-04-06 23:47, Kent Overstreet wrote:
>> On Fri, Apr 05, 2024 at 03:54:45PM +0200, Klara Modin wrote:
>>> Hi,
>>>
>>> On 2024-03-21 17:36, Suren Baghdasaryan wrote:
>>>> Introduce CONFIG_MEM_ALLOC_PROFILING which provides definitions to
>>>> easily
>>>> instrument memory allocators. It registers an "alloc_tags" codetag type
>>>> with /proc/allocinfo interface to output allocation tag information
>>>> when
>>>> the feature is enabled.
>>>> CONFIG_MEM_ALLOC_PROFILING_DEBUG is provided for debugging the memory
>>>> allocation profiling instrumentation.
>>>> Memory allocation profiling can be enabled or disabled at runtime using
>>>> /proc/sys/vm/mem_profiling sysctl when
>>>> CONFIG_MEM_ALLOC_PROFILING_DEBUG=n.
>>>> CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT enables memory allocation
>>>> profiling by default.
>>>>
>>>> Signed-off-by: Suren Baghdasaryan <surenb@google.com>
>>>> Co-developed-by: Kent Overstreet <kent.overstreet@linux.dev>
>>>> Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
>>>
>>> With this commit (9e2dcefa791e9d14006b360fba3455510fd3325d in
>>> next-20240404), randconfig with KCONFIG_SEED=0xE6264236 fails to build
>>> with the attached error. The following patch fixes the build error
>>> for me,
>>> but I don't know if it's correct.
>>
>> Looks good - if you sound out an official patch I'll ack it.
>>
>
> I gave it a try and sent out a patch [1]. This is my first time doing
> that and it's likely not without mistakes.
>
> 1.
> https://lore.kernel.org/lkml/20240407133252.173636-1-klarasmodin@gmail.com/T/#u
linux/smp.h may be needed as well. I tried cross-compiling the
randconfig for riscv which complains of missing raw_smp_processor_id()
and including linux/smp.h resolves that.
Does this look reasonable, and if so, should I send it as well?
diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
index afc9e259a2d3..7fe1cbdab0b0 100644
--- a/include/linux/alloc_tag.h
+++ b/include/linux/alloc_tag.h
@@ -13,6 +13,7 @@
#include <linux/cpumask.h>
#include <linux/static_key.h>
#include <linux/irqflags.h>
+#include <linux/smp.h>
struct alloc_tag_counters {
u64 bytes;
[-- Attachment #2: randconfig-riscv.gz --]
[-- Type: application/gzip, Size: 42191 bytes --]
[-- Attachment #3: riscv-alloc_tag-error --]
[-- Type: text/plain, Size: 3328 bytes --]
In file included from ././include/linux/compiler_types.h:151,
from <command-line>:
./include/linux/alloc_tag.h: In function ‘__alloc_tag_ref_set’:
./include/asm-generic/percpu.h:31:40: error: implicit declaration of function ‘raw_smp_processor_id’ [-Wimplicit-function-declaration]
31 | #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
| ^~~~~~~~~~~~~~~~~~~~
./include/linux/compiler-gcc.h:35:33: note: in definition of macro ‘RELOC_HIDE’
35 | (typeof(ptr)) (__ptr + (off)); \
| ^~~
./include/asm-generic/percpu.h:44:31: note: in expansion of macro ‘SHIFT_PERCPU_PTR’
44 | #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
| ^~~~~~~~~~~~~~~~
./include/asm-generic/percpu.h:31:25: note: in expansion of macro ‘per_cpu_offset’
31 | #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
| ^~~~~~~~~~~~~~
./include/asm-generic/percpu.h:44:53: note: in expansion of macro ‘__my_cpu_offset’
44 | #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
| ^~~~~~~~~~~~~~~
./include/linux/percpu-defs.h:242:9: note: in expansion of macro ‘arch_raw_cpu_ptr’
242 | arch_raw_cpu_ptr(ptr); \
| ^~~~~~~~~~~~~~~~
./include/asm-generic/percpu.h:72:10: note: in expansion of macro ‘raw_cpu_ptr’
72 | *raw_cpu_ptr(&(pcp)) op val; \
| ^~~~~~~~~~~
./include/asm-generic/percpu.h:156:9: note: in expansion of macro ‘raw_cpu_generic_to_op’
156 | raw_cpu_generic_to_op(pcp, val, op); \
| ^~~~~~~~~~~~~~~~~~~~~
./include/asm-generic/percpu.h:401:41: note: in expansion of macro ‘this_cpu_generic_to_op’
401 | #define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
| ^~~~~~~~~~~~~~~~~~~~~~
./include/linux/percpu-defs.h:365:25: note: in expansion of macro ‘this_cpu_add_1’
365 | case 1: stem##1(variable, __VA_ARGS__);break; \
| ^~~~
./include/linux/percpu-defs.h:491:41: note: in expansion of macro ‘__pcpu_size_call’
491 | #define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
| ^~~~~~~~~~~~~~~~
./include/linux/percpu-defs.h:501:41: note: in expansion of macro ‘this_cpu_add’
501 | #define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
| ^~~~~~~~~~~~
./include/linux/alloc_tag.h:146:9: note: in expansion of macro ‘this_cpu_inc’
146 | this_cpu_inc(tag->counters->calls);
| ^~~~~~~~~~~~
make[4]: *** [scripts/Makefile.build:244: arch/riscv/kernel/irq.o] Error 1
make[3]: *** [scripts/Makefile.build:485: arch/riscv/kernel] Error 2
make[2]: *** [scripts/Makefile.build:485: arch/riscv] Error 2
make[1]: *** [/home/klara/git/linux/Makefile:1919: .] Error 2
make: *** [Makefile:240: __sub-make] Error 2
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH v6 00/37] Memory allocation profiling
@ 2024-03-21 16:36 Suren Baghdasaryan
2024-03-21 16:36 ` [PATCH v6 13/37] lib: add allocation tagging support for memory " Suren Baghdasaryan
0 siblings, 1 reply; 5+ messages in thread
From: Suren Baghdasaryan @ 2024-03-21 16:36 UTC (permalink / raw)
To: akpm
Cc: kent.overstreet, mhocko, vbabka, hannes, roman.gushchin, mgorman,
dave, willy, liam.howlett, penguin-kernel, corbet, void, peterz,
juri.lelli, catalin.marinas, will, arnd, tglx, mingo, dave.hansen,
x86, peterx, david, axboe, mcgrof, masahiroy, nathan, dennis,
jhubbard, tj, muchun.song, rppt, paulmck, pasha.tatashin,
yosryahmed, yuzhao, dhowells, hughd, andreyknvl, keescook,
ndesaulniers, vvvvvv, gregkh, ebiggers, ytcoode, vincent.guittot,
dietmar.eggemann, rostedt, bsegall, bristot, vschneid, cl,
penberg, iamjoonsoo.kim, 42.hyeyoo, glider, elver, dvyukov,
songmuchun, jbaron, aliceryhl, rientjes, minchan, kaleshsingh,
surenb, kernel-team, linux-doc, linux-kernel, iommu, linux-arch,
linux-fsdevel, linux-mm, linux-modules, kasan-dev, cgroups
Overview:
Low overhead [1] per-callsite memory allocation profiling. Not just for
debug kernels, overhead low enough to be deployed in production.
Example output:
root@moria-kvm:~# sort -rn /proc/allocinfo
127664128 31168 mm/page_ext.c:270 func:alloc_page_ext
56373248 4737 mm/slub.c:2259 func:alloc_slab_page
14880768 3633 mm/readahead.c:247 func:page_cache_ra_unbounded
14417920 3520 mm/mm_init.c:2530 func:alloc_large_system_hash
13377536 234 block/blk-mq.c:3421 func:blk_mq_alloc_rqs
11718656 2861 mm/filemap.c:1919 func:__filemap_get_folio
9192960 2800 kernel/fork.c:307 func:alloc_thread_stack_node
4206592 4 net/netfilter/nf_conntrack_core.c:2567 func:nf_ct_alloc_hashtable
4136960 1010 drivers/staging/ctagmod/ctagmod.c:20 [ctagmod] func:ctagmod_start
3940352 962 mm/memory.c:4214 func:alloc_anon_folio
2894464 22613 fs/kernfs/dir.c:615 func:__kernfs_new_node
...
Since v5 [2]:
- Added Reviewed-by and Acked-by, per Vlastimil Babka and Miguel Ojeda
- Changed pgalloc_tag_{add|sub} to use number of pages instead of order, per Matthew Wilcox
- Changed pgalloc_tag_sub_bytes to pgalloc_tag_sub_pages and adjusted the usage, per Matthew Wilcox
- Moved static key check before prepare_slab_obj_exts_hook(), per Vlastimil Babka
- Fixed RUST helper, per Miguel Ojeda
- Fixed documentation, per Randy Dunlap
- Rebased over mm-unstable
Usage:
kconfig options:
- CONFIG_MEM_ALLOC_PROFILING
- CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
- CONFIG_MEM_ALLOC_PROFILING_DEBUG
adds warnings for allocations that weren't accounted because of a
missing annotation
sysctl:
/proc/sys/vm/mem_profiling
Runtime info:
/proc/allocinfo
Notes:
[1]: Overhead
To measure the overhead we are comparing the following configurations:
(1) Baseline with CONFIG_MEMCG_KMEM=n
(2) Disabled by default (CONFIG_MEM_ALLOC_PROFILING=y &&
CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n)
(3) Enabled by default (CONFIG_MEM_ALLOC_PROFILING=y &&
CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=y)
(4) Enabled at runtime (CONFIG_MEM_ALLOC_PROFILING=y &&
CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n && /proc/sys/vm/mem_profiling=1)
(5) Baseline with CONFIG_MEMCG_KMEM=y && allocating with __GFP_ACCOUNT
(6) Disabled by default (CONFIG_MEM_ALLOC_PROFILING=y &&
CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n) && CONFIG_MEMCG_KMEM=y
(7) Enabled by default (CONFIG_MEM_ALLOC_PROFILING=y &&
CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=y) && CONFIG_MEMCG_KMEM=y
Performance overhead:
To evaluate performance we implemented an in-kernel test executing
multiple get_free_page/free_page and kmalloc/kfree calls with allocation
sizes growing from 8 to 240 bytes with CPU frequency set to max and CPU
affinity set to a specific CPU to minimize the noise. Below are results
from running the test on Ubuntu 22.04.2 LTS with 6.8.0-rc1 kernel on
56 core Intel Xeon:
kmalloc pgalloc
(1 baseline) 6.764s 16.902s
(2 default disabled) 6.793s (+0.43%) 17.007s (+0.62%)
(3 default enabled) 7.197s (+6.40%) 23.666s (+40.02%)
(4 runtime enabled) 7.405s (+9.48%) 23.901s (+41.41%)
(5 memcg) 13.388s (+97.94%) 48.460s (+186.71%)
(6 def disabled+memcg) 13.332s (+97.10%) 48.105s (+184.61%)
(7 def enabled+memcg) 13.446s (+98.78%) 54.963s (+225.18%)
Memory overhead:
Kernel size:
text data bss dec diff
(1) 26515311 18890222 17018880 62424413
(2) 26524728 19423818 16740352 62688898 264485
(3) 26524724 19423818 16740352 62688894 264481
(4) 26524728 19423818 16740352 62688898 264485
(5) 26541782 18964374 16957440 62463596 39183
Memory consumption on a 56 core Intel CPU with 125GB of memory:
Code tags: 192 kB
PageExts: 262144 kB (256MB)
SlabExts: 9876 kB (9.6MB)
PcpuExts: 512 kB (0.5MB)
Total overhead is 0.2% of total memory.
Benchmarks:
Hackbench tests run 100 times:
hackbench -s 512 -l 200 -g 15 -f 25 -P
baseline disabled profiling enabled profiling
avg 0.3543 0.3559 (+0.0016) 0.3566 (+0.0023)
stdev 0.0137 0.0188 0.0077
hackbench -l 10000
baseline disabled profiling enabled profiling
avg 6.4218 6.4306 (+0.0088) 6.5077 (+0.0859)
stdev 0.0933 0.0286 0.0489
stress-ng tests:
stress-ng --class memory --seq 4 -t 60
stress-ng --class cpu --seq 4 -t 60
Results posted at: https://evilpiepirate.org/~kent/memalloc_prof_v4_stress-ng/
[2] https://lore.kernel.org/all/20240306182440.2003814-1-surenb@google.com/
Kent Overstreet (13):
fix missing vmalloc.h includes
asm-generic/io.h: Kill vmalloc.h dependency
mm/slub: Mark slab_free_freelist_hook() __always_inline
scripts/kallysms: Always include __start and __stop symbols
fs: Convert alloc_inode_sb() to a macro
rust: Add a rust helper for krealloc()
mempool: Hook up to memory allocation profiling
mm: percpu: Introduce pcpuobj_ext
mm: percpu: Add codetag reference into pcpuobj_ext
mm: vmalloc: Enable memory allocation profiling
rhashtable: Plumb through alloc tag
MAINTAINERS: Add entries for code tagging and memory allocation
profiling
memprofiling: Documentation
Suren Baghdasaryan (24):
mm: introduce slabobj_ext to support slab object extensions
mm: introduce __GFP_NO_OBJ_EXT flag to selectively prevent slabobj_ext
creation
mm/slab: introduce SLAB_NO_OBJ_EXT to avoid obj_ext creation
slab: objext: introduce objext_flags as extension to
page_memcg_data_flags
lib: code tagging framework
lib: code tagging module support
lib: prevent module unloading if memory is not freed
lib: add allocation tagging support for memory allocation profiling
lib: introduce support for page allocation tagging
lib: introduce early boot parameter to avoid page_ext memory overhead
mm: percpu: increase PERCPU_MODULE_RESERVE to accommodate allocation
tags
change alloc_pages name in dma_map_ops to avoid name conflicts
mm: enable page allocation tagging
mm: create new codetag references during page splitting
mm: fix non-compound multi-order memory accounting in __free_pages
mm/page_ext: enable early_page_ext when
CONFIG_MEM_ALLOC_PROFILING_DEBUG=y
lib: add codetag reference into slabobj_ext
mm/slab: add allocation accounting into slab allocation and free paths
mm/slab: enable slab allocation tagging for kmalloc and friends
mm: percpu: enable per-cpu allocation tagging
lib: add memory allocations report in show_mem()
codetag: debug: skip objext checking when it's for objext itself
codetag: debug: mark codetags for reserved pages as empty
codetag: debug: introduce OBJEXTS_ALLOC_FAIL to mark failed slab_ext
allocations
Documentation/admin-guide/sysctl/vm.rst | 16 +
Documentation/filesystems/proc.rst | 29 ++
Documentation/mm/allocation-profiling.rst | 100 ++++++
Documentation/mm/index.rst | 1 +
MAINTAINERS | 17 +
arch/alpha/kernel/pci_iommu.c | 2 +-
arch/alpha/lib/checksum.c | 1 +
arch/alpha/lib/fpreg.c | 1 +
arch/alpha/lib/memcpy.c | 1 +
arch/arm/kernel/irq.c | 1 +
arch/arm/kernel/traps.c | 1 +
arch/arm64/kernel/efi.c | 1 +
arch/loongarch/include/asm/kfence.h | 1 +
arch/mips/jazz/jazzdma.c | 2 +-
arch/powerpc/kernel/dma-iommu.c | 2 +-
arch/powerpc/kernel/iommu.c | 1 +
arch/powerpc/mm/mem.c | 1 +
arch/powerpc/platforms/ps3/system-bus.c | 4 +-
arch/powerpc/platforms/pseries/vio.c | 2 +-
arch/riscv/kernel/elf_kexec.c | 1 +
arch/riscv/kernel/probes/kprobes.c | 1 +
arch/s390/kernel/cert_store.c | 1 +
arch/s390/kernel/ipl.c | 1 +
arch/x86/include/asm/io.h | 1 +
arch/x86/kernel/amd_gart_64.c | 2 +-
arch/x86/kernel/cpu/sgx/main.c | 1 +
arch/x86/kernel/irq_64.c | 1 +
arch/x86/mm/fault.c | 1 +
drivers/accel/ivpu/ivpu_mmu_context.c | 1 +
drivers/gpu/drm/gma500/mmu.c | 1 +
drivers/gpu/drm/i915/gem/i915_gem_pages.c | 1 +
.../gpu/drm/i915/gem/selftests/mock_dmabuf.c | 1 +
drivers/gpu/drm/i915/gt/shmem_utils.c | 1 +
drivers/gpu/drm/i915/gvt/firmware.c | 1 +
drivers/gpu/drm/i915/gvt/gtt.c | 1 +
drivers/gpu/drm/i915/gvt/handlers.c | 1 +
drivers/gpu/drm/i915/gvt/mmio.c | 1 +
drivers/gpu/drm/i915/gvt/vgpu.c | 1 +
drivers/gpu/drm/i915/intel_gvt.c | 1 +
drivers/gpu/drm/imagination/pvr_vm_mips.c | 1 +
drivers/gpu/drm/mediatek/mtk_drm_gem.c | 1 +
drivers/gpu/drm/omapdrm/omap_gem.c | 1 +
drivers/gpu/drm/v3d/v3d_bo.c | 1 +
drivers/gpu/drm/vmwgfx/vmwgfx_binding.c | 1 +
drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c | 1 +
drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c | 1 +
drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 1 +
drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c | 1 +
drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c | 1 +
drivers/gpu/drm/xen/xen_drm_front_gem.c | 1 +
drivers/hwtracing/coresight/coresight-trbe.c | 1 +
drivers/iommu/dma-iommu.c | 2 +-
.../marvell/octeon_ep/octep_pfvf_mbox.c | 1 +
.../marvell/octeon_ep_vf/octep_vf_mbox.c | 1 +
.../net/ethernet/microsoft/mana/hw_channel.c | 1 +
drivers/parisc/ccio-dma.c | 2 +-
drivers/parisc/sba_iommu.c | 2 +-
drivers/platform/x86/uv_sysfs.c | 1 +
drivers/scsi/mpi3mr/mpi3mr_transport.c | 2 +
drivers/staging/media/atomisp/pci/hmm/hmm.c | 2 +-
drivers/vfio/pci/pds/dirty.c | 1 +
drivers/virt/acrn/mm.c | 1 +
drivers/virtio/virtio_mem.c | 1 +
drivers/xen/grant-dma-ops.c | 2 +-
drivers/xen/swiotlb-xen.c | 2 +-
include/asm-generic/codetag.lds.h | 14 +
include/asm-generic/io.h | 1 -
include/asm-generic/vmlinux.lds.h | 3 +
include/linux/alloc_tag.h | 205 +++++++++++
include/linux/codetag.h | 81 +++++
include/linux/dma-map-ops.h | 2 +-
include/linux/fortify-string.h | 5 +-
include/linux/fs.h | 6 +-
include/linux/gfp.h | 126 ++++---
include/linux/gfp_types.h | 11 +
include/linux/memcontrol.h | 56 ++-
include/linux/mempool.h | 73 ++--
include/linux/mm.h | 9 +
include/linux/mm_types.h | 4 +-
include/linux/page_ext.h | 1 -
include/linux/pagemap.h | 9 +-
include/linux/pds/pds_common.h | 2 +
include/linux/percpu.h | 27 +-
include/linux/pgalloc_tag.h | 134 +++++++
include/linux/rhashtable-types.h | 11 +-
include/linux/sched.h | 24 ++
include/linux/slab.h | 179 +++++-----
include/linux/string.h | 4 +-
include/linux/vmalloc.h | 60 +++-
include/rdma/rdmavt_qp.h | 1 +
init/Kconfig | 4 +
kernel/dma/mapping.c | 4 +-
kernel/kallsyms_selftest.c | 2 +-
kernel/module/main.c | 29 +-
lib/Kconfig.debug | 31 ++
lib/Makefile | 3 +
lib/alloc_tag.c | 243 +++++++++++++
lib/codetag.c | 283 +++++++++++++++
lib/rhashtable.c | 28 +-
mm/compaction.c | 7 +-
mm/debug_vm_pgtable.c | 1 +
mm/filemap.c | 6 +-
mm/huge_memory.c | 2 +
mm/kfence/core.c | 14 +-
mm/kfence/kfence.h | 4 +-
mm/memcontrol.c | 56 +--
mm/mempolicy.c | 52 +--
mm/mempool.c | 36 +-
mm/mm_init.c | 13 +-
mm/nommu.c | 64 ++--
mm/page_alloc.c | 71 ++--
mm/page_ext.c | 13 +
mm/page_owner.c | 2 +-
mm/percpu-internal.h | 26 +-
mm/percpu.c | 120 +++----
mm/show_mem.c | 26 ++
mm/slab.h | 51 ++-
mm/slab_common.c | 6 +-
mm/slub.c | 327 +++++++++++++++---
mm/util.c | 44 +--
mm/vmalloc.c | 88 ++---
rust/helpers.c | 8 +
scripts/kallsyms.c | 13 +
scripts/module.lds.S | 7 +
sound/pci/hda/cs35l41_hda.c | 1 +
125 files changed, 2319 insertions(+), 652 deletions(-)
create mode 100644 Documentation/mm/allocation-profiling.rst
create mode 100644 include/asm-generic/codetag.lds.h
create mode 100644 include/linux/alloc_tag.h
create mode 100644 include/linux/codetag.h
create mode 100644 include/linux/pgalloc_tag.h
create mode 100644 lib/alloc_tag.c
create mode 100644 lib/codetag.c
base-commit: a824831a082f1d8f9b51a4c0598e633d38555fcf
--
2.44.0.291.gc1ea87d7ee-goog
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH v6 13/37] lib: add allocation tagging support for memory allocation profiling
2024-03-21 16:36 [PATCH v6 00/37] Memory " Suren Baghdasaryan
@ 2024-03-21 16:36 ` Suren Baghdasaryan
0 siblings, 0 replies; 5+ messages in thread
From: Suren Baghdasaryan @ 2024-03-21 16:36 UTC (permalink / raw)
To: akpm
Cc: kent.overstreet, mhocko, vbabka, hannes, roman.gushchin, mgorman,
dave, willy, liam.howlett, penguin-kernel, corbet, void, peterz,
juri.lelli, catalin.marinas, will, arnd, tglx, mingo, dave.hansen,
x86, peterx, david, axboe, mcgrof, masahiroy, nathan, dennis,
jhubbard, tj, muchun.song, rppt, paulmck, pasha.tatashin,
yosryahmed, yuzhao, dhowells, hughd, andreyknvl, keescook,
ndesaulniers, vvvvvv, gregkh, ebiggers, ytcoode, vincent.guittot,
dietmar.eggemann, rostedt, bsegall, bristot, vschneid, cl,
penberg, iamjoonsoo.kim, 42.hyeyoo, glider, elver, dvyukov,
songmuchun, jbaron, aliceryhl, rientjes, minchan, kaleshsingh,
surenb, kernel-team, linux-doc, linux-kernel, iommu, linux-arch,
linux-fsdevel, linux-mm, linux-modules, kasan-dev, cgroups
Introduce CONFIG_MEM_ALLOC_PROFILING which provides definitions to easily
instrument memory allocators. It registers an "alloc_tags" codetag type
with /proc/allocinfo interface to output allocation tag information when
the feature is enabled.
CONFIG_MEM_ALLOC_PROFILING_DEBUG is provided for debugging the memory
allocation profiling instrumentation.
Memory allocation profiling can be enabled or disabled at runtime using
/proc/sys/vm/mem_profiling sysctl when CONFIG_MEM_ALLOC_PROFILING_DEBUG=n.
CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT enables memory allocation
profiling by default.
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Co-developed-by: Kent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
---
Documentation/admin-guide/sysctl/vm.rst | 16 +++
Documentation/filesystems/proc.rst | 29 +++++
include/asm-generic/codetag.lds.h | 14 +++
include/asm-generic/vmlinux.lds.h | 3 +
include/linux/alloc_tag.h | 145 +++++++++++++++++++++++
include/linux/sched.h | 24 ++++
lib/Kconfig.debug | 25 ++++
lib/Makefile | 2 +
lib/alloc_tag.c | 149 ++++++++++++++++++++++++
scripts/module.lds.S | 7 ++
10 files changed, 414 insertions(+)
create mode 100644 include/asm-generic/codetag.lds.h
create mode 100644 include/linux/alloc_tag.h
create mode 100644 lib/alloc_tag.c
diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst
index c59889de122b..e86c968a7a0e 100644
--- a/Documentation/admin-guide/sysctl/vm.rst
+++ b/Documentation/admin-guide/sysctl/vm.rst
@@ -43,6 +43,7 @@ Currently, these files are in /proc/sys/vm:
- legacy_va_layout
- lowmem_reserve_ratio
- max_map_count
+- mem_profiling (only if CONFIG_MEM_ALLOC_PROFILING=y)
- memory_failure_early_kill
- memory_failure_recovery
- min_free_kbytes
@@ -425,6 +426,21 @@ e.g., up to one or two maps per allocation.
The default value is 65530.
+mem_profiling
+==============
+
+Enable memory profiling (when CONFIG_MEM_ALLOC_PROFILING=y)
+
+1: Enable memory profiling.
+
+0: Disable memory profiling.
+
+Enabling memory profiling introduces a small performance overhead for all
+memory allocations.
+
+The default value depends on CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT.
+
+
memory_failure_early_kill:
==========================
diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst
index c6a6b9df2104..5d2fc58b5b1f 100644
--- a/Documentation/filesystems/proc.rst
+++ b/Documentation/filesystems/proc.rst
@@ -688,6 +688,7 @@ files are there, and which are missing.
============ ===============================================================
File Content
============ ===============================================================
+ allocinfo Memory allocations profiling information
apm Advanced power management info
bootconfig Kernel command line obtained from boot config,
and, if there were kernel parameters from the
@@ -953,6 +954,34 @@ also be allocatable although a lot of filesystem metadata may have to be
reclaimed to achieve this.
+allocinfo
+~~~~~~~
+
+Provides information about memory allocations at all locations in the code
+base. Each allocation in the code is identified by its source file, line
+number, module (if originates from a loadable module) and the function calling
+the allocation. The number of bytes allocated and number of calls at each
+location are reported.
+
+Example output.
+
+::
+
+ > sort -rn /proc/allocinfo
+ 127664128 31168 mm/page_ext.c:270 func:alloc_page_ext
+ 56373248 4737 mm/slub.c:2259 func:alloc_slab_page
+ 14880768 3633 mm/readahead.c:247 func:page_cache_ra_unbounded
+ 14417920 3520 mm/mm_init.c:2530 func:alloc_large_system_hash
+ 13377536 234 block/blk-mq.c:3421 func:blk_mq_alloc_rqs
+ 11718656 2861 mm/filemap.c:1919 func:__filemap_get_folio
+ 9192960 2800 kernel/fork.c:307 func:alloc_thread_stack_node
+ 4206592 4 net/netfilter/nf_conntrack_core.c:2567 func:nf_ct_alloc_hashtable
+ 4136960 1010 drivers/staging/ctagmod/ctagmod.c:20 [ctagmod] func:ctagmod_start
+ 3940352 962 mm/memory.c:4214 func:alloc_anon_folio
+ 2894464 22613 fs/kernfs/dir.c:615 func:__kernfs_new_node
+ ...
+
+
meminfo
~~~~~~~
diff --git a/include/asm-generic/codetag.lds.h b/include/asm-generic/codetag.lds.h
new file mode 100644
index 000000000000..64f536b80380
--- /dev/null
+++ b/include/asm-generic/codetag.lds.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __ASM_GENERIC_CODETAG_LDS_H
+#define __ASM_GENERIC_CODETAG_LDS_H
+
+#define SECTION_WITH_BOUNDARIES(_name) \
+ . = ALIGN(8); \
+ __start_##_name = .; \
+ KEEP(*(_name)) \
+ __stop_##_name = .;
+
+#define CODETAG_SECTIONS() \
+ SECTION_WITH_BOUNDARIES(alloc_tags)
+
+#endif /* __ASM_GENERIC_CODETAG_LDS_H */
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index f7749d0f2562..3e4497b5135a 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -50,6 +50,8 @@
* [__nosave_begin, __nosave_end] for the nosave data
*/
+#include <asm-generic/codetag.lds.h>
+
#ifndef LOAD_OFFSET
#define LOAD_OFFSET 0
#endif
@@ -366,6 +368,7 @@
. = ALIGN(8); \
BOUNDED_SECTION_BY(__dyndbg_classes, ___dyndbg_classes) \
BOUNDED_SECTION_BY(__dyndbg, ___dyndbg) \
+ CODETAG_SECTIONS() \
LIKELY_PROFILE() \
BRANCH_PROFILE() \
TRACE_PRINTKS() \
diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
new file mode 100644
index 000000000000..b970ff1c80dc
--- /dev/null
+++ b/include/linux/alloc_tag.h
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * allocation tagging
+ */
+#ifndef _LINUX_ALLOC_TAG_H
+#define _LINUX_ALLOC_TAG_H
+
+#include <linux/bug.h>
+#include <linux/codetag.h>
+#include <linux/container_of.h>
+#include <linux/preempt.h>
+#include <asm/percpu.h>
+#include <linux/cpumask.h>
+#include <linux/static_key.h>
+
+struct alloc_tag_counters {
+ u64 bytes;
+ u64 calls;
+};
+
+/*
+ * An instance of this structure is created in a special ELF section at every
+ * allocation callsite. At runtime, the special section is treated as
+ * an array of these. Embedded codetag utilizes codetag framework.
+ */
+struct alloc_tag {
+ struct codetag ct;
+ struct alloc_tag_counters __percpu *counters;
+} __aligned(8);
+
+#ifdef CONFIG_MEM_ALLOC_PROFILING
+
+static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
+{
+ return container_of(ct, struct alloc_tag, ct);
+}
+
+#ifdef ARCH_NEEDS_WEAK_PER_CPU
+/*
+ * When percpu variables are required to be defined as weak, static percpu
+ * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).
+ */
+#error "Memory allocation profiling is incompatible with ARCH_NEEDS_WEAK_PER_CPU"
+#endif
+
+#define DEFINE_ALLOC_TAG(_alloc_tag) \
+ static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr); \
+ static struct alloc_tag _alloc_tag __used __aligned(8) \
+ __section("alloc_tags") = { \
+ .ct = CODE_TAG_INIT, \
+ .counters = &_alloc_tag_cntr };
+
+DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
+ mem_alloc_profiling_key);
+
+static inline bool mem_alloc_profiling_enabled(void)
+{
+ return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
+ &mem_alloc_profiling_key);
+}
+
+static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
+{
+ struct alloc_tag_counters v = { 0, 0 };
+ struct alloc_tag_counters *counter;
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ counter = per_cpu_ptr(tag->counters, cpu);
+ v.bytes += counter->bytes;
+ v.calls += counter->calls;
+ }
+
+ return v;
+}
+
+#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
+static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag)
+{
+ WARN_ONCE(ref && ref->ct,
+ "alloc_tag was not cleared (got tag for %s:%u)\n",
+ ref->ct->filename, ref->ct->lineno);
+
+ WARN_ONCE(!tag, "current->alloc_tag not set");
+}
+
+static inline void alloc_tag_sub_check(union codetag_ref *ref)
+{
+ WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
+}
+#else
+static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
+static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
+#endif
+
+/* Caller should verify both ref and tag to be valid */
+static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
+{
+ ref->ct = &tag->ct;
+ /*
+ * We need in increment the call counter every time we have a new
+ * allocation or when we split a large allocation into smaller ones.
+ * Each new reference for every sub-allocation needs to increment call
+ * counter because when we free each part the counter will be decremented.
+ */
+ this_cpu_inc(tag->counters->calls);
+}
+
+static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)
+{
+ alloc_tag_add_check(ref, tag);
+ if (!ref || !tag)
+ return;
+
+ __alloc_tag_ref_set(ref, tag);
+ this_cpu_add(tag->counters->bytes, bytes);
+}
+
+static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)
+{
+ struct alloc_tag *tag;
+
+ alloc_tag_sub_check(ref);
+ if (!ref || !ref->ct)
+ return;
+
+ tag = ct_to_alloc_tag(ref->ct);
+
+ this_cpu_sub(tag->counters->bytes, bytes);
+ this_cpu_dec(tag->counters->calls);
+
+ ref->ct = NULL;
+}
+
+#else /* CONFIG_MEM_ALLOC_PROFILING */
+
+#define DEFINE_ALLOC_TAG(_alloc_tag)
+static inline bool mem_alloc_profiling_enabled(void) { return false; }
+static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
+ size_t bytes) {}
+static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
+
+#endif /* CONFIG_MEM_ALLOC_PROFILING */
+
+#endif /* _LINUX_ALLOC_TAG_H */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 3c2abbc587b4..4118b3f959c3 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -770,6 +770,10 @@ struct task_struct {
unsigned int flags;
unsigned int ptrace;
+#ifdef CONFIG_MEM_ALLOC_PROFILING
+ struct alloc_tag *alloc_tag;
+#endif
+
#ifdef CONFIG_SMP
int on_cpu;
struct __call_single_node wake_entry;
@@ -810,6 +814,7 @@ struct task_struct {
struct task_group *sched_task_group;
#endif
+
#ifdef CONFIG_UCLAMP_TASK
/*
* Clamp values requested for a scheduling entity.
@@ -2187,4 +2192,23 @@ static inline int sched_core_idle_cpu(int cpu) { return idle_cpu(cpu); }
extern void sched_set_stop_task(int cpu, struct task_struct *stop);
+#ifdef CONFIG_MEM_ALLOC_PROFILING
+static inline struct alloc_tag *alloc_tag_save(struct alloc_tag *tag)
+{
+ swap(current->alloc_tag, tag);
+ return tag;
+}
+
+static inline void alloc_tag_restore(struct alloc_tag *tag, struct alloc_tag *old)
+{
+#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
+ WARN(current->alloc_tag != tag, "current->alloc_tag was changed:\n");
+#endif
+ current->alloc_tag = old;
+}
+#else
+#define alloc_tag_save(_tag) NULL
+#define alloc_tag_restore(_tag, _old) do {} while (0)
+#endif
+
#endif
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index d2dbdd45fd9a..d9a6477afdb1 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -972,6 +972,31 @@ config CODE_TAGGING
bool
select KALLSYMS
+config MEM_ALLOC_PROFILING
+ bool "Enable memory allocation profiling"
+ default n
+ depends on PROC_FS
+ depends on !DEBUG_FORCE_WEAK_PER_CPU
+ select CODE_TAGGING
+ help
+ Track allocation source code and record total allocation size
+ initiated at that code location. The mechanism can be used to track
+ memory leaks with a low performance and memory impact.
+
+config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
+ bool "Enable memory allocation profiling by default"
+ default y
+ depends on MEM_ALLOC_PROFILING
+
+config MEM_ALLOC_PROFILING_DEBUG
+ bool "Memory allocation profiler debugging"
+ default n
+ depends on MEM_ALLOC_PROFILING
+ select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
+ help
+ Adds warnings with helpful error messages for memory allocation
+ profiling.
+
source "lib/Kconfig.kasan"
source "lib/Kconfig.kfence"
source "lib/Kconfig.kmsan"
diff --git a/lib/Makefile b/lib/Makefile
index 910335da8f13..2f4e17bfb299 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -234,6 +234,8 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
obj-$(CONFIG_CODE_TAGGING) += codetag.o
+obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o
+
lib-$(CONFIG_GENERIC_BUG) += bug.o
obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
new file mode 100644
index 000000000000..f09c8a422bc2
--- /dev/null
+++ b/lib/alloc_tag.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/alloc_tag.h>
+#include <linux/fs.h>
+#include <linux/gfp.h>
+#include <linux/module.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_buf.h>
+#include <linux/seq_file.h>
+
+static struct codetag_type *alloc_tag_cttype;
+
+DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
+ mem_alloc_profiling_key);
+
+static void *allocinfo_start(struct seq_file *m, loff_t *pos)
+{
+ struct codetag_iterator *iter;
+ struct codetag *ct;
+ loff_t node = *pos;
+
+ iter = kzalloc(sizeof(*iter), GFP_KERNEL);
+ m->private = iter;
+ if (!iter)
+ return NULL;
+
+ codetag_lock_module_list(alloc_tag_cttype, true);
+ *iter = codetag_get_ct_iter(alloc_tag_cttype);
+ while ((ct = codetag_next_ct(iter)) != NULL && node)
+ node--;
+
+ return ct ? iter : NULL;
+}
+
+static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
+{
+ struct codetag_iterator *iter = (struct codetag_iterator *)arg;
+ struct codetag *ct = codetag_next_ct(iter);
+
+ (*pos)++;
+ if (!ct)
+ return NULL;
+
+ return iter;
+}
+
+static void allocinfo_stop(struct seq_file *m, void *arg)
+{
+ struct codetag_iterator *iter = (struct codetag_iterator *)m->private;
+
+ if (iter) {
+ codetag_lock_module_list(alloc_tag_cttype, false);
+ kfree(iter);
+ }
+}
+
+static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
+{
+ struct alloc_tag *tag = ct_to_alloc_tag(ct);
+ struct alloc_tag_counters counter = alloc_tag_read(tag);
+ s64 bytes = counter.bytes;
+
+ seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
+ codetag_to_text(out, ct);
+ seq_buf_putc(out, ' ');
+ seq_buf_putc(out, '\n');
+}
+
+static int allocinfo_show(struct seq_file *m, void *arg)
+{
+ struct codetag_iterator *iter = (struct codetag_iterator *)arg;
+ char *bufp;
+ size_t n = seq_get_buf(m, &bufp);
+ struct seq_buf buf;
+
+ seq_buf_init(&buf, bufp, n);
+ alloc_tag_to_text(&buf, iter->ct);
+ seq_commit(m, seq_buf_used(&buf));
+ return 0;
+}
+
+static const struct seq_operations allocinfo_seq_op = {
+ .start = allocinfo_start,
+ .next = allocinfo_next,
+ .stop = allocinfo_stop,
+ .show = allocinfo_show,
+};
+
+static void __init procfs_init(void)
+{
+ proc_create_seq("allocinfo", 0444, NULL, &allocinfo_seq_op);
+}
+
+static bool alloc_tag_module_unload(struct codetag_type *cttype,
+ struct codetag_module *cmod)
+{
+ struct codetag_iterator iter = codetag_get_ct_iter(cttype);
+ struct alloc_tag_counters counter;
+ bool module_unused = true;
+ struct alloc_tag *tag;
+ struct codetag *ct;
+
+ for (ct = codetag_next_ct(&iter); ct; ct = codetag_next_ct(&iter)) {
+ if (iter.cmod != cmod)
+ continue;
+
+ tag = ct_to_alloc_tag(ct);
+ counter = alloc_tag_read(tag);
+
+ if (WARN(counter.bytes,
+ "%s:%u module %s func:%s has %llu allocated at module unload",
+ ct->filename, ct->lineno, ct->modname, ct->function, counter.bytes))
+ module_unused = false;
+ }
+
+ return module_unused;
+}
+
+static struct ctl_table memory_allocation_profiling_sysctls[] = {
+ {
+ .procname = "mem_profiling",
+ .data = &mem_alloc_profiling_key,
+#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
+ .mode = 0444,
+#else
+ .mode = 0644,
+#endif
+ .proc_handler = proc_do_static_key,
+ },
+ { }
+};
+
+static int __init alloc_tag_init(void)
+{
+ const struct codetag_type_desc desc = {
+ .section = "alloc_tags",
+ .tag_size = sizeof(struct alloc_tag),
+ .module_unload = alloc_tag_module_unload,
+ };
+
+ alloc_tag_cttype = codetag_register_type(&desc);
+ if (IS_ERR_OR_NULL(alloc_tag_cttype))
+ return PTR_ERR(alloc_tag_cttype);
+
+ register_sysctl_init("vm", memory_allocation_profiling_sysctls);
+ procfs_init();
+
+ return 0;
+}
+module_init(alloc_tag_init);
diff --git a/scripts/module.lds.S b/scripts/module.lds.S
index bf5bcf2836d8..45c67a0994f3 100644
--- a/scripts/module.lds.S
+++ b/scripts/module.lds.S
@@ -9,6 +9,8 @@
#define DISCARD_EH_FRAME *(.eh_frame)
#endif
+#include <asm-generic/codetag.lds.h>
+
SECTIONS {
/DISCARD/ : {
*(.discard)
@@ -47,12 +49,17 @@ SECTIONS {
.data : {
*(.data .data.[0-9a-zA-Z_]*)
*(.data..L*)
+ CODETAG_SECTIONS()
}
.rodata : {
*(.rodata .rodata.[0-9a-zA-Z_]*)
*(.rodata..L*)
}
+#else
+ .data : {
+ CODETAG_SECTIONS()
+ }
#endif
}
--
2.44.0.291.gc1ea87d7ee-goog
^ permalink raw reply related [flat|nested] 5+ messages in thread
end of thread, other threads:[~2024-04-07 16:51 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-04-05 13:54 [PATCH v6 13/37] lib: add allocation tagging support for memory allocation profiling Klara Modin
2024-04-06 21:47 ` Kent Overstreet
2024-04-07 13:44 ` Klara Modin
2024-04-07 16:50 ` Klara Modin
-- strict thread matches above, loose matches on Subject: below --
2024-03-21 16:36 [PATCH v6 00/37] Memory " Suren Baghdasaryan
2024-03-21 16:36 ` [PATCH v6 13/37] lib: add allocation tagging support for memory " Suren Baghdasaryan
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).