* [7.1.y 1/3] mm/slab: decouple SLAB_NO_SHEAVES from SLAB_NO_OBJ_EXT
2026-08-01 12:25 [7.1.y 0/3] mm/slab: fix unbounded recursion in free path with memalloc profiling Harry Yoo
@ 2026-08-01 12:25 ` Harry Yoo
2026-08-01 12:25 ` [7.1.y 2/3] lib/alloc_tag: introduce mem_alloc_profiling_permanently_disabled() Harry Yoo
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Harry Yoo @ 2026-08-01 12:25 UTC (permalink / raw)
To: Danielle Costantino, Shakeel Butt, Suren Baghdasaryan, Hao Ge,
Vlastimil Babka, Harry Yoo, Andrew Morton, Hao Li,
Christoph Lameter, David Rientjes, Roman Gushchin, Kees Cook,
Pedro Falcato, Liam R . Howlett
Cc: linux-mm, linux-kernel, stable
From: "Harry Yoo (Oracle)" <harry@kernel.org>
commit 982e31382d9a1a3c8c4e6a13702a53711f4efe9f upstream.
Bootstrap caches are created with SLAB_NO_OBJ_EXT to disallow sheaves
and obj_exts.
To allow disabling obj_exts while allowing sheaves, decouple
SLAB_NO_SHEAVES from SLAB_NO_OBJ_EXT. Bootstrap caches now have both
SLAB_NO_SHEAVES and SLAB_NO_OBJ_EXT.
No functional change intended.
Reviewed-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Signed-off-by: Harry Yoo (Oracle) <harry@kernel.org>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Link: https://patch.msgid.link/20260713-kmalloc-no-objext-v3-2-47c7bd138de7@kernel.org
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Signed-off-by: Harry Yoo <harry@kernel.org>
---
include/linux/slab.h | 13 +++++++++++--
mm/slub.c | 10 ++++++----
2 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/include/linux/slab.h b/include/linux/slab.h
index 1a69255bb87f..fda7e24a762b 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -58,10 +58,13 @@ enum _slab_flag_bits {
#endif
_SLAB_OBJECT_POISON,
_SLAB_CMPXCHG_DOUBLE,
+#ifdef CONFIG_SLAB_OBJ_EXT
_SLAB_NO_OBJ_EXT,
-#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
+#ifdef CONFIG_64BIT
_SLAB_OBJ_EXT_IN_OBJ,
#endif
+#endif
+ _SLAB_NO_SHEAVES,
_SLAB_FLAGS_LAST_BIT
};
@@ -239,8 +242,14 @@ enum _slab_flag_bits {
#endif
#define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
-/* Slab created using create_boot_cache */
+/* Slab caches without obj_exts array */
+#ifdef CONFIG_SLAB_OBJ_EXT
#define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT)
+#else
+#define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED
+#endif
+
+#define SLAB_NO_SHEAVES __SLAB_FLAG_BIT(_SLAB_NO_SHEAVES)
#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
#define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ)
diff --git a/mm/slub.c b/mm/slub.c
index fa24b985a783..33e64f21ecc1 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -7726,12 +7726,12 @@ static unsigned int calculate_sheaf_capacity(struct kmem_cache *s,
return 0;
/*
- * Bootstrap caches can't have sheaves for now (SLAB_NO_OBJ_EXT).
+ * Bootstrap caches can't have sheaves for now (SLAB_NO_SHEAVES).
* SLAB_NOLEAKTRACE caches (e.g., kmemleak's object_cache) must not
* have sheaves to avoid recursion when sheaf allocation triggers
* kmemleak tracking.
*/
- if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
+ if (s->flags & (SLAB_NO_SHEAVES | SLAB_NOLEAKTRACE))
return 0;
/*
@@ -8513,7 +8513,8 @@ void __init kmem_cache_init(void)
create_boot_cache(kmem_cache_node, "kmem_cache_node",
sizeof(struct kmem_cache_node),
- SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
+ SLAB_HWCACHE_ALIGN | SLAB_NO_SHEAVES | SLAB_NO_OBJ_EXT,
+ 0, 0);
hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
@@ -8523,7 +8524,8 @@ void __init kmem_cache_init(void)
create_boot_cache(kmem_cache, "kmem_cache",
offsetof(struct kmem_cache, per_node) +
nr_node_ids * sizeof(struct kmem_cache_per_node_ptrs),
- SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
+ SLAB_HWCACHE_ALIGN | SLAB_NO_SHEAVES | SLAB_NO_OBJ_EXT,
+ 0, 0);
kmem_cache = bootstrap(&boot_kmem_cache);
kmem_cache_node = bootstrap(&boot_kmem_cache_node);
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [7.1.y 2/3] lib/alloc_tag: introduce mem_alloc_profiling_permanently_disabled()
2026-08-01 12:25 [7.1.y 0/3] mm/slab: fix unbounded recursion in free path with memalloc profiling Harry Yoo
2026-08-01 12:25 ` [7.1.y 1/3] mm/slab: decouple SLAB_NO_SHEAVES from SLAB_NO_OBJ_EXT Harry Yoo
@ 2026-08-01 12:25 ` Harry Yoo
2026-08-01 12:25 ` [7.1.y 3/3] mm/slab: prevent unbounded recursion in free path with new kmalloc type Harry Yoo
2026-08-01 12:46 ` [7.1.y 0/3] mm/slab: fix unbounded recursion in free path with memalloc profiling Harry Yoo
3 siblings, 0 replies; 5+ messages in thread
From: Harry Yoo @ 2026-08-01 12:25 UTC (permalink / raw)
To: Danielle Costantino, Shakeel Butt, Suren Baghdasaryan, Hao Ge,
Vlastimil Babka, Harry Yoo, Andrew Morton, Hao Li,
Christoph Lameter, David Rientjes, Roman Gushchin, Kees Cook,
Pedro Falcato, Liam R . Howlett
Cc: linux-mm, linux-kernel, stable
From: "Harry Yoo (Oracle)" <harry@kernel.org>
commit a37b0066a10aabf3c968b4566706fb866eaf9a85 upstream.
mem_alloc_profiling_enabled() tells whether memalloc profiling is
currently enabled. However, even when this function returns false,
it can be enabled later.
However, this is not enough. Some optimizations can be applied only when
memalloc profiling is permanently disabled. For example, to skip the
creation of KMALLOC_NO_OBJ_EXT caches at boot time, mem_profiling must
be set to "never", "0" w/ debugging on, or have been shutdown so that
it can no longer be enabled.
Introduce mem_alloc_profiling_permanently_disabled() for this purpose.
Signed-off-by: Harry Yoo (Oracle) <harry@kernel.org>
Acked-by: Suren Baghdasaryan <surenb@google.com>
Link: https://patch.msgid.link/20260713-kmalloc-no-objext-v3-3-47c7bd138de7@kernel.org
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Signed-off-by: Harry Yoo <harry@kernel.org>
---
include/linux/alloc_tag.h | 3 +++
lib/alloc_tag.c | 9 +++++++++
2 files changed, 12 insertions(+)
diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
index 02de2ede560f..7e7cdc7612be 100644
--- a/include/linux/alloc_tag.h
+++ b/include/linux/alloc_tag.h
@@ -134,6 +134,8 @@ static inline bool mem_alloc_profiling_enabled(void)
&mem_alloc_profiling_key);
}
+bool mem_alloc_profiling_permanently_disabled(void);
+
static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
{
struct alloc_tag_counters v = { 0, 0 };
@@ -239,6 +241,7 @@ static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag)
#define DEFINE_ALLOC_TAG(_alloc_tag)
static inline bool mem_alloc_profiling_enabled(void) { return false; }
+static inline bool mem_alloc_profiling_permanently_disabled(void) { return true; }
static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
size_t bytes) {}
static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
index a9ab88f416b9..e3a923594604 100644
--- a/lib/alloc_tag.c
+++ b/lib/alloc_tag.c
@@ -26,6 +26,15 @@ static bool mem_profiling_support = true;
static bool mem_profiling_support;
#endif
+/*
+ * Memory allocation profiling is permanently disabled and cannot be enabled.
+ * Must be called after setup_early_mem_profiling().
+ */
+bool mem_alloc_profiling_permanently_disabled(void)
+{
+ return !mem_profiling_support;
+}
+
static struct codetag_type *alloc_tag_cttype;
#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [7.1.y 3/3] mm/slab: prevent unbounded recursion in free path with new kmalloc type
2026-08-01 12:25 [7.1.y 0/3] mm/slab: fix unbounded recursion in free path with memalloc profiling Harry Yoo
2026-08-01 12:25 ` [7.1.y 1/3] mm/slab: decouple SLAB_NO_SHEAVES from SLAB_NO_OBJ_EXT Harry Yoo
2026-08-01 12:25 ` [7.1.y 2/3] lib/alloc_tag: introduce mem_alloc_profiling_permanently_disabled() Harry Yoo
@ 2026-08-01 12:25 ` Harry Yoo
2026-08-01 12:46 ` [7.1.y 0/3] mm/slab: fix unbounded recursion in free path with memalloc profiling Harry Yoo
3 siblings, 0 replies; 5+ messages in thread
From: Harry Yoo @ 2026-08-01 12:25 UTC (permalink / raw)
To: Danielle Costantino, Shakeel Butt, Suren Baghdasaryan, Hao Ge,
Vlastimil Babka, Harry Yoo, Andrew Morton, Hao Li,
Christoph Lameter, David Rientjes, Roman Gushchin, Kees Cook,
Pedro Falcato, Liam R . Howlett
Cc: linux-mm, linux-kernel, stable
From: "Harry Yoo (Oracle)" <harry@kernel.org>
commit d9e6a7623938968e3752b67e37eaff097e559a54 upstream.
Commit 280ea9c3154b ("mm/slab: avoid allocating slabobj_ext array from
its own slab") avoided recursive allocation of obj_exts from kmalloc
caches of the same size, by bumping the obj_exts array's allocation
size whenever the array size equals the size of the object being
allocated.
However, as reported by Danielle Costantino and Shakeel Butt,
even slabs from kmalloc caches of different sizes can form a cycle
by allocating obj_exts arrays from each other [1]:
What happened: a KMALLOC_NORMAL slab's obj_exts array (used by
allocation profiling / memcg accounting) is itself kmalloc()'d from a
KMALLOC_NORMAL cache, so the "slab holds another slab's obj_exts array"
relation can form cycles. With sizeof(struct slabobj_ext) == 16 and
the host's geometry:
- kmalloc-512 has 64 objects/slab -> array is 64*16 == 1024 bytes,
served from kmalloc-1k;
- kmalloc-1k has 32 objects/slab -> array is 32*16 == 512 bytes,
served from kmalloc-512.
A kmalloc-512 slab and a kmalloc-1k slab therefore hold each other's
obj_exts array. Discarding one frees the other's array, which empties
and discards that slab, which frees the first's array, and so on:
__free_slab() -> free_slab_obj_exts() -> kfree() -> discard_slab() ->
__free_slab() recurses along the cycle until the stack is exhausted.
With memory allocation profiling, this allows unbounded recursion
in the free path and led to a stack overflow on a production host in
the Meta fleet [1]:
BUG: TASK stack guard page was hit
Oops: stack guard page
RIP: 0010:kfree+0x8/0x5d0
Call Trace:
__free_slab+0x66/0xc0
kfree+0x3f0/0x5d0
... ( ~125x __free_slab <-> kfree ) ...
<kernel driver freeing a resource>
do_syscall_64
It is proposed [1] to resolve this issue by always serving the obj_exts
array allocation from kmalloc caches (or large kmalloc) of sizes larger
than the object size. However, as pointed out by Vlastimil Babka [2],
this can waste an excessive amount of memory as slabs from large
kmalloc sizes (e.g. kmalloc-8k) generally need obj_exts arrays much
smaller than the object size.
Therefore, rather than bumping the size, let us take a different
approach; disallow formation of cycles between kmalloc types when
allocating obj_exts arrays. Currently, all obj_exts arrays are served
from normal kmalloc caches. Cycles cannot be created if obj_exts arrays
of normal kmalloc caches are served from a special kmalloc type that can
never have obj_exts arrays.
To achieve this, create a new kmalloc type called KMALLOC_NO_OBJ_EXT.
KMALLOC_NO_OBJ_EXT caches are created with SLAB_NO_OBJ_EXT flag when
either 1) memory allocation profiling is not permanently disabled,
or 2) kmalloc types with a priority higher than KMALLOC_CGROUP are
aliased with KMALLOC_NORMAL.
Sheaf bootstrapping for KMALLOC_NO_OBJ_EXT caches now must be deferred
because allocation of a barn can trigger obj_exts array allocation of
normal kmalloc caches when the KMALLOC_NO_OBJ_EXT cache for that size
is not ready yet. For simplicity, perform bootstrapping of sheaves for
all kmalloc caches later.
Introduce a new slab alloc flag, SLAB_ALLOC_NO_OBJ_EXT, to prevent
allocation of obj_exts arrays, and let kmalloc_slab() override the type
to KMALLOC_NO_OBJ_EXT when specified. Note that kmalloc_type() remains
unchanged because kmalloc_flags() bypasses the kmalloc fastpath.
Do not pass SLAB_ALLOC_NO_RECURSE to kmalloc_flags() in
alloc_slab_obj_exts() and instead use SLAB_ALLOC_NO_OBJ_EXT only when
the objects are allocated from normal kmalloc caches. While this
prevents unbounded recursive allocation of obj_exts, it allows
KMALLOC_NO_OBJ_EXT caches to have sheaves.
Since sheaf allocations specify SLAB_ALLOC_NO_RECURSE that prevents
allocation of both sheaves and obj_exts arrays, the recursion depth
is bounded.
obj_exts arrays for non-kmalloc-normal caches can now have a valid tag.
Do not call mark_obj_codetag_empty() when freeing an obj_exts array to
avoid false warnings. KMALLOC_NO_OBJ_EXT don't need this as they never
allocate those arrays.
Reported-by: Danielle Costantino <dcostantino@meta.com>
Reported-by: Shakeel Butt <shakeel.butt@linux.dev>
Closes: https://lore.kernel.org/linux-mm/20260625230029.703750-1-shakeel.butt@linux.dev [1]
Fixes: 4b8736964640 ("mm/slab: add allocation accounting into slab allocation and free paths")
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/linux-mm/c5c4208d-a6f0-413e-bad9-49be12f12d55@kernel.org [2]
Signed-off-by: Harry Yoo (Oracle) <harry@kernel.org>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Link: https://patch.msgid.link/20260713-kmalloc-no-objext-v3-4-47c7bd138de7@kernel.org
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
[harry@kernel.org: Backport notes:
- Fix a minor conflict due to missing partitioned
kmalloc caches in 7.1.
- Use __GFP_NO_OBJ_EXT instead of SLAB_ALLOC_NO_OBJ_EXT
since slab's internal alloc_flags do not exist in 7.1.
- Since there is no way to distinguish between no obj_exts vs.
no recursion in 7.1 due to lack of SLAB_ALLOC_* flags, sheaves for
normal kmalloc caches are allocated from kmalloc-no-objext-* unlike
upstream.
Also, allocations from kmalloc-no-objext cannot allocate sheaves and
may observe slightly higher contention. Only 7.1 has this problem as
kmalloc caches don't have sheaves in LTS kernels.
- Apply the __GFP_NO_OBJ_EXT flag to the !allow_spin path in
alloc_slab_obj_exts(). ]
Signed-off-by: Harry Yoo <harry@kernel.org>
---
include/linux/slab.h | 6 +++
mm/slab.h | 28 +++++++++++++-
mm/slab_common.c | 13 +++++++
mm/slub.c | 87 +++++++++++++++-----------------------------
4 files changed, 75 insertions(+), 59 deletions(-)
diff --git a/include/linux/slab.h b/include/linux/slab.h
index fda7e24a762b..739f474c5d92 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -642,6 +642,9 @@ enum kmalloc_cache_type {
#endif
#ifndef CONFIG_MEMCG
KMALLOC_CGROUP = KMALLOC_NORMAL,
+#endif
+#ifndef CONFIG_SLAB_OBJ_EXT
+ KMALLOC_NO_OBJ_EXT = KMALLOC_NORMAL,
#endif
KMALLOC_RANDOM_START = KMALLOC_NORMAL,
KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR,
@@ -655,6 +658,9 @@ enum kmalloc_cache_type {
#endif
#ifdef CONFIG_MEMCG
KMALLOC_CGROUP,
+#endif
+#ifdef CONFIG_SLAB_OBJ_EXT
+ KMALLOC_NO_OBJ_EXT,
#endif
NR_KMALLOC_TYPES
};
diff --git a/mm/slab.h b/mm/slab.h
index bf2f87acf5e3..50414bd2b919 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -365,9 +365,13 @@ static inline struct kmem_cache *
kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, unsigned long caller)
{
unsigned int index;
+ enum kmalloc_cache_type type = kmalloc_type(flags, caller);
+
+ if (flags & __GFP_NO_OBJ_EXT)
+ type = KMALLOC_NO_OBJ_EXT;
if (!b)
- b = &kmalloc_caches[kmalloc_type(flags, caller)];
+ b = &kmalloc_caches[type];
if (size <= 192)
index = kmalloc_size_index[size_index_elem(size)];
else
@@ -402,7 +406,8 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s)
{
if (!is_kmalloc_cache(s))
return false;
- return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT));
+
+ return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT));
}
bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj);
@@ -505,6 +510,25 @@ static inline void metadata_access_disable(void)
kasan_enable_current();
}
+/*
+ * Return true if KMALLOC_NORMAL caches may need obj_exts arrays.
+ *
+ * Memory allocation profiling requires obj_exts for all caches.
+ * Memcg usually doesn't need them for normal kmalloc caches, but kmalloc types
+ * with a priority higher than KMALLOC_CGROUP can be aliased with KMALLOC_NORMAL.
+ */
+static inline bool need_kmalloc_no_objext(void)
+{
+ if (!mem_alloc_profiling_permanently_disabled())
+ return true;
+
+ if (!mem_cgroup_kmem_disabled() &&
+ (KMALLOC_NORMAL == KMALLOC_RECLAIM))
+ return true;
+
+ return false;
+}
+
#ifdef CONFIG_SLAB_OBJ_EXT
/*
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 8b661fff5eed..214374233fb8 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -843,6 +843,12 @@ EXPORT_SYMBOL(kmalloc_size_roundup);
#define KMALLOC_RANDOM_NAME(N, sz)
#endif
+#ifdef CONFIG_SLAB_OBJ_EXT
+#define KMALLOC_NO_OBJ_EXT_NAME(sz) .name[KMALLOC_NO_OBJ_EXT] = "kmalloc-no-objext-" #sz,
+#else
+#define KMALLOC_NO_OBJ_EXT_NAME(sz)
+#endif
+
#define INIT_KMALLOC_INFO(__size, __short_size) \
{ \
.name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \
@@ -850,6 +856,7 @@ EXPORT_SYMBOL(kmalloc_size_roundup);
KMALLOC_CGROUP_NAME(__short_size) \
KMALLOC_DMA_NAME(__short_size) \
KMALLOC_RANDOM_NAME(RANDOM_KMALLOC_CACHES_NR, __short_size) \
+ KMALLOC_NO_OBJ_EXT_NAME(__short_size) \
.size = __size, \
}
@@ -957,6 +964,12 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type)
return;
}
flags |= SLAB_ACCOUNT;
+ } else if (IS_ENABLED(CONFIG_SLAB_OBJ_EXT) && type == KMALLOC_NO_OBJ_EXT) {
+ if (!need_kmalloc_no_objext()) {
+ kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
+ return;
+ }
+ flags |= SLAB_NO_OBJ_EXT | SLAB_NO_MERGE;
} else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) {
flags |= SLAB_CACHE_DMA;
}
diff --git a/mm/slub.c b/mm/slub.c
index 33e64f21ecc1..d83c3622ccde 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2106,42 +2106,6 @@ static inline void init_slab_obj_exts(struct slab *slab)
slab->obj_exts = 0;
}
-/*
- * Calculate the allocation size for slabobj_ext array.
- *
- * When memory allocation profiling is enabled, the obj_exts array
- * could be allocated from the same slab cache it's being allocated for.
- * This would prevent the slab from ever being freed because it would
- * always contain at least one allocated object (its own obj_exts array).
- *
- * To avoid this, increase the allocation size when we detect the array
- * may come from the same cache, forcing it to use a different cache.
- */
-static inline size_t obj_exts_alloc_size(struct kmem_cache *s,
- struct slab *slab, gfp_t gfp)
-{
- size_t sz = sizeof(struct slabobj_ext) * slab->objects;
- struct kmem_cache *obj_exts_cache;
-
- if (sz > KMALLOC_MAX_CACHE_SIZE)
- return sz;
-
- if (!is_kmalloc_normal(s))
- return sz;
-
- obj_exts_cache = kmalloc_slab(sz, NULL, gfp, 0);
- /*
- * We can't simply compare s with obj_exts_cache, because random kmalloc
- * caches have multiple caches per size, selected by caller address.
- * Since caller address may differ between kmalloc_slab() and actual
- * allocation, bump size when sizes are equal.
- */
- if (s->object_size == obj_exts_cache->object_size)
- return obj_exts_cache->object_size + 1;
-
- return sz;
-}
-
int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp_t gfp, bool new_slab)
{
@@ -2150,13 +2114,17 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
unsigned long new_exts;
unsigned long old_exts;
struct slabobj_ext *vec;
- size_t sz;
+ size_t sz = sizeof(struct slabobj_ext) * slab->objects;
gfp &= ~OBJCGS_CLEAR_MASK;
- /* Prevent recursive extension vector allocation */
- gfp |= __GFP_NO_OBJ_EXT;
- sz = obj_exts_alloc_size(s, slab, gfp);
+ /*
+ * In most cases, obj_exts arrays are allocated from normal kmalloc.
+ * However, normal kmalloc caches must allocate them from
+ * KMALLOC_NO_OBJ_EXT caches to prevent recursion.
+ */
+ if (is_kmalloc_normal(s))
+ gfp |= __GFP_NO_OBJ_EXT;
/*
* Note that allow_spin may be false during early boot and its
@@ -2165,10 +2133,11 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
* very early allocations on those.
*/
if (unlikely(!allow_spin))
- vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT,
+ vec = kmalloc_nolock(sz, __GFP_ZERO | (gfp & __GFP_NO_OBJ_EXT),
slab_nid(slab));
else
- vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab));
+ vec = kmalloc_node(sz, gfp | __GFP_ZERO,
+ slab_nid(slab));
if (!vec) {
/*
@@ -2183,8 +2152,21 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
return -ENOMEM;
}
- VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL &&
- virt_to_slab(vec)->slab_cache == s);
+ if (IS_ENABLED(CONFIG_DEBUG_VM)) {
+ struct kmem_cache *exts_cache;
+ struct slab *exts_slab;
+
+ exts_slab = virt_to_slab(vec);
+ if (exts_slab) {
+ /*
+ * The vector must be allocated from either normal or
+ * KMALLOC_NO_OBJ_EXT kmalloc caches to avoid cycles.
+ */
+ exts_cache = exts_slab->slab_cache;
+ WARN_ON_ONCE(!is_kmalloc_normal(exts_cache) &&
+ !(exts_cache->flags & SLAB_NO_OBJ_EXT));
+ }
+ }
new_exts = (unsigned long)vec;
#ifdef CONFIG_MEMCG
@@ -2207,7 +2189,6 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
* assign slabobj_exts in parallel. In this case the existing
* objcg vector should be reused.
*/
- mark_obj_codetag_empty(vec);
if (unlikely(!allow_spin))
kfree_nolock(vec);
else
@@ -2243,14 +2224,6 @@ static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin)
return;
}
- /*
- * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its
- * corresponding extension will be NULL. alloc_tag_sub() will throw a
- * warning if slab has extensions but the extension of an object is
- * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that
- * the extension for obj_exts is expected to be NULL.
- */
- mark_obj_codetag_empty(obj_exts);
if (allow_spin)
kfree(obj_exts);
else
@@ -7906,10 +7879,10 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
s->allocflags |= __GFP_RECLAIMABLE;
/*
- * For KMALLOC_NORMAL caches we enable sheaves later by
- * bootstrap_kmalloc_sheaves() to avoid recursion
+ * For kmalloc caches we enable sheaves later by
+ * bootstrap_kmalloc_sheaves() to avoid recursion.
*/
- if (!is_kmalloc_normal(s))
+ if (!is_kmalloc_cache(s))
s->sheaf_capacity = calculate_sheaf_capacity(s, args);
/*
@@ -8475,7 +8448,7 @@ static void __init bootstrap_kmalloc_sheaves(void)
{
enum kmalloc_cache_type type;
- for (type = KMALLOC_NORMAL; type <= KMALLOC_RANDOM_END; type++) {
+ for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) {
for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
struct kmem_cache *s = kmalloc_caches[type][idx];
--
2.53.0
^ permalink raw reply related [flat|nested] 5+ messages in thread