* [merged mm-nonmm-stable] fork-define-a-local-gfp_vmap_stack.patch removed from -mm tree
@ 2025-05-21 17:49 Andrew Morton
0 siblings, 0 replies; 2+ messages in thread
From: Andrew Morton @ 2025-05-21 17:49 UTC (permalink / raw)
To: mm-commits, pasha.tatashin, mjguzik, linus.walleij, akpm
The quilt patch titled
Subject: fork: define a local GFP_VMAP_STACK
has been removed from the -mm tree. Its filename was
fork-define-a-local-gfp_vmap_stack.patch
This patch was dropped because it was merged into the mm-nonmm-stable branch
of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
------------------------------------------------------
From: Linus Walleij <linus.walleij@linaro.org>
Subject: fork: define a local GFP_VMAP_STACK
Date: Fri, 09 May 2025 09:25:09 +0200
The current allocation of VMAP stack memory is using (THREADINFO_GFP &
~__GFP_ACCOUNT) which is a complicated way of saying (GFP_KERNEL |
__GFP_ZERO):
<linux/thread_info.h>:
define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_ZERO)
<linux/gfp_types.h>:
define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT)
This is an unfortunate side-effect of independent changes blurring the
picture:
commit 19809c2da28aee5860ad9a2eff760730a0710df0 changed (THREADINFO_GFP |
__GFP_HIGHMEM) to just THREADINFO_GFP since highmem became implicit.
commit 9b6f7e163cd0f468d1b9696b785659d3c27c8667 then added stack caching
and rewrote the allocation to (THREADINFO_GFP & ~__GFP_ACCOUNT) as cached
stacks need to be accounted separately. However that code, when it
eventually accounts the memory does this:
ret = memcg_kmem_charge(vm->pages[i], GFP_KERNEL, 0)
so the memory is charged as a GFP_KERNEL allocation.
Define a unique GFP_VMAP_STACK to use
GFP_KERNEL | __GFP_ZERO and move the comment there.
Link: https://lkml.kernel.org/r/20250509-gfp-stack-v1-1-82f6f7efc210@linaro.org
Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
Reported-by: Mateusz Guzik <mjguzik@gmail.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---
kernel/fork.c | 88 ++++++++++++++++++++++++------------------------
1 file changed, 45 insertions(+), 43 deletions(-)
--- a/kernel/fork.c~fork-define-a-local-gfp_vmap_stack
+++ a/kernel/fork.c
@@ -185,7 +185,13 @@ static inline void free_task_struct(stru
kmem_cache_free(task_struct_cachep, tsk);
}
-#ifdef CONFIG_VMAP_STACK
+/*
+ * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
+ * kmemcache based allocator.
+ */
+# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
+
+# ifdef CONFIG_VMAP_STACK
/*
* vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
* flush. Try to minimize the number of calls by caching stacks.
@@ -198,14 +204,14 @@ struct vm_stack {
struct vm_struct *stack_vm_area;
};
-static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area)
+static bool try_release_thread_stack_to_cache(struct vm_struct *vm)
{
unsigned int i;
for (i = 0; i < NR_CACHED_STACKS; i++) {
struct vm_struct *tmp = NULL;
- if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm_area))
+ if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm))
return true;
}
return false;
@@ -214,12 +220,11 @@ static bool try_release_thread_stack_to_
static void thread_stack_free_rcu(struct rcu_head *rh)
{
struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu);
- struct vm_struct *vm_area = vm_stack->stack_vm_area;
if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area))
return;
- vfree(vm_area->addr);
+ vfree(vm_stack);
}
static void thread_stack_delayed_free(struct task_struct *tsk)
@@ -232,32 +237,32 @@ static void thread_stack_delayed_free(st
static int free_vm_stack_cache(unsigned int cpu)
{
- struct vm_struct **cached_vm_stack_areas = per_cpu_ptr(cached_stacks, cpu);
+ struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);
int i;
for (i = 0; i < NR_CACHED_STACKS; i++) {
- struct vm_struct *vm_area = cached_vm_stack_areas[i];
+ struct vm_struct *vm_stack = cached_vm_stacks[i];
- if (!vm_area)
+ if (!vm_stack)
continue;
- vfree(vm_area->addr);
- cached_vm_stack_areas[i] = NULL;
+ vfree(vm_stack->addr);
+ cached_vm_stacks[i] = NULL;
}
return 0;
}
-static int memcg_charge_kernel_stack(struct vm_struct *vm_area)
+static int memcg_charge_kernel_stack(struct vm_struct *vm)
{
int i;
int ret;
int nr_charged = 0;
- BUG_ON(vm_area->nr_pages != THREAD_SIZE / PAGE_SIZE);
+ BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
- ret = memcg_kmem_charge_page(vm_area->pages[i], GFP_KERNEL, 0);
+ ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, 0);
if (ret)
goto err;
nr_charged++;
@@ -265,35 +270,38 @@ static int memcg_charge_kernel_stack(str
return 0;
err:
for (i = 0; i < nr_charged; i++)
- memcg_kmem_uncharge_page(vm_area->pages[i], 0);
+ memcg_kmem_uncharge_page(vm->pages[i], 0);
return ret;
}
static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
- struct vm_struct *vm_area;
+ struct vm_struct *vm;
void *stack;
int i;
for (i = 0; i < NR_CACHED_STACKS; i++) {
- vm_area = this_cpu_xchg(cached_stacks[i], NULL);
- if (!vm_area)
- continue;
+ struct vm_struct *s;
- if (memcg_charge_kernel_stack(vm_area)) {
- vfree(vm_area->addr);
- return -ENOMEM;
- }
+ s = this_cpu_xchg(cached_stacks[i], NULL);
+
+ if (!s)
+ continue;
/* Reset stack metadata. */
- kasan_unpoison_range(vm_area->addr, THREAD_SIZE);
+ kasan_unpoison_range(s->addr, THREAD_SIZE);
- stack = kasan_reset_tag(vm_area->addr);
+ stack = kasan_reset_tag(s->addr);
/* Clear stale pointers from reused stack. */
memset(stack, 0, THREAD_SIZE);
- tsk->stack_vm_area = vm_area;
+ if (memcg_charge_kernel_stack(s)) {
+ vfree(s->addr);
+ return -ENOMEM;
+ }
+
+ tsk->stack_vm_area = s;
tsk->stack = stack;
return 0;
}
@@ -309,8 +317,8 @@ static int alloc_thread_stack_node(struc
if (!stack)
return -ENOMEM;
- vm_area = find_vm_area(stack);
- if (memcg_charge_kernel_stack(vm_area)) {
+ vm = find_vm_area(stack);
+ if (memcg_charge_kernel_stack(vm)) {
vfree(stack);
return -ENOMEM;
}
@@ -319,7 +327,7 @@ static int alloc_thread_stack_node(struc
* free_thread_stack() can be called in interrupt context,
* so cache the vm_struct.
*/
- tsk->stack_vm_area = vm_area;
+ tsk->stack_vm_area = vm;
stack = kasan_reset_tag(stack);
tsk->stack = stack;
return 0;
@@ -334,13 +342,7 @@ static void free_thread_stack(struct tas
tsk->stack_vm_area = NULL;
}
-#else /* !CONFIG_VMAP_STACK */
-
-/*
- * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
- * kmemcache based allocator.
- */
-#if THREAD_SIZE >= PAGE_SIZE
+# else /* !CONFIG_VMAP_STACK */
static void thread_stack_free_rcu(struct rcu_head *rh)
{
@@ -372,7 +374,8 @@ static void free_thread_stack(struct tas
tsk->stack = NULL;
}
-#else /* !(THREAD_SIZE >= PAGE_SIZE) */
+# endif /* CONFIG_VMAP_STACK */
+# else /* !(THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)) */
static struct kmem_cache *thread_stack_cache;
@@ -411,8 +414,7 @@ void thread_stack_cache_init(void)
BUG_ON(thread_stack_cache == NULL);
}
-#endif /* THREAD_SIZE >= PAGE_SIZE */
-#endif /* CONFIG_VMAP_STACK */
+# endif /* THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) */
/* SLAB cache for signal_struct structures (tsk->signal) */
static struct kmem_cache *signal_cachep;
@@ -515,11 +517,11 @@ void vm_area_free(struct vm_area_struct
static void account_kernel_stack(struct task_struct *tsk, int account)
{
if (IS_ENABLED(CONFIG_VMAP_STACK)) {
- struct vm_struct *vm_area = task_stack_vm_area(tsk);
+ struct vm_struct *vm = task_stack_vm_area(tsk);
int i;
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
- mod_lruvec_page_state(vm_area->pages[i], NR_KERNEL_STACK_KB,
+ mod_lruvec_page_state(vm->pages[i], NR_KERNEL_STACK_KB,
account * (PAGE_SIZE / 1024));
} else {
void *stack = task_stack_page(tsk);
@@ -535,12 +537,12 @@ void exit_task_stack_account(struct task
account_kernel_stack(tsk, -1);
if (IS_ENABLED(CONFIG_VMAP_STACK)) {
- struct vm_struct *vm_area;
+ struct vm_struct *vm;
int i;
- vm_area = task_stack_vm_area(tsk);
+ vm = task_stack_vm_area(tsk);
for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
- memcg_kmem_uncharge_page(vm_area->pages[i], 0);
+ memcg_kmem_uncharge_page(vm->pages[i], 0);
}
}
_
Patches currently in -mm which might be from linus.walleij@linaro.org are
^ permalink raw reply [flat|nested] 2+ messages in thread
* [merged mm-nonmm-stable] fork-define-a-local-gfp_vmap_stack.patch removed from -mm tree
@ 2025-07-10 5:59 Andrew Morton
0 siblings, 0 replies; 2+ messages in thread
From: Andrew Morton @ 2025-07-10 5:59 UTC (permalink / raw)
To: mm-commits, rppt, pasha.tatashin, mjguzik, linus.walleij, akpm
The quilt patch titled
Subject: fork: define a local GFP_VMAP_STACK
has been removed from the -mm tree. Its filename was
fork-define-a-local-gfp_vmap_stack.patch
This patch was dropped because it was merged into the mm-nonmm-stable branch
of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
------------------------------------------------------
From: Linus Walleij <linus.walleij@linaro.org>
Subject: fork: define a local GFP_VMAP_STACK
Date: Fri, 09 May 2025 09:25:09 +0200
The current allocation of VMAP stack memory is using (THREADINFO_GFP &
~__GFP_ACCOUNT) which is a complicated way of saying (GFP_KERNEL |
__GFP_ZERO):
<linux/thread_info.h>:
define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_ZERO)
<linux/gfp_types.h>:
define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT)
This is an unfortunate side-effect of independent changes blurring the
picture:
commit 19809c2da28aee5860ad9a2eff760730a0710df0 changed (THREADINFO_GFP |
__GFP_HIGHMEM) to just THREADINFO_GFP since highmem became implicit.
commit 9b6f7e163cd0f468d1b9696b785659d3c27c8667 then added stack caching
and rewrote the allocation to (THREADINFO_GFP & ~__GFP_ACCOUNT) as cached
stacks need to be accounted separately. However that code, when it
eventually accounts the memory does this:
ret = memcg_kmem_charge(vm->pages[i], GFP_KERNEL, 0)
so the memory is charged as a GFP_KERNEL allocation.
Define a unique GFP_VMAP_STACK to use
GFP_KERNEL | __GFP_ZERO and move the comment there.
Link: https://lkml.kernel.org/r/20250509-gfp-stack-v1-1-82f6f7efc210@linaro.org
Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
Reported-by: Mateusz Guzik <mjguzik@gmail.com>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---
kernel/fork.c | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
--- a/kernel/fork.c~fork-define-a-local-gfp_vmap_stack
+++ a/kernel/fork.c
@@ -201,6 +201,12 @@ static inline void free_task_struct(stru
*/
#define NR_CACHED_STACKS 2
static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
+/*
+ * Allocated stacks are cached and later reused by new threads, so memcg
+ * accounting is performed by the code assigning/releasing stacks to tasks.
+ * We need a zeroed memory without __GFP_ACCOUNT.
+ */
+#define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO)
struct vm_stack {
struct rcu_head rcu;
@@ -307,13 +313,8 @@ static int alloc_thread_stack_node(struc
return 0;
}
- /*
- * Allocated stacks are cached and later reused by new threads,
- * so memcg accounting is performed manually on assigning/releasing
- * stacks to tasks. Drop __GFP_ACCOUNT.
- */
stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,
- THREADINFO_GFP & ~__GFP_ACCOUNT,
+ GFP_VMAP_STACK,
node, __builtin_return_address(0));
if (!stack)
return -ENOMEM;
_
Patches currently in -mm which might be from linus.walleij@linaro.org are
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2025-07-10 5:59 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2025-05-21 17:49 [merged mm-nonmm-stable] fork-define-a-local-gfp_vmap_stack.patch removed from -mm tree Andrew Morton
-- strict thread matches above, loose matches on Subject: below --
2025-07-10 5:59 Andrew Morton
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.