* [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
@ 2026-10-16 15:53 Puranjay Mohan
2026-10-17 15:54 ` sashiko-bot
` (4 more replies)
0 siblings, 5 replies; 8+ messages in thread
From: Puranjay Mohan @ 2026-10-16 15:53 UTC (permalink / raw)
To: linux-mm; +Cc: Puranjay Mohan, Vlastimil Babka, Harry Yoo, Andrew Morton
__kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
extend deferred free mechanism to handle rcu sheaves") added a fallback:
when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
per-CPU llist and submitted to call_rcu() later from irq_work.
call_rcu() now handles this itself. When it is called with IRQs disabled
(from NMI, or re-entered from instrumentation), it stages the callback on
a per-CPU list and re-issues it from irq_work. Callers no longer need to
guard against that, so always call call_rcu() directly and remove the
rcu_sheaves llist and its irq_work handling.
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
mm/slub.c | 28 +++-------------------------
1 file changed, 3 insertions(+), 25 deletions(-)
diff --git a/mm/slub.c b/mm/slub.c
index 8fbc6bf8ebc51..6c44f4495db5a 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4145,7 +4145,6 @@ static void flush_all(struct kmem_cache *s)
struct deferred_percpu_work {
struct llist_head objects;
struct llist_head objects_by_rcu;
- struct llist_head rcu_sheaves;
struct irq_work work;
};
@@ -4154,7 +4153,6 @@ static void deferred_percpu_work_fn(struct irq_work *work);
static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
.objects = LLIST_HEAD_INIT(objects),
.objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
- .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
.work = IRQ_WORK_INIT(deferred_percpu_work_fn),
};
@@ -6308,22 +6306,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
* we flush before local_unlock to make sure a racing
* flush_all_rcu_sheaves() doesn't miss this sheaf
*/
- if (rcu_sheaf) {
- /*
- * With !allow_spin, we might have interrupted call_rcu()'s
- * IRQ-disabled critical section. If IRQs are not disabled,
- * we know that's not the case.
- */
- if (unlikely(!allow_spin && irqs_disabled())) {
- struct deferred_percpu_work *dpw;
-
- dpw = this_cpu_ptr(&deferred_percpu_work);
- if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
- irq_work_queue(&dpw->work);
- } else {
- call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
- }
- }
+ if (rcu_sheaf)
+ call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
local_unlock(&s->cpu_sheaves->lock);
@@ -6532,12 +6516,10 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
static void deferred_percpu_work_fn(struct irq_work *work)
{
struct deferred_percpu_work *dpw;
- struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
+ struct llist_head *objs, *objs_by_rcu;
struct llist_node *llnode, *pos, *t;
- struct slab_sheaf *sheaf, *next;
dpw = container_of(work, struct deferred_percpu_work, work);
- rcu_sheaves = &dpw->rcu_sheaves;
objs = &dpw->objects;
objs_by_rcu = &dpw->objects_by_rcu;
@@ -6571,10 +6553,6 @@ static void deferred_percpu_work_fn(struct irq_work *work)
kvfree_call_rcu(head, objp);
}
-
- llnode = llist_del_all(rcu_sheaves);
- llist_for_each_entry_safe(sheaf, next, llnode, llnode)
- call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
}
static void defer_free(struct kmem_cache *s, void *head)
base-commit: e52c7c28dbece00f54dff26e1554df2fe59b02f3
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
@ 2026-10-17 15:54 ` sashiko-bot
2026-10-20 9:23 ` Vlastimil Babka (SUSE)
` (3 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: sashiko-bot @ 2026-10-17 15:54 UTC (permalink / raw)
To: Puranjay Mohan, linux-mm; +Cc: Vlastimil Babka, Harry Yoo, Andrew Morton
Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [High] mm: call_rcu() list corruption due to reentrancy from NMI
- [Low] mm: unused llnode field in struct slab_sheaf
--
commit 81da114bf5411864e59d873df170399156acac6d
Author: Puranjay Mohan <puranjay@kernel.org>
mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
This patch removes the deferred irq_work fallback for rcu_sheaves and
instead calls call_rcu() directly from __kfree_rcu_sheaf() in all contexts.
The commit message states that call_rcu() now natively handles reentrancy
from NMIs.
> diff --git a/mm/slub.c b/mm/slub.c
> index 8fbc6bf8ebc51..6c44f4495db5a 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4145,7 +4145,6 @@ static void flush_all(struct kmem_cache *s)
> struct deferred_percpu_work {
> struct llist_head objects;
> struct llist_head objects_by_rcu;
> - struct llist_head rcu_sheaves;
> struct irq_work work;
> };
[Severity: Low]
This isn't a bug, but since this patch removes the rcu_sheaves list and all
its associated logic, should the llnode field in struct slab_sheaf also be
removed?
It appears the llnode field in struct slab_sheaf was only used to defer
this call_rcu() invocation.
[ ... ]
> @@ -6308,22 +6306,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
> * we flush before local_unlock to make sure a racing
> * flush_all_rcu_sheaves() doesn't miss this sheaf
> */
> - if (rcu_sheaf) {
> - /*
> - * With !allow_spin, we might have interrupted call_rcu()'s
> - * IRQ-disabled critical section. If IRQs are not disabled,
> - * we know that's not the case.
> - */
> - if (unlikely(!allow_spin && irqs_disabled())) {
> - struct deferred_percpu_work *dpw;
> -
> - dpw = this_cpu_ptr(&deferred_percpu_work);
> - if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
> - irq_work_queue(&dpw->work);
> - } else {
> - call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
> - }
> - }
> + if (rcu_sheaf)
> + call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
[Severity: High]
The commit message states: "When it is called with IRQs disabled (from NMI,
or re-entered from instrumentation), it stages the callback on a per-CPU
list and re-issues it from irq_work."
Looking at the implementation of call_rcu() and __call_rcu_common() in
kernel/rcu/tree.c, it disables IRQs and synchronously manipulates the
callback list. It does not appear to contain an NMI/irq_work fallback.
If __kfree_rcu_sheaf() is called from an NMI or tracing context (such as
from bpf or perf) while the CPU is already executing call_rcu() or
rcu_do_batch(), could this direct call to call_rcu() re-enter the list
manipulation and corrupt the RCU callback lists?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261016155340.3304319-1-puranjay@kernel.org?part=1
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
2026-10-17 15:54 ` sashiko-bot
@ 2026-10-20 9:23 ` Vlastimil Babka (SUSE)
2026-10-20 10:06 ` Puranjay Mohan
2026-11-02 15:15 ` Harry Yoo
` (2 subsequent siblings)
4 siblings, 1 reply; 8+ messages in thread
From: Vlastimil Babka (SUSE) @ 2026-10-20 9:23 UTC (permalink / raw)
To: Puranjay Mohan, linux-mm; +Cc: Harry Yoo, Andrew Morton, Hao Li, RCU
On 10/16/26 17:53, Puranjay Mohan wrote:
> __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> extend deferred free mechanism to handle rcu sheaves") added a fallback:
> when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> per-CPU llist and submitted to call_rcu() later from irq_work.
>
> call_rcu() now handles this itself. When it is called with IRQs disabled
That's great! can you reference which commit does that? I assume it will be
merged in this 7.4 merge window?
Thanks,
Vlastimil
> (from NMI, or re-entered from instrumentation), it stages the callback on
> a per-CPU list and re-issues it from irq_work. Callers no longer need to
> guard against that, so always call call_rcu() directly and remove the
> rcu_sheaves llist and its irq_work handling.
>
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---
> mm/slub.c | 28 +++-------------------------
> 1 file changed, 3 insertions(+), 25 deletions(-)
>
> diff --git a/mm/slub.c b/mm/slub.c
> index 8fbc6bf8ebc51..6c44f4495db5a 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4145,7 +4145,6 @@ static void flush_all(struct kmem_cache *s)
> struct deferred_percpu_work {
> struct llist_head objects;
> struct llist_head objects_by_rcu;
> - struct llist_head rcu_sheaves;
> struct irq_work work;
> };
>
> @@ -4154,7 +4153,6 @@ static void deferred_percpu_work_fn(struct irq_work *work);
> static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
> .objects = LLIST_HEAD_INIT(objects),
> .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
> - .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
> .work = IRQ_WORK_INIT(deferred_percpu_work_fn),
> };
>
> @@ -6308,22 +6306,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
> * we flush before local_unlock to make sure a racing
> * flush_all_rcu_sheaves() doesn't miss this sheaf
> */
> - if (rcu_sheaf) {
> - /*
> - * With !allow_spin, we might have interrupted call_rcu()'s
> - * IRQ-disabled critical section. If IRQs are not disabled,
> - * we know that's not the case.
> - */
> - if (unlikely(!allow_spin && irqs_disabled())) {
> - struct deferred_percpu_work *dpw;
> -
> - dpw = this_cpu_ptr(&deferred_percpu_work);
> - if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
> - irq_work_queue(&dpw->work);
> - } else {
> - call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
> - }
> - }
> + if (rcu_sheaf)
> + call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
>
> local_unlock(&s->cpu_sheaves->lock);
>
> @@ -6532,12 +6516,10 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
> static void deferred_percpu_work_fn(struct irq_work *work)
> {
> struct deferred_percpu_work *dpw;
> - struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
> + struct llist_head *objs, *objs_by_rcu;
> struct llist_node *llnode, *pos, *t;
> - struct slab_sheaf *sheaf, *next;
>
> dpw = container_of(work, struct deferred_percpu_work, work);
> - rcu_sheaves = &dpw->rcu_sheaves;
> objs = &dpw->objects;
> objs_by_rcu = &dpw->objects_by_rcu;
>
> @@ -6571,10 +6553,6 @@ static void deferred_percpu_work_fn(struct irq_work *work)
>
> kvfree_call_rcu(head, objp);
> }
> -
> - llnode = llist_del_all(rcu_sheaves);
> - llist_for_each_entry_safe(sheaf, next, llnode, llnode)
> - call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
> }
>
> static void defer_free(struct kmem_cache *s, void *head)
>
> base-commit: e52c7c28dbece00f54dff26e1554df2fe59b02f3
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
2026-10-20 9:23 ` Vlastimil Babka (SUSE)
@ 2026-10-20 10:06 ` Puranjay Mohan
2026-10-20 16:18 ` Paul E. McKenney
0 siblings, 1 reply; 8+ messages in thread
From: Puranjay Mohan @ 2026-10-20 10:06 UTC (permalink / raw)
To: Vlastimil Babka (SUSE); +Cc: linux-mm, Harry Yoo, Andrew Morton, Hao Li, RCU
On Tue, Oct 20, 2026 at 10:23 AM Vlastimil Babka (SUSE)
<vbabka@kernel.org> wrote:
>
> On 10/16/26 17:53, Puranjay Mohan wrote:
> > __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> > kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> > CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> > extend deferred free mechanism to handle rcu sheaves") added a fallback:
> > when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> > per-CPU llist and submitted to call_rcu() later from irq_work.
> >
> > call_rcu() now handles this itself. When it is called with IRQs disabled
>
> That's great! can you reference which commit does that? I assume it will be
> merged in this 7.4 merge window?
It is part of the RCU pull request for 7.4:
https://lore.kernel.org/all/720d335d-17f1-48e7-b5e6-49352712cdca@paulmck-laptop/
Thanks,
Puranjay
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
2026-10-20 10:06 ` Puranjay Mohan
@ 2026-10-20 16:18 ` Paul E. McKenney
0 siblings, 0 replies; 8+ messages in thread
From: Paul E. McKenney @ 2026-10-20 16:18 UTC (permalink / raw)
To: Puranjay Mohan
Cc: Vlastimil Babka (SUSE), linux-mm, Harry Yoo, Andrew Morton,
Hao Li, RCU
On Tue, Oct 20, 2026 at 11:06:23AM +0100, Puranjay Mohan wrote:
> On Tue, Oct 20, 2026 at 10:23 AM Vlastimil Babka (SUSE)
> <vbabka@kernel.org> wrote:
> >
> > On 10/16/26 17:53, Puranjay Mohan wrote:
> > > __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> > > kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> > > CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> > > extend deferred free mechanism to handle rcu sheaves") added a fallback:
> > > when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> > > per-CPU llist and submitted to call_rcu() later from irq_work.
> > >
> > > call_rcu() now handles this itself. When it is called with IRQs disabled
> >
> > That's great! can you reference which commit does that? I assume it will be
> > merged in this 7.4 merge window?
Glad you like it, and thanks to Puranjay for making this happen!
> It is part of the RCU pull request for 7.4:
> https://lore.kernel.org/all/720d335d-17f1-48e7-b5e6-49352712cdca@paulmck-laptop/
More specifically, these guys:
af78eca13b035 rcu: Make call_rcu() safe to call from any context
0ed912d346fdb rcu: Make Tiny call_rcu() safe to call from any context
35a26d87042b7 srcu: Make call_srcu() safe to call from any context
a04b2854bedc7 srcu: Make Tiny call_srcu() safe to call from any context
d23e30d76c32d rcutorture: Disable fragile readers during overload testing
813f2ad270032 rcutorture: Exercise ->call() from NMI context
0464f587c61af selftests/bpf: Add a call_srcu() re-entry reproducer
Thanx, Paul
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
2026-10-17 15:54 ` sashiko-bot
2026-10-20 9:23 ` Vlastimil Babka (SUSE)
@ 2026-11-02 15:15 ` Harry Yoo
2026-11-02 15:31 ` Harry Yoo (Meta)
2026-11-04 8:24 ` Vlastimil Babka (SUSE)
4 siblings, 0 replies; 8+ messages in thread
From: Harry Yoo @ 2026-11-02 15:15 UTC (permalink / raw)
To: Puranjay Mohan; +Cc: linux-mm, Vlastimil Babka, Andrew Morton
On Fri, Oct 16, 2026 at 08:53:39AM -0700, Puranjay Mohan wrote:
> __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> extend deferred free mechanism to handle rcu sheaves") added a fallback:
> when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> per-CPU llist and submitted to call_rcu() later from irq_work.
>
> call_rcu() now handles this itself. When it is called with IRQs disabled
> (from NMI, or re-entered from instrumentation), it stages the callback on
> a per-CPU list and re-issues it from irq_work. Callers no longer need to
> guard against that, so always call call_rcu() directly and remove the
> rcu_sheaves llist and its irq_work handling.
>
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---
Looks good to me,
Reviewed-by: Harry Yoo (Meta) <harry@kernel.org>
Thanks Puranjay for cleaning this up.
--
Cheers,
Harry / Hyeonggon
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
` (2 preceding siblings ...)
2026-11-02 15:15 ` Harry Yoo
@ 2026-11-02 15:31 ` Harry Yoo (Meta)
2026-11-04 8:24 ` Vlastimil Babka (SUSE)
4 siblings, 0 replies; 8+ messages in thread
From: Harry Yoo (Meta) @ 2026-11-02 15:31 UTC (permalink / raw)
To: linux-mm, Puranjay Mohan; +Cc: kernel-team, Vlastimil Babka, Andrew Morton
On Fri, 16 Oct 2026 08:53:39 -0700, Puranjay Mohan wrote:
> __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> extend deferred free mechanism to handle rcu sheaves") added a fallback:
> when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> per-CPU llist and submitted to call_rcu() later from irq_work.
>
> [...]
Applied to mm/slab.git slab/for-next, thanks!
[1/1] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
commit: 55130199fc98d6f5177b4624c44ba47ca44f61c7
I adjusted two things:
1. Mentioned the commits that made call_rcu() safe from any context, as they landed
v7.4-rc1
2. Adjusted a nit sashiko feedback to drop slab_sheaf.llnode field.
range-diff before/after:
1: 81b3beb05dc3 ! 1: 55130199fc98 mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
@@ Commit message
when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
per-CPU llist and submitted to call_rcu() later from irq_work.
- call_rcu() now handles this itself. When it is called with IRQs disabled
- (from NMI, or re-entered from instrumentation), it stages the callback on
- a per-CPU list and re-issues it from irq_work. Callers no longer need to
- guard against that, so always call call_rcu() directly and remove the
- rcu_sheaves llist and its irq_work handling.
+ Since commit af78eca13b03 ("rcu: Make call_rcu() safe to call from any
+ context") and commit 0ed912d346fdb2ec ("rcu: Make Tiny call_rcu() safe
+ to call from any context"), call_rcu() now handles this itself.
+
+ When it is called with IRQs disabled (from NMI, or re-entered from
+ instrumentation), it stages the callback on a per-CPU list and re-issues
+ it from irq_work. Callers no longer need to guard against that, so
+ always call call_rcu() directly and remove the rcu_sheaves llist and
+ its irq_work handling.
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
+ Link: https://patch.msgid.link/20261016155340.3304319-1-puranjay@kernel.org
+ Link: https://sashiko.dev/#/patchset/20261016155340.3304319-1-puranjay%40kernel.org
Reviewed-by: Harry Yoo (Meta) <harry@kernel.org>
+ [ harry@kernel.org: mention the commits that made call_rcu() safe to
+ call from any context and remove unused struct slab_sheaf.llnode field
+ per sashiko feedback ]
+ Signed-off-by: Harry Yoo (Meta) <harry@kernel.org>
## mm/slub.c ##
+@@ mm/slub.c: struct slab_sheaf {
+ union {
+ struct rcu_head rcu_head;
+ struct list_head barn_list;
+- /* only used to defer call_rcu() in unknown context */
+- struct llist_node llnode;
+ /* only used for prefilled sheafs */
+ struct {
+ unsigned int capacity;
@@ mm/slub.c: static void flush_all(struct kmem_cache *s)
struct deferred_percpu_work {
struct llist_head objects;
--
Cheers,
Harry / Hyeonggon
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
` (3 preceding siblings ...)
2026-11-02 15:31 ` Harry Yoo (Meta)
@ 2026-11-04 8:24 ` Vlastimil Babka (SUSE)
4 siblings, 0 replies; 8+ messages in thread
From: Vlastimil Babka (SUSE) @ 2026-11-04 8:24 UTC (permalink / raw)
To: Puranjay Mohan, linux-mm; +Cc: Harry Yoo, Andrew Morton
On 10/16/26 17:53, Puranjay Mohan wrote:
> __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> extend deferred free mechanism to handle rcu sheaves") added a fallback:
> when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> per-CPU llist and submitted to call_rcu() later from irq_work.
>
> call_rcu() now handles this itself. When it is called with IRQs disabled
> (from NMI, or re-entered from instrumentation), it stages the callback on
> a per-CPU list and re-issues it from irq_work. Callers no longer need to
> guard against that, so always call call_rcu() directly and remove the
> rcu_sheaves llist and its irq_work handling.
>
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
Reviewed-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
> ---
> mm/slub.c | 28 +++-------------------------
> 1 file changed, 3 insertions(+), 25 deletions(-)
>
> diff --git a/mm/slub.c b/mm/slub.c
> index 8fbc6bf8ebc51..6c44f4495db5a 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4145,7 +4145,6 @@ static void flush_all(struct kmem_cache *s)
> struct deferred_percpu_work {
> struct llist_head objects;
> struct llist_head objects_by_rcu;
> - struct llist_head rcu_sheaves;
> struct irq_work work;
> };
>
> @@ -4154,7 +4153,6 @@ static void deferred_percpu_work_fn(struct irq_work *work);
> static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
> .objects = LLIST_HEAD_INIT(objects),
> .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
> - .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
> .work = IRQ_WORK_INIT(deferred_percpu_work_fn),
> };
>
> @@ -6308,22 +6306,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
> * we flush before local_unlock to make sure a racing
> * flush_all_rcu_sheaves() doesn't miss this sheaf
> */
> - if (rcu_sheaf) {
> - /*
> - * With !allow_spin, we might have interrupted call_rcu()'s
> - * IRQ-disabled critical section. If IRQs are not disabled,
> - * we know that's not the case.
> - */
> - if (unlikely(!allow_spin && irqs_disabled())) {
> - struct deferred_percpu_work *dpw;
> -
> - dpw = this_cpu_ptr(&deferred_percpu_work);
> - if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
> - irq_work_queue(&dpw->work);
> - } else {
> - call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
> - }
> - }
> + if (rcu_sheaf)
> + call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
>
> local_unlock(&s->cpu_sheaves->lock);
>
> @@ -6532,12 +6516,10 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
> static void deferred_percpu_work_fn(struct irq_work *work)
> {
> struct deferred_percpu_work *dpw;
> - struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
> + struct llist_head *objs, *objs_by_rcu;
> struct llist_node *llnode, *pos, *t;
> - struct slab_sheaf *sheaf, *next;
>
> dpw = container_of(work, struct deferred_percpu_work, work);
> - rcu_sheaves = &dpw->rcu_sheaves;
> objs = &dpw->objects;
> objs_by_rcu = &dpw->objects_by_rcu;
>
> @@ -6571,10 +6553,6 @@ static void deferred_percpu_work_fn(struct irq_work *work)
>
> kvfree_call_rcu(head, objp);
> }
> -
> - llnode = llist_del_all(rcu_sheaves);
> - llist_for_each_entry_safe(sheaf, next, llnode, llnode)
> - call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
> }
>
> static void defer_free(struct kmem_cache *s, void *head)
>
> base-commit: e52c7c28dbece00f54dff26e1554df2fe59b02f3
^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2026-11-04 8:24 UTC | newest]
Thread overview: 8+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
2026-10-17 15:54 ` sashiko-bot
2026-10-20 9:23 ` Vlastimil Babka (SUSE)
2026-10-20 10:06 ` Puranjay Mohan
2026-10-20 16:18 ` Paul E. McKenney
2026-11-02 15:15 ` Harry Yoo
2026-11-02 15:31 ` Harry Yoo (Meta)
2026-11-04 8:24 ` Vlastimil Babka (SUSE)
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.