All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
@ 2026-10-16 15:53 Puranjay Mohan
  2026-10-17 15:54 ` sashiko-bot
                   ` (4 more replies)
  0 siblings, 5 replies; 8+ messages in thread
From: Puranjay Mohan @ 2026-10-16 15:53 UTC (permalink / raw)
  To: linux-mm; +Cc: Puranjay Mohan, Vlastimil Babka, Harry Yoo, Andrew Morton

__kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
extend deferred free mechanism to handle rcu sheaves") added a fallback:
when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
per-CPU llist and submitted to call_rcu() later from irq_work.

call_rcu() now handles this itself. When it is called with IRQs disabled
(from NMI, or re-entered from instrumentation), it stages the callback on
a per-CPU list and re-issues it from irq_work. Callers no longer need to
guard against that, so always call call_rcu() directly and remove the
rcu_sheaves llist and its irq_work handling.

Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
 mm/slub.c | 28 +++-------------------------
 1 file changed, 3 insertions(+), 25 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 8fbc6bf8ebc51..6c44f4495db5a 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4145,7 +4145,6 @@ static void flush_all(struct kmem_cache *s)
 struct deferred_percpu_work {
 	struct llist_head objects;
 	struct llist_head objects_by_rcu;
-	struct llist_head rcu_sheaves;
 	struct irq_work work;
 };
 
@@ -4154,7 +4153,6 @@ static void deferred_percpu_work_fn(struct irq_work *work);
 static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
 	.objects = LLIST_HEAD_INIT(objects),
 	.objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
-	.rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
 	.work = IRQ_WORK_INIT(deferred_percpu_work_fn),
 };
 
@@ -6308,22 +6306,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
 	 * we flush before local_unlock to make sure a racing
 	 * flush_all_rcu_sheaves() doesn't miss this sheaf
 	 */
-	if (rcu_sheaf) {
-		/*
-		 * With !allow_spin, we might have interrupted call_rcu()'s
-		 * IRQ-disabled critical section. If IRQs are not disabled,
-		 * we know that's not the case.
-		 */
-		if (unlikely(!allow_spin && irqs_disabled())) {
-			struct deferred_percpu_work *dpw;
-
-			dpw = this_cpu_ptr(&deferred_percpu_work);
-			if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
-				irq_work_queue(&dpw->work);
-		} else {
-			call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
-		}
-	}
+	if (rcu_sheaf)
+		call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
 
 	local_unlock(&s->cpu_sheaves->lock);
 
@@ -6532,12 +6516,10 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
 static void deferred_percpu_work_fn(struct irq_work *work)
 {
 	struct deferred_percpu_work *dpw;
-	struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
+	struct llist_head *objs, *objs_by_rcu;
 	struct llist_node *llnode, *pos, *t;
-	struct slab_sheaf *sheaf, *next;
 
 	dpw = container_of(work, struct deferred_percpu_work, work);
-	rcu_sheaves = &dpw->rcu_sheaves;
 	objs = &dpw->objects;
 	objs_by_rcu = &dpw->objects_by_rcu;
 
@@ -6571,10 +6553,6 @@ static void deferred_percpu_work_fn(struct irq_work *work)
 
 		kvfree_call_rcu(head, objp);
 	}
-
-	llnode = llist_del_all(rcu_sheaves);
-	llist_for_each_entry_safe(sheaf, next, llnode, llnode)
-		call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
 }
 
 static void defer_free(struct kmem_cache *s, void *head)

base-commit: e52c7c28dbece00f54dff26e1554df2fe59b02f3
-- 
2.53.0-Meta



^ permalink raw reply related	[flat|nested] 8+ messages in thread

* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
  2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
@ 2026-10-17 15:54 ` sashiko-bot
  2026-10-20  9:23 ` Vlastimil Babka (SUSE)
                   ` (3 subsequent siblings)
  4 siblings, 0 replies; 8+ messages in thread
From: sashiko-bot @ 2026-10-17 15:54 UTC (permalink / raw)
  To: Puranjay Mohan, linux-mm; +Cc: Vlastimil Babka, Harry Yoo, Andrew Morton

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [High] mm: call_rcu() list corruption due to reentrancy from NMI
- [Low] mm: unused llnode field in struct slab_sheaf
--

commit 81da114bf5411864e59d873df170399156acac6d
Author: Puranjay Mohan <puranjay@kernel.org>

mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context

This patch removes the deferred irq_work fallback for rcu_sheaves and
instead calls call_rcu() directly from __kfree_rcu_sheaf() in all contexts.
The commit message states that call_rcu() now natively handles reentrancy
from NMIs.

> diff --git a/mm/slub.c b/mm/slub.c
> index 8fbc6bf8ebc51..6c44f4495db5a 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4145,7 +4145,6 @@ static void flush_all(struct kmem_cache *s)
>  struct deferred_percpu_work {
>  	struct llist_head objects;
>  	struct llist_head objects_by_rcu;
> -	struct llist_head rcu_sheaves;
>  	struct irq_work work;
>  };

[Severity: Low]
This isn't a bug, but since this patch removes the rcu_sheaves list and all
its associated logic, should the llnode field in struct slab_sheaf also be
removed?

It appears the llnode field in struct slab_sheaf was only used to defer
this call_rcu() invocation.

[ ... ]

> @@ -6308,22 +6306,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
>  	 * we flush before local_unlock to make sure a racing
>  	 * flush_all_rcu_sheaves() doesn't miss this sheaf
>  	 */
> -	if (rcu_sheaf) {
> -		/*
> -		 * With !allow_spin, we might have interrupted call_rcu()'s
> -		 * IRQ-disabled critical section. If IRQs are not disabled,
> -		 * we know that's not the case.
> -		 */
> -		if (unlikely(!allow_spin && irqs_disabled())) {
> -			struct deferred_percpu_work *dpw;
> -
> -			dpw = this_cpu_ptr(&deferred_percpu_work);
> -			if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
> -				irq_work_queue(&dpw->work);
> -		} else {
> -			call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
> -		}
> -	}
> +	if (rcu_sheaf)
> +		call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);

[Severity: High]
The commit message states: "When it is called with IRQs disabled (from NMI,
or re-entered from instrumentation), it stages the callback on a per-CPU
list and re-issues it from irq_work."

Looking at the implementation of call_rcu() and __call_rcu_common() in
kernel/rcu/tree.c, it disables IRQs and synchronously manipulates the
callback list. It does not appear to contain an NMI/irq_work fallback.

If __kfree_rcu_sheaf() is called from an NMI or tracing context (such as
from bpf or perf) while the CPU is already executing call_rcu() or
rcu_do_batch(), could this direct call to call_rcu() re-enter the list
manipulation and corrupt the RCU callback lists?

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261016155340.3304319-1-puranjay@kernel.org?part=1


^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
  2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
  2026-10-17 15:54 ` sashiko-bot
@ 2026-10-20  9:23 ` Vlastimil Babka (SUSE)
  2026-10-20 10:06   ` Puranjay Mohan
  2026-11-02 15:15 ` Harry Yoo
                   ` (2 subsequent siblings)
  4 siblings, 1 reply; 8+ messages in thread
From: Vlastimil Babka (SUSE) @ 2026-10-20  9:23 UTC (permalink / raw)
  To: Puranjay Mohan, linux-mm; +Cc: Harry Yoo, Andrew Morton, Hao Li, RCU

On 10/16/26 17:53, Puranjay Mohan wrote:
> __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> extend deferred free mechanism to handle rcu sheaves") added a fallback:
> when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> per-CPU llist and submitted to call_rcu() later from irq_work.
> 
> call_rcu() now handles this itself. When it is called with IRQs disabled

That's great! can you reference which commit does that? I assume it will be
merged in this 7.4 merge window?

Thanks,
Vlastimil

> (from NMI, or re-entered from instrumentation), it stages the callback on
> a per-CPU list and re-issues it from irq_work. Callers no longer need to
> guard against that, so always call call_rcu() directly and remove the
> rcu_sheaves llist and its irq_work handling.
> 
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---
>  mm/slub.c | 28 +++-------------------------
>  1 file changed, 3 insertions(+), 25 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 8fbc6bf8ebc51..6c44f4495db5a 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4145,7 +4145,6 @@ static void flush_all(struct kmem_cache *s)
>  struct deferred_percpu_work {
>  	struct llist_head objects;
>  	struct llist_head objects_by_rcu;
> -	struct llist_head rcu_sheaves;
>  	struct irq_work work;
>  };
>  
> @@ -4154,7 +4153,6 @@ static void deferred_percpu_work_fn(struct irq_work *work);
>  static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
>  	.objects = LLIST_HEAD_INIT(objects),
>  	.objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
> -	.rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
>  	.work = IRQ_WORK_INIT(deferred_percpu_work_fn),
>  };
>  
> @@ -6308,22 +6306,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
>  	 * we flush before local_unlock to make sure a racing
>  	 * flush_all_rcu_sheaves() doesn't miss this sheaf
>  	 */
> -	if (rcu_sheaf) {
> -		/*
> -		 * With !allow_spin, we might have interrupted call_rcu()'s
> -		 * IRQ-disabled critical section. If IRQs are not disabled,
> -		 * we know that's not the case.
> -		 */
> -		if (unlikely(!allow_spin && irqs_disabled())) {
> -			struct deferred_percpu_work *dpw;
> -
> -			dpw = this_cpu_ptr(&deferred_percpu_work);
> -			if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
> -				irq_work_queue(&dpw->work);
> -		} else {
> -			call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
> -		}
> -	}
> +	if (rcu_sheaf)
> +		call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
>  
>  	local_unlock(&s->cpu_sheaves->lock);
>  
> @@ -6532,12 +6516,10 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
>  static void deferred_percpu_work_fn(struct irq_work *work)
>  {
>  	struct deferred_percpu_work *dpw;
> -	struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
> +	struct llist_head *objs, *objs_by_rcu;
>  	struct llist_node *llnode, *pos, *t;
> -	struct slab_sheaf *sheaf, *next;
>  
>  	dpw = container_of(work, struct deferred_percpu_work, work);
> -	rcu_sheaves = &dpw->rcu_sheaves;
>  	objs = &dpw->objects;
>  	objs_by_rcu = &dpw->objects_by_rcu;
>  
> @@ -6571,10 +6553,6 @@ static void deferred_percpu_work_fn(struct irq_work *work)
>  
>  		kvfree_call_rcu(head, objp);
>  	}
> -
> -	llnode = llist_del_all(rcu_sheaves);
> -	llist_for_each_entry_safe(sheaf, next, llnode, llnode)
> -		call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
>  }
>  
>  static void defer_free(struct kmem_cache *s, void *head)
> 
> base-commit: e52c7c28dbece00f54dff26e1554df2fe59b02f3



^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
  2026-10-20  9:23 ` Vlastimil Babka (SUSE)
@ 2026-10-20 10:06   ` Puranjay Mohan
  2026-10-20 16:18     ` Paul E. McKenney
  0 siblings, 1 reply; 8+ messages in thread
From: Puranjay Mohan @ 2026-10-20 10:06 UTC (permalink / raw)
  To: Vlastimil Babka (SUSE); +Cc: linux-mm, Harry Yoo, Andrew Morton, Hao Li, RCU

On Tue, Oct 20, 2026 at 10:23 AM Vlastimil Babka (SUSE)
<vbabka@kernel.org> wrote:
>
> On 10/16/26 17:53, Puranjay Mohan wrote:
> > __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> > kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> > CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> > extend deferred free mechanism to handle rcu sheaves") added a fallback:
> > when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> > per-CPU llist and submitted to call_rcu() later from irq_work.
> >
> > call_rcu() now handles this itself. When it is called with IRQs disabled
>
> That's great! can you reference which commit does that? I assume it will be
> merged in this 7.4 merge window?

It is part of the RCU pull request for 7.4:
https://lore.kernel.org/all/720d335d-17f1-48e7-b5e6-49352712cdca@paulmck-laptop/

Thanks,
Puranjay


^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
  2026-10-20 10:06   ` Puranjay Mohan
@ 2026-10-20 16:18     ` Paul E. McKenney
  0 siblings, 0 replies; 8+ messages in thread
From: Paul E. McKenney @ 2026-10-20 16:18 UTC (permalink / raw)
  To: Puranjay Mohan
  Cc: Vlastimil Babka (SUSE), linux-mm, Harry Yoo, Andrew Morton,
	Hao Li, RCU

On Tue, Oct 20, 2026 at 11:06:23AM +0100, Puranjay Mohan wrote:
> On Tue, Oct 20, 2026 at 10:23 AM Vlastimil Babka (SUSE)
> <vbabka@kernel.org> wrote:
> >
> > On 10/16/26 17:53, Puranjay Mohan wrote:
> > > __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> > > kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> > > CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> > > extend deferred free mechanism to handle rcu sheaves") added a fallback:
> > > when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> > > per-CPU llist and submitted to call_rcu() later from irq_work.
> > >
> > > call_rcu() now handles this itself. When it is called with IRQs disabled
> >
> > That's great! can you reference which commit does that? I assume it will be
> > merged in this 7.4 merge window?

Glad you like it, and thanks to Puranjay for making this happen!

> It is part of the RCU pull request for 7.4:
> https://lore.kernel.org/all/720d335d-17f1-48e7-b5e6-49352712cdca@paulmck-laptop/

More specifically, these guys:

af78eca13b035 rcu: Make call_rcu() safe to call from any context
0ed912d346fdb rcu: Make Tiny call_rcu() safe to call from any context
35a26d87042b7 srcu: Make call_srcu() safe to call from any context
a04b2854bedc7 srcu: Make Tiny call_srcu() safe to call from any context
d23e30d76c32d rcutorture:  Disable fragile readers during overload testing
813f2ad270032 rcutorture: Exercise ->call() from NMI context
0464f587c61af selftests/bpf: Add a call_srcu() re-entry reproducer

							Thanx, Paul


^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
  2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
  2026-10-17 15:54 ` sashiko-bot
  2026-10-20  9:23 ` Vlastimil Babka (SUSE)
@ 2026-11-02 15:15 ` Harry Yoo
  2026-11-02 15:31 ` Harry Yoo (Meta)
  2026-11-04  8:24 ` Vlastimil Babka (SUSE)
  4 siblings, 0 replies; 8+ messages in thread
From: Harry Yoo @ 2026-11-02 15:15 UTC (permalink / raw)
  To: Puranjay Mohan; +Cc: linux-mm, Vlastimil Babka, Andrew Morton

On Fri, Oct 16, 2026 at 08:53:39AM -0700, Puranjay Mohan wrote:
> __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> extend deferred free mechanism to handle rcu sheaves") added a fallback:
> when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> per-CPU llist and submitted to call_rcu() later from irq_work.
> 
> call_rcu() now handles this itself. When it is called with IRQs disabled
> (from NMI, or re-entered from instrumentation), it stages the callback on
> a per-CPU list and re-issues it from irq_work. Callers no longer need to
> guard against that, so always call call_rcu() directly and remove the
> rcu_sheaves llist and its irq_work handling.
> 
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---

Looks good to me,
Reviewed-by: Harry Yoo (Meta) <harry@kernel.org>

Thanks Puranjay for cleaning this up.

-- 
Cheers,
Harry / Hyeonggon


^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
  2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
                   ` (2 preceding siblings ...)
  2026-11-02 15:15 ` Harry Yoo
@ 2026-11-02 15:31 ` Harry Yoo (Meta)
  2026-11-04  8:24 ` Vlastimil Babka (SUSE)
  4 siblings, 0 replies; 8+ messages in thread
From: Harry Yoo (Meta) @ 2026-11-02 15:31 UTC (permalink / raw)
  To: linux-mm, Puranjay Mohan; +Cc: kernel-team, Vlastimil Babka, Andrew Morton

On Fri, 16 Oct 2026 08:53:39 -0700, Puranjay Mohan wrote:
> __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> extend deferred free mechanism to handle rcu sheaves") added a fallback:
> when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> per-CPU llist and submitted to call_rcu() later from irq_work.
> 
> [...]

Applied to mm/slab.git slab/for-next, thanks!

[1/1] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
      commit: 55130199fc98d6f5177b4624c44ba47ca44f61c7

I adjusted two things:

1. Mentioned the commits that made call_rcu() safe from any context, as they landed
   v7.4-rc1
2. Adjusted a nit sashiko feedback to drop slab_sheaf.llnode field.

range-diff before/after:

1:  81b3beb05dc3 ! 1:  55130199fc98 mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
    @@ Commit message
         when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
         per-CPU llist and submitted to call_rcu() later from irq_work.
     
    -    call_rcu() now handles this itself. When it is called with IRQs disabled
    -    (from NMI, or re-entered from instrumentation), it stages the callback on
    -    a per-CPU list and re-issues it from irq_work. Callers no longer need to
    -    guard against that, so always call call_rcu() directly and remove the
    -    rcu_sheaves llist and its irq_work handling.
    +    Since commit af78eca13b03 ("rcu: Make call_rcu() safe to call from any
    +    context") and commit 0ed912d346fdb2ec ("rcu: Make Tiny call_rcu() safe
    +    to call from any context"), call_rcu() now handles this itself.
    +
    +    When it is called with IRQs disabled (from NMI, or re-entered from
    +    instrumentation), it stages the callback on a per-CPU list and re-issues
    +    it from irq_work. Callers no longer need to guard against that, so
    +    always call call_rcu() directly and remove the rcu_sheaves llist and
    +    its irq_work handling.
     
         Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
    +    Link: https://patch.msgid.link/20261016155340.3304319-1-puranjay@kernel.org
    +    Link: https://sashiko.dev/#/patchset/20261016155340.3304319-1-puranjay%40kernel.org
         Reviewed-by: Harry Yoo (Meta) <harry@kernel.org>
    +    [ harry@kernel.org: mention the commits that made call_rcu() safe to
    +      call from any context and remove unused struct slab_sheaf.llnode field
    +      per sashiko feedback ]
    +    Signed-off-by: Harry Yoo (Meta) <harry@kernel.org>
     
      ## mm/slub.c ##
    +@@ mm/slub.c: struct slab_sheaf {
    + 	union {
    + 		struct rcu_head rcu_head;
    + 		struct list_head barn_list;
    +-		/* only used to defer call_rcu() in unknown context */
    +-		struct llist_node llnode;
    + 		/* only used for prefilled sheafs */
    + 		struct {
    + 			unsigned int capacity;
     @@ mm/slub.c: static void flush_all(struct kmem_cache *s)
      struct deferred_percpu_work {
      	struct llist_head objects;


-- 
Cheers,
Harry / Hyeonggon



^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context
  2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
                   ` (3 preceding siblings ...)
  2026-11-02 15:31 ` Harry Yoo (Meta)
@ 2026-11-04  8:24 ` Vlastimil Babka (SUSE)
  4 siblings, 0 replies; 8+ messages in thread
From: Vlastimil Babka (SUSE) @ 2026-11-04  8:24 UTC (permalink / raw)
  To: Puranjay Mohan, linux-mm; +Cc: Harry Yoo, Andrew Morton

On 10/16/26 17:53, Puranjay Mohan wrote:
> __kfree_rcu_sheaf() can run with allow_spin == false, e.g. from
> kfree_rcu_nolock(), where it may interrupt call_rcu() itself on the same
> CPU. call_rcu() was not reentrant, so commit 69abda97a09b2 ("mm/slab:
> extend deferred free mechanism to handle rcu sheaves") added a fallback:
> when !allow_spin and IRQs are disabled, the full rcu sheaf is put on a
> per-CPU llist and submitted to call_rcu() later from irq_work.
> 
> call_rcu() now handles this itself. When it is called with IRQs disabled
> (from NMI, or re-entered from instrumentation), it stages the callback on
> a per-CPU list and re-issues it from irq_work. Callers no longer need to
> guard against that, so always call call_rcu() directly and remove the
> rcu_sheaves llist and its irq_work handling.
> 
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>

Reviewed-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>

> ---
>  mm/slub.c | 28 +++-------------------------
>  1 file changed, 3 insertions(+), 25 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 8fbc6bf8ebc51..6c44f4495db5a 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -4145,7 +4145,6 @@ static void flush_all(struct kmem_cache *s)
>  struct deferred_percpu_work {
>  	struct llist_head objects;
>  	struct llist_head objects_by_rcu;
> -	struct llist_head rcu_sheaves;
>  	struct irq_work work;
>  };
>  
> @@ -4154,7 +4153,6 @@ static void deferred_percpu_work_fn(struct irq_work *work);
>  static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
>  	.objects = LLIST_HEAD_INIT(objects),
>  	.objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
> -	.rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
>  	.work = IRQ_WORK_INIT(deferred_percpu_work_fn),
>  };
>  
> @@ -6308,22 +6306,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
>  	 * we flush before local_unlock to make sure a racing
>  	 * flush_all_rcu_sheaves() doesn't miss this sheaf
>  	 */
> -	if (rcu_sheaf) {
> -		/*
> -		 * With !allow_spin, we might have interrupted call_rcu()'s
> -		 * IRQ-disabled critical section. If IRQs are not disabled,
> -		 * we know that's not the case.
> -		 */
> -		if (unlikely(!allow_spin && irqs_disabled())) {
> -			struct deferred_percpu_work *dpw;
> -
> -			dpw = this_cpu_ptr(&deferred_percpu_work);
> -			if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
> -				irq_work_queue(&dpw->work);
> -		} else {
> -			call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
> -		}
> -	}
> +	if (rcu_sheaf)
> +		call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
>  
>  	local_unlock(&s->cpu_sheaves->lock);
>  
> @@ -6532,12 +6516,10 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
>  static void deferred_percpu_work_fn(struct irq_work *work)
>  {
>  	struct deferred_percpu_work *dpw;
> -	struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
> +	struct llist_head *objs, *objs_by_rcu;
>  	struct llist_node *llnode, *pos, *t;
> -	struct slab_sheaf *sheaf, *next;
>  
>  	dpw = container_of(work, struct deferred_percpu_work, work);
> -	rcu_sheaves = &dpw->rcu_sheaves;
>  	objs = &dpw->objects;
>  	objs_by_rcu = &dpw->objects_by_rcu;
>  
> @@ -6571,10 +6553,6 @@ static void deferred_percpu_work_fn(struct irq_work *work)
>  
>  		kvfree_call_rcu(head, objp);
>  	}
> -
> -	llnode = llist_del_all(rcu_sheaves);
> -	llist_for_each_entry_safe(sheaf, next, llnode, llnode)
> -		call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
>  }
>  
>  static void defer_free(struct kmem_cache *s, void *head)
> 
> base-commit: e52c7c28dbece00f54dff26e1554df2fe59b02f3



^ permalink raw reply	[flat|nested] 8+ messages in thread

end of thread, other threads:[~2026-11-04  8:24 UTC | newest]

Thread overview: 8+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-10-16 15:53 [PATCH] mm/slab: Call call_rcu() directly from __kfree_rcu_sheaf() in any context Puranjay Mohan
2026-10-17 15:54 ` sashiko-bot
2026-10-20  9:23 ` Vlastimil Babka (SUSE)
2026-10-20 10:06   ` Puranjay Mohan
2026-10-20 16:18     ` Paul E. McKenney
2026-11-02 15:15 ` Harry Yoo
2026-11-02 15:31 ` Harry Yoo (Meta)
2026-11-04  8:24 ` Vlastimil Babka (SUSE)

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.