[PATCH bpf-next v2 0/7] bpf: introduce and use rcu_read_lock_dont_migrate

[PATCH bpf-next v2 0/7] bpf: introduce and use rcu_read_lock_dont_migrate

[Menglong Dong]

migrate_disable() and rcu_read_lock() are used to together in many case in
bpf. However, when PREEMPT_RCU is not enabled, rcu_read_lock() will
disable preemption, which indicate migrate_disable(), so we don't need to
call it in this case.

In this series, we introduce rcu_read_lock_dont_migrate and
rcu_read_unlock_migrate, which will call migrate_disable and
migrate_enable only when PREEMPT_RCU enabled. And use
rcu_read_lock_dont_migrate in bpf subsystem.

Not sure if I should send the 1st patch to the RCU subsystem :/

Changes since V1:
* introduce rcu_read_lock_dont_migrate() instead of
  rcu_migrate_disable() + rcu_read_lock()

Menglong Dong (7):
  rcu: add rcu_read_lock_dont_migrate()
  bpf: use rcu_read_lock_dont_migrate() for bpf_cgrp_storage_free()
  bpf: use rcu_read_lock_dont_migrate() for bpf_inode_storage_free()
  bpf: use rcu_read_lock_dont_migrate() for bpf_iter_run_prog()
  bpf: use rcu_read_lock_dont_migrate() for bpf_task_storage_free()
  bpf: use rcu_read_lock_dont_migrate() for bpf_prog_run_array_cg()
  bpf: use rcu_read_lock_dont_migrate() for trampoline.c

 include/linux/rcupdate.h       | 24 ++++++++++++++++++++++++
 kernel/bpf/bpf_cgrp_storage.c  |  6 ++----
 kernel/bpf/bpf_inode_storage.c |  6 ++----
 kernel/bpf/bpf_iter.c          |  6 ++----
 kernel/bpf/bpf_task_storage.c  |  6 ++----
 kernel/bpf/cgroup.c            |  6 ++----
 kernel/bpf/trampoline.c        | 18 ++++++------------
 7 files changed, 40 insertions(+), 32 deletions(-)

-- 
2.50.1

[PATCH bpf-next v2 1/7] rcu: add rcu_read_lock_dont_migrate()

[Menglong Dong]

migrate_disable() is called to disable migration in the kernel, and it is
often used together with rcu_read_lock().

However, with PREEMPT_RCU disabled, it's unnecessary, as rcu_read_lock()
will always disable preemption, which will also disable migration.

Introduce rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate(),
which will do the migration enable and disable only when !PREEMPT_RCU.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
v2:
- introduce rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
---
 include/linux/rcupdate.h | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 120536f4c6eb..8918b911911f 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -962,6 +962,30 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
 	preempt_enable_notrace();
 }
 
+#ifdef CONFIG_PREEMPT_RCU
+static __always_inline void rcu_read_lock_dont_migrate(void)
+{
+	migrate_disable();
+	rcu_read_lock();
+}
+
+static inline void rcu_read_unlock_migrate(void)
+{
+	rcu_read_unlock();
+	migrate_enable();
+}
+#else
+static __always_inline void rcu_read_lock_dont_migrate(void)
+{
+	rcu_read_lock();
+}
+
+static inline void rcu_read_unlock_migrate(void)
+{
+	rcu_read_unlock();
+}
+#endif
+
 /**
  * RCU_INIT_POINTER() - initialize an RCU protected pointer
  * @p: The pointer to be initialized.
-- 
2.50.1

[PATCH bpf-next v2 2/7] bpf: use rcu_read_lock_dont_migrate() for bpf_cgrp_storage_free()

[Menglong Dong]

Use rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate() in
bpf_cgrp_storage_free to obtain better performance when PREEMPT_RCU is
not enabled.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
v2:
- use rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
---
 kernel/bpf/bpf_cgrp_storage.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/kernel/bpf/bpf_cgrp_storage.c b/kernel/bpf/bpf_cgrp_storage.c
index 148da8f7ff36..0687a760974a 100644
--- a/kernel/bpf/bpf_cgrp_storage.c
+++ b/kernel/bpf/bpf_cgrp_storage.c
@@ -45,8 +45,7 @@ void bpf_cgrp_storage_free(struct cgroup *cgroup)
 {
 	struct bpf_local_storage *local_storage;
 
-	migrate_disable();
-	rcu_read_lock();
+	rcu_read_lock_dont_migrate();
 	local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
 	if (!local_storage)
 		goto out;
@@ -55,8 +54,7 @@ void bpf_cgrp_storage_free(struct cgroup *cgroup)
 	bpf_local_storage_destroy(local_storage);
 	bpf_cgrp_storage_unlock();
 out:
-	rcu_read_unlock();
-	migrate_enable();
+	rcu_read_unlock_migrate();
 }
 
 static struct bpf_local_storage_data *
-- 
2.50.1

[PATCH bpf-next v2 3/7] bpf: use rcu_read_lock_dont_migrate() for bpf_inode_storage_free()

[Menglong Dong]

Use rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate() in
bpf_inode_storage_free to obtain better performance when PREEMPT_RCU is
not enabled.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
v2:
- use rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
---
 kernel/bpf/bpf_inode_storage.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c
index 15a3eb9b02d9..e54cce2b9175 100644
--- a/kernel/bpf/bpf_inode_storage.c
+++ b/kernel/bpf/bpf_inode_storage.c
@@ -62,8 +62,7 @@ void bpf_inode_storage_free(struct inode *inode)
 	if (!bsb)
 		return;
 
-	migrate_disable();
-	rcu_read_lock();
+	rcu_read_lock_dont_migrate();
 
 	local_storage = rcu_dereference(bsb->storage);
 	if (!local_storage)
@@ -71,8 +70,7 @@ void bpf_inode_storage_free(struct inode *inode)
 
 	bpf_local_storage_destroy(local_storage);
 out:
-	rcu_read_unlock();
-	migrate_enable();
+	rcu_read_unlock_migrate();
 }
 
 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
-- 
2.50.1

[PATCH bpf-next v2 4/7] bpf: use rcu_read_lock_dont_migrate() for bpf_iter_run_prog()

[Menglong Dong]

Use rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate() in
bpf_iter_run_prog to obtain better performance when PREEMPT_RCU is
not enabled.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
v2:
- use rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
---
 kernel/bpf/bpf_iter.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c
index 0cbcae727079..6ac35430c573 100644
--- a/kernel/bpf/bpf_iter.c
+++ b/kernel/bpf/bpf_iter.c
@@ -705,13 +705,11 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 		migrate_enable();
 		rcu_read_unlock_trace();
 	} else {
-		rcu_read_lock();
-		migrate_disable();
+		rcu_read_lock_dont_migrate();
 		old_run_ctx = bpf_set_run_ctx(&run_ctx);
 		ret = bpf_prog_run(prog, ctx);
 		bpf_reset_run_ctx(old_run_ctx);
-		migrate_enable();
-		rcu_read_unlock();
+		rcu_read_unlock_migrate();
 	}
 
 	/* bpf program can only return 0 or 1:
-- 
2.50.1

[PATCH bpf-next v2 5/7] bpf: use rcu_read_lock_dont_migrate() for bpf_task_storage_free()

[Menglong Dong]

Use rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate() in
bpf_task_storage_free to obtain better performance when PREEMPT_RCU is
not enabled.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
v2:
- use rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
---
 kernel/bpf/bpf_task_storage.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c
index 1109475953c0..a1dc1bf0848a 100644
--- a/kernel/bpf/bpf_task_storage.c
+++ b/kernel/bpf/bpf_task_storage.c
@@ -70,8 +70,7 @@ void bpf_task_storage_free(struct task_struct *task)
 {
 	struct bpf_local_storage *local_storage;
 
-	migrate_disable();
-	rcu_read_lock();
+	rcu_read_lock_dont_migrate();
 
 	local_storage = rcu_dereference(task->bpf_storage);
 	if (!local_storage)
@@ -81,8 +80,7 @@ void bpf_task_storage_free(struct task_struct *task)
 	bpf_local_storage_destroy(local_storage);
 	bpf_task_storage_unlock();
 out:
-	rcu_read_unlock();
-	migrate_enable();
+	rcu_read_unlock_migrate();
 }
 
 static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)
-- 
2.50.1

[PATCH bpf-next v2 6/7] bpf: use rcu_read_lock_dont_migrate() for bpf_prog_run_array_cg()

[Menglong Dong]

Use rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate() in
bpf_prog_run_array_cg to obtain better performance when PREEMPT_RCU is
not enabled.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
v2:
- use rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
---
 kernel/bpf/cgroup.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 180b630279b9..9912c7b9a266 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -71,8 +71,7 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,
 	u32 func_ret;
 
 	run_ctx.retval = retval;
-	migrate_disable();
-	rcu_read_lock();
+	rcu_read_lock_dont_migrate();
 	array = rcu_dereference(cgrp->effective[atype]);
 	item = &array->items[0];
 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
@@ -88,8 +87,7 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,
 		item++;
 	}
 	bpf_reset_run_ctx(old_run_ctx);
-	rcu_read_unlock();
-	migrate_enable();
+	rcu_read_unlock_migrate();
 	return run_ctx.retval;
 }
 
-- 
2.50.1

[PATCH bpf-next v2 7/7] bpf: use rcu_read_lock_dont_migrate() for trampoline.c

[Menglong Dong]

Use rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate() in
trampoline.c to obtain better performance when PREEMPT_RCU is not enabled.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
v2:
- use rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
---
 kernel/bpf/trampoline.c | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 0e364614c3a2..5949095e51c3 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -899,8 +899,7 @@ static __always_inline u64 notrace bpf_prog_start_time(void)
 static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
 	__acquires(RCU)
 {
-	rcu_read_lock();
-	migrate_disable();
+	rcu_read_lock_dont_migrate();
 
 	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
 
@@ -949,8 +948,7 @@ static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,
 
 	update_prog_stats(prog, start);
 	this_cpu_dec(*(prog->active));
-	migrate_enable();
-	rcu_read_unlock();
+	rcu_read_unlock_migrate();
 }
 
 static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
@@ -960,8 +958,7 @@ static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
 	/* Runtime stats are exported via actual BPF_LSM_CGROUP
 	 * programs, not the shims.
 	 */
-	rcu_read_lock();
-	migrate_disable();
+	rcu_read_lock_dont_migrate();
 
 	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
 
@@ -974,8 +971,7 @@ static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
 {
 	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
 
-	migrate_enable();
-	rcu_read_unlock();
+	rcu_read_unlock_migrate();
 }
 
 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
@@ -1033,8 +1029,7 @@ static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,
 				    struct bpf_tramp_run_ctx *run_ctx)
 	__acquires(RCU)
 {
-	rcu_read_lock();
-	migrate_disable();
+	rcu_read_lock_dont_migrate();
 
 	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
 
@@ -1048,8 +1043,7 @@ static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,
 	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
 
 	update_prog_stats(prog, start);
-	migrate_enable();
-	rcu_read_unlock();
+	rcu_read_unlock_migrate();
 }
 
 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)
-- 
2.50.1

Re: [PATCH bpf-next v2 1/7] rcu: add rcu_read_lock_dont_migrate()

[Paul E. McKenney]

On Tue, Aug 19, 2025 at 05:34:18PM +0800, Menglong Dong wrote:
> migrate_disable() is called to disable migration in the kernel, and it is
> often used together with rcu_read_lock().
> 
> However, with PREEMPT_RCU disabled, it's unnecessary, as rcu_read_lock()
> will always disable preemption, which will also disable migration.
> 
> Introduce rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate(),
> which will do the migration enable and disable only when !PREEMPT_RCU.
> 
> Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>

This works, but could be made much more compact with no performance
degradation.  Please see below.

						Thanx, Paul

> ---
> v2:
> - introduce rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
> ---
>  include/linux/rcupdate.h | 24 ++++++++++++++++++++++++
>  1 file changed, 24 insertions(+)
> 
> diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
> index 120536f4c6eb..8918b911911f 100644
> --- a/include/linux/rcupdate.h
> +++ b/include/linux/rcupdate.h
> @@ -962,6 +962,30 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
>  	preempt_enable_notrace();
>  }
>  
> +#ifdef CONFIG_PREEMPT_RCU
> +static __always_inline void rcu_read_lock_dont_migrate(void)
> +{

Why not use IS_ENABLED(CONFIG_PREEMPT_RCU) to collapse the two sets of
definitions together?

> +	migrate_disable();
> +	rcu_read_lock();
> +}
> +
> +static inline void rcu_read_unlock_migrate(void)
> +{
> +	rcu_read_unlock();
> +	migrate_enable();
> +}
> +#else
> +static __always_inline void rcu_read_lock_dont_migrate(void)
> +{
> +	rcu_read_lock();
> +}
> +
> +static inline void rcu_read_unlock_migrate(void)
> +{
> +	rcu_read_unlock();
> +}
> +#endif
> +
>  /**
>   * RCU_INIT_POINTER() - initialize an RCU protected pointer
>   * @p: The pointer to be initialized.
> -- 
> 2.50.1
> 

Re: [PATCH bpf-next v2 1/7] rcu: add rcu_read_lock_dont_migrate()

[Menglong Dong]

On Tue, Aug 19, 2025 at 10:58 PM Paul E. McKenney <paulmck@kernel.org> wrote:
>
> On Tue, Aug 19, 2025 at 05:34:18PM +0800, Menglong Dong wrote:
> > migrate_disable() is called to disable migration in the kernel, and it is
> > often used together with rcu_read_lock().
> >
> > However, with PREEMPT_RCU disabled, it's unnecessary, as rcu_read_lock()
> > will always disable preemption, which will also disable migration.
> >
> > Introduce rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate(),
> > which will do the migration enable and disable only when !PREEMPT_RCU.
> >
> > Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
>
> This works, but could be made much more compact with no performance
> degradation.  Please see below.
>
>                                                 Thanx, Paul
>
> > ---
> > v2:
> > - introduce rcu_read_lock_dont_migrate() instead of rcu_migrate_disable()
> > ---
> >  include/linux/rcupdate.h | 24 ++++++++++++++++++++++++
> >  1 file changed, 24 insertions(+)
> >
> > diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
> > index 120536f4c6eb..8918b911911f 100644
> > --- a/include/linux/rcupdate.h
> > +++ b/include/linux/rcupdate.h
> > @@ -962,6 +962,30 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
> >       preempt_enable_notrace();
> >  }
> >
> > +#ifdef CONFIG_PREEMPT_RCU
> > +static __always_inline void rcu_read_lock_dont_migrate(void)
> > +{
>
> Why not use IS_ENABLED(CONFIG_PREEMPT_RCU) to collapse the two sets of
> definitions together?

Yeah, that's a good idea, which makes the code much simpler.

Thanks!
Menglong Dong

>
> > +     migrate_disable();
> > +     rcu_read_lock();
> > +}
> > +
> > +static inline void rcu_read_unlock_migrate(void)
> > +{
> > +     rcu_read_unlock();
> > +     migrate_enable();
> > +}
> > +#else
> > +static __always_inline void rcu_read_lock_dont_migrate(void)
> > +{
> > +     rcu_read_lock();
> > +}
> > +
> > +static inline void rcu_read_unlock_migrate(void)
> > +{
> > +     rcu_read_unlock();
> > +}
> > +#endif
> > +
> >  /**
> >   * RCU_INIT_POINTER() - initialize an RCU protected pointer
> >   * @p: The pointer to be initialized.
> > --
> > 2.50.1
> >