[PATCH bpf-next 0/4] bpf: add and use migrate_enable_rcu

[PATCH bpf-next 0/4] bpf: add and use migrate_enable_rcu

[Menglong Dong]

For now, we can use rcu_read_lock_dont_migrate() for the case that call
both rcu_read_lock() and migrate_disable(). However, sometimes they can be
called separately.

Therefore, we factor out migrate_enable_rcu and migrate_disable_rcu from
rcu_read_lock_dont_migrate and rcu_read_unlock_migrate.

And we introduce the function bpf_prog_run_pin_on_cpu_rcu(), which is
similar to bpf_prog_run_pin_on_cpu() but use
migrate_disable_rcu/migrate_enable_rcu instead.

The function bpf_prog_run_pin_on_cpu_rcu() is used in following functions:

  sk_psock_msg_verdict
  sk_psock_tls_strp_read
  sk_psock_strp_read
  sk_psock_strp_parse
  sk_psock_verdict_recv
  bpf_prog_run_clear_cb

Menglong Dong (4):
  rcu: factor out migrate_enable_rcu and migrate_disable_rcu
  bpf: introduce bpf_prog_run_pin_on_cpu_rcu()
  bpf: use bpf_prog_run_pin_on_cpu_rcu() in skmsg.c
  bpf: use bpf_prog_run_pin_on_cpu_rcu() in bpf_prog_run_clear_cb

 include/linux/filter.h   | 16 +++++++++++++++-
 include/linux/rcupdate.h | 18 +++++++++++++++---
 net/core/skmsg.c         | 10 +++++-----
 3 files changed, 35 insertions(+), 9 deletions(-)

-- 
2.51.0

[PATCH bpf-next 1/4] rcu: factor out migrate_enable_rcu and migrate_disable_rcu

[Menglong Dong]

Factor out migrate_enable_rcu/migrate_disable_rcu from
rcu_read_lock_dont_migrate/rcu_read_unlock_migrate.

These functions will be used in the following patches.

It's a little weird to define them in rcupdate.h. Maybe we should move
them to sched.h?

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
 include/linux/rcupdate.h | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index c5b30054cd01..43626ccc07e2 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -988,18 +988,32 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
 	preempt_enable_notrace();
 }
 
-static __always_inline void rcu_read_lock_dont_migrate(void)
+/* This can only be used with rcu_read_lock held */
+static inline void migrate_enable_rcu(void)
+{
+	WARN_ON_ONCE(!rcu_read_lock_held());
+	if (IS_ENABLED(CONFIG_PREEMPT_RCU))
+		migrate_enable();
+}
+
+/* This can only be used with rcu_read_lock held */
+static inline void migrate_disable_rcu(void)
 {
+	WARN_ON_ONCE(!rcu_read_lock_held());
 	if (IS_ENABLED(CONFIG_PREEMPT_RCU))
 		migrate_disable();
+}
+
+static __always_inline void rcu_read_lock_dont_migrate(void)
+{
 	rcu_read_lock();
+	migrate_disable_rcu();
 }
 
 static inline void rcu_read_unlock_migrate(void)
 {
+	migrate_enable_rcu();
 	rcu_read_unlock();
-	if (IS_ENABLED(CONFIG_PREEMPT_RCU))
-		migrate_enable();
 }
 
 /**
-- 
2.51.0

[PATCH bpf-next 2/4] bpf: introduce bpf_prog_run_pin_on_cpu_rcu()

[Menglong Dong]

Introduce bpf_prog_run_pin_on_cpu_rcu(), which will be called with
rcu_read_lock. migrate_disable_rcu and migrate_enable_rcu are used to get
better performance when CONFIG_PREEMPT_RCU is not enabled.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
 include/linux/filter.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/include/linux/filter.h b/include/linux/filter.h
index f5c859b8131a..48eb42358543 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -747,6 +747,18 @@ static inline u32 bpf_prog_run_pin_on_cpu(const struct bpf_prog *prog,
 	return ret;
 }
 
+/* The same as bpf_prog_run_pin_on_cpu, except rcu_read_lock should be held */
+static inline u32 bpf_prog_run_pin_on_cpu_rcu(const struct bpf_prog *prog,
+					      const void *ctx)
+{
+	u32 ret;
+
+	migrate_disable_rcu();
+	ret = bpf_prog_run(prog, ctx);
+	migrate_enable_rcu();
+	return ret;
+}
+
 #define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
 
 struct bpf_skb_data_end {
-- 
2.51.0

[PATCH bpf-next 3/4] bpf: use bpf_prog_run_pin_on_cpu_rcu() in skmsg.c

[Menglong Dong]

Replace bpf_prog_run_pin_on_cpu() with bpf_prog_run_pin_on_cpu_rcu() in
following functions to obtain better performance:

  sk_psock_msg_verdict
  sk_psock_tls_strp_read
  sk_psock_strp_read
  sk_psock_strp_parse
  sk_psock_verdict_recv

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
 net/core/skmsg.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/net/core/skmsg.c b/net/core/skmsg.c
index 2ac7731e1e0a..1d3f47b07659 100644
--- a/net/core/skmsg.c
+++ b/net/core/skmsg.c
@@ -908,7 +908,7 @@ int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
 
 	sk_msg_compute_data_pointers(msg);
 	msg->sk = sk;
-	ret = bpf_prog_run_pin_on_cpu(prog, msg);
+	ret = bpf_prog_run_pin_on_cpu_rcu(prog, msg);
 	ret = sk_psock_map_verd(ret, msg->sk_redir);
 	psock->apply_bytes = msg->apply_bytes;
 	if (ret == __SK_REDIRECT) {
@@ -993,7 +993,7 @@ int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)
 		skb->sk = psock->sk;
 		skb_dst_drop(skb);
 		skb_bpf_redirect_clear(skb);
-		ret = bpf_prog_run_pin_on_cpu(prog, skb);
+		ret = bpf_prog_run_pin_on_cpu_rcu(prog, skb);
 		ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
 		skb->sk = NULL;
 	}
@@ -1101,7 +1101,7 @@ static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb)
 		skb->sk = sk;
 		skb_dst_drop(skb);
 		skb_bpf_redirect_clear(skb);
-		ret = bpf_prog_run_pin_on_cpu(prog, skb);
+		ret = bpf_prog_run_pin_on_cpu_rcu(prog, skb);
 		skb_bpf_set_strparser(skb);
 		ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
 		skb->sk = NULL;
@@ -1126,7 +1126,7 @@ static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb)
 	prog = READ_ONCE(psock->progs.stream_parser);
 	if (likely(prog)) {
 		skb->sk = psock->sk;
-		ret = bpf_prog_run_pin_on_cpu(prog, skb);
+		ret = bpf_prog_run_pin_on_cpu_rcu(prog, skb);
 		skb->sk = NULL;
 	}
 	rcu_read_unlock();
@@ -1230,7 +1230,7 @@ static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb)
 	if (likely(prog)) {
 		skb_dst_drop(skb);
 		skb_bpf_redirect_clear(skb);
-		ret = bpf_prog_run_pin_on_cpu(prog, skb);
+		ret = bpf_prog_run_pin_on_cpu_rcu(prog, skb);
 		ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
 	}
 	ret = sk_psock_verdict_apply(psock, skb, ret);
-- 
2.51.0

[PATCH bpf-next 4/4] bpf: use bpf_prog_run_pin_on_cpu_rcu() in bpf_prog_run_clear_cb

[Menglong Dong]

All the calling of bpf_prog_run_clear_cb() is protected with
rcu_read_lock, so we can replace bpf_prog_run_pin_on_cpu() with
bpf_prog_run_pin_on_cpu_rcu() for it.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
 include/linux/filter.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 48eb42358543..5ec5b16538f4 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -995,7 +995,7 @@ static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog,
 	if (unlikely(prog->cb_access))
 		memset(cb_data, 0, BPF_SKB_CB_LEN);
 
-	res = bpf_prog_run_pin_on_cpu(prog, skb);
+	res = bpf_prog_run_pin_on_cpu_rcu(prog, skb);
 	return res;
 }
 
-- 
2.51.0

Re: [PATCH bpf-next 1/4] rcu: factor out migrate_enable_rcu and migrate_disable_rcu

[Alexei Starovoitov]

On Tue, Oct 14, 2025 at 4:27 AM Menglong Dong <menglong8.dong@gmail.com> wrote:
>
> Factor out migrate_enable_rcu/migrate_disable_rcu from
> rcu_read_lock_dont_migrate/rcu_read_unlock_migrate.
>
> These functions will be used in the following patches.
>
> It's a little weird to define them in rcupdate.h. Maybe we should move
> them to sched.h?
>
> Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
> ---
>  include/linux/rcupdate.h | 20 +++++++++++++++++---
>  1 file changed, 17 insertions(+), 3 deletions(-)
>
> diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
> index c5b30054cd01..43626ccc07e2 100644
> --- a/include/linux/rcupdate.h
> +++ b/include/linux/rcupdate.h
> @@ -988,18 +988,32 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
>         preempt_enable_notrace();
>  }
>
> -static __always_inline void rcu_read_lock_dont_migrate(void)
> +/* This can only be used with rcu_read_lock held */
> +static inline void migrate_enable_rcu(void)
> +{
> +       WARN_ON_ONCE(!rcu_read_lock_held());
> +       if (IS_ENABLED(CONFIG_PREEMPT_RCU))
> +               migrate_enable();
> +}
> +
> +/* This can only be used with rcu_read_lock held */
> +static inline void migrate_disable_rcu(void)
>  {
> +       WARN_ON_ONCE(!rcu_read_lock_held());
>         if (IS_ENABLED(CONFIG_PREEMPT_RCU))
>                 migrate_disable();
> +}
> +
> +static __always_inline void rcu_read_lock_dont_migrate(void)
> +{
>         rcu_read_lock();
> +       migrate_disable_rcu();
>  }
>
>  static inline void rcu_read_unlock_migrate(void)
>  {
> +       migrate_enable_rcu();
>         rcu_read_unlock();
> -       if (IS_ENABLED(CONFIG_PREEMPT_RCU))
> -               migrate_enable();
>  }

Sorry. I don't like any of it. It obfuscates the code
without adding any benefits.

pw-bot: cr

Re: [PATCH bpf-next 1/4] rcu: factor out migrate_enable_rcu and migrate_disable_rcu

[Menglong Dong]

On 2025/10/14 22:59, Alexei Starovoitov wrote:
> On Tue, Oct 14, 2025 at 4:27 AM Menglong Dong <menglong8.dong@gmail.com> wrote:
> >
> > Factor out migrate_enable_rcu/migrate_disable_rcu from
> > rcu_read_lock_dont_migrate/rcu_read_unlock_migrate.
> >
> > These functions will be used in the following patches.
> >
> > It's a little weird to define them in rcupdate.h. Maybe we should move
> > them to sched.h?
> >
> > Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
> > ---
> >  include/linux/rcupdate.h | 20 +++++++++++++++++---
> >  1 file changed, 17 insertions(+), 3 deletions(-)
> >
> > diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
> > index c5b30054cd01..43626ccc07e2 100644
> > --- a/include/linux/rcupdate.h
> > +++ b/include/linux/rcupdate.h
> > @@ -988,18 +988,32 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
> >         preempt_enable_notrace();
> >  }
> >
> > -static __always_inline void rcu_read_lock_dont_migrate(void)
> > +/* This can only be used with rcu_read_lock held */
> > +static inline void migrate_enable_rcu(void)
> > +{
> > +       WARN_ON_ONCE(!rcu_read_lock_held());
> > +       if (IS_ENABLED(CONFIG_PREEMPT_RCU))
> > +               migrate_enable();
> > +}
> > +
> > +/* This can only be used with rcu_read_lock held */
> > +static inline void migrate_disable_rcu(void)
> >  {
> > +       WARN_ON_ONCE(!rcu_read_lock_held());
> >         if (IS_ENABLED(CONFIG_PREEMPT_RCU))
> >                 migrate_disable();
> > +}
> > +
> > +static __always_inline void rcu_read_lock_dont_migrate(void)
> > +{
> >         rcu_read_lock();
> > +       migrate_disable_rcu();
> >  }
> >
> >  static inline void rcu_read_unlock_migrate(void)
> >  {
> > +       migrate_enable_rcu();
> >         rcu_read_unlock();
> > -       if (IS_ENABLED(CONFIG_PREEMPT_RCU))
> > -               migrate_enable();
> >  }
> 
> Sorry. I don't like any of it. It obfuscates the code
> without adding any benefits.

It has a slight performance improving for some BPF type, such as
SK_SKB, SK_MSG.

Hmm, after we make migrate_disable() inline, the performance
improving here is extremely slight. And you are right, it do obfuscate
the code :/

Thanks!
Menglong Dong

> 
> pw-bot: cr
> 
>