From: Kuniyuki Iwashima <kuniyu@google.com>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>,
Martin KaFai Lau <martin.lau@linux.dev>,
Eduard Zingerman <eddyz87@gmail.com>,
Kumar Kartikeya Dwivedi <memxor@gmail.com>
Cc: "Amery Hung" <ameryhung@gmail.com>,
"Yonghong Song" <yonghong.song@linux.dev>,
"John Fastabend" <john.fastabend@gmail.com>,
"Stanislav Fomichev" <sdf@fomichev.me>,
"Eric Dumazet" <edumazet@kernel.org>,
"Neal Cardwell" <ncardwell@google.com>,
"Willem de Bruijn" <willemb@google.com>,
"Tenzin Ukyab" <ukyab@berkeley.edu>,
"Clément Léger" <cleger@meta.com>,
"Kuniyuki Iwashima" <kuniyu@google.com>,
"Kuniyuki Iwashima" <kuni1840@gmail.com>,
bpf@vger.kernel.org, netdev@vger.kernel.org
Subject: [PATCH v5 bpf-next 04/10] bpf: tcp: Guard fast-path bpf_tcp_ops_call() under per-socket flag.
Date: Thu, 8 Oct 2026 03:15:24 +0000 [thread overview]
Message-ID: <20261008031604.256498-5-kuniyu@google.com> (raw)
In-Reply-To: <20261008031604.256498-1-kuniyu@google.com>
bpf_tcp_ops.{parse_hdr,hdr_opt_len} are called for every
incoming / outgoing skb.
bpf_tcp_ops.rtt is called once per RTT, which is every
incoming skb in ping-pong workloads like tcp_rr.
Even attaching NULL callbacks in the fast path hurts performance.
Let's guard them (and write_hdr_opt) with the new per-socket flags.
__bpf_tcp_ops_call() and bpf_tcp_ops_call_flag() are added to check
cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) first and avoid accessing
tp->bpf_tcp_ops_flags when no bpf_tcp_ops is attached.
Note that bpf_tcp_ops_hdr_opt_len() has 3 callers and previously
checked the static key both in tcp_established_options() and inside
bpf_tcp_ops_call(). Now the static key is checked once at the
beginning of bpf_tcp_ops_hdr_opt_len(), and the redundant check in
tcp_established_options() is removed.
tcp_bpf_hdr_opt_len() must use __always_inline, not just
inline, otherwise Clang inlines bpf_tcp_ops_hdr_opt_len() to
tcp_bpf_hdr_opt_len() instead.
$ llvm-objdump -S -D --disassemble=tcp_established_options vmlinux
...
; if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp,
ffffffff8251270e: 41 f6 87 20 0c 00 00 40 testb $0x40, 0xc20(%r15)
ffffffff82512716: 75 47 jne 0xffffffff8251275f <tcp_established_options+0x1ff>
; asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]")
ffffffff82512718: 0f 1f 44 00 00 nopl (%rax,%rax)
; return size;
ffffffff8251271d: 44 89 e0 movl %r12d, %eax
ffffffff82512720: 5b popq %rbx
ffffffff82512721: 41 5c popq %r12
ffffffff82512723: 41 5e popq %r14
ffffffff82512725: 41 5f popq %r15
ffffffff82512727: 5d popq %rbp
ffffffff82512728: 2e e9 42 03 43 00 jmp 0xffffffff82942a70 <__x86_return_thunk>
bpf_skops_write_hdr_opt() is still not free, but it can be
optimised later if needed.
Signed-off-by: Kuniyuki Iwashima <kuniyu@google.com>
---
v5: Always inline static key to tcp_established_options()
v4: Check static key first and then flags
---
include/net/tcp.h | 44 ++++++++++++++++++++++------------
net/ipv4/tcp_input.c | 12 +++++++++-
net/ipv4/tcp_output.c | 55 ++++++++++++++++++++++++++-----------------
3 files changed, 74 insertions(+), 37 deletions(-)
diff --git a/include/net/tcp.h b/include/net/tcp.h
index 55ad9db99596..b7c0f1a8797a 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -3064,22 +3064,33 @@ struct bpf_tcp_ops {
u32 opt_off);
};
-#define bpf_tcp_ops_call(op, sk, ...) \
+#define __bpf_tcp_ops_call(op, sk, ...) \
do { \
- if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { \
- const struct bpf_prog_array_item *item; \
- const struct bpf_tcp_ops *tcp_ops; \
- struct cgroup *cgrp; \
+ const struct bpf_prog_array_item *item; \
+ const struct bpf_tcp_ops *tcp_ops; \
+ struct cgroup *cgrp; \
\
- cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \
- rcu_read_lock_dont_migrate(); \
- bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \
- CGROUP_TCP_SOCK_OPS) { \
- if (tcp_ops->op) \
- tcp_ops->op(sk, ##__VA_ARGS__); \
- } \
- rcu_read_unlock_migrate(); \
+ cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \
+ rcu_read_lock_dont_migrate(); \
+ bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \
+ CGROUP_TCP_SOCK_OPS) { \
+ if (tcp_ops->op) \
+ tcp_ops->op(sk, ##__VA_ARGS__); \
} \
+ rcu_read_unlock_migrate(); \
+} while (0)
+
+#define bpf_tcp_ops_call(op, sk, ...) \
+do { \
+ if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) \
+ __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \
+} while (0)
+
+#define bpf_tcp_ops_call_flag(op, flag, sk, ...) \
+do { \
+ if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) && \
+ BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), flag)) \
+ __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \
} while (0)
#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) \
@@ -3115,7 +3126,9 @@ do { \
})
#else
-#define bpf_tcp_ops_call(op, sk, ...) do { } while (0)
+#define __bpf_tcp_ops_call(op, sk, ...) do { } while (0)
+#define bpf_tcp_ops_call(op, sk, ...) do { } while (0)
+#define bpf_tcp_ops_call_flag(op, flag, sk, ...) do { } while (0)
#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) (init_retval)
#endif
@@ -3150,7 +3163,8 @@ static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt)
{
if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt);
- bpf_tcp_ops_call(rtt, sk, mrtt, srtt);
+
+ bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt);
}
#if IS_ENABLED(CONFIG_SMC)
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 209db8effcc4..4478d3f3d4b0 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -210,6 +210,11 @@ static void bpf_skops_established(struct sock *sk, int bpf_op,
static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb)
{
+ const struct tcp_sock *tp;
+
+ if (!cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS))
+ return;
+
switch (sk->sk_state) {
case TCP_SYN_RECV:
case TCP_SYN_SENT:
@@ -217,7 +222,12 @@ static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb)
return;
}
- bpf_tcp_ops_call(parse_hdr, sk, skb);
+ tp = tcp_sk(sk);
+
+ if ((tp->rx_opt.saw_unknown &&
+ BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_UNKNOWN)) ||
+ BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_ALL))
+ __bpf_tcp_ops_call(parse_hdr, sk, skb);
}
static __cold void tcp_gro_dev_warn(const struct sock *sk, const struct sk_buff *skb,
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 3ac465513bf5..313dfe70a386 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -581,8 +581,8 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
* writer's bytes). The writer finds the append point by scanning from
* first_opt_off + nr_written to the first NOP.
*/
- bpf_tcp_ops_call(write_hdr_opt, sk, skb, req, syn_skb, synack_type,
- first_opt_off + nr_written);
+ bpf_tcp_ops_call_flag(write_hdr_opt, WRITE_HDR_OPT, sk, skb, req,
+ syn_skb, synack_type, first_opt_off + nr_written);
}
#else
static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
@@ -613,11 +613,12 @@ static u32 bpf_tcp_ops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
{
unsigned int remaining_out = remaining, reserved;
- if (!remaining)
- return 0;
+ if (!BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), WRITE_HDR_OPT) ||
+ !remaining)
+ return remaining;
/* bpf_tcp_ops_reserve_hdr_opt() reserves space via remaining_out */
- bpf_tcp_ops_call(hdr_opt_len, sk, skb, req, syn_skb, synack_type, &remaining_out);
+ __bpf_tcp_ops_call(hdr_opt_len, sk, skb, req, syn_skb, synack_type, &remaining_out);
reserved = remaining - remaining_out;
if (!reserved)
@@ -630,6 +631,21 @@ static u32 bpf_tcp_ops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
return remaining - reserved;
}
+static __always_inline u32 tcp_bpf_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
+ struct request_sock *req,
+ struct sk_buff *syn_skb,
+ enum tcp_synack_type synack_type,
+ struct tcp_out_options *opts,
+ u32 remaining)
+{
+ if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS))
+ remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, req, syn_skb,
+ synack_type, opts,
+ remaining);
+
+ return remaining;
+}
+
static __be32 *process_tcp_ao_options(struct tcp_sock *tp,
const struct tcp_request_sock *tcprsk,
struct tcp_out_options *opts,
@@ -1089,8 +1105,8 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
remaining);
- remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
- remaining);
+ remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
+ remaining);
return MAX_TCP_OPTION_SPACE - remaining;
}
@@ -1179,8 +1195,8 @@ static unsigned int tcp_synack_options(const struct sock *sk,
remaining = bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
synack_type, opts, remaining);
- remaining = bpf_tcp_ops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
- synack_type, opts, remaining);
+ remaining = tcp_bpf_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
+ synack_type, opts, remaining);
return MAX_TCP_OPTION_SPACE - remaining;
}
@@ -1193,8 +1209,9 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb
struct tcp_key *key)
{
struct tcp_sock *tp = tcp_sk(sk);
- unsigned int size = 0;
unsigned int eff_sacks;
+ unsigned int remaining;
+ unsigned int size = 0;
opts->options = 0;
opts->bpf_opt_len = 0;
@@ -1224,10 +1241,10 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb
* left.
*/
if (sk_is_mptcp(sk)) {
- unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
bool has_ts = opts->options & OPTION_TS;
int opt_size;
+ remaining = MAX_TCP_OPTION_SPACE - size;
opts->mptcp.drop_ts = 0;
opt_size = mptcp_established_options(sk, skb, remaining, has_ts,
@@ -1244,7 +1261,8 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb
eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
if (unlikely(eff_sacks)) {
- const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
+ remaining = MAX_TCP_OPTION_SPACE - size;
+
if (likely(remaining >= TCPOLEN_SACK_BASE_ALIGNED +
TCPOLEN_SACK_PERBLOCK)) {
opts->num_sack_blocks =
@@ -1277,22 +1295,17 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb
if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp,
BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) {
- unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
-
+ remaining = MAX_TCP_OPTION_SPACE - size;
remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
remaining);
size = MAX_TCP_OPTION_SPACE - remaining;
}
- if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) {
- unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
+ remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
+ MAX_TCP_OPTION_SPACE - size);
- remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
- remaining);
-
- size = MAX_TCP_OPTION_SPACE - remaining;
- }
+ size = MAX_TCP_OPTION_SPACE - remaining;
return size;
}
--
2.56.0.360.g66cac248cb-goog
next prev parent reply other threads:[~2026-10-08 3:16 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-08 3:15 [PATCH v5 bpf-next 00/10] bpf: Add bpf_tcp_ops hooks for TCP AutoLOWAT Kuniyuki Iwashima
2026-10-08 3:15 ` [PATCH v5 bpf-next 01/10] bpf: tcp: Convert deny-list for bpf_{get,set}sockopt() to allow-list Kuniyuki Iwashima
2026-10-08 3:15 ` [PATCH v5 bpf-next 02/10] bpf: tcp: Add a new per-socket flag and kfunc for bpf_tcp_ops Kuniyuki Iwashima
2026-10-08 3:15 ` [PATCH v5 bpf-next 03/10] selftest: bpf: Use bpf_tcp_ops_set_flags() in bpf_tcp_ops_hdr.c Kuniyuki Iwashima
2026-10-08 3:15 ` Kuniyuki Iwashima [this message]
2026-10-08 3:33 ` [PATCH v5 bpf-next 04/10] bpf: tcp: Guard fast-path bpf_tcp_ops_call() under per-socket flag sashiko-bot
2026-10-08 4:02 ` bot+bpf-ci
2026-10-08 22:11 ` Stanislav Fomichev
2026-10-08 3:15 ` [PATCH v5 bpf-next 05/10] bpf: tcp: Check BPF_SOCK_OPS_TEST_FLAG() after cgroup_bpf_enabled(CGROUP_SOCK_OPS) Kuniyuki Iwashima
2026-10-08 16:56 ` Amery Hung
2026-10-08 22:11 ` Stanislav Fomichev
2026-10-08 3:15 ` [PATCH v5 bpf-next 06/10] bpf: tcp: Introduce bpf_tcp_ops.{enqueue,dequeue}_rcvq() Kuniyuki Iwashima
2026-10-08 3:15 ` [PATCH v5 bpf-next 07/10] tcp: Split out __tcp_set_rcvlowat() Kuniyuki Iwashima
2026-10-08 3:15 ` [PATCH v5 bpf-next 08/10] bpf: mptcp: Don't support BPF_TCP_OPS_FLAG_RCVQ Kuniyuki Iwashima
2026-10-08 3:15 ` [PATCH v5 bpf-next 09/10] bpf: tcp: Add kfunc to adjust sk->sk_rcvlowat Kuniyuki Iwashima
2026-10-08 3:43 ` sashiko-bot
2026-10-08 3:15 ` [PATCH v5 bpf-next 10/10] selftest: bpf: Add test for bpf_tcp_ops.{enqueue,dequeue}_rcvq() Kuniyuki Iwashima
2026-10-08 17:16 ` Amery Hung
2026-10-10 16:40 ` [PATCH v5 bpf-next 00/10] bpf: Add bpf_tcp_ops hooks for TCP AutoLOWAT patchwork-bot+netdevbpf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261008031604.256498-5-kuniyu@google.com \
--to=kuniyu@google.com \
--cc=ameryhung@gmail.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=cleger@meta.com \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=edumazet@kernel.org \
--cc=john.fastabend@gmail.com \
--cc=kuni1840@gmail.com \
--cc=martin.lau@linux.dev \
--cc=memxor@gmail.com \
--cc=ncardwell@google.com \
--cc=netdev@vger.kernel.org \
--cc=sdf@fomichev.me \
--cc=ukyab@berkeley.edu \
--cc=willemb@google.com \
--cc=yonghong.song@linux.dev \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.