From: Yonghong Song <yonghong.song@linux.dev>
To: bpf@vger.kernel.org
Cc: Alexei Starovoitov <ast@kernel.org>,
Andrii Nakryiko <andrii@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Eduard Zingerman <eddyz87@gmail.com>,
kernel-team@fb.com
Subject: [PATCH bpf-next v5 04/21] bpf: Add lookups for exception cleanup resumes and landing pads
Date: Tue, 22 Sep 2026 21:59:06 -0700 [thread overview]
Message-ID: <20260923045906.2416312-1-yonghong.song@linux.dev> (raw)
In-Reply-To: <20260923045846.2414643-1-yonghong.song@linux.dev>
Add two new files, exception.h and exception.c, to host the exception
handling code. Only a few helpers so far: recognising a call to
bpf_unwind_resume(), and asking which landing pad, if any, a call site
unwinds to.
The pad of a call site is kept in insn_aux_data, so the two places that
move instructions around -- bpf_patch_insn_data() and
verifier_remove_insns() -- learn to keep it in step.
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
include/linux/bpf_verifier.h | 6 ++++++
kernel/bpf/Makefile | 2 +-
kernel/bpf/exception.c | 23 +++++++++++++++++++++++
kernel/bpf/exception.h | 12 ++++++++++++
kernel/bpf/fixups.c | 26 +++++++++++++++++++++++++-
5 files changed, 67 insertions(+), 2 deletions(-)
create mode 100644 kernel/bpf/exception.c
create mode 100644 kernel/bpf/exception.h
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 2b6b480c059e..666c73308b32 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -710,6 +710,11 @@ struct bpf_insn_aux_data {
u64 jump_target:1;
unsigned int orig_idx; /* original instruction index, initialized once */
+ /*
+ * 1 + the instruction index of the exception cleanup landing pad
+ * this call site unwinds to, or 0 for none.
+ */
+ u32 cleanup_pad;
/*
* CFG strongly connected component this instruction belongs to,
* zero if it is a singleton SCC.
@@ -1519,6 +1524,7 @@ u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t);
int bpf_find_subprog(struct bpf_verifier_env *env, int off);
bool bpf_is_throw_kfunc(struct bpf_insn *insn);
+bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn);
int bpf_compute_const_regs(struct bpf_verifier_env *env);
int bpf_prune_dead_branches(struct bpf_verifier_env *env);
int bpf_check_cfg(struct bpf_verifier_env *env);
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 9a92c348bbda..af9bc60428ad 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -11,7 +11,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
-obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o
+obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o exception.o
obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
obj-$(CONFIG_BPF_JIT) += trampoline.o
diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c
new file mode 100644
index 000000000000..6989f31aa6c9
--- /dev/null
+++ b/kernel/bpf/exception.c
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+#include <linux/btf.h>
+#include <linux/btf_ids.h>
+#include <linux/filter.h>
+#include "exception.h"
+
+BTF_ID_LIST_SINGLE(bpf_unwind_resume_id, func, bpf_unwind_resume)
+
+bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn)
+{
+ return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
+ insn->imm == bpf_unwind_resume_id[0];
+}
+
+int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx)
+{
+ u32 pad = env->insn_aux_data[idx].cleanup_pad;
+
+ return pad ? (int)pad - 1 : -1;
+}
diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h
new file mode 100644
index 000000000000..634cb2c15ffc
--- /dev/null
+++ b/kernel/bpf/exception.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+#ifndef _LINUX_BPF_EXCEPTION_H
+#define _LINUX_BPF_EXCEPTION_H
+
+#include <linux/types.h>
+
+struct bpf_verifier_env;
+
+int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx);
+
+#endif /* _LINUX_BPF_EXCEPTION_H */
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 2add8001c3ec..e3459123803c 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -256,11 +256,18 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
data[i].non_stack_access =
data[off + cnt - 1].non_stack_access;
data[off + cnt - 1].non_stack_access = false;
+ data[i].cleanup_pad = data[off + cnt - 1].cleanup_pad;
+ data[off + cnt - 1].cleanup_pad = 0;
} else if (bpf_is_mem_insn(insn + i)) {
data[i].non_stack_access = true;
}
}
+ if (env->cleanup_info_cnt)
+ for (i = 0; i < prog_len; i++)
+ if (data[i].cleanup_pad > off + 1)
+ data[i].cleanup_pad += cnt - 1;
+
/*
* Last slot instruction could be a newly generated
* BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access
@@ -549,6 +556,7 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
unsigned int orig_prog_len = env->prog->len;
int err;
+ u32 i;
if (bpf_prog_is_offloaded(env->prog->aux))
bpf_prog_offload_remove_insns(env, off, cnt);
@@ -573,6 +581,17 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
sizeof(*aux_data) * (orig_prog_len - off - cnt));
env->insn_aux_data_len -= cnt;
+ if (env->cleanup_info_cnt) {
+ for (i = 0; i < env->insn_aux_data_len; i++) {
+ u32 pad = aux_data[i].cleanup_pad;
+
+ if (pad > off + cnt)
+ aux_data[i].cleanup_pad = pad - cnt;
+ else if (pad > off)
+ aux_data[i].cleanup_pad = 0;
+ }
+ }
+
return 0;
}
@@ -648,7 +667,7 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env)
struct bpf_insn *insn = env->prog->insnsi;
int insn_cnt = env->prog->len;
bool is_may_goto_0, is_ja;
- int i, err;
+ int i, j, err;
for (i = 0; i < insn_cnt; i++) {
is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0));
@@ -657,6 +676,11 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env)
if (!is_may_goto_0 && !is_ja)
continue;
+ if (env->cleanup_info_cnt)
+ for (j = 0; j < insn_cnt; j++)
+ if (env->insn_aux_data[j].cleanup_pad == i + 1)
+ env->insn_aux_data[j].cleanup_pad = i + 2;
+
err = verifier_remove_insns(env, i, 1);
if (err)
return err;
--
2.53.0-Meta
next prev parent reply other threads:[~2026-09-23 4:59 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-23 4:58 [PATCH bpf-next v5 00/21] bpf: Run exception cleanup landing pads when bpf_throw() unwinds Yonghong Song
2026-09-23 4:58 ` [PATCH bpf-next v5 01/21] bpf: Pack bpf_insn_aux_data flags into bit fields Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 19:16 ` Eduard Zingerman
2026-09-23 4:58 ` [PATCH bpf-next v5 02/21] bpf: Accept the compiler's exception cleanup table at program load Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 03/21] bpf: Add the bpf_unwind_resume() kfunc Yonghong Song
2026-09-23 6:13 ` Kumar Kartikeya Dwivedi
2026-09-23 4:59 ` Yonghong Song [this message]
2026-09-23 4:59 ` [PATCH bpf-next v5 05/21] bpf: Prepare for an exception cleanup table before the CFG walk Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 4:59 ` [PATCH bpf-next v5 06/21] bpf: Make exception landing pads reachable in the CFG Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 19:25 ` Eduard Zingerman
2026-09-23 4:59 ` [PATCH bpf-next v5 07/21] bpf: Explore the landing pads no call site reaches Yonghong Song
2026-09-23 16:26 ` Eduard Zingerman
2026-09-23 17:27 ` Yonghong Song
2026-09-23 18:06 ` Eduard Zingerman
2026-09-23 18:09 ` Yonghong Song
2026-09-23 21:28 ` Alexei Starovoitov
2026-09-23 22:19 ` Eduard Zingerman
2026-09-23 23:20 ` Alexei Starovoitov
2026-09-24 0:13 ` Kumar Kartikeya Dwivedi
2026-09-24 1:53 ` Alexei Starovoitov
2026-09-24 6:12 ` Kumar Kartikeya Dwivedi
2026-09-23 23:09 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 08/21] bpf: Walk the exception unwind in the verifier Yonghong Song
2026-09-23 16:34 ` Eduard Zingerman
2026-09-23 18:01 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 09/21] bpf: Refuse exception cleanup shapes bpf_throw() cannot dispatch Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 10/21] bpf: Refuse a private stack for a program with an exception cleanup table Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 11/21] bpf: Dispatch exception cleanup pads from bpf_throw() Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 12/21] bpf, x86: Dispatch exception cleanup pads at run time Yonghong Song
2026-09-23 6:45 ` Eduard Zingerman
2026-09-23 20:45 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 13/21] bpf, arm64: " Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 14/21] libbpf: Resolve the compiler's _Unwind_Resume to the kernel's kfunc Yonghong Song
2026-09-23 8:36 ` Kumar Kartikeya Dwivedi
2026-09-23 5:00 ` [PATCH bpf-next v5 15/21] libbpf: Add cleanup_info to bpf_prog_load_opts Yonghong Song
2026-09-23 5:00 ` [PATCH bpf-next v5 16/21] libbpf: Collect .bpf_cleanup records and pass them to the kernel Yonghong Song
2026-09-23 5:52 ` Kumar Kartikeya Dwivedi
2026-09-23 5:00 ` [PATCH bpf-next v5 17/21] libbpf: Carry the exception cleanup table through the light skeleton Yonghong Song
2026-09-23 5:00 ` [PATCH bpf-next v5 18/21] libbpf: Let the static linker carry .bpf_cleanup relocations Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 5:00 ` [PATCH bpf-next v5 19/21] selftests/bpf: Add an end-to-end .bpf_cleanup exception test Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 5:00 ` [PATCH bpf-next v5 20/21] selftests/bpf: Cover the exception cleanup shapes the chain does not reach Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 7:50 ` Eduard Zingerman
2026-09-23 5:00 ` [PATCH bpf-next v5 21/21] selftests/bpf: Load an exception cleanup program from a light skeleton Yonghong Song
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260923045906.2416312-1-yonghong.song@linux.dev \
--to=yonghong.song@linux.dev \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=kernel-team@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox