[PATCH bpf-next 0/7] bpf: Inline IRQ enter/exit instructions

[PATCH bpf-next 0/7] bpf: Inline IRQ enter/exit instructions

[Emil Tsalapatis]

The functions bpf_local_irq_save and bpf_local_irq_restore are only
a couple instructions long, since they are thin wrappers around
local_irq_save()/local_irq_restore(). However, they still require
a full call along with caller register save and restore. Since these
functions are used extensively in performance-critical code like
spinlocks, it is desireable to reduce their overhead.

Add inlining for those functions. The inlining happens in two stages.
First replace the call itself with a special BPF instruction denoting
an IRQ disable-and save, or restore-and-enable. Those instructions
are translated by the architecture-specific JIT into the corresponding
operation. There is no functional difference between the emitted code
and local_irq_save/restore(). Inlining reduces the overall latency of
a no-op critical sections from 47ns to 29ns on a high-end Zen 5 machine.

Add support for this inlining to the x86 and aarch64 JITs. The fixup
does not happen for JITs without explicit support, or for kernel
configurations that change the code run during local_irq_save/restore().

Mark the functions as fastcalls to remove caller register pushing and
popping. The calls are only marked as fastcalls when they are valid
for inlining, so the calls are not marked if not eligible or in
architectures that do not support inlining.

Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>

Emil Tsalapatis (7):
  bpf: Add internal IRQ save and restore instructions
  x86, bpf: JIT internal IRQ save and restore instructions
  arm64, bpf: JIT internal IRQ save and restore instructions
  bpf: Inline local IRQ kfunc calls during verifier fixup
  selftests/bpf: Test local IRQ kfunc inlining
  bpf: Enable fastcall for inlined local IRQ kfuncs
  selftests/bpf: Verify register preservation for IRQ fastcalls

 arch/arm64/net/bpf_jit.h                      |   8 ++
 arch/arm64/net/bpf_jit_comp.c                 |  34 +++++
 arch/x86/net/bpf_jit_comp.c                   |  22 +++
 include/linux/filter.h                        |  13 ++
 kernel/bpf/core.c                             |   5 +
 kernel/bpf/disasm.c                           |   8 ++
 kernel/bpf/helpers.c                          |   4 +-
 kernel/bpf/verifier.c                         |  17 +++
 .../selftests/bpf/prog_tests/irq_fastcall.c   |  71 ++++++++++
 .../selftests/bpf/prog_tests/irq_jit.c        | 128 ++++++++++++++++++
 tools/testing/selftests/bpf/progs/irq.c       |  24 ++++
 .../selftests/bpf/progs/irq_fastcall.c        |  56 ++++++++
 tools/testing/selftests/bpf/progs/irq_jit.c   |  55 ++++++++
 13 files changed, 443 insertions(+), 2 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/irq_fastcall.c
 create mode 100644 tools/testing/selftests/bpf/prog_tests/irq_jit.c
 create mode 100644 tools/testing/selftests/bpf/progs/irq_fastcall.c
 create mode 100644 tools/testing/selftests/bpf/progs/irq_jit.c

-- 
2.52.0

[PATCH bpf-next 1/7] bpf: Add internal IRQ save and restore instructions

[Emil Tsalapatis]

Introduce a kernel-private BPF_ST | BPF_IRQ instruction with save and
restore operations. Encode the flags-pointer register in dst_reg and the
operation in imm, leaving src_reg and off reserved as zero.

Add a JIT capability hook whose default implementation declines support,
plus disassembler output for both operations. Architecture backends opt
in separately, and the verifier starts emitting the instructions in a
later patch.

Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
 include/linux/filter.h | 13 +++++++++++++
 kernel/bpf/core.c      |  5 +++++
 kernel/bpf/disasm.c    |  8 ++++++++
 3 files changed, 26 insertions(+)

diff --git a/include/linux/filter.h b/include/linux/filter.h
index e0218e677a1d..bf67dfa19c17 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -90,6 +90,10 @@ struct ctl_table_header;
  */
 #define BPF_NOSPEC	0xc0
 
+#define BPF_IRQ		0xe0
+#define BPF_IRQ_SAVE	0
+#define BPF_IRQ_RESTORE	1
+
 /* As per nm, we expose JITed images as text (code) section for
  * kallsyms. That way, tools like perf can find it to match
  * addresses.
@@ -598,6 +602,14 @@ static inline int bpf_atomic_load_reg(const struct bpf_insn *insn)
 		.off   = 0,					\
 		.imm   = 0 })
 
+#define BPF_IRQ_INSN(OP, FLAGS_REG)				\
+	((struct bpf_insn) {					\
+		.code  = BPF_ST | BPF_IRQ,			\
+		.dst_reg = FLAGS_REG,				\
+		.src_reg = 0,					\
+		.off   = 0,					\
+		.imm   = OP })
+
 /* Internal classic blocks for direct assignment */
 
 #define __BPF_STMT(CODE, K)					\
@@ -1277,6 +1289,7 @@ bool bpf_jit_needs_zext(void);
 bool bpf_jit_inlines_helper_call(s32 imm);
 bool bpf_jit_supports_subprog_tailcalls(void);
 bool bpf_jit_supports_percpu_insn(void);
+bool bpf_jit_supports_irq_save_restore(void);
 bool bpf_jit_supports_kfunc_call(void);
 bool bpf_jit_supports_callx(void);
 bool bpf_jit_supports_kfunc_ret_reg_pair(void);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 3030beb531d9..0ab9a210e1e8 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -3383,6 +3383,11 @@ bool __weak bpf_jit_supports_percpu_insn(void)
 	return false;
 }
 
+bool __weak bpf_jit_supports_irq_save_restore(void)
+{
+	return false;
+}
+
 bool __weak bpf_jit_supports_kfunc_call(void)
 {
 	return false;
diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c
index 889fc074496f..125e2c5754e9 100644
--- a/kernel/bpf/disasm.c
+++ b/kernel/bpf/disasm.c
@@ -9,6 +9,9 @@
 
 /* Only defined by the non-UAPI linux/filter.h, which this file cannot use. */
 #define BPF_PROBE_ATOMIC 0xe0
+#define BPF_IRQ 0xe0
+#define BPF_IRQ_SAVE 0
+#define BPF_IRQ_RESTORE 1
 
 #define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x)
 static const char * const func_id_str[] = {
@@ -294,6 +297,11 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
 				insn->off, insn->imm);
 		} else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) {
 			verbose(cbs->private_data, "(%02x) nospec", insn->code);
+		} else if (insn->code == (BPF_ST | BPF_IRQ) &&
+			   (insn->imm == BPF_IRQ_SAVE || insn->imm == BPF_IRQ_RESTORE)) {
+			verbose(cbs->private_data, "(%02x) bpf_local_irq_%s(r%d)",
+				insn->code, insn->imm == BPF_IRQ_SAVE ? "save" : "restore",
+				insn->dst_reg);
 		} else {
 			verbose(cbs->private_data, "BUG_st_%02x", insn->code);
 		}
-- 
2.52.0

[PATCH bpf-next 2/7] x86, bpf: JIT internal IRQ save and restore instructions

[Emil Tsalapatis]

Implement the internal IRQ save/restore instruction in the x86-64 JIT,
using the encoded destination register as the flags pointer.

Save RFLAGS, disable interrupts and store the flags through that pointer.
Restore loads the saved flags and enables interrupts only when the saved
interrupt-enable bit is set.

Advertise support only when IRQ tracing and paravirtualized IRQ
operations do not require additional work.

Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
 arch/x86/net/bpf_jit_comp.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 6a2fe3d0039d..62833c0f7c97 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3089,6 +3089,23 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 			EMIT_LFENCE();
 			break;
 
+		case BPF_ST | BPF_IRQ:
+			if (imm32 == BPF_IRQ_SAVE) {
+				EMIT1(0x9c);
+				EMIT2(0x41, add_1reg(0x58, BPF_REG_AX));
+				EMIT1(0xfa);
+				emit_stx(&prog, BPF_DW, dst_reg, BPF_REG_AX, 0);
+			} else if (imm32 == BPF_IRQ_RESTORE) {
+				emit_ldx(&prog, BPF_DW, BPF_REG_AX, dst_reg, 0);
+				maybe_emit_1mod(&prog, BPF_REG_AX, true);
+				EMIT2_off32(0xf7, add_1reg(0xc0, BPF_REG_AX), X86_EFLAGS_IF);
+				EMIT2(X86_JE, 1);
+				EMIT1(0xfb);
+			} else {
+				return -EINVAL;
+			}
+			break;
+
 		case BPF_ST | BPF_MEM | BPF_B:
 		case BPF_ST | BPF_MEM | BPF_H:
 		case BPF_ST | BPF_MEM | BPF_W:
@@ -5116,6 +5133,11 @@ bool bpf_jit_supports_kfunc_call(void)
 	return true;
 }
 
+bool bpf_jit_supports_irq_save_restore(void)
+{
+	return !IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && !IS_ENABLED(CONFIG_PARAVIRT_XXL);
+}
+
 bool bpf_jit_supports_kfunc_ret_reg_pair(void)
 {
 	return true;
-- 
2.52.0

[PATCH bpf-next 3/7] arm64, bpf: JIT internal IRQ save and restore instructions

[Emil Tsalapatis]

Implement the internal IRQ save/restore instruction in the arm64 JIT,
using the encoded destination register as the flags pointer.

Support both DAIF masking and GIC priority masking. Preserve the existing
priority-mask synchronization requirements when restoring interrupt
state, and decline inlining when IRQ tracing or priority-mask debugging
is enabled.

Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
 arch/arm64/net/bpf_jit.h      |  8 ++++++++
 arch/arm64/net/bpf_jit_comp.c | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h
index b2fe6e6dcf44..23cbc38510a0 100644
--- a/arch/arm64/net/bpf_jit.h
+++ b/arch/arm64/net/bpf_jit.h
@@ -8,6 +8,7 @@
 #define _BPF_JIT_H
 
 #include <asm/insn.h>
+#include <asm/sysreg.h>
 
 /* 5-bit Register Operand */
 #define A64_R(x)	AARCH64_INSN_REG_##x
@@ -338,6 +339,13 @@
 #define A64_MRS_SP_EL0(Rt) \
 	aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_SP_EL0)
 
+#define A64_MRS_SYSREG(Rt, SysReg) \
+	(aarch64_insn_get_mrs_value() | (SysReg) | (Rt))
+#define A64_MSR_SYSREG(Rt, SysReg) \
+	(aarch64_insn_get_msr_reg_value() | (SysReg) | (Rt))
+#define A64_MSR_DAIFSET(Imm) \
+	(aarch64_insn_get_msr_imm_value() | pstate_field(3, 6) | ((Imm) << PSTATE_Imm_shift))
+
 /* Barriers */
 #define A64_SB aarch64_insn_get_sb_value()
 #define A64_DSB_NSH (aarch64_insn_get_dsb_base_value() | 0x7 << 8)
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 326e9e1dea0a..b7f4e4b68e9d 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -22,6 +22,7 @@
 #include <asm/cpufeature.h>
 #include <asm/debug-monitors.h>
 #include <asm/insn.h>
+#include <asm/irqflags.h>
 #include <asm/text-patching.h>
 
 #include "bpf_jit.h"
@@ -2031,6 +2032,33 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 		}
 		break;
 
+	case BPF_ST | BPF_IRQ:
+	{
+		bool pmr = system_uses_irq_prio_masking();
+		u32 sysreg = pmr ? SYS_ICC_PMR_EL1 : sys_reg(3, 3, 4, 2, 1);
+
+		if (imm == BPF_IRQ_SAVE) {
+			emit(A64_MRS_SYSREG(tmp, sysreg), ctx);
+			if (pmr) {
+				emit(A64_CMP_I(1, tmp, GIC_PRIO_IRQON), ctx);
+				emit(A64_B_(A64_COND_NE, 3), ctx);
+				emit_a64_mov_i(1, tmp2, GIC_PRIO_IRQOFF, ctx);
+				emit(A64_MSR_SYSREG(tmp2, sysreg), ctx);
+			} else {
+				emit(A64_MSR_DAIFSET(3), ctx);
+			}
+			emit(A64_STR64I(tmp, dst, 0), ctx);
+		} else if (imm == BPF_IRQ_RESTORE) {
+			emit(A64_LDR64I(tmp, dst, 0), ctx);
+			emit(A64_MSR_SYSREG(tmp, sysreg), ctx);
+			if (pmr && !alternative_has_cap_likely(ARM64_HAS_GIC_PRIO_RELAXED_SYNC))
+				emit(aarch64_insn_gen_dsb(AARCH64_INSN_MB_SY), ctx);
+		} else {
+			return -EINVAL;
+		}
+		break;
+	}
+
 	/* ST: *(size *)(dst + off) = imm */
 	case BPF_ST | BPF_MEM | BPF_W:
 	case BPF_ST | BPF_MEM | BPF_H:
@@ -3537,6 +3565,12 @@ bool bpf_jit_supports_percpu_insn(void)
 	return true;
 }
 
+bool bpf_jit_supports_irq_save_restore(void)
+{
+	return !IS_ENABLED(CONFIG_TRACE_IRQFLAGS) &&
+	       !IS_ENABLED(CONFIG_ARM64_DEBUG_PRIORITY_MASKING);
+}
+
 bool bpf_jit_bypass_spec_v4(void)
 {
 	/* In case of arm64, we rely on the firmware mitigation of Speculative
-- 
2.52.0

[PATCH bpf-next 4/7] bpf: Inline local IRQ kfunc calls during verifier fixup

[Emil Tsalapatis]

Replace calls to the core bpf_local_irq_save() and
bpf_local_irq_restore() kfuncs with the corresponding internal IRQ
instruction when the selected JIT advertises support. Encode R1, the
flags-pointer argument register, as the instruction destination.

This runs after the original call has been checked. The existing
process_irq_flag() validation checks the stack flags operand and tracks
its initialization and IRQ-region state, so the fixup does not accept new
operand types or bypass those checks.

Keep calls unchanged for unsupported backends and configurations.

Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
 kernel/bpf/verifier.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5601d2056218..aff7080b5c85 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -23372,6 +23372,16 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			     insn->imm);
 		return -EFAULT;
 	}
+	if (!desc->offset && bpf_jit_supports_irq_save_restore() &&
+	    (desc->func_id == special_kfunc_list[KF_bpf_local_irq_save] ||
+	     desc->func_id == special_kfunc_list[KF_bpf_local_irq_restore])) {
+		int irq_op = desc->func_id == special_kfunc_list[KF_bpf_local_irq_save] ?
+			     BPF_IRQ_SAVE : BPF_IRQ_RESTORE;
+
+		insn_buf[0] = BPF_IRQ_INSN(irq_op, BPF_REG_1);
+		*cnt = 1;
+		return 0;
+	}
 	base_desc_idx = desc - env->prog->aux->kfunc_tab->descs;
 
 	near_call = !bpf_jit_supports_far_kfunc_call();
-- 
2.52.0

[PATCH bpf-next 5/7] selftests/bpf: Test local IRQ kfunc inlining

[Emil Tsalapatis]

Check that userspace cannot submit the kernel-private IRQ save and
restore opcodes directly.

Add nested IRQ save/restore tests in the main program and across a BPF
subprogram call. Inspect translated instructions for the expected
operation counts, flags register and reserved fields. Check the native
x86-64 or arm64 instruction sequences when inlining is enabled, and the
kfunc fallback when the kernel configuration disables it.

Run the JIT tests with hardening enabled and restore the original sysctl
setting afterwards. The selftests are discovered automatically by the
existing build rules.

Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
 .../selftests/bpf/prog_tests/irq_jit.c        | 128 ++++++++++++++++++
 tools/testing/selftests/bpf/progs/irq.c       |  24 ++++
 tools/testing/selftests/bpf/progs/irq_jit.c   |  55 ++++++++
 3 files changed, 207 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/irq_jit.c
 create mode 100644 tools/testing/selftests/bpf/progs/irq_jit.c

diff --git a/tools/testing/selftests/bpf/prog_tests/irq_jit.c b/tools/testing/selftests/bpf/prog_tests/irq_jit.c
new file mode 100644
index 000000000000..3d5dc3c77edb
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/irq_jit.c
@@ -0,0 +1,128 @@
+#include <test_progs.h>
+#include <network_helpers.h>
+#include <sysctl_helpers.h>
+#include "irq_jit.skel.h"
+
+static void check_irq_native(int prog_fd, __u32 jited_len)
+{
+#if defined(__x86_64__)
+	static const __u8 save[] = { 0x9c, 0x41, 0x5a, 0xfa, 0x4c, 0x89, 0x17 };
+	static const __u8 restore[] = {
+		0x4c, 0x8b, 0x17, 0x49, 0xf7, 0xc2, 0x00, 0x02, 0x00, 0x00, 0x74, 0x01, 0xfb,
+	};
+#elif defined(__aarch64__)
+	static const __u8 save[] = { 0xdf, 0x43, 0x03, 0xd5 };
+	static const __u8 restore[] = { 0x0a, 0x00, 0x40, 0xf9, 0x2a, 0x42, 0x1b, 0xd5 };
+	static const __u8 pmr_save[] = { 0x0b, 0x46, 0x18, 0xd5 };
+	static const __u8 pmr_restore[] = { 0x0a, 0x00, 0x40, 0xf9, 0x0a, 0x46, 0x18, 0xd5 };
+#else
+	return;
+#endif
+#if defined(__x86_64__) || defined(__aarch64__)
+	struct bpf_prog_info info = {};
+	__u32 info_len = sizeof(info);
+	__u8 *image;
+	int err;
+
+	if (!ASSERT_GT(jited_len, 0, "jited_len"))
+		return;
+	image = malloc(jited_len);
+	if (!ASSERT_OK_PTR(image, "jited_image"))
+		return;
+	info.jited_prog_len = jited_len;
+	info.jited_prog_insns = ptr_to_u64(image);
+	err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
+	if (!ASSERT_OK(err, "jited_prog_info"))
+		goto out;
+#if defined(__x86_64__)
+	ASSERT_TRUE(memmem(image, jited_len, save, sizeof(save)), "native_irq_save");
+	ASSERT_TRUE(memmem(image, jited_len, restore, sizeof(restore)), "native_irq_restore");
+#else
+	ASSERT_TRUE(memmem(image, jited_len, save, sizeof(save)) ||
+		    memmem(image, jited_len, pmr_save, sizeof(pmr_save)), "native_irq_save");
+	ASSERT_TRUE(memmem(image, jited_len, restore, sizeof(restore)) ||
+		    memmem(image, jited_len, pmr_restore, sizeof(pmr_restore)), "native_irq_restore");
+#endif
+out:
+	free(image);
+#endif
+}
+
+static void check_irq_insns(int prog_fd, bool expect_inline)
+{
+	struct bpf_prog_info info = {};
+	__u32 info_len = sizeof(info);
+	struct bpf_insn *insns;
+	__u32 xlated_len;
+	int save_count = 0, restore_count = 0;
+	int err;
+
+	err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
+	if (!ASSERT_OK(err, "prog_info"))
+		return;
+	xlated_len = info.xlated_prog_len;
+	insns = malloc(xlated_len);
+	if (!ASSERT_OK_PTR(insns, "xlated_insns"))
+		return;
+	memset(&info, 0, sizeof(info));
+	info.xlated_prog_len = xlated_len;
+	info.xlated_prog_insns = ptr_to_u64(insns);
+	err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
+	if (!ASSERT_OK(err, "xlated_prog_info"))
+		goto out;
+	for (size_t insn_idx = 0; insn_idx < info.xlated_prog_len / sizeof(*insns); insn_idx++) {
+		struct bpf_insn *insn = &insns[insn_idx];
+
+		if (insn->code != (BPF_ST | 0xe0))
+			continue;
+		ASSERT_EQ((__u32)insn->dst_reg, BPF_REG_1, "flags_reg");
+		ASSERT_EQ((__u32)insn->src_reg, 0, "reserved_src_reg");
+		ASSERT_EQ(insn->off, 0, "reserved_off");
+		if (insn->imm == 0)
+			save_count++;
+		else if (insn->imm == 1)
+			restore_count++;
+		else
+			ASSERT_FAIL("unexpected IRQ operation %d", insn->imm);
+	}
+	ASSERT_EQ(save_count, expect_inline ? 2 : 0, "save_count");
+	ASSERT_EQ(restore_count, expect_inline ? 2 : 0, "restore_count");
+	if (expect_inline)
+		check_irq_native(prog_fd, info.jited_prog_len);
+out:
+	free(insns);
+}
+
+void test_irq_jit(void)
+{
+	struct irq_jit *skel;
+	const char *harden = "/proc/sys/net/core/bpf_jit_harden";
+	char old_harden[32];
+	int err;
+
+	if (sysctl_set_or_fail(harden, old_harden, "2"))
+		return;
+	skel = irq_jit__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "irq_jit_open_and_load"))
+		goto restore_harden;
+	for (int prog_idx = 0; prog_idx < 2; prog_idx++) {
+		struct bpf_program *prog = prog_idx ? skel->progs.irq_jit_subprog :
+						    skel->progs.irq_jit_basic;
+		LIBBPF_OPTS(bpf_test_run_opts, opts,
+			.data_in = &pkt_v4,
+			.data_size_in = sizeof(pkt_v4),
+			.repeat = 16,
+		);
+
+		if (!test__start_subtest(bpf_program__name(prog)))
+			continue;
+		err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
+		if (!ASSERT_OK(err, "test_run"))
+			continue;
+		ASSERT_LE(opts.retval, 1, "test_retval");
+		check_irq_insns(bpf_program__fd(prog), opts.retval == 0);
+	}
+	irq_jit__destroy(skel);
+restore_harden:
+	sysctl_set_or_fail(harden, NULL, old_harden);
+}
diff --git a/tools/testing/selftests/bpf/progs/irq.c b/tools/testing/selftests/bpf/progs/irq.c
index 50727fa9b11d..ffb74d3c55c0 100644
--- a/tools/testing/selftests/bpf/progs/irq.c
+++ b/tools/testing/selftests/bpf/progs/irq.c
@@ -11,6 +11,30 @@ extern void bpf_local_irq_save(unsigned long *) __weak __ksym;
 extern void bpf_local_irq_restore(unsigned long *) __weak __ksym;
 extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym;
 
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define IRQ_FLAGS_REG_BYTE 0x01
+#else
+#define IRQ_FLAGS_REG_BYTE 0x10
+#endif
+
+SEC("?tc")
+__failure __msg("unknown opcode e2")
+__naked void irq_internal_save_rejected(void)
+{
+	asm volatile (".byte 0xe2, %[flags_reg]; .short 0; .long 0; r0 = 0; exit;"
+		      : : __imm_const(flags_reg, IRQ_FLAGS_REG_BYTE)
+		      : __clobber_all);
+}
+
+SEC("?tc")
+__failure __msg("unknown opcode e2")
+__naked void irq_internal_restore_rejected(void)
+{
+	asm volatile (".byte 0xe2, %[flags_reg]; .short 0; .long 1; r0 = 0; exit;"
+		      : : __imm_const(flags_reg, IRQ_FLAGS_REG_BYTE)
+		      : __clobber_all);
+}
+
 struct bpf_res_spin_lock lockA __hidden SEC(".data.A");
 struct bpf_res_spin_lock lockB __hidden SEC(".data.B");
 
diff --git a/tools/testing/selftests/bpf/progs/irq_jit.c b/tools/testing/selftests/bpf/progs/irq_jit.c
new file mode 100644
index 000000000000..45834448333b
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/irq_jit.c
@@ -0,0 +1,55 @@
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+
+extern void bpf_local_irq_save(unsigned long *) __ksym;
+extern void bpf_local_irq_restore(unsigned long *) __ksym;
+extern bool CONFIG_TRACE_IRQFLAGS __kconfig __weak;
+extern bool CONFIG_PARAVIRT_XXL __kconfig __weak;
+extern bool CONFIG_ARM64_DEBUG_PRIORITY_MASKING __kconfig __weak;
+
+static __always_inline int irq_jit_result(void)
+{
+	bool fallback = CONFIG_TRACE_IRQFLAGS;
+
+#if defined(__TARGET_ARCH_x86)
+	fallback |= CONFIG_PARAVIRT_XXL;
+#elif defined(__TARGET_ARCH_arm64)
+	fallback |= CONFIG_ARM64_DEBUG_PRIORITY_MASKING;
+#else
+	fallback = true;
+#endif
+	return fallback;
+}
+
+SEC("tc")
+int irq_jit_basic(struct __sk_buff *ctx)
+{
+	unsigned long outer_flags, inner_flags;
+
+	bpf_local_irq_save(&outer_flags);
+	bpf_local_irq_save(&inner_flags);
+	bpf_local_irq_restore(&inner_flags);
+	bpf_local_irq_restore(&outer_flags);
+	return ctx->len ? irq_jit_result() : 2;
+}
+
+static __noinline void irq_jit_nested(void)
+{
+	unsigned long flags;
+
+	bpf_local_irq_save(&flags);
+	bpf_local_irq_restore(&flags);
+}
+
+SEC("tc")
+int irq_jit_subprog(struct __sk_buff *ctx)
+{
+	unsigned long flags;
+
+	bpf_local_irq_save(&flags);
+	irq_jit_nested();
+	bpf_local_irq_restore(&flags);
+	return ctx->len ? irq_jit_result() : 2;
+}
+
+char _license[] SEC("license") = "GPL";
-- 
2.52.0

[PATCH bpf-next 6/7] bpf: Enable fastcall for inlined local IRQ kfuncs

[Emil Tsalapatis]

Mark bpf_local_irq_save() and bpf_local_irq_restore() with KF_FASTCALL
so the verifier can remove BPF caller compatibility spills and reloads
around calls that are replaced with internal IRQ instructions.

Only enable this optimization when bpf_jit_supports_irq_save_restore()
reports support. Backends and configurations that keep the native kfunc
calls must retain the compatibility spills, since those functions use
the ordinary native calling convention.

Share the IRQ kfunc identification helper between the call summary and
the late fixup so the fastcall decision follows the same backend gate
as inlining. Leave the native function implementations unchanged.

Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
 kernel/bpf/helpers.c  |  4 ++--
 kernel/bpf/verifier.c | 11 +++++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 648abe1ba63a..26b6a9df0d45 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -5197,8 +5197,8 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
-BTF_ID_FLAGS(func, bpf_local_irq_save)
-BTF_ID_FLAGS(func, bpf_local_irq_restore)
+BTF_ID_FLAGS(func, bpf_local_irq_save, KF_FASTCALL)
+BTF_ID_FLAGS(func, bpf_local_irq_restore, KF_FASTCALL)
 #ifdef CONFIG_BPF_EVENTS
 BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr, KF_PERFMON)
 BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr, KF_PERFMON)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index aff7080b5c85..b1ea1564e959 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -19605,6 +19605,12 @@ bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
 	}
 }
 
+static bool is_bpf_local_irq_kfunc(u32 func_id)
+{
+	return func_id == special_kfunc_list[KF_bpf_local_irq_save] ||
+	       func_id == special_kfunc_list[KF_bpf_local_irq_restore];
+}
+
 /* If @call is a kfunc or helper call, fills @cs and returns true,
  * otherwise returns false.
  */
@@ -19641,6 +19647,8 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
 			return false;
 		cs->arg_slot_cnt = btf_proto_slots(meta.btf, meta.func_proto);
 		cs->fastcall = meta.kfunc_flags & KF_FASTCALL;
+		if (!call->off && is_bpf_local_irq_kfunc(meta.func_id))
+			cs->fastcall &= bpf_jit_supports_irq_save_restore();
 		cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type));
 		return true;
 	}
@@ -23373,8 +23381,7 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		return -EFAULT;
 	}
 	if (!desc->offset && bpf_jit_supports_irq_save_restore() &&
-	    (desc->func_id == special_kfunc_list[KF_bpf_local_irq_save] ||
-	     desc->func_id == special_kfunc_list[KF_bpf_local_irq_restore])) {
+	    is_bpf_local_irq_kfunc(desc->func_id)) {
 		int irq_op = desc->func_id == special_kfunc_list[KF_bpf_local_irq_save] ?
 			     BPF_IRQ_SAVE : BPF_IRQ_RESTORE;
 
-- 
2.52.0

[PATCH bpf-next 7/7] selftests/bpf: Verify register preservation for IRQ fastcalls

[Emil Tsalapatis]

Keep distinct runtime-derived values in R0 and R2-R5 across local IRQ
save and restore calls, and check all ten captured register values.
These void, single-argument kfuncs must preserve those registers when
their BPF caller compatibility spills are removed during inlining.

Provide matching stack spills and reloads around each call so the same
program remains correct on configurations that use native kfunc calls.
Check that the translated program either contains two internal IRQ
instructions and no compatibility memory instructions, or retains all
twenty compatibility instructions when the calls are not inlined.

Run the program with context marks zero and one and sixteen repeats
per mark. Always check the translated instructions and captured register
values, without changing the system's JIT hardening configuration.

Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
 .../selftests/bpf/prog_tests/irq_fastcall.c   | 71 +++++++++++++++++++
 .../selftests/bpf/progs/irq_fastcall.c        | 56 +++++++++++++++
 2 files changed, 127 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/irq_fastcall.c
 create mode 100644 tools/testing/selftests/bpf/progs/irq_fastcall.c

diff --git a/tools/testing/selftests/bpf/prog_tests/irq_fastcall.c b/tools/testing/selftests/bpf/prog_tests/irq_fastcall.c
new file mode 100644
index 000000000000..3c1073bc1e49
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/irq_fastcall.c
@@ -0,0 +1,71 @@
+#include <test_progs.h>
+#include <network_helpers.h>
+#include "irq_fastcall.skel.h"
+
+static void check_spills(int prog_fd)
+{
+	struct bpf_prog_info info = {};
+	__u32 info_len = sizeof(info), xlated_len;
+	struct bpf_insn *insns;
+	int err, spill_count = 0, irq_count = 0;
+
+	err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
+	if (!ASSERT_OK(err, "prog_info"))
+		return;
+	xlated_len = info.xlated_prog_len;
+	insns = malloc(xlated_len);
+	if (!ASSERT_OK_PTR(insns, "insns"))
+		return;
+	memset(&info, 0, sizeof(info));
+	info.xlated_prog_len = xlated_len;
+	info.xlated_prog_insns = ptr_to_u64(insns);
+	err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
+	if (!ASSERT_OK(err, "xlated_info"))
+		goto out;
+	for (size_t insn_idx = 0; insn_idx < xlated_len / sizeof(*insns); insn_idx++) {
+		struct bpf_insn *insn = &insns[insn_idx];
+
+		if (insn->code == (BPF_ST | 0xe0))
+			irq_count++;
+		if ((insn->code == (BPF_STX | BPF_MEM | BPF_DW) &&
+		     insn->dst_reg == BPF_REG_10 && insn->off <= -16) ||
+		    (insn->code == (BPF_LDX | BPF_MEM | BPF_DW) &&
+		     insn->src_reg == BPF_REG_10 && insn->off <= -16))
+			spill_count++;
+	}
+	ASSERT_TRUE(irq_count == 0 || irq_count == 2, "irq_inlining");
+	ASSERT_EQ(spill_count, irq_count ? 0 : 20, "compatibility_spills");
+out:
+	free(insns);
+}
+
+void test_irq_fastcall(void)
+{
+	struct irq_fastcall *skel;
+	int prog_fd, err;
+
+	skel = irq_fastcall__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
+		return;
+	prog_fd = bpf_program__fd(skel->progs.irq_fastcall_preserve);
+	check_spills(prog_fd);
+	for (int mark = 0; mark < 2; mark++) {
+		struct __sk_buff ctx = { .mark = mark };
+
+		LIBBPF_OPTS(bpf_test_run_opts, opts,
+			.data_in = &pkt_v4,
+			.data_size_in = sizeof(pkt_v4),
+			.ctx_in = &ctx,
+			.ctx_size_in = sizeof(ctx),
+			.repeat = 16,
+		);
+
+		err = bpf_prog_test_run_opts(prog_fd, &opts);
+		ASSERT_OK(err, "test_run");
+		ASSERT_EQ(opts.retval, 0, "retval");
+		for (int reg_idx = 0; reg_idx < 10; reg_idx++)
+			ASSERT_EQ(skel->bss->saved_regs[reg_idx],
+				  mark + 17 * (reg_idx % 5 + 1), "preserved_register");
+	}
+	irq_fastcall__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/progs/irq_fastcall.c b/tools/testing/selftests/bpf/progs/irq_fastcall.c
new file mode 100644
index 000000000000..3988e489c14f
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/irq_fastcall.c
@@ -0,0 +1,56 @@
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+extern void bpf_local_irq_save(unsigned long *) __ksym __bpf_fastcall;
+extern void bpf_local_irq_restore(unsigned long *) __ksym __bpf_fastcall;
+
+__u64 saved_regs[10];
+
+void __kfunc_btf_root(void)
+{
+	asm volatile ("" : : "r"(&bpf_local_irq_save), "r"(&bpf_local_irq_restore));
+}
+
+#define IRQ_SET_REGS \
+	"r0 = r7; r0 += 17; r2 = r7; r2 += 34;" \
+	"r3 = r7; r3 += 51; r4 = r7; r4 += 68; r5 = r7; r5 += 85;"
+#define IRQ_SPILLS \
+	"*(u64 *)(r10 - 16) = r0;" \
+	"*(u64 *)(r10 - 24) = r2;" \
+	"*(u64 *)(r10 - 32) = r3;" \
+	"*(u64 *)(r10 - 40) = r4;" \
+	"*(u64 *)(r10 - 48) = r5;"
+#define IRQ_FILLS \
+	"r5 = *(u64 *)(r10 - 48);" \
+	"r4 = *(u64 *)(r10 - 40);" \
+	"r3 = *(u64 *)(r10 - 32);" \
+	"r2 = *(u64 *)(r10 - 24);" \
+	"r0 = *(u64 *)(r10 - 16);"
+#define IRQ_CAPTURE_REGS \
+	"*(u64 *)(r6 + 0) = r0;" \
+	"*(u64 *)(r6 + 8) = r2;" \
+	"*(u64 *)(r6 + 16) = r3;" \
+	"*(u64 *)(r6 + 24) = r4;" \
+	"*(u64 *)(r6 + 32) = r5;"
+
+SEC("tc")
+__naked int irq_fastcall_preserve(void)
+{
+	asm volatile (
+		"r7 = *(u32 *)(r1 + 8); r6 = %[saved_regs] ll;"
+		"r1 = r10; r1 += -8;"
+		IRQ_SET_REGS IRQ_SPILLS
+		"call %[bpf_local_irq_save];"
+		IRQ_FILLS IRQ_CAPTURE_REGS
+		"r6 += 40; r1 = r10; r1 += -8;"
+		IRQ_SET_REGS IRQ_SPILLS
+		"call %[bpf_local_irq_restore];"
+		IRQ_FILLS IRQ_CAPTURE_REGS
+		"r0 = 0; exit;"
+		:
+		: __imm(bpf_local_irq_save), __imm(bpf_local_irq_restore), __imm_addr(saved_regs)
+		: __clobber_all);
+}
+
+char _license[] SEC("license") = "GPL";
-- 
2.52.0

Re: [PATCH bpf-next 5/7] selftests/bpf: Test local IRQ kfunc inlining

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [Medium] selftests/bpf: parallel race on bpf_jit_harden in test_irq_jit()
- [Medium] selftests/bpf: missing is_jit_enabled() skip check in test_irq_jit()
--

commit c1826dafea259f7db660c28b5f90f57c88d53332
Author: Emil Tsalapatis <emil@etsalapatis.com>

selftests/bpf: Test local IRQ kfunc inlining

This commit adds BPF selftests for local IRQ save and restore kfunc inlining,
checking that userspace cannot submit the kernel-private opcodes directly and
verifying the translated and JITed instructions with bpf_jit_harden enabled.

> diff --git a/tools/testing/selftests/bpf/prog_tests/irq_jit.c b/tools/testing/selftests/bpf/prog_tests/irq_jit.c
> new file mode 100644
> index 0000000000000..3d5dc3c77edbc
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/prog_tests/irq_jit.c

[ ... ]

> +void test_irq_jit(void)
> +{
> +	struct irq_jit *skel;
> +	const char *harden = "/proc/sys/net/core/bpf_jit_harden";
> +	char old_harden[32];
> +	int err;

[Severity: Medium]
Should test_irq_jit() be declared as serial_test_irq_jit() instead?

Because /proc/sys/net/core/bpf_jit_harden modifies the global bpf_jit_harden
sysctl rather than per-netns state, running test_irq_jit() in parallel mode
(test_progs -j) enables constant blinding and disables BPF kallsyms globally
while other worker processes are running.

Concurrent tests that assert on unblinded xlated instruction offsets (such as
check_one_to_one_mapping(), check_simple(), and check_deletions() in
tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c) or look up JITed BPF
symbols in /proc/kallsyms (such as test_stacktrace_ips) can fail spuriously
while bpf_jit_harden is set to 2.

In addition, test_irq_jit() races with other parallel tests that read and
write bpf_jit_harden (such as toggle_jit_harden() in
tools/testing/selftests/bpf/prog_tests/subprogs.c and check_blindness() in
tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c). A concurrent writer
can clear bpf_jit_harden before irq_jit__open_and_load() runs, or cause
old_harden to sample and restore "2", leaving hardening permanently enabled
for subsequent tests like serial_test_btf_rust().

Other tests that set /proc/sys/net/core/bpf_jit_harden to "2" via
sysctl_set_or_fail(), such as serial_test_arena_scalar_blinded(),
serial_test_jit_insns_blinded(), and serial_test_kasan(), use the
serial_test_ prefix to avoid this race.

> +
> +	if (sysctl_set_or_fail(harden, old_harden, "2"))
> +		return;
> +	skel = irq_jit__open_and_load();
> +	if (!ASSERT_OK_PTR(skel, "irq_jit_open_and_load"))
> +		goto restore_harden;

[Severity: Medium]
What happens in test_irq_jit() when the kernel is built without
CONFIG_BPF_JIT or when the BPF JIT is disabled at runtime
(/proc/sys/net/core/bpf_jit_enable == 0)?

When CONFIG_BPF_JIT is not enabled, /proc/sys/net/core/bpf_jit_harden does
not exist and sysctl_set_or_fail() fails the test with ENOENT. When the JIT
is compiled in but disabled at runtime, add_kfunc_call() in
kernel/bpf/verifier.c rejects the kfunc calls with -ENOTSUPP:

    if (!env->prog->jit_requested) {
        verbose(env, "JIT is required for calling kernel function\n");
        return -ENOTSUPP;
    }

causing irq_jit__open_and_load() to fail the test instead of skipping.

Could test_irq_jit() check is_jit_enabled() and call test__skip() before
touching bpf_jit_harden, similar to serial_test_arena_scalar_blinded() in
tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c:

    if (!is_jit_enabled()) {
        test__skip();
        return;
    }

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20261027162820.10887-1-emil@etsalapatis.com?part=5

Re: [PATCH bpf-next 0/7] bpf: Inline IRQ enter/exit instructions

[Alexei Starovoitov]

On Tue, Oct 27, 2026 at 09:28 AM Emil Tsalapatis <emil@etsalapatis.com> wrote:
> Add inlining for those functions. The inlining happens in two stages.
> First replace the call itself with a special BPF instruction denoting
> an IRQ disable-and save, or restore-and-enable. Those instructions
> are translated by the architecture-specific JIT into the corresponding
> operation.

No. That doesn't scale. We cannot add new insns for every to-be-inlined kfunc.
See bpf_jit_inlines_helper_call().

Like:

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 6a2fe3d..c1c9a1c 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3495,6 +3495,17 @@ populate_extable:
 			}
 			if (!imm32)
 				return -EINVAL;
+			if (src_reg == BPF_PSEUDO_KFUNC_CALL && bpf_jit_inlines_kfunc_call(func)) {
+				if (func == (u8 *)bpf_local_irq_save) {
+					/* pushfq; pop [rdi]; cli */
+					EMIT4(0x9C, 0x8F, 0x07, 0xFA);
+				} else {
+					/* test dword ptr [rdi], X86_EFLAGS_IF; je 1f; sti; 1: */
+					EMIT2_off32(0xF7, 0x07, X86_EFLAGS_IF);
+					EMIT3(X86_JE, 1, 0xFB);
+				}
+				break;
+			}
 			if (src_reg == BPF_PSEUDO_KFUNC_CALL) {
 				fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
 				if (!fm)
@@ -5116,6 +5127,14 @@ bool bpf_jit_supports_kfunc_call(void)
 	return true;
 }

+bool bpf_jit_inlines_kfunc_call(void *func)
+{
+	if (!IS_ENABLED(CONFIG_BPF_SYSCALL) || IS_ENABLED(CONFIG_TRACE_IRQFLAGS) ||
+	    cpu_feature_enabled(X86_FEATURE_XENPV))
+		return false;
+	return func == bpf_local_irq_save || func == bpf_local_irq_restore;
+}

...

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 648abe1..26b6a9d 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -5197,8 +5197,8 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
-BTF_ID_FLAGS(func, bpf_local_irq_save)
-BTF_ID_FLAGS(func, bpf_local_irq_restore)
+BTF_ID_FLAGS(func, bpf_local_irq_save, KF_FASTCALL)
+BTF_ID_FLAGS(func, bpf_local_irq_restore, KF_FASTCALL)

and corresponding bits in bpf_get_call_summary()...

but before doing any of that there gotta be performance numbers that it's worth doing.

pw-bot: cr

Re: [PATCH bpf-next 3/7] arm64, bpf: JIT internal IRQ save and restore instructions

[bot+bpf-ci]

[-- Attachment #1: Type: text/plain, Size: 1891 bytes --]

> arm64, bpf: JIT internal IRQ save and restore instructions

TL;DR: Nit only, no bug found: in the arm64 IRQ save/restore JIT, the raw
sys_reg(3, 3, 4, 2, 1) and pstate_field(3, 6) tuples could get names like
SYS_DAIF or PSTATE_DAIFSET, or a comment.

> diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h
> index b2fe6e6dcf44..23cbc38510a0 100644
> --- a/arch/arm64/net/bpf_jit.h
> +++ b/arch/arm64/net/bpf_jit.h

[ ... ]

> @@ -338,6 +339,13 @@
>  #define A64_MRS_SP_EL0(Rt) \
>  	aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_SP_EL0)
>  
> +#define A64_MRS_SYSREG(Rt, SysReg) \
> +	(aarch64_insn_get_mrs_value() | (SysReg) | (Rt))
> +#define A64_MSR_SYSREG(Rt, SysReg) \
> +	(aarch64_insn_get_msr_reg_value() | (SysReg) | (Rt))
> +#define A64_MSR_DAIFSET(Imm) \
> +	(aarch64_insn_get_msr_imm_value() | pstate_field(3, 6) | ((Imm) << PSTATE_Imm_shift))

[ ... ]

> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index aaa34c739004..aed84f0d8728 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c

[ ... ]

> @@ -2026,6 +2027,33 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
>  		}
>  		break;
>  
> +	case BPF_ST | BPF_IRQ:
> +	{
> +		bool pmr = system_uses_irq_prio_masking();
> +		u32 sysreg = pmr ? SYS_ICC_PMR_EL1 : sys_reg(3, 3, 4, 2, 1);

this isn't a bug, but could sys_reg(3, 3, 4, 2, 1) and pstate_field(3, 6)
get names, such as a SYS_DAIF or PSTATE_DAIFSET style define (or at least
a comment)?

SYS_ICC_PMR_EL1 is named on the same line, but a reader has to decode both
tuples by hand to see that they mean DAIF and DAIFSet.


---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

CI run summary: https://github.com/kernel-patches/bpf/actions/runs/40544258296

Re: [PATCH bpf-next 7/7] selftests/bpf: Verify register preservation for IRQ fastcalls

[bot+bpf-ci]

[-- Attachment #1: Type: text/plain, Size: 1906 bytes --]

> selftests/bpf: Verify register preservation for IRQ fastcalls

TL;DR: Nits only: the two new irq_fastcall selftest files (prog_tests and
progs) lack the SPDX license identifier line that nearly all neighbouring
files carry.

> diff --git a/tools/testing/selftests/bpf/prog_tests/irq_fastcall.c b/tools/testing/selftests/bpf/prog_tests/irq_fastcall.c
> new file mode 100644
> index 000000000000..3c1073bc1e49
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/prog_tests/irq_fastcall.c
> @@ -0,0 +1,71 @@
> +#include <test_progs.h>
> +#include <network_helpers.h>
> +#include "irq_fastcall.skel.h"

Neither new file starts with an SPDX license identifier.  This one begins
with '#include <test_progs.h>' and progs/irq_fastcall.c begins with
'#include <vmlinux.h>'.

The char _license[] SEC("license") = "GPL" in the BPF object is the program
license that the verifier checks, not the file's SPDX header.

Most files in prog_tests/ carry the tag (482 of 487), and checkpatch.pl
warns about a missing SPDX-License-Identifier on the first line of a new .c
file.  irq_jit.c from earlier in this series is one of the few that lack it.

Should both new files start with:

  // SPDX-License-Identifier: GPL-2.0

like the other files in prog_tests/ and progs/?

[ ... ]

> diff --git a/tools/testing/selftests/bpf/progs/irq_fastcall.c b/tools/testing/selftests/bpf/progs/irq_fastcall.c
> new file mode 100644
> index 000000000000..3988e489c14f
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/progs/irq_fastcall.c
> @@ -0,0 +1,56 @@
> +#include <vmlinux.h>
> +#include <bpf/bpf_helpers.h>
> +#include "bpf_misc.h"

The same SPDX line seems to be missing here as well.


---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

CI run summary: https://github.com/kernel-patches/bpf/actions/runs/40544258296

Re: [PATCH bpf-next 5/7] selftests/bpf: Test local IRQ kfunc inlining

[bot+bpf-ci]

[-- Attachment #1: Type: text/plain, Size: 2467 bytes --]

> selftests/bpf: Test local IRQ kfunc inlining

TL;DR: Test issue in irq_jit.c: ASSERT_FAIL() in the "unexpected IRQ
operation" branch appears never to fail the test, so that guard is
ineffective; PRINT_FAIL() may be intended.

> diff --git a/tools/testing/selftests/bpf/prog_tests/irq_jit.c b/tools/testing/selftests/bpf/prog_tests/irq_jit.c
> new file mode 100644
> index 000000000000..3d5dc3c77edb
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/prog_tests/irq_jit.c
> @@ -0,0 +1,128 @@

[ ... ]

> +static void check_irq_insns(int prog_fd, bool expect_inline)
> +{

[ ... ]

> +	for (size_t insn_idx = 0; insn_idx < info.xlated_prog_len / sizeof(*insns); insn_idx++) {
> +		struct bpf_insn *insn = &insns[insn_idx];
> +
> +		if (insn->code != (BPF_ST | 0xe0))
> +			continue;
> +		ASSERT_EQ((__u32)insn->dst_reg, BPF_REG_1, "flags_reg");
> +		ASSERT_EQ((__u32)insn->src_reg, 0, "reserved_src_reg");
> +		ASSERT_EQ(insn->off, 0, "reserved_off");
> +		if (insn->imm == 0)
> +			save_count++;
> +		else if (insn->imm == 1)
> +			restore_count++;
> +		else
> +			ASSERT_FAIL("unexpected IRQ operation %d", insn->imm);
> +	}
> +	ASSERT_EQ(save_count, expect_inline ? 2 : 0, "save_count");
> +	ASSERT_EQ(restore_count, expect_inline ? 2 : 0, "restore_count");

A subsystem pattern flags this as potentially concerning: ASSERT_FAIL() as
the only thing that marks a failure.

In test_progs.h, ASSERT_FAIL() expands to CHECK(false, "", fmt"\n", ##args).
Since the condition is false, _CHECK() takes its pass branch, so test__fail()
is not called and the format string is not printed.

If the else branch is reached for an unexpected imm value, neither
save_count nor restore_count is touched.  An unknown operation sitting
next to two valid saves and two valid restores would still satisfy the
ASSERT_EQ(save_count, ...) and ASSERT_EQ(restore_count, ...) checks
above.

Can the "unexpected IRQ operation" check ever fail the test as written?
Would PRINT_FAIL(), which calls test__fail() unconditionally, be the
intended macro here?

For context, the kernel currently emits only imm 0 or 1 for these
instructions, so this branch only matters as a guard against a future
encoding change.  That guard is what does not work today.


---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

CI run summary: https://github.com/kernel-patches/bpf/actions/runs/40544258296

Re: [PATCH bpf-next 0/7] bpf: Inline IRQ enter/exit instructions

[Emil Tsalapatis]

On Tue, Oct 27, 2026 at 4:53 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Tue, Oct 27, 2026 at 09:28 AM Emil Tsalapatis <emil@etsalapatis.com> wrote:
> > Add inlining for those functions. The inlining happens in two stages.
> > First replace the call itself with a special BPF instruction denoting
> > an IRQ disable-and save, or restore-and-enable. Those instructions
> > are translated by the architecture-specific JIT into the corresponding
> > operation.
>
> No. That doesn't scale. We cannot add new insns for every to-be-inlined kfunc.
> See bpf_jit_inlines_helper_call().
>

Ack, the code below seems strictly better than reserving an instruction.

> Like:
>
> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
> index 6a2fe3d..c1c9a1c 100644
> --- a/arch/x86/net/bpf_jit_comp.c
> +++ b/arch/x86/net/bpf_jit_comp.c
> @@ -3495,6 +3495,17 @@ populate_extable:
>                         }
>                         if (!imm32)
>                                 return -EINVAL;
> +                       if (src_reg == BPF_PSEUDO_KFUNC_CALL && bpf_jit_inlines_kfunc_call(func)) {
> +                               if (func == (u8 *)bpf_local_irq_save) {
> +                                       /* pushfq; pop [rdi]; cli */
> +                                       EMIT4(0x9C, 0x8F, 0x07, 0xFA);
> +                               } else {
> +                                       /* test dword ptr [rdi], X86_EFLAGS_IF; je 1f; sti; 1: */
> +                                       EMIT2_off32(0xF7, 0x07, X86_EFLAGS_IF);
> +                                       EMIT3(X86_JE, 1, 0xFB);
> +                               }
> +                               break;
> +                       }
>                         if (src_reg == BPF_PSEUDO_KFUNC_CALL) {
>                                 fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
>                                 if (!fm)
> @@ -5116,6 +5127,14 @@ bool bpf_jit_supports_kfunc_call(void)
>         return true;
>  }
>
> +bool bpf_jit_inlines_kfunc_call(void *func)
> +{
> +       if (!IS_ENABLED(CONFIG_BPF_SYSCALL) || IS_ENABLED(CONFIG_TRACE_IRQFLAGS) ||
> +           cpu_feature_enabled(X86_FEATURE_XENPV))
> +               return false;
> +       return func == bpf_local_irq_save || func == bpf_local_irq_restore;
> +}
>
> ...
>
> diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
> index 648abe1..26b6a9d 100644
> --- a/kernel/bpf/helpers.c
> +++ b/kernel/bpf/helpers.c
> @@ -5197,8 +5197,8 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON)
>  BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE)
>  BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)
>  BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
> -BTF_ID_FLAGS(func, bpf_local_irq_save)
> -BTF_ID_FLAGS(func, bpf_local_irq_restore)
> +BTF_ID_FLAGS(func, bpf_local_irq_save, KF_FASTCALL)
> +BTF_ID_FLAGS(func, bpf_local_irq_restore, KF_FASTCALL)
>
> and corresponding bits in bpf_get_call_summary()...
>
> but before doing any of that there gotta be performance numbers that it's worth doing.
>

Makes sense, I assume you mean more macro numbers from
production-grade programs apart from ubench numbers.

> pw-bot: cr

Re: [PATCH bpf-next 0/7] bpf: Inline IRQ enter/exit instructions

[Kumar Kartikeya Dwivedi]

On Wed Oct 28, 2026 at 2:55 AM CET, Emil Tsalapatis wrote:
> On Tue, Oct 27, 2026 at 4:53 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
>>
>> On Tue, Oct 27, 2026 at 09:28 AM Emil Tsalapatis <emil@etsalapatis.com> wrote:
>> > Add inlining for those functions. The inlining happens in two stages.
>> > First replace the call itself with a special BPF instruction denoting
>> > an IRQ disable-and save, or restore-and-enable. Those instructions
>> > are translated by the architecture-specific JIT into the corresponding
>> > operation.
>>
>> No. That doesn't scale. We cannot add new insns for every to-be-inlined kfunc.
>> See bpf_jit_inlines_helper_call().
>>
>
> Ack, the code below seems strictly better than reserving an instruction.
>
>> Like:
>>
>> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
>> index 6a2fe3d..c1c9a1c 100644
>> --- a/arch/x86/net/bpf_jit_comp.c
>> +++ b/arch/x86/net/bpf_jit_comp.c
>> @@ -3495,6 +3495,17 @@ populate_extable:
>>                         }
>>                         if (!imm32)
>>                                 return -EINVAL;
>> +                       if (src_reg == BPF_PSEUDO_KFUNC_CALL && bpf_jit_inlines_kfunc_call(func)) {
>> +                               if (func == (u8 *)bpf_local_irq_save) {
>> +                                       /* pushfq; pop [rdi]; cli */
>> +                                       EMIT4(0x9C, 0x8F, 0x07, 0xFA);
>> +                               } else {
>> +                                       /* test dword ptr [rdi], X86_EFLAGS_IF; je 1f; sti; 1: */
>> +                                       EMIT2_off32(0xF7, 0x07, X86_EFLAGS_IF);
>> +                                       EMIT3(X86_JE, 1, 0xFB);
>> +                               }
>> +                               break;
>> +                       }
>>                         if (src_reg == BPF_PSEUDO_KFUNC_CALL) {
>>                                 fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
>>                                 if (!fm)
>> @@ -5116,6 +5127,14 @@ bool bpf_jit_supports_kfunc_call(void)
>>         return true;
>>  }
>>
>> +bool bpf_jit_inlines_kfunc_call(void *func)
>> +{
>> +       if (!IS_ENABLED(CONFIG_BPF_SYSCALL) || IS_ENABLED(CONFIG_TRACE_IRQFLAGS) ||
>> +           cpu_feature_enabled(X86_FEATURE_XENPV))
>> +               return false;
>> +       return func == bpf_local_irq_save || func == bpf_local_irq_restore;
>> +}
>>
>> ...
>>
>> diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
>> index 648abe1..26b6a9d 100644
>> --- a/kernel/bpf/helpers.c
>> +++ b/kernel/bpf/helpers.c
>> @@ -5197,8 +5197,8 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON)
>>  BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE)
>>  BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)
>>  BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
>> -BTF_ID_FLAGS(func, bpf_local_irq_save)
>> -BTF_ID_FLAGS(func, bpf_local_irq_restore)
>> +BTF_ID_FLAGS(func, bpf_local_irq_save, KF_FASTCALL)
>> +BTF_ID_FLAGS(func, bpf_local_irq_restore, KF_FASTCALL)
>>
>> and corresponding bits in bpf_get_call_summary()...
>>
>> but before doing any of that there gotta be performance numbers that it's worth doing.
>>
>
> Makes sense, I assume you mean more macro numbers from
> production-grade programs apart from ubench numbers.
>

While you are it, it would be useful for bpf_preempt_{disable,enable}() to get
similar treatment.

>> pw-bot: cr