* [PATCH v4 1/4] LoongArch: bpf: Fix memory ordering for value-returning atomics
2026-07-29 2:28 [PATCH v4 0/4] LoongArch bpf kptr xchg inline support Chenguang Zhao
@ 2026-07-29 2:28 ` Chenguang Zhao
2026-07-29 2:28 ` [PATCH v4 2/4] LoongArch: bpf: Advertise JIT support for kptr xchg inline Chenguang Zhao
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Chenguang Zhao @ 2026-07-29 2:28 UTC (permalink / raw)
To: chenhuacai, kernel, ast, daniel, andrii, martin.lau, eddyz87,
memxor, yangtiezhu
Cc: hengqi.chen, song, yonghong.song, jolsa, shuah, loongarch, bpf,
linux-kselftest, chenguang.zhao, Chenguang Zhao
From: Chenguang Zhao <zhaochenguang@kylinos.cn>
Per the Linux Kernel Memory Model, value-returning atomic RMW operations
must provide sequentially consistent ordering (a full memory barrier). On
LoongArch, plain AMO instructions and bare ll/sc loops do not satisfy this
requirement by themselves.
Update emit_atomic_rmw() to emit barrier-carrying instructions for all
value-returning BPF atomics:
- BPF_FETCH (ADD/AND/OR/XOR): use am*_db.{b,h,w,d}
- BPF_XCHG: use amswap_db.{b,h,w,d}
- BPF_CMPXCHG: emit dbar 0x700 after the ll/sc loop, matching
__WEAK_LLSC_MB in cmpxchg.h
Add the corresponding instruction encodings and emit helpers to inst.h.
Non-value-returning RMW ops (plain BPF_ADD, BPF_AND, etc.) are left as
weakly ordered, consistent with LKMM.
Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
---
arch/loongarch/include/asm/inst.h | 18 +++++++++++++++++
arch/loongarch/net/bpf_jit.c | 32 +++++++++++++++++--------------
2 files changed, 36 insertions(+), 14 deletions(-)
diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h
index 76b723590023..bdbc17d07110 100644
--- a/arch/loongarch/include/asm/inst.h
+++ b/arch/loongarch/include/asm/inst.h
@@ -199,6 +199,10 @@ enum reg3_op {
amswaph_op = 0x70b9,
amaddb_op = 0x70ba,
amaddh_op = 0x70bb,
+ amswapdbb_op = 0x70bc,
+ amswapdbh_op = 0x70bd,
+ amadddbb_op = 0x70be,
+ amadddbh_op = 0x70bf,
amswapw_op = 0x70c0,
amswapd_op = 0x70c1,
amaddw_op = 0x70c2,
@@ -783,6 +787,20 @@ DEF_EMIT_REG3_FORMAT(amswapb, amswapb_op)
DEF_EMIT_REG3_FORMAT(amswaph, amswaph_op)
DEF_EMIT_REG3_FORMAT(amswapw, amswapw_op)
DEF_EMIT_REG3_FORMAT(amswapd, amswapd_op)
+DEF_EMIT_REG3_FORMAT(amswapdbb, amswapdbb_op)
+DEF_EMIT_REG3_FORMAT(amswapdbh, amswapdbh_op)
+DEF_EMIT_REG3_FORMAT(amadddbb, amadddbb_op)
+DEF_EMIT_REG3_FORMAT(amadddbh, amadddbh_op)
+DEF_EMIT_REG3_FORMAT(amadddbw, amadddbw_op)
+DEF_EMIT_REG3_FORMAT(amadddbd, amadddbd_op)
+DEF_EMIT_REG3_FORMAT(amanddbw, amanddbw_op)
+DEF_EMIT_REG3_FORMAT(amanddbd, amanddbd_op)
+DEF_EMIT_REG3_FORMAT(amordbw, amordbw_op)
+DEF_EMIT_REG3_FORMAT(amordbd, amordbd_op)
+DEF_EMIT_REG3_FORMAT(amxordbw, amxordbw_op)
+DEF_EMIT_REG3_FORMAT(amxordbd, amxordbd_op)
+DEF_EMIT_REG3_FORMAT(amswapdbw, amswapdbw_op)
+DEF_EMIT_REG3_FORMAT(amswapdbd, amswapdbd_op)
#define DEF_EMIT_REG3SA2_FORMAT(NAME, OP) \
static inline void emit_##NAME(union loongarch_instruction *insn, \
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 29c281bef28e..5417a5624327 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -8,6 +8,9 @@
#include <asm/asm-offsets.h>
#include "bpf_jit.h"
+/* dbar hint for ll/sc completion ordering, see __WEAK_LLSC_MB */
+#define DBAR_LLSC_MB 0x700
+
#define LOONGARCH_MAX_REG_ARGS 8
#define LOONGARCH_SAVE_RA_NINSNS 1
@@ -433,7 +436,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amadd.b instruction is not supported\n");
return -EINVAL;
}
- emit_insn(ctx, amaddb, src, t1, t3);
+ emit_insn(ctx, amadddbb, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_H:
@@ -441,39 +444,39 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amadd.h instruction is not supported\n");
return -EINVAL;
}
- emit_insn(ctx, amaddh, src, t1, t3);
+ emit_insn(ctx, amadddbh, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_W:
- emit_insn(ctx, amaddw, src, t1, t3);
+ emit_insn(ctx, amadddbw, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_DW:
- emit_insn(ctx, amaddd, src, t1, t3);
+ emit_insn(ctx, amadddbd, src, t1, t3);
break;
}
break;
case BPF_AND | BPF_FETCH:
if (isdw) {
- emit_insn(ctx, amandd, src, t1, t3);
+ emit_insn(ctx, amanddbd, src, t1, t3);
} else {
- emit_insn(ctx, amandw, src, t1, t3);
+ emit_insn(ctx, amanddbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
case BPF_OR | BPF_FETCH:
if (isdw) {
- emit_insn(ctx, amord, src, t1, t3);
+ emit_insn(ctx, amordbd, src, t1, t3);
} else {
- emit_insn(ctx, amorw, src, t1, t3);
+ emit_insn(ctx, amordbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
case BPF_XOR | BPF_FETCH:
if (isdw) {
- emit_insn(ctx, amxord, src, t1, t3);
+ emit_insn(ctx, amxordbd, src, t1, t3);
} else {
- emit_insn(ctx, amxorw, src, t1, t3);
+ emit_insn(ctx, amxordbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
@@ -485,7 +488,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amswap.b instruction is not supported\n");
return -EINVAL;
}
- emit_insn(ctx, amswapb, src, t1, t3);
+ emit_insn(ctx, amswapdbb, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_H:
@@ -493,15 +496,15 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amswap.h instruction is not supported\n");
return -EINVAL;
}
- emit_insn(ctx, amswaph, src, t1, t3);
+ emit_insn(ctx, amswapdbh, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_W:
- emit_insn(ctx, amswapw, src, t1, t3);
+ emit_insn(ctx, amswapdbw, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_DW:
- emit_insn(ctx, amswapd, src, t1, t3);
+ emit_insn(ctx, amswapdbd, src, t1, t3);
break;
}
break;
@@ -524,6 +527,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
emit_zext_32(ctx, r0, true);
}
+ emit_insn(ctx, dbar, DBAR_LLSC_MB);
break;
default:
pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm);
--
2.25.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH v4 4/4] selftests/bpf: Add kptr-xchg benchmark
2026-07-29 2:28 [PATCH v4 0/4] LoongArch bpf kptr xchg inline support Chenguang Zhao
` (2 preceding siblings ...)
2026-07-29 2:28 ` [PATCH v4 3/4] selftests/bpf: Enable kptr_xchg_inline test on LoongArch Chenguang Zhao
@ 2026-07-29 2:28 ` Chenguang Zhao
3 siblings, 0 replies; 5+ messages in thread
From: Chenguang Zhao @ 2026-07-29 2:28 UTC (permalink / raw)
To: chenhuacai, kernel, ast, daniel, andrii, martin.lau, eddyz87,
memxor, yangtiezhu
Cc: hengqi.chen, song, yonghong.song, jolsa, shuah, loongarch, bpf,
linux-kselftest, chenguang.zhao, Chenguang Zhao
From: Chenguang Zhao <zhaochenguang@kylinos.cn>
Add a bpf selftest benchmark that exercises bpf_kptr_xchg() in a tight loop
so helper vs inlined JIT paths can be compared on supported architectures.
Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
---
tools/testing/selftests/bpf/Makefile | 2 +
tools/testing/selftests/bpf/bench.c | 2 +
.../selftests/bpf/benchs/bench_kptr_xchg.c | 96 +++++++++++++++++++
.../selftests/bpf/progs/kptr_xchg_bench.c | 49 ++++++++++
4 files changed, 149 insertions(+)
create mode 100644 tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c
create mode 100644 tools/testing/selftests/bpf/progs/kptr_xchg_bench.c
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index b642ee489ea6..6dfc81748584 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -980,6 +980,7 @@ $(OUTPUT)/bench_lpm_trie_map.o: $(OUTPUT)/lpm_trie_bench.skel.h $(OUTPUT)/lpm_tr
$(OUTPUT)/bench_bpf_nop.o: $(OUTPUT)/bpf_nop_bench.skel.h bench_bpf_timing.h
$(OUTPUT)/bench_xdp_lb.o: $(OUTPUT)/xdp_lb_bench.skel.h bench_bpf_timing.h
$(OUTPUT)/bench_bpf_timing.o: bench_bpf_timing.h
+$(OUTPUT)/bench_kptr_xchg.o: $(OUTPUT)/kptr_xchg_bench.skel.h
$(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ)
$(OUTPUT)/bench: LDLIBS += -lm
$(OUTPUT)/bench: $(OUTPUT)/bench.o \
@@ -1005,6 +1006,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \
$(OUTPUT)/bench_bpf_timing.o \
$(OUTPUT)/bench_bpf_nop.o \
$(OUTPUT)/bench_xdp_lb.o \
+ $(OUTPUT)/bench_kptr_xchg.o \
$(OUTPUT)/usdt_1.o \
$(OUTPUT)/usdt_2.o \
#
diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c
index 3d9d2cd7764b..550290d6701a 100644
--- a/tools/testing/selftests/bpf/bench.c
+++ b/tools/testing/selftests/bpf/bench.c
@@ -582,6 +582,7 @@ extern const struct bench bench_lpm_trie_delete;
extern const struct bench bench_lpm_trie_free;
extern const struct bench bench_bpf_nop;
extern const struct bench bench_xdp_lb;
+extern const struct bench bench_kptr_xchg;
static const struct bench *benchs[] = {
&bench_count_global,
@@ -665,6 +666,7 @@ static const struct bench *benchs[] = {
&bench_lpm_trie_free,
&bench_bpf_nop,
&bench_xdp_lb,
+ &bench_kptr_xchg,
};
static void find_benchmark(void)
diff --git a/tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c b/tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c
new file mode 100644
index 000000000000..b8a0d346fda6
--- /dev/null
+++ b/tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2026. Loongson Technology Corporation Limited */
+#include <argp.h>
+#include "bench.h"
+#include "kptr_xchg_bench.skel.h"
+
+static struct ctx {
+ struct kptr_xchg_bench *skel;
+} ctx;
+
+static struct {
+ __u32 nr_loops;
+} args = {
+ .nr_loops = 256,
+};
+
+enum {
+ ARG_NR_LOOPS = 7000,
+};
+
+static const struct argp_option opts[] = {
+ { "nr_loops", ARG_NR_LOOPS, "nr_loops", 0,
+ "Set number of bpf_kptr_xchg() calls per trigger"},
+ {},
+};
+
+static error_t parse_arg(int key, char *arg, struct argp_state *state)
+{
+ switch (key) {
+ case ARG_NR_LOOPS:
+ args.nr_loops = strtol(arg, NULL, 10);
+ break;
+ default:
+ return ARGP_ERR_UNKNOWN;
+ }
+
+ return 0;
+}
+
+static const struct argp bench_kptr_xchg_argp = {
+ .options = opts,
+ .parser = parse_arg,
+};
+
+static void validate(void)
+{
+ if (env.consumer_cnt != 0) {
+ fprintf(stderr, "benchmark doesn't support consumer!\n");
+ exit(1);
+ }
+}
+
+static void *producer(void *input)
+{
+ while (true)
+ syscall(__NR_getpgid);
+
+ return NULL;
+}
+
+static void measure(struct bench_res *res)
+{
+ res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
+}
+
+static void setup(void)
+{
+ struct bpf_link *link;
+
+ setup_libbpf();
+
+ ctx.skel = kptr_xchg_bench__open_and_load();
+ if (!ctx.skel) {
+ fprintf(stderr, "failed to open skeleton\n");
+ exit(1);
+ }
+
+ ctx.skel->data->nr_loops = args.nr_loops;
+
+ link = bpf_program__attach(ctx.skel->progs.benchmark);
+ if (!link) {
+ fprintf(stderr, "failed to attach program!\n");
+ exit(1);
+ }
+}
+
+const struct bench bench_kptr_xchg = {
+ .name = "kptr-xchg",
+ .argp = &bench_kptr_xchg_argp,
+ .validate = validate,
+ .setup = setup,
+ .producer_thread = producer,
+ .measure = measure,
+ .report_progress = ops_report_progress,
+ .report_final = ops_report_final,
+};
diff --git a/tools/testing/selftests/bpf/progs/kptr_xchg_bench.c b/tools/testing/selftests/bpf/progs/kptr_xchg_bench.c
new file mode 100644
index 000000000000..363883073e2c
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/kptr_xchg_bench.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2026. Loongson Technology Corporation Limited */
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+
+#include "bpf_experimental.h"
+#include "bpf_misc.h"
+
+char _license[] SEC("license") = "GPL";
+
+#define MAX_XCHG_LOOPS 4096
+
+struct bin_data {
+ char blob[32];
+};
+
+#define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8)))
+
+private(kptr) struct bin_data __kptr *ptr;
+u32 nr_loops = 256;
+long hits;
+
+SEC("fentry/" SYS_PREFIX "sys_getpgid")
+int benchmark(void *ctx)
+{
+ struct bin_data *old;
+ u32 i;
+
+ for (i = 0; i < MAX_XCHG_LOOPS; i++) {
+ if (i >= nr_loops)
+ break;
+
+ old = bpf_kptr_xchg(&ptr, NULL);
+ if (old)
+ bpf_obj_drop(old);
+ }
+
+ __sync_add_and_fetch(&hits, i);
+ return 0;
+}
+
+/*
+ * BTF FUNC records are not generated for kfuncs referenced only through
+ * optimized paths. Keep bpf_obj_drop() visible to libbpf's kfunc linker.
+ */
+void __btf_root(void)
+{
+ bpf_obj_drop(NULL);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 5+ messages in thread