* [PATCH bpf-next v2 3/3] selftests/bpf: Follow-up to JIT instruction tests
2026-10-16 10:18 [PATCH bpf-next v2 1/3] bpf, s390: Mask the count of 32-bit shifts by register Daniel Borkmann
2026-10-16 10:18 ` [PATCH bpf-next v2 2/3] selftests/bpf: Restore the je <-> jmp distance in convergence check Daniel Borkmann
@ 2026-10-16 10:18 ` Daniel Borkmann
2026-10-16 10:58 ` bot+bpf-ci
2026-10-16 17:50 ` [PATCH bpf-next v2 1/3] bpf, s390: Mask the count of 32-bit shifts by register patchwork-bot+netdevbpf
2 siblings, 1 reply; 6+ messages in thread
From: Daniel Borkmann @ 2026-10-16 10:18 UTC (permalink / raw)
To: alexei.starovoitov; +Cc: memxor, bpf
Close gaps in the JIT instruction tests found by earlier Sashiko review,
by comparing the programs with what reaches the JIT, and by a sweep
through RFC9669 to construct tests for corner cases. Non-exhaustive list:
- The staggered jumps loaded their operands once, so the verifier decided
the later guards of ja and gotol and the later jumps of jeq_k and
jlt32_k, and a landing short of the target passed with r0 at zero.
Reload the operands per region and let r0 carry the region number
from the jump to its target.
- The matching 32-bit cmpxchg compared an r0 the mismatching one had zero
extended. Reload r0 with the full value first and check that the source
register keeps its upper half, as test_bpf did.
- The 32-bit atomics wrote and read back only their word, so a JIT touching
all 64 bits passed when the other half was zero. Fill and read back the
whole slot.
- The sub-word loads of the zero offset, offsets and probe_mem tests landed
in a register holding nothing in particular. Set it to all ones first.
- Let probe_mem load every size, sign extending too, and run ld_imm64 into
every register, not only r1.
- Drop the load-acquire probe: a JIT without the instruction fails like for
any other one instead of skipping.
# LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t jit_insns
[...]
#191/1 jit_insns/arena access with zero offset:OK
#191/2 jit_insns/cast to a user address:OK
#191/3 jit_insns/alu_imm:OK
#191/4 jit_insns/alu_reg:OK
#191/5 jit_insns/alu_imm_regs:OK
#191/6 jit_insns/div_const:OK
#191/7 jit_insns/alu_reg_pairs:OK
#191/8 jit_insns/alu_edges:OK
#191/9 jit_insns/shifts:OK
#191/10 jit_insns/shift_counts:OK
#191/11 jit_insns/atomics:OK
#191/12 jit_insns/atomic_reg_pairs:OK
#191/13 jit_insns/xadd_chain:OK
#191/14 jit_insns/ld_imm64:OK
#191/15 jit_insns/jmp_imm:OK
#191/16 jit_insns/jmp_reg:OK
#191/17 jit_insns/jmp_reg_pairs:OK
#191/18 jit_insns/jmp_edges:OK
#191/19 jit_insns/staggered_jumps:OK
#191/20 jit_insns/max_jmp:OK
#191/21 jit_insns/convergence:OK
#191/22 jit_insns/calls:OK
#191/23 jit_insns/helper_calls:OK
#191/24 jit_insns/ldst_zero_off:OK
#191/25 jit_insns/ldst_offsets:OK
#191/26 jit_insns/acq_rel:OK
#191/27 jit_insns/movsx:OK
#191/28 jit_insns/jset_imm:OK
#191/29 jit_insns/cmp_imm:OK
#191/30 jit_insns/alu_consts:OK
#191/31 jit_insns/bswap:OK
#191/32 jit_insns/probe_mem:OK
#191 jit_insns:OK
#192/1 jit_insns_blinded/alu_imm:OK
#192/2 jit_insns_blinded/alu_imm_regs:OK
#192/3 jit_insns_blinded/alu_edges:OK
#192/4 jit_insns_blinded/div_const:OK
#192/5 jit_insns_blinded/shifts:OK
#192/6 jit_insns_blinded/xadd_chain:OK
#192/7 jit_insns_blinded/ld_imm64:OK
#192/8 jit_insns_blinded/jmp_imm:OK
#192/9 jit_insns_blinded/jmp_edges:OK
#192/10 jit_insns_blinded/ldst_zero_off:OK
#192/11 jit_insns_blinded/ldst_offsets:OK
#192/12 jit_insns_blinded/jset_imm:OK
#192/13 jit_insns_blinded/cmp_imm:OK
#192/14 jit_insns_blinded/alu_consts:OK
#192 jit_insns_blinded:OK
Summary: 2/46 PASSED, 0 SKIPPED, 0/0 FAILED
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
---
.../selftests/bpf/prog_tests/jit_insns.c | 1104 ++++++++++++++---
1 file changed, 913 insertions(+), 191 deletions(-)
diff --git a/tools/testing/selftests/bpf/prog_tests/jit_insns.c b/tools/testing/selftests/bpf/prog_tests/jit_insns.c
index dabf8a4abdfe..dabefdb8a353 100644
--- a/tools/testing/selftests/bpf/prog_tests/jit_insns.c
+++ b/tools/testing/selftests/bpf/prog_tests/jit_insns.c
@@ -16,18 +16,18 @@
struct sweep_ctx {
__u64 in[NR_CASES][2];
- __u64 out[NR_CASES][3];
+ __u64 out[NR_CASES][4];
};
#define IN_OFF(k, j) (offsetof(struct sweep_ctx, in) + ((k) * 2 + (j)) * 8)
-#define OUT_OFF(k, j) (offsetof(struct sweep_ctx, out) + ((k) * 3 + (j)) * 8)
+#define OUT_OFF(k, j) (offsetof(struct sweep_ctx, out) + ((k) * 4 + (j)) * 8)
/* The cases of one program: operands in, results out, expected results */
struct batch {
struct bpf_insn *insns;
int cnt, cap;
struct sweep_ctx ctx;
- __u64 exp[NR_CASES][3];
+ __u64 exp[NR_CASES][4];
__u8 nr_out[NR_CASES];
int nr;
char desc[80];
@@ -149,6 +149,65 @@ static int scratch_reg(int r1, int r2)
return BPF_REG_9;
}
+static int size_bits(int size)
+{
+ switch (size) {
+ case BPF_B:
+ return 8;
+ case BPF_H:
+ return 16;
+ case BPF_W:
+ return 32;
+ default:
+ return 64;
+ }
+}
+
+/* The first size bytes of v in memory, as a load of that size reads them */
+static __u64 head_bytes(__u64 v, int size)
+{
+ int bits = size_bits(size);
+
+ if (bits == 64)
+ return v;
+#if __BYTE_ORDER == __BIG_ENDIAN
+ return v >> (64 - bits);
+#else
+ return v & ((1ULL << bits) - 1);
+#endif
+}
+
+/* v with its first size bytes in memory replaced by those of w */
+static __u64 head_merge(__u64 v, __u64 w, int size)
+{
+ int bits = size_bits(size);
+ __u64 mask;
+
+ if (bits == 64)
+ return w;
+ mask = (1ULL << bits) - 1;
+#if __BYTE_ORDER == __BIG_ENDIAN
+ return (v & ~(mask << (64 - bits))) | ((w & mask) << (64 - bits));
+#else
+ return (v & ~mask) | (w & mask);
+#endif
+}
+
+/* v as a sign extending load of that size reads it */
+static __s64 sign_extend(__u64 v, int size)
+{
+ switch (size) {
+ case BPF_B:
+ return (__s8)v;
+ case BPF_H:
+ return (__s16)v;
+ case BPF_W:
+ return (__s32)v;
+ default:
+ return v;
+ }
+}
+
static const char *alu_name(int op)
{
switch (BPF_OP(op)) {
@@ -189,12 +248,13 @@ static const char *jmp_name(int op)
}
}
-/* ALU result computation, false when the operands are not valid */
+/* ALU result computation, with the ISA result for every operand */
enum { F_ALU32 = 1, F_SIGNED = 2 };
-static bool alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags)
+static void alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags)
{
bool is_signed = flags & F_SIGNED;
+ int mask = flags & F_ALU32 ? 31 : 63;
/* Narrow operands for ALU32 */
if (flags & F_ALU32) {
@@ -225,12 +285,15 @@ static bool alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags)
*res = v1 ^ v2;
break;
case BPF_LSH:
- *res = v1 << v2;
+ *res = v1 << (v2 & mask);
break;
case BPF_RSH:
- *res = v1 >> v2;
+ *res = v1 >> (v2 & mask);
break;
case BPF_ARSH:
+ if (flags & F_ALU32)
+ v1 = (__u64)(__s32)v1;
+ v2 &= mask;
*res = v1 >> v2;
if (v2 > 0 && v1 > INT64_MAX)
*res |= ~0ULL << (64 - v2);
@@ -244,30 +307,28 @@ static bool alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags)
case BPF_MUL:
*res = v1 * v2;
break;
+ /* By zero: the quotient is zero and the dividend keeps, by -1: negation and zero */
case BPF_DIV:
if (v2 == 0)
- return false;
- if (!is_signed) {
+ *res = 0;
+ else if (!is_signed)
*res = v1 / v2;
- } else {
- if ((__s64)v2 == -1) /* Handled by the verifier */
- return false;
+ else if ((__s64)v2 == -1)
+ *res = -v1;
+ else
*res = (__u64)((__s64)v1 / (__s64)v2);
- }
break;
case BPF_MOD:
if (v2 == 0)
- return false;
- if (!is_signed) {
+ *res = v1;
+ else if (!is_signed)
*res = v1 % v2;
- } else {
- if ((__s64)v2 == -1)
- return false;
+ else if ((__s64)v2 == -1)
+ *res = 0;
+ else
*res = (__u64)((__s64)v1 % (__s64)v2);
- }
break;
}
- return true;
}
/*
@@ -311,6 +372,27 @@ static void fill_pattern(struct batch *b, void *arg, int dbits, int sbits,
batch_flush(b);
}
+/* Operands at the edges: zero, one and the limits of every width */
+static const __u64 edges[] = {
+ 0, 1, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000,
+ 0xffffffff, 0x100000000ULL, 0x7fffffffffffffffULL, 0x8000000000000000ULL,
+ 0xffffffff80000000ULL, 0xffffffff7fffffffULL, 0xffffffffffffffffULL,
+};
+
+static void fill_edges(struct batch *b, void *arg, bool imm, emit_fn emit)
+{
+ int i, j;
+
+ for (i = 0; i < ARRAY_SIZE(edges); i++)
+ for (j = 0; j < ARRAY_SIZE(edges); j++) {
+ /* The immediate has to fit its 32 bits */
+ if (imm && (__s64)edges[j] != (__s32)edges[j])
+ continue;
+ emit(b, arg, edges[i], edges[j]);
+ }
+ batch_flush(b);
+}
+
/* ALU operations with an immediate and with a register for all magnitudes */
static const int alu_ops[] = {
BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD,
@@ -328,9 +410,16 @@ static void emit_alu_imm(struct batch *b, void *arg, __s64 dst, __s64 imm)
__u64 res;
int k;
- if (!alu_result(&res, dst, (__s32)imm, a->op, a->flags))
+ /* The verifier rejects a zero divisor and a shift count beyond the width */
+ if ((a->op == BPF_DIV || a->op == BPF_MOD) && !imm)
+ return;
+ if ((a->op == BPF_LSH || a->op == BPF_RSH || a->op == BPF_ARSH) &&
+ (__u64)imm >= (alu32 ? 32 : 64))
return;
- k = batch_case(b, BPF_REG_3, dst, 0);
+ if (a->op == BPF_NEG)
+ imm = 0;
+ alu_result(&res, dst, (__s32)imm, a->op, a->flags);
+ k = batch_case(b, BPF_REG_3, dst, (__s32)imm);
LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1);
EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(a->op) | BPF_K,
BPF_REG_1, 0, a->flags & F_SIGNED ? 1 : 0, imm));
@@ -345,8 +434,11 @@ static void emit_alu_reg(struct batch *b, void *arg, __s64 dst, __s64 src)
__u64 res;
int k;
- if (!alu_result(&res, dst, src, a->op, a->flags))
+ /* Shift counts beyond the width have their own subtest */
+ if ((a->op == BPF_LSH || a->op == BPF_RSH || a->op == BPF_ARSH) &&
+ (alu32 ? (__u32)src : (__u64)src) >= (alu32 ? 32 : 64))
return;
+ alu_result(&res, dst, src, a->op, a->flags);
k = batch_case(b, BPF_REG_3, dst, src);
LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1);
LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2);
@@ -398,6 +490,41 @@ static void test_alu_reg(void)
test_alu_magnitudes(true);
}
+static void test_alu_edges(void)
+{
+ static const struct alu_arg ops[] = {
+ { BPF_MOV }, { BPF_AND }, { BPF_OR }, { BPF_XOR }, { BPF_LSH }, { BPF_RSH },
+ { BPF_ARSH }, { BPF_ADD }, { BPF_SUB }, { BPF_MUL }, { BPF_DIV }, { BPF_MOD },
+ { BPF_DIV, F_SIGNED }, { BPF_MOD, F_SIGNED }, { BPF_NEG },
+ };
+ struct batch b;
+ int o, f, r, i;
+
+ if (!batch_init(&b))
+ return;
+ for (f = 0; f < 2; f++) {
+ for (o = 0; o < ARRAY_SIZE(ops); o++) {
+ struct alu_arg a = { ops[o].op, ops[o].flags | (f ? F_ALU32 : 0) };
+
+ for (r = 0; r < (a.op == BPF_NEG ? 1 : 2); r++) {
+ snprintf(b.desc, sizeof(b.desc), "alu%d_%s%s_%c edges",
+ f ? 32 : 64, a.flags & F_SIGNED ? "s" : "", alu_name(a.op),
+ r ? 'x' : 'k');
+ if (a.op == BPF_NEG) {
+ for (i = 0; i < ARRAY_SIZE(edges); i++)
+ emit_alu_imm(&b, &a, edges[i], 0);
+ batch_flush(&b);
+ } else if (r) {
+ fill_edges(&b, &a, false, emit_alu_reg);
+ } else {
+ fill_edges(&b, &a, true, emit_alu_imm);
+ }
+ }
+ }
+ }
+ batch_fini(&b);
+}
+
/* ALU operations with an immediate for every destination register */
static const int alu_reg_ops[] = {
BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_LSH, BPF_RSH, BPF_ARSH,
@@ -516,11 +643,7 @@ static __u64 div_result(int op, __u32 flags, __s32 imm, __u64 n)
{
__u64 res;
- /* LLONG_MIN / -1 == LLONG_MIN, LLONG_MIN % -1 == 0, same for INT_MIN */
- if ((flags & F_SIGNED) && imm == -1)
- res = op == BPF_MOD ? 0 : -n;
- else
- alu_result(&res, n, (__s32)imm, op, flags);
+ alu_result(&res, n, (__s32)imm, op, flags);
return flags & F_ALU32 ? (__u32)res : res;
}
@@ -570,37 +693,46 @@ static void test_div_const(void)
/* ALU operations for all combinations of operand registers */
static void alu_reg_pairs(struct batch *b, int op, __u32 flags)
{
- bool alu32 = flags & F_ALU32;
- __u64 dst, src, res, same;
- int rd, rs, rc, k;
+ bool alu32 = flags & F_ALU32, divmod = op == BPF_DIV || op == BPF_MOD;
+ int code = (alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_X;
+ int off = flags & F_SIGNED ? 1 : 0, rd, rs, rc, k, n, nr = 1;
+ __u64 dst, srcs[3], src, res, same;
if (flags & F_SIGNED)
dst = alu32 ? (__u64)-76543210LL : (__u64)-7654321076543210LL;
else
dst = alu32 ? 0x76543210ULL : 0x7edcba9876543210ULL;
- src = alu32 ? 0x01234567ULL : 0x0123456789abcdefULL;
+ srcs[0] = alu32 ? 0x01234567ULL : 0x0123456789abcdefULL;
if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH)
- src &= 31;
- alu_result(&res, dst, src, op, flags);
- alu_result(&same, src, src, op, flags);
- if (alu32) {
- res = (__u32)res;
- same = (__u32)same;
+ srcs[0] &= 31;
+ /* A zero divisor, in 32 bits also one with only the upper half set */
+ if (divmod) {
+ srcs[nr++] = 0;
+ if (alu32)
+ srcs[nr++] = 1ULL << 32;
}
snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_x register pairs", alu32 ? 32 : 64,
flags & F_SIGNED ? "s" : "", alu_name(op));
- for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) {
- for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) {
- rc = scratch_reg(rd, rs);
- k = batch_case(b, rc, rd == rs ? src : dst, src);
- LD_IN(b, k, 0, rc, rd);
- if (rs != rd)
- LD_IN(b, k, 1, rc, rs);
- EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_X,
- rd, rs, flags & F_SIGNED ? 1 : 0, 0));
- ST_OUT(b, k, 0, rc, rd);
- batch_expect(b, k, 0, rd == rs ? same : res);
+ for (n = 0; n < nr; n++) {
+ src = srcs[n];
+ alu_result(&res, dst, src, op, flags);
+ alu_result(&same, src, src, op, flags);
+ if (alu32) {
+ res = (__u32)res;
+ same = (__u32)same;
+ }
+ for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) {
+ for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) {
+ rc = scratch_reg(rd, rs);
+ k = batch_case(b, rc, rd == rs ? src : dst, src);
+ LD_IN(b, k, 0, rc, rd);
+ if (rs != rd)
+ LD_IN(b, k, 1, rc, rs);
+ EMIT(b, BPF_RAW_INSN(code, rd, rs, off, 0));
+ ST_OUT(b, k, 0, rc, rd);
+ batch_expect(b, k, 0, rd == rs ? same : res);
+ }
}
}
batch_flush(b);
@@ -622,16 +754,17 @@ static void test_alu_reg_pairs(void)
batch_fini(&b);
}
+static const __s64 shift_vals[] = {
+ 0x0123456789abcdefLL, /* dword > 0, word < 0 */
+ 0xfedcba9876543210LL, /* dword < 0, word > 0 */
+ 0xfedcba0198765432LL, /* dword < 0, word < 0 */
+ 0x0123458967abcdefLL, /* dword > 0, word > 0 */
+};
+static const int shift_ops[] = { BPF_LSH, BPF_RSH, BPF_ARSH };
+
/* Shifts for all shift counts, by immediate, by register and by the shifted register */
static void test_shifts(void)
{
- static const __s64 vals[] = {
- 0x0123456789abcdefLL, /* dword > 0, word < 0 */
- 0xfedcba9876543210LL, /* dword < 0, word > 0 */
- 0xfedcba0198765432LL, /* dword < 0, word < 0 */
- 0x0123458967abcdefLL, /* dword > 0, word > 0 */
- };
- static const int ops[] = { BPF_LSH, BPF_RSH, BPF_ARSH };
struct batch b;
int o, f, v, imm, k;
@@ -640,14 +773,14 @@ static void test_shifts(void)
for (f = 0; f < 2; f++) {
int bits = f ? 32 : 64;
- for (o = 0; o < ARRAY_SIZE(ops); o++) {
- int op = ops[o];
+ for (o = 0; o < ARRAY_SIZE(shift_ops); o++) {
+ int op = shift_ops[o];
snprintf(b.desc, sizeof(b.desc), "alu%d_%s all shift counts", bits,
alu_name(op));
- for (v = 0; v < ARRAY_SIZE(vals); v++) {
+ for (v = 0; v < ARRAY_SIZE(shift_vals); v++) {
for (imm = 0; imm < bits; imm++) {
- __u64 reg = vals[v], res;
+ __u64 reg = shift_vals[v], res;
if (f)
reg = op == BPF_ARSH ? (__u64)(__s32)reg :
@@ -656,7 +789,7 @@ static void test_shifts(void)
if (f)
res = (__u32)res;
- k = batch_case(&b, BPF_REG_3, vals[v], imm);
+ k = batch_case(&b, BPF_REG_3, shift_vals[v], imm);
LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1);
if (f)
EMIT(&b, BPF_ALU32_IMM(op, BPF_REG_1, imm));
@@ -665,7 +798,7 @@ static void test_shifts(void)
ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1);
batch_expect(&b, k, 0, res);
- k = batch_case(&b, BPF_REG_3, vals[v], imm);
+ k = batch_case(&b, BPF_REG_3, shift_vals[v], imm);
LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1);
LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2);
if (f)
@@ -696,6 +829,54 @@ static void test_shifts(void)
batch_fini(&b);
}
+/*
+ * Shifts by register with counts at and above the width and negative ones,
+ * which the ISA masks by the width. Apart in a subtest of its own, as not
+ * every JIT does so.
+ */
+static void test_shift_counts(void)
+{
+ static const __s64 counts[] = { 32, 33, 63, 64, 65, 127, -1, -3, 1LL << 32 | 3 };
+ struct batch b;
+ int o, f, v, c, k;
+
+ if (!batch_init(&b))
+ return;
+ for (f = 0; f < 2; f++) {
+ int bits = f ? 32 : 64;
+
+ for (o = 0; o < ARRAY_SIZE(shift_ops); o++) {
+ int op = shift_ops[o];
+
+ snprintf(b.desc, sizeof(b.desc), "alu%d_%s_x counts beyond the width",
+ bits, alu_name(op));
+ for (c = 0; c < ARRAY_SIZE(counts); c++) {
+ __u64 cnt = f ? (__u32)counts[c] : counts[c];
+
+ for (v = 0; v < ARRAY_SIZE(shift_vals); v++) {
+ __u64 reg = shift_vals[v], res;
+
+ if (f)
+ reg = op == BPF_ARSH ? (__u64)(__s32)reg :
+ (__u32)reg;
+ alu_result(&res, reg, cnt & (bits - 1), op, 0);
+ k = batch_case(&b, BPF_REG_3, shift_vals[v], counts[c]);
+ LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1);
+ LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2);
+ if (f)
+ EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_2));
+ else
+ EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_2));
+ ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1);
+ batch_expect(&b, k, 0, f ? (__u32)res : res);
+ }
+ }
+ batch_flush(&b);
+ }
+ }
+ batch_fini(&b);
+}
+
/* Atomic operations on the stack for all magnitudes of memory and source operand */
static const int atomic_ops[] = {
BPF_ADD, BPF_AND, BPF_OR, BPF_XOR,
@@ -723,16 +904,18 @@ static void emit_atomic(struct batch *b, void *arg, __s64 dst, __s64 src)
fetch = a->op & BPF_FETCH ? (w ? (__u32)dst : dst) : src;
k = batch_case(b, BPF_REG_3, dst, src);
+ /* The kept value fills the slot, a 32-bit operation leaves the other half alone */
+ emit_ld_imm64(b, BPF_REG_0, keep);
+ EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT));
LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1);
EMIT(b, BPF_STX_MEM(a->size, BPF_REG_10, BPF_REG_1, MEM_SLOT));
LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2);
- emit_ld_imm64(b, BPF_REG_0, keep);
EMIT(b, BPF_ATOMIC_OP(a->size, a->op, BPF_REG_10, BPF_REG_2, MEM_SLOT));
- EMIT(b, BPF_LDX_MEM(a->size, BPF_REG_1, BPF_REG_10, MEM_SLOT));
+ EMIT(b, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, MEM_SLOT));
ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1);
ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_2);
ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0);
- batch_expect(b, k, 0, w ? (__u32)res : res);
+ batch_expect(b, k, 0, head_merge(keep, w ? (__u32)res : res, a->size));
batch_expect(b, k, 1, fetch);
batch_expect(b, k, 2, keep);
}
@@ -746,21 +929,30 @@ static void emit_cmpxchg(struct batch *b, void *arg, __s64 dst, __s64 src)
k = batch_case(b, BPF_REG_3, dst, src);
LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1);
+ /* The inverted value fills the slot, a 32-bit operation leaves the other half alone */
+ EMIT(b, BPF_MOV64_REG(BPF_REG_0, BPF_REG_1));
+ EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1));
+ EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT));
EMIT(b, BPF_STX_MEM(size, BPF_REG_10, BPF_REG_1, MEM_SLOT));
LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2);
/* Mismatch: the memory keeps its value, r0 gets it */
- EMIT(b, BPF_MOV64_REG(BPF_REG_0, BPF_REG_1));
- EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1));
EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT));
ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_0);
- /* Match: the memory gets the source, r0 the old value */
+ /*
+ * Match: the memory gets the source, r0 the old value. r0 holds the
+ * full value, so the 32-bit compare has to ignore its upper half, and
+ * the source register has to keep its own.
+ */
+ LD_IN(b, k, 0, BPF_REG_3, BPF_REG_0);
EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT));
- EMIT(b, BPF_LDX_MEM(size, BPF_REG_1, BPF_REG_10, MEM_SLOT));
+ EMIT(b, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, MEM_SLOT));
ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_1);
ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0);
+ ST_OUT(b, k, 3, BPF_REG_3, BPF_REG_2);
batch_expect(b, k, 0, old);
- batch_expect(b, k, 1, w ? (__u32)src : src);
+ batch_expect(b, k, 1, head_merge(~(__u64)dst, w ? (__u32)src : src, size));
batch_expect(b, k, 2, old);
+ batch_expect(b, k, 3, src);
}
static void test_atomics(void)
@@ -797,7 +989,7 @@ static void atomic_reg_pairs(struct batch *b, int size, int op)
int rd, rs, rc, k, n;
if (w) {
- mem = 0x01234567U;
+ mem = 0xfedcba9801234567ULL;
upd = 0x76543210U;
} else {
mem = 0x0123456789abcdefULL;
@@ -809,18 +1001,19 @@ static void atomic_reg_pairs(struct batch *b, int size, int op)
op & BPF_FETCH ? "_fetch" : "");
for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) {
for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) {
- __u64 regs[10], old = mem, new, src, cmp;
+ __u64 regs[10], old = w ? (__u32)mem : mem, new, src, cmp;
/* The source would be the pointer, r0 the compare value */
if (rs == rd || (op == BPF_CMPXCHG && rd == BPF_REG_0))
continue;
rc = scratch_reg(rd, rs);
k = batch_case(b, rc, mem, upd);
- /* Memory from in0, r0 inverted, source from in1, pointer */
- LD_IN(b, k, 0, rc, rs);
- EMIT(b, BPF_STX_MEM(size, BPF_REG_10, rs, MEM_SLOT));
+ /* r0 inverted into the slot, memory from in0, source from in1, pointer */
LD_IN(b, k, 0, rc, BPF_REG_0);
EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1));
+ EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT));
+ LD_IN(b, k, 0, rc, rs);
+ EMIT(b, BPF_STX_MEM(size, BPF_REG_10, rs, MEM_SLOT));
LD_IN(b, k, 1, rc, rs);
EMIT(b, BPF_MOV64_REG(rd, BPF_REG_10));
EMIT(b, BPF_ATOMIC_OP(size, op, rd, rs, MEM_SLOT));
@@ -829,7 +1022,7 @@ static void atomic_reg_pairs(struct batch *b, int size, int op)
if (rd != BPF_REG_0)
ST_OUT(b, k, n++, rc, BPF_REG_0);
ST_OUT(b, k, n++, rc, rs);
- EMIT(b, BPF_LDX_MEM(size, rs, BPF_REG_10, MEM_SLOT));
+ EMIT(b, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_10, MEM_SLOT));
ST_OUT(b, k, n, rc, rs);
/* Model with the register aliasing of r0 */
@@ -854,7 +1047,7 @@ static void atomic_reg_pairs(struct batch *b, int size, int op)
if (rd != BPF_REG_0)
batch_expect(b, k, n++, regs[BPF_REG_0]);
batch_expect(b, k, n++, regs[rs]);
- batch_expect(b, k, n, new);
+ batch_expect(b, k, n, head_merge(~mem, new, size));
}
}
batch_flush(b);
@@ -905,13 +1098,14 @@ static void test_xadd_chain(void)
free(insns);
}
-/* 64-bit immediate loads for all magnitudes and byte patterns */
-static void ld_imm64_case(struct batch *b, __u64 imm)
+/* 64-bit immediate loads for all magnitudes, byte patterns and registers */
+static void ld_imm64_case(struct batch *b, int rd, __u64 imm)
{
- int k = batch_case(b, BPF_REG_3, imm, 0);
+ int rc = scratch_reg(rd, rd);
+ int k = batch_case(b, rc, imm, 0);
- emit_ld_imm64(b, BPF_REG_1, imm);
- ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1);
+ emit_ld_imm64(b, rd, imm);
+ ST_OUT(b, k, 0, rc, rd);
batch_expect(b, k, 0, imm);
}
@@ -925,7 +1119,13 @@ static void test_ld_imm64(void)
{ 1, 0x81, 0, 0xff }, /* positive and zero */
{ 0x80, 0x80, 0, 0xff }, /* negative and zero */
};
- int block = 64, bit, adj, sign, p, pattern, index;
+ /* Zero, a 32-bit, a sign extended 32-bit and a full immediate for every register */
+ static const __u64 imms[] = {
+ 0, 1, 0x7fffffff, 0x80000000, 0xffffffff, 0x100000000ULL,
+ 0xffffffffffffffffULL, 0xffffffff80000000ULL, 0xffffffff7fffffffULL,
+ 0x0123456789abcdefULL,
+ };
+ int block = 64, bit, adj, sign, p, pattern, index, rd, i;
struct batch b;
if (!batch_init(&b))
@@ -934,7 +1134,7 @@ static void test_ld_imm64(void)
for (bit = 0; bit <= 62; bit++)
for (adj = -block / 2; adj < block / 2; adj++)
for (sign = -1; sign <= 1; sign += 2)
- ld_imm64_case(&b, sign * ((1LL << bit) + adj));
+ ld_imm64_case(&b, BPF_REG_1, sign * ((1LL << bit) + adj));
batch_flush(&b);
for (p = 0; p < ARRAY_SIZE(bytes); p++) {
@@ -956,10 +1156,15 @@ static void test_ld_imm64(void)
imm = (imm << 8) | (__u8)byte;
}
rand = rand * 1664525 + 1013904223;
- ld_imm64_case(&b, imm);
+ ld_imm64_case(&b, BPF_REG_1, imm);
}
batch_flush(&b);
}
+ snprintf(b.desc, sizeof(b.desc), "ld_imm64 registers");
+ for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++)
+ for (i = 0; i < ARRAY_SIZE(imms); i++)
+ ld_imm64_case(&b, rd, imms[i]);
+ batch_flush(&b);
batch_fini(&b);
}
@@ -1066,6 +1271,27 @@ static void test_jmp_reg(void)
test_jmp_magnitudes(true);
}
+static void test_jmp_edges(void)
+{
+ struct batch b;
+ int o, f, r;
+
+ if (!batch_init(&b))
+ return;
+ for (f = 0; f < 2; f++) {
+ for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) {
+ for (r = 0; r < 2; r++) {
+ struct jmp_arg a = { jmp_ops[o], f, r };
+
+ snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_%c edges", f ? "32" : "",
+ jmp_name(a.op), r ? 'x' : 'k');
+ fill_edges(&b, &a, !r, emit_jmp);
+ }
+ }
+ }
+ batch_fini(&b);
+}
+
/* A conditional jump on two registers, with the operands swapped for n == 1 */
static void jmp_reg_pair(struct batch *b, bool is32, int op, int rd, int rs, int n)
{
@@ -1136,10 +1362,14 @@ static struct bpf_insn jmp_insn(bool ja32)
/*
* Taken jumps over regions of doubling length, forward or backward, up to
* the maximal offset of the jump: 16 bits, or 2^17 for the 32-bit
- * unconditional jump. The not taken path of a conditional jump and the
- * guard of an unconditional one lead to an exit with the region number, so
- * the verifier sees every instruction reachable and a jump landing anywhere
- * else than on its target fails the run.
+ * unconditional jump. The operands are reloaded in every region, so that
+ * the verifier cannot decide the jumps from the bounds of the region
+ * before and fold the regions away. r0 gains the region number before the
+ * jump and loses it again only on the target, so the not taken path of a
+ * conditional jump, the guard of an unconditional one and a jump landing
+ * short of its target, or forward past it, exit with a nonzero r0. A
+ * backward jump overshooting its target comes back to itself and hangs
+ * the run instead.
*/
static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool backward,
const char *desc)
@@ -1152,18 +1382,20 @@ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool
.ctx_size_in = sizeof(ctx),
);
int max = ja32 ? 1 << 17 : INT16_MAX;
- int len = 128 + 2 * max, i = 0, n = 0, gap, fd;
+ int len = 256 + 2 * max, i = 0, n = 0, gap, fd;
struct bpf_insn *insns;
insns = malloc(len * sizeof(*insns));
if (!ASSERT_OK_PTR(insns, "insns"))
return;
- insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8);
- insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
+ insns[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0);
for (gap = backward ? 4 : 2; gap <= max; gap = gap < max / 2 ? gap * 2 : max) {
int k, a, t, x, g, j;
+ insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 0);
+ insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_6, 8);
+ insns[i++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, ++n);
if (backward) {
/*
* Over the region to its guard and jump, which comes back
@@ -1172,7 +1404,8 @@ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool
*/
a = i++;
t = i;
- for (k = 0; k < gap - 3; k++)
+ insns[i++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_0, n);
+ for (k = 0; k < gap - 3 - ja; k++)
insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0);
x = i++;
g = i;
@@ -1181,7 +1414,6 @@ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool
j = i;
insns[i] = *jmp;
set_jmp_off(&insns[i++], ja32, t - (j + 1));
- insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, ++n);
insns[i++] = BPF_EXIT_INSN();
insns[a] = jmp_insn(ja32);
set_jmp_off(&insns[a], ja32, g - (a + 1));
@@ -1193,10 +1425,10 @@ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool
insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 1);
insns[i] = *jmp;
set_jmp_off(&insns[i++], ja32, gap);
- insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, ++n);
- for (k = 0; k < gap - 2; k++)
+ for (k = 0; k < gap - 1; k++)
insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0);
insns[i++] = BPF_EXIT_INSN();
+ insns[i++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_0, n);
}
if (gap == max)
break;
@@ -1334,50 +1566,6 @@ static void prog_run(int fd, __u64 in0, __u64 in1, __u64 expected, const char *d
ASSERT_EQ(ctx.res, expected, buf);
}
-static int size_bits(int size)
-{
- switch (size) {
- case BPF_B:
- return 8;
- case BPF_H:
- return 16;
- case BPF_W:
- return 32;
- default:
- return 64;
- }
-}
-
-/* The first size bytes of v in memory, as a load of that size reads them */
-static __u64 head_bytes(__u64 v, int size)
-{
- int bits = size_bits(size);
-
- if (bits == 64)
- return v;
-#if __BYTE_ORDER == __BIG_ENDIAN
- return v >> (64 - bits);
-#else
- return v & ((1ULL << bits) - 1);
-#endif
-}
-
-/* v with its first size bytes in memory replaced by those of w */
-static __u64 head_merge(__u64 v, __u64 w, int size)
-{
- int bits = size_bits(size);
- __u64 mask;
-
- if (bits == 64)
- return w;
- mask = (1ULL << bits) - 1;
-#if __BYTE_ORDER == __BIG_ENDIAN
- return (v & ~(mask << (64 - bits))) | ((w & mask) << (64 - bits));
-#else
- return (v & ~mask) | (w & mask);
-#endif
-}
-
static void test_ldst_zero_off(void)
{
static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW };
@@ -1398,8 +1586,9 @@ static void test_ldst_zero_off(void)
for (k = 0; k < NR_PAIRS; k++) {
int rd = pair_reg(b, k);
- /* Load with a zero offset, also into the base */
+ /* Load with a zero offset over all ones, also into the base */
prog_start(&p);
+ EMIT(&p, BPF_MOV64_IMM(rd, -1));
EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_6));
EMIT(&p, BPF_LDX_MEM(size, rd, rb, 0));
prog_end(&p, rd);
@@ -1431,8 +1620,7 @@ static void test_ldst_zero_off(void)
for (j = 0; j < ARRAY_SIZE(inputs); j++) {
__s64 v = head_bytes(inputs[j], size);
- v = size == BPF_B ? (__s8)v :
- size == BPF_H ? (__s16)v : (__s32)v;
+ v = sign_extend(v, size);
prog_run(fd, inputs[j], 0, v, desc);
}
close(fd);
@@ -1614,6 +1802,12 @@ static int other_reg(int r1, int r2)
return BPF_REG_9;
}
+static void off_prog_ld_imm64(struct off_prog *p, int reg, __u64 imm)
+{
+ EMIT(p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, reg, 0, 0, (__u32)imm));
+ EMIT(p, BPF_RAW_INSN(0, 0, 0, 0, imm >> 32));
+}
+
/* Look up the value of key into r0, exit with 1 when the lookup fails */
static void off_prog_lookup(struct off_prog *p, int map_fd, int key)
{
@@ -1700,8 +1894,11 @@ static void load_offsets(struct off_mem *m, int kind, int size, int rb, int rd,
off_prog_start(&p, map_fd, rb);
for (i = 0; i < nr; i++) {
+ /* The base gets reloaded when it is the destination, all ones otherwise */
if (rd == rb)
EMIT(&p, BPF_LDX_MEM(BPF_DW, rb, BPF_REG_10, SLOT_BASE));
+ else
+ EMIT(&p, BPF_MOV64_IMM(rd, -1));
if (kind == K_LDX)
EMIT(&p, BPF_LDX_MEM(size, rd, rb, off[i]));
else
@@ -1722,7 +1919,7 @@ static void load_offsets(struct off_mem *m, int kind, int size, int rb, int rd,
__u64 v = load_bytes(m->mem + MEM_BASE + off[i], size);
if (kind == K_LDSX)
- v = size == BPF_B ? (__s8)v : size == BPF_H ? (__s16)v : (__s32)v;
+ v = sign_extend(v, size);
if (res[i] != v) {
snprintf(desc, sizeof(desc), "%s size=%d base=r%d dst=r%d off=%d",
kind == K_LDX ? "ldx" : "ldsx", size, rb, rd, off[i]);
@@ -1873,7 +2070,7 @@ static void test_movsx(void)
}
/* Branch on dst and imm, with 1 in res when taken and 0 otherwise */
-static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm)
+static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm, int off)
{
int rd = pat_regs[d], rt = pair_reg(d, 1);
@@ -1881,9 +2078,9 @@ static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm)
EMIT(p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0));
EMIT(p, BPF_MOV64_IMM(rt, 1));
if (is64)
- EMIT(p, BPF_JMP_IMM(op, rd, imm, 1));
+ EMIT(p, BPF_JMP_IMM(op, rd, imm, off));
else
- EMIT(p, BPF_JMP32_IMM(op, rd, imm, 1));
+ EMIT(p, BPF_JMP32_IMM(op, rd, imm, off));
EMIT(p, BPF_MOV64_IMM(rt, 0));
prog_end(p, rt);
}
@@ -1903,7 +2100,7 @@ static void test_jset_imm(void)
for (is64 = false; ; is64 = true) {
for (m = 0; m < ARRAY_SIZE(masks); m++) {
- jmp_prog(&p, is64, BPF_JSET, d, masks[m]);
+ jmp_prog(&p, is64, BPF_JSET, d, masks[m], 1);
snprintf(desc, sizeof(desc), "jset%d dst=r%d mask=0x%x",
is64 ? 64 : 32, rd, masks[m]);
fd = prog_load(&p, desc);
@@ -1975,7 +2172,7 @@ static void test_cmp_imm(void)
for (is64 = false; ; is64 = true) {
for (o = 0; o < ARRAY_SIZE(ops); o++) {
for (m = 0; m < ARRAY_SIZE(imms); m++) {
- jmp_prog(&p, is64, ops[o], d, imms[m]);
+ jmp_prog(&p, is64, ops[o], d, imms[m], 1);
snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d imm=%d",
is64 ? 64 : 32, ops[o], rd, imms[m]);
fd = prog_load(&p, desc);
@@ -1990,6 +2187,18 @@ static void test_cmp_imm(void)
prog_run(fd, inputs[j], 0, taken, desc);
}
close(fd);
+ if (m)
+ continue;
+ /* A jump over nothing lands on the same path either way */
+ jmp_prog(&p, is64, ops[o], d, imms[m], 0);
+ snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d off=0",
+ is64 ? 64 : 32, ops[o], rd);
+ fd = prog_load(&p, desc);
+ if (fd < 0)
+ return;
+ for (j = 0; j < ARRAY_SIZE(inputs); j++)
+ prog_run(fd, inputs[j], 0, 0, desc);
+ close(fd);
}
}
if (is64)
@@ -2122,7 +2331,15 @@ static void test_bswap(void)
static void test_probe_mem(void)
{
- int kfunc_id, type_id, b, k, j, fd;
+ static const struct {
+ int size;
+ bool sx;
+ } loads[] = {
+ { BPF_B, false }, { BPF_H, false }, { BPF_W, false }, { BPF_DW, false },
+ { BPF_B, true }, { BPF_H, true }, { BPF_W, true },
+ };
+ int kfunc_id, type_id, b, k, l, j, fd;
+ struct bpf_insn call;
struct btf *btf;
struct prog p;
char desc[64];
@@ -2136,30 +2353,46 @@ static void test_probe_mem(void)
if (!ASSERT_GT(kfunc_id, 0, "bpf_rdonly_cast id") ||
!ASSERT_GT(type_id, 0, "timespec64 id"))
return;
+ call = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, kfunc_id);
- /* The kernel copy of the ctx through an untrusted pointer in every register */
+ /*
+ * The kernel copy of the ctx through an untrusted pointer in every
+ * register, in every size, zero and sign extending over all ones
+ */
for (b = 0; b < ARRAY_SIZE(pat_regs); b++) {
int rb = pat_regs[b];
for (k = 0; k < NR_PAIRS; k++) {
int rd = pair_reg(b, k);
- prog_start(&p);
- EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_6));
- EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id));
- EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL,
- 0, kfunc_id));
- if (rb != BPF_REG_0)
- EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_0));
- EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, rb, 0));
- prog_end(&p, rd);
- snprintf(desc, sizeof(desc), "probe_mem base=r%d dst=r%d", rb, rd);
- fd = prog_load(&p, desc);
- if (fd < 0)
- return;
- for (j = 0; j < ARRAY_SIZE(inputs); j++)
- prog_run(fd, inputs[j], 0, inputs[j], desc);
- close(fd);
+ for (l = 0; l < ARRAY_SIZE(loads); l++) {
+ int size = loads[l].size, mode = loads[l].sx ? BPF_MEMSX : BPF_MEM;
+
+ prog_start(&p);
+ EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_6));
+ EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id));
+ EMIT(&p, call);
+ if (rb != BPF_REG_0)
+ EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_0));
+ if (rd != rb)
+ EMIT(&p, BPF_MOV64_IMM(rd, -1));
+ EMIT(&p, BPF_RAW_INSN(BPF_LDX | mode | size, rd, rb, 0, 0));
+ prog_end(&p, rd);
+ snprintf(desc, sizeof(desc),
+ "probe_mem%s size=%d base=r%d dst=r%d",
+ loads[l].sx ? "sx" : "", size, rb, rd);
+ fd = prog_load(&p, desc);
+ if (fd < 0)
+ return;
+ for (j = 0; j < ARRAY_SIZE(inputs); j++) {
+ __s64 v = head_bytes(inputs[j], size);
+
+ if (loads[l].sx)
+ v = sign_extend(v, size);
+ prog_run(fd, inputs[j], 0, v, desc);
+ }
+ close(fd);
+ }
}
}
@@ -2171,8 +2404,7 @@ static void test_probe_mem(void)
EMIT(&p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_1, 0, 0, (__u32)addr));
EMIT(&p, BPF_RAW_INSN(0, 0, 0, 0, addr >> 32));
EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id));
- EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL,
- 0, kfunc_id));
+ EMIT(&p, call);
EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0));
prog_end(&p, BPF_REG_3);
snprintf(desc, sizeof(desc), "probe_mem addr=0x%llx", addr);
@@ -2184,24 +2416,6 @@ static void test_probe_mem(void)
}
}
-/* Whether the JIT supports load-acquire and store-release */
-static bool acq_rel_ok(void)
-{
- LIBBPF_OPTS(bpf_prog_load_opts, lopts, .prog_flags = BPF_F_SLEEPABLE);
- struct prog p;
- int fd;
-
- prog_start(&p);
- EMIT(&p, BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0));
- EMIT(&p, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8));
- prog_end(&p, BPF_REG_0);
- fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p.insns, p.cnt, &lopts);
- if (fd < 0)
- return false;
- close(fd);
- return true;
-}
-
/*
* Load-acquire and store-release of every size with every register as
* base, on a stack slot holding in0: the loads zero extend like the plain
@@ -2215,10 +2429,6 @@ static void test_acq_rel(void)
char desc[64];
int b, k, s, j, fd;
- if (!acq_rel_ok()) {
- test__skip();
- return;
- }
for (b = 0; b < ARRAY_SIZE(pat_regs); b++) {
int rb = pat_regs[b];
@@ -2340,6 +2550,502 @@ static void test_max_jmp(void)
max_jmp(true, false);
}
+/*
+ * Fillers of two and three bytes of native code on x86-64: a 32-bit and a
+ * 64-bit move between two low registers, which give a region every length
+ * from two bytes on and which constant blinding leaves alone.
+ */
+static int fill_bytes(struct bpf_insn *insns, int n, int bytes)
+{
+ if (bytes & 1) {
+ insns[n++] = BPF_MOV64_REG(BPF_REG_2, BPF_REG_3);
+ bytes -= 3;
+ }
+ for (; bytes > 0; bytes -= 2)
+ insns[n++] = BPF_MOV32_REG(BPF_REG_2, BPF_REG_3);
+ return n;
+}
+
+static void prog_result(struct bpf_insn *insns, int *n)
+{
+ insns[(*n)++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 10);
+ insns[(*n)++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES);
+ insns[(*n)++] = BPF_MOV64_IMM(BPF_REG_0, 0);
+ insns[(*n)++] = BPF_EXIT_INSN();
+}
+
+/* A region of x86-64 bytes through fill_bytes(), or of so many 64 and 32-bit moves */
+struct fill {
+ int bytes, wide, narrow;
+};
+
+static int fill_moves(struct bpf_insn *insns, int n, int wide, int narrow)
+{
+ while (wide-- > 0)
+ insns[n++] = BPF_MOV64_REG(BPF_REG_2, BPF_REG_3);
+ while (narrow-- > 0)
+ insns[n++] = BPF_MOV32_REG(BPF_REG_2, BPF_REG_3);
+ return n;
+}
+
+static int fill_region(struct bpf_insn *insns, int n, const struct fill *f)
+{
+ if (f->wide || f->narrow)
+ return fill_moves(insns, n, f->wide, f->narrow);
+ return fill_bytes(insns, n, f->bytes);
+}
+
+static int fill_insns(const struct fill *f)
+{
+ return f->wide + f->narrow + f->bytes / 2 + 1;
+}
+
+/*
+ * Jumps whose lengths depend on each other: a conditional jump forward
+ * over a region, a jump back from its end to its start or to the jump in
+ * front of it, and an unconditional jump out of it. The region takes every
+ * length around the short branch limit, in front of the exit and behind
+ * it, so the three jumps cross the limit at different lengths and in both
+ * directions. A chain of nop jumps in it, gone one per pass, shifts the
+ * phase in which the cycle meets the padding passes, as settling jumps
+ * elsewhere do. This is generic to any JIT with short and long branch
+ * forms: the ranges of the running one are probed, see find_ranges(), and
+ * the cycle swept across each of them. r0 goes in with 0 to 3 and picks
+ * one of the four paths through the jumps.
+ */
+static void jump_cycle(bool back_cond, bool to_start, int inner, const struct fill *front,
+ const struct fill *back, const char *desc)
+{
+ struct bpf_insn *insns;
+ int n = 0, disp, head, chain, to_back[2], out, i, fd;
+
+ insns = malloc((32 + 2 * inner + fill_insns(front) + fill_insns(back)) * sizeof(*insns));
+ if (!ASSERT_OK_PTR(insns, desc))
+ return;
+ insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0);
+ insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1);
+ insns[n++] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3);
+ insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0);
+ /* The dispatch into the chain, see nop_chain() */
+ disp = n;
+ n += inner;
+ head = n++;
+ n = fill_region(insns, n, front);
+ chain = n;
+ for (i = 0; i < inner; i++) {
+ insns[disp + i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0x1000 + i, chain - disp - 1);
+ insns[n++] = BPF_JMP_IMM(BPF_JA, 0, 0, inner - 1 - i);
+ }
+ to_back[0] = n++;
+ to_back[1] = n++;
+ out = n++;
+ insns[to_back[0]] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 1, n - to_back[0] - 1);
+ insns[to_back[1]] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 3, n - to_back[1] - 1);
+ insns[n++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1);
+ n = fill_region(insns, n, back);
+ /* Taken with 2 and not with 4, so the verifier keeps the condition */
+ if (back_cond)
+ insns[n] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 4, head + to_start - n - 1);
+ else
+ insns[n] = BPF_JMP_IMM(BPF_JA, 0, 0, head + to_start - n - 1);
+ n++;
+ insns[head] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, n - head - 1);
+ insns[out] = BPF_JMP_IMM(BPF_JA, 0, 0, n - out - 1);
+ prog_result(insns, &n);
+
+ fd = load_prog(insns, n, desc);
+ free(insns);
+ if (fd < 0)
+ return;
+ prog_run(fd, 0, 0, 10, desc);
+ prog_run(fd, 1, 0, 12, desc);
+ prog_run(fd, 2, 0, 12, desc);
+ prog_run(fd, 3, 0, 14, desc);
+ close(fd);
+}
+
+/*
+ * A region which ends in a chain of jumps to its end: the last one jumps
+ * to the next instruction, which the JIT drops as a nop, whereupon the one
+ * before it becomes one, so the chain goes away one jump per pass and the
+ * jump over the region shrinks late, in the passes which pad the image.
+ * The filler in front of the chain puts that jump around the short branch
+ * limit, as a conditional and as an unconditional one, over a chain of ja
+ * and one of gotol. Each jump of the chain is the target of a conditional
+ * jump in front of the region on a value the verifier cannot see, as the
+ * chain is dead behind its first jump otherwise.
+ */
+static void nop_chain(bool over_cond, bool ja32, int len, int fill)
+{
+ struct bpf_insn insns[256];
+ int n = 0, disp, over, chain, i, fd;
+ char desc[64];
+
+ insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0);
+ insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1);
+ insns[n++] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3);
+ insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0);
+ /* The dispatch into the chain, filled in once the chain is placed */
+ disp = n;
+ n += len;
+ if (!over_cond)
+ insns[n++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, 1);
+ over = n++;
+ n = fill_bytes(insns, n, fill);
+ chain = n;
+ for (i = 0; i < len; i++) {
+ insns[disp + i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0x1000 + i, chain - disp - 1);
+ insns[n++] = ja32 ? BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, len - 1 - i) :
+ BPF_JMP_IMM(BPF_JA, 0, 0, len - 1 - i);
+ }
+ insns[n++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1);
+ if (over_cond)
+ insns[over] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 2, n - over - 1);
+ else
+ insns[over] = BPF_JMP_IMM(BPF_JA, 0, 0, n - over - 1);
+ prog_result(insns, &n);
+
+ snprintf(desc, sizeof(desc), "nop chain of %s under %s len=%d fill=%d",
+ ja32 ? "gotol" : "ja", over_cond ? "jcc" : "ja", len, fill);
+ fd = load_prog(insns, n, desc);
+ if (fd < 0)
+ return;
+ prog_run(fd, 0, 0, 10, desc);
+ prog_run(fd, 1, 0, 11, desc);
+ prog_run(fd, 2, 0, 13, desc);
+ prog_run(fd, 3, 0, 13, desc);
+ close(fd);
+}
+
+/* The native length of a program's image, 0 without a JIT */
+static __u32 jited_len(int fd)
+{
+ struct bpf_prog_info info = {};
+ __u32 len = sizeof(info);
+
+ if (!ASSERT_OK(bpf_prog_get_info_by_fd(fd, &info, &len), "prog info"))
+ return 0;
+ return info.jited_prog_len;
+}
+
+/* A conditional jump over the moves, for the length of its image */
+static int probe_len(int wide, int narrow, __u32 *len)
+{
+ struct bpf_insn *insns;
+ int n = 0, fd;
+
+ insns = malloc((wide + narrow + 8) * sizeof(*insns));
+ if (!ASSERT_OK_PTR(insns, "probe insns"))
+ return -1;
+ insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0);
+ insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0);
+ insns[n++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, wide + narrow);
+ n = fill_moves(insns, n, wide, narrow);
+ insns[n++] = BPF_MOV64_IMM(BPF_REG_0, 0);
+ insns[n++] = BPF_EXIT_INSN();
+ fd = load_prog(insns, n, "branch range probe");
+ free(insns);
+ if (fd < 0)
+ return -1;
+ *len = jited_len(fd);
+ close(fd);
+ return 0;
+}
+
+/*
+ * The short branch ranges of the running JIT, as counts of 64-bit moves a
+ * conditional jump reaches over, and the native lengths of the two moves.
+ * The image of the probe grows by the moves alone until the jump does not
+ * reach over them any more and takes a longer form, which adds the
+ * difference of the forms: find the first count that does so in every
+ * power of two of moves up to the 16-bit offset of the jump, as a JIT may
+ * have several forms. Without a JIT or with fixed length branches there is
+ * none.
+ */
+static int find_ranges(int *ranges, int max, int *wide_bytes, int *narrow_bytes)
+{
+ __u32 base, len;
+ int excess, plateau = 0, c, lo, hi, mid, k = 0;
+
+ if (probe_len(8, 0, &base) || !base)
+ return 0;
+ if (probe_len(24, 0, &len))
+ return 0;
+ *wide_bytes = (len - base) / 16;
+ if (probe_len(8, 16, &len))
+ return 0;
+ *narrow_bytes = (len - base) / 16;
+ if (!*wide_bytes || !*narrow_bytes)
+ return 0;
+ for (c = 32; k < max && c <= 16384; c *= 2) {
+ if (probe_len(c, 0, &len))
+ return k;
+ excess = len - base - (c - 8) * *wide_bytes;
+ if (excess <= plateau)
+ continue;
+ lo = c / 2;
+ hi = c;
+ while (hi - lo > 1) {
+ mid = (lo + hi) / 2;
+ if (probe_len(mid, 0, &len))
+ return k;
+ if (len - base - (mid - 8) * *wide_bytes > plateau)
+ hi = mid;
+ else
+ lo = mid;
+ }
+ ranges[k++] = hi;
+ plateau = excess;
+ }
+ return k;
+}
+
+static int gcd(int a, int b)
+{
+ while (b) {
+ int t = a % b;
+
+ a = b;
+ b = t;
+ }
+ return a;
+}
+
+/* The jump cycle with the region in front of the exit and then behind it */
+static void cycle_bytes(int kind, int inner, int bytes)
+{
+ struct fill front = { .bytes = bytes }, back = {};
+ char desc[96];
+
+ snprintf(desc, sizeof(desc), "jump cycle %s back to %s inner=%d front=%d",
+ kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", inner, bytes);
+ jump_cycle(kind & 1, kind & 2, inner, &front, &back, desc);
+ front.bytes = 0;
+ back.bytes = bytes;
+ snprintf(desc, sizeof(desc), "jump cycle %s back to %s inner=%d back=%d",
+ kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", inner, bytes);
+ jump_cycle(kind & 1, kind & 2, inner, &front, &back, desc);
+}
+
+/* The jump cycle with a region of so many native bytes of moves */
+static void cycle_moves(int kind, int range, int bytes, int wide, int narrow)
+{
+ struct fill front = {}, back = {};
+ char desc[96];
+ int b;
+
+ for (b = 0; b * narrow <= bytes; b++)
+ if ((bytes - b * narrow) % wide == 0)
+ break;
+ if (b * narrow > bytes)
+ return;
+ front.wide = (bytes - b * narrow) / wide;
+ front.narrow = b;
+ snprintf(desc, sizeof(desc), "jump cycle %s back to %s range %d at %d bytes",
+ kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", range, bytes);
+ jump_cycle(kind & 1, kind & 2, 0, &front, &back, desc);
+}
+
+static void test_convergence(void)
+{
+ /* The chain gone, the jump over it is at and below the limit */
+ static const int fills[] = { 100, 110, 116, 120 };
+ /* The last jump of a chain is a nop the verifier already removes */
+ static const int inners[] = { 0, 2, 3, 4, 5 };
+ int ranges[4], wide, narrow, g, nr, bytes, lo, hi, c, f, i, l, k;
+
+ /* The x86-64 imm8 range, byte by byte */
+ for (c = 0; c < 4; c++)
+ for (i = 0; i < ARRAY_SIZE(inners); i++)
+ for (f = 100; f <= 132; f++)
+ cycle_bytes(c, inners[i], f);
+ for (c = 0; c < 4; c++)
+ for (f = 0; f < ARRAY_SIZE(fills); f++)
+ for (l = 2; l <= 26; l++)
+ nop_chain(c & 1, c & 2, l, fills[f]);
+
+ /*
+ * The short branch ranges of this JIT, which the fillers of 64 and
+ * 32-bit moves approach at the granularity their lengths allow
+ */
+ nr = find_ranges(ranges, ARRAY_SIZE(ranges), &wide, &narrow);
+ for (k = 0; k < nr; k++) {
+ printf("short branch range at %d bytes, moves of %d and %d bytes\n",
+ ranges[k] * wide, wide, narrow);
+ g = gcd(wide, narrow);
+ lo = ranges[k] * wide - 128;
+ hi = ranges[k] * wide + 32;
+ for (c = 0; c < 4; c++)
+ for (bytes = lo; bytes <= hi; bytes += g)
+ cycle_moves(c, ranges[k] * wide, bytes, wide, narrow);
+ }
+}
+
+/*
+ * Calls into a subprogram with all five arguments, which clobbers a subset
+ * of r6-r9 and uses the deepest stack slot the budget leaves, directly or
+ * through another one clobbering the rest, so the JIT has to save and
+ * restore what it uses.
+ */
+/* Values to keep in r6-r9 across calls */
+static const __u64 kept[] = {
+ 0x1111111111111111ULL, 0x2222222222222222ULL,
+ 0x3333333333333333ULL, 0x4444444444444444ULL,
+};
+
+static void test_calls(void)
+{
+ LIBBPF_OPTS(bpf_test_run_opts, opts);
+ __u64 expected = 1 + 2 + 3 + 4 + 5;
+ struct bpf_insn insns[64];
+ struct pat_ctx ctx;
+ char desc[64];
+ int nested, mask, n, call, r, fd, leaf;
+
+ /* The frames above the leaf take up to 32 bytes each of the budget */
+ leaf = (is_large_stack_supported() ? 2048 : 512) - 2 * 32;
+ for (r = 0; r < 4; r++)
+ expected += kept[r];
+ opts.ctx_in = &ctx;
+ opts.ctx_size_in = sizeof(ctx);
+ for (nested = 0; nested < 2; nested++) {
+ for (mask = 0; mask < 16; mask++) {
+ n = 0;
+ insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -8);
+ for (r = 0; r < 4; r++) {
+ insns[n++] = BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_6 + r,
+ 0, 0, (__u32)kept[r]);
+ insns[n++] = BPF_RAW_INSN(0, 0, 0, 0, kept[r] >> 32);
+ }
+ for (r = 1; r <= 5; r++)
+ insns[n++] = BPF_MOV64_IMM(r, r);
+ call = n++;
+ for (r = 6; r <= 9; r++)
+ insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r);
+ insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -8);
+ insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES);
+ insns[n++] = BPF_MOV64_IMM(BPF_REG_0, 0);
+ insns[n++] = BPF_EXIT_INSN();
+ insns[call] = BPF_CALL_REL(n - call - 1);
+ if (nested) {
+ /* Clobber the subset, pass the arguments on */
+ for (r = 0; r < 4; r++)
+ if (mask & (1 << r))
+ insns[n++] = BPF_MOV64_IMM(BPF_REG_6 + r, 0);
+ call = n++;
+ insns[n++] = BPF_EXIT_INSN();
+ insns[call] = BPF_CALL_REL(n - call - 1);
+ }
+ /* Clobber the rest, sum the arguments through the deepest slot */
+ for (r = 0; r < 4; r++)
+ if ((nested ? ~mask : mask) & (1 << r))
+ insns[n++] = BPF_MOV64_IMM(BPF_REG_6 + r, 0);
+ insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -leaf);
+ insns[n++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2);
+ for (r = 3; r <= 5; r++)
+ insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r);
+ insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -leaf);
+ insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
+ insns[n++] = BPF_EXIT_INSN();
+
+ snprintf(desc, sizeof(desc), "call%s clobbering 0x%x",
+ nested ? " nested" : "", mask);
+ fd = load_prog(insns, n, desc);
+ if (fd < 0)
+ return;
+ memset(&ctx, 0, sizeof(ctx));
+ if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc))
+ ASSERT_EQ(ctx.res, expected, desc);
+ close(fd);
+ }
+ }
+}
+
+/*
+ * Helper calls with up to five arguments and a kfunc call: a dynptr over a
+ * map value, a read from it into the stack and its size, with r6-r9 live
+ * across the calls. The returns, the bytes read, the size and r6-r9 add up.
+ */
+static void test_helper_calls(void)
+{
+ static const int offs[] = { 0, 8, 1000, 4088 };
+ struct off_prog p;
+ struct btf *btf;
+ __u8 mem[4096];
+ __u64 expected;
+ __u32 key = 0;
+ char desc[64];
+ int map_fd, kfunc_id, i, r, fd;
+
+ btf = btf__load_vmlinux_btf();
+ if (!ASSERT_OK_PTR(btf, "vmlinux btf"))
+ return;
+ kfunc_id = btf__find_by_name_kind(btf, "bpf_dynptr_size", BTF_KIND_FUNC);
+ btf__free(btf);
+ if (!ASSERT_GT(kfunc_id, 0, "bpf_dynptr_size id"))
+ return;
+ for (i = 0; i < sizeof(mem); i++)
+ mem[i] = i * 7 + 3;
+ map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "helper_mem", sizeof(key), sizeof(mem), 1,
+ NULL);
+ if (!ASSERT_GE(map_fd, 0, "helper_mem map"))
+ return;
+ if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, mem, 0), "helper_mem update"))
+ goto out;
+
+ for (i = 0; i < ARRAY_SIZE(offs); i++) {
+ p.cnt = 0;
+ EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -64));
+ for (r = 0; r < 4; r++)
+ off_prog_ld_imm64(&p, BPF_REG_6 + r, kept[r]);
+ off_prog_lookup(&p, map_fd, key);
+ /* The dynptr at -80 over the value, the returns summed at -96 */
+ EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_0));
+ EMIT(&p, BPF_MOV64_IMM(BPF_REG_2, sizeof(mem)));
+ EMIT(&p, BPF_MOV64_IMM(BPF_REG_3, 0));
+ EMIT(&p, BPF_MOV64_REG(BPF_REG_4, BPF_REG_10));
+ EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -80));
+ EMIT(&p, BPF_EMIT_CALL(BPF_FUNC_dynptr_from_mem));
+ EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -96));
+ /* Eight bytes from the offset into -88 */
+ EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
+ EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -88));
+ EMIT(&p, BPF_MOV64_IMM(BPF_REG_2, 8));
+ EMIT(&p, BPF_MOV64_REG(BPF_REG_3, BPF_REG_10));
+ EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -80));
+ EMIT(&p, BPF_MOV64_IMM(BPF_REG_4, offs[i]));
+ EMIT(&p, BPF_MOV64_IMM(BPF_REG_5, 0));
+ EMIT(&p, BPF_EMIT_CALL(BPF_FUNC_dynptr_read));
+ EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -96));
+ EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1));
+ EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -96));
+ /* The size through the kfunc */
+ EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
+ EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -80));
+ EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0,
+ kfunc_id));
+ EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -88));
+ EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1));
+ EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -96));
+ EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1));
+ for (r = BPF_REG_6; r <= BPF_REG_9; r++)
+ EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r));
+ EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -64));
+ EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES));
+ snprintf(desc, sizeof(desc), "helper calls off=%d", offs[i]);
+ fd = off_prog_load(&p, desc, false);
+ if (fd < 0)
+ goto out;
+ expected = sizeof(mem) + load_bytes(mem + offs[i], BPF_DW);
+ for (r = 0; r < 4; r++)
+ expected += kept[r];
+ prog_run(fd, 0, 0, expected, desc);
+ close(fd);
+ }
+out:
+ close(map_fd);
+}
+
void test_jit_insns(void)
{
RUN_TESTS(jit_insns_arena);
@@ -2353,8 +3059,12 @@ void test_jit_insns(void)
test_div_const();
if (test__start_subtest("alu_reg_pairs"))
test_alu_reg_pairs();
+ if (test__start_subtest("alu_edges"))
+ test_alu_edges();
if (test__start_subtest("shifts"))
test_shifts();
+ if (test__start_subtest("shift_counts"))
+ test_shift_counts();
if (test__start_subtest("atomics"))
test_atomics();
if (test__start_subtest("atomic_reg_pairs"))
@@ -2369,10 +3079,18 @@ void test_jit_insns(void)
test_jmp_reg();
if (test__start_subtest("jmp_reg_pairs"))
test_jmp_reg_pairs();
+ if (test__start_subtest("jmp_edges"))
+ test_jmp_edges();
if (test__start_subtest("staggered_jumps"))
test_staggered_jumps();
if (test__start_subtest("max_jmp"))
test_max_jmp();
+ if (test__start_subtest("convergence"))
+ test_convergence();
+ if (test__start_subtest("calls"))
+ test_calls();
+ if (test__start_subtest("helper_calls"))
+ test_helper_calls();
if (test__start_subtest("ldst_zero_off"))
test_ldst_zero_off();
if (test__start_subtest("ldst_offsets"))
@@ -2409,6 +3127,8 @@ void serial_test_jit_insns_blinded(void)
test_alu_imm();
if (test__start_subtest("alu_imm_regs"))
test_alu_imm_regs();
+ if (test__start_subtest("alu_edges"))
+ test_alu_edges();
if (test__start_subtest("div_const"))
test_div_const();
if (test__start_subtest("shifts"))
@@ -2419,6 +3139,8 @@ void serial_test_jit_insns_blinded(void)
test_ld_imm64();
if (test__start_subtest("jmp_imm"))
test_jmp_imm();
+ if (test__start_subtest("jmp_edges"))
+ test_jmp_edges();
if (test__start_subtest("ldst_zero_off"))
test_ldst_zero_off();
if (test__start_subtest("ldst_offsets"))
--
2.43.0
^ permalink raw reply related [flat|nested] 6+ messages in thread