* [PATCH net-next 08/13] ARM: net: bpf: use immediate forms of instructions where possible
From: Russell King @ 2018-07-10 12:36 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
In-Reply-To: <20180710123340.GK17271@n2100.armlinux.org.uk>
Rather than moving constants to a register and then using them in a
subsequent instruction, use them directly in the desired instruction
cutting out the "middle" register. This removes two instructions from
the tail call code path.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
arch/arm/net/bpf_jit_32.c | 16 +++++++++-------
arch/arm/net/bpf_jit_32.h | 3 +++
2 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index 753b5b2b2e3d..c7591877c350 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -256,7 +256,7 @@ static u32 arm_bpf_ldst_imm12(u32 op, u8 rt, u8 rn, s16 imm12)
op |= ARM_INST_LDST__U;
else
imm12 = -imm12;
- return op | (imm12 & 0xfff);
+ return op | (imm12 & ARM_INST_LDST__IMM12);
}
static u32 arm_bpf_ldst_imm8(u32 op, u8 rt, u8 rn, s16 imm8)
@@ -1012,11 +1012,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
/* if (index >= array->map.max_entries)
* goto out;
*/
+ BUILD_BUG_ON(offsetof(struct bpf_array, map.max_entries) >
+ ARM_INST_LDST__IMM12);
off = offsetof(struct bpf_array, map.max_entries);
/* array->map.max_entries */
- emit_a32_mov_i(tmp[1], off, ctx);
r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx);
- emit(ARM_LDR_R(tmp[1], r_array, tmp[1]), ctx);
+ emit(ARM_LDR_I(tmp[1], r_array, off), ctx);
/* index is 32-bit for arrays */
r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx);
/* index >= array->map.max_entries */
@@ -1041,10 +1042,10 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
* if (prog == NULL)
* goto out;
*/
+ BUILD_BUG_ON(offsetof(struct bpf_array, ptrs) > ARM_ALU_IMM);
off = offsetof(struct bpf_array, ptrs);
- emit_a32_mov_i(tmp[1], off, ctx);
r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx);
- emit(ARM_ADD_R(tmp[1], r_array, tmp[1]), ctx);
+ emit(ARM_ADD_I(tmp[1], r_array, off), ctx);
r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx);
emit(ARM_MOV_SI(tmp[0], r_index, SRTYPE_ASL, 2), ctx);
emit(ARM_LDR_R(tmp[1], tmp[1], tmp[0]), ctx);
@@ -1052,9 +1053,10 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
_emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
/* goto *(prog->bpf_func + prologue_size); */
+ BUILD_BUG_ON(offsetof(struct bpf_prog, bpf_func) >
+ ARM_INST_LDST__IMM12);
off = offsetof(struct bpf_prog, bpf_func);
- emit_a32_mov_i(tmp2[1], off, ctx);
- emit(ARM_LDR_R(tmp[1], tmp[1], tmp2[1]), ctx);
+ emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx);
emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx);
emit_bx_r(tmp[1], ctx);
diff --git a/arch/arm/net/bpf_jit_32.h b/arch/arm/net/bpf_jit_32.h
index c55bc39d3e22..ca9f1f5589f4 100644
--- a/arch/arm/net/bpf_jit_32.h
+++ b/arch/arm/net/bpf_jit_32.h
@@ -78,6 +78,7 @@
#define ARM_INST_EOR_I 0x02200000
#define ARM_INST_LDST__U 0x00800000
+#define ARM_INST_LDST__IMM12 0x00000fff
#define ARM_INST_LDRB_I 0x05500000
#define ARM_INST_LDRB_R 0x07d00000
#define ARM_INST_LDRH_I 0x015000b0
@@ -154,6 +155,8 @@
*/
#define ARM_INST_UDF 0xe7fddef1
+#define ARM_ALU_IMM 0xff
+
/* register */
#define _AL3_R(op, rd, rn, rm) ((op ## _R) | (rd) << 12 | (rn) << 16 | (rm))
/* immediate */
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 07/13] ARM: net: bpf: access eBPF scratch space using ARM FP register
From: Russell King @ 2018-07-10 12:36 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
In-Reply-To: <20180710123340.GK17271@n2100.armlinux.org.uk>
Access the eBPF scratch space using the frame pointer rather than our
stack pointer, as the offsets from the ARM frame pointer are constant
across all eBPF programs.
Since we no longer reference the scratch space registers from the stack
pointer, this simplifies emit_push_r64() as it no longer needs to know
how many words are pushed onto the stack.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
arch/arm/net/bpf_jit_32.c | 44 +++++++++++++++++++++++++-------------------
1 file changed, 25 insertions(+), 19 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index 45a3599e94a4..753b5b2b2e3d 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -108,6 +108,12 @@ enum {
#define STACK_OFFSET(k) (-4 - (k) * 4)
#define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4)
+#ifdef CONFIG_FRAME_POINTER
+#define EBPF_SCRATCH_TO_ARM_FP(x) ((x) - 4 * hweight16(CALLEE_PUSH_MASK) - 4)
+#else
+#define EBPF_SCRATCH_TO_ARM_FP(x) (x)
+#endif
+
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */
#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */
#define TCALL_CNT (MAX_BPF_JIT_REG + 2) /* Tail Call Count */
@@ -294,9 +300,6 @@ static void jit_fill_hole(void *area, unsigned int size)
#define _STACK_SIZE (ctx->prog->aux->stack_depth + SCRATCH_SIZE)
#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
-/* Get the offset of eBPF REGISTERs stored on scratch space. */
-#define STACK_VAR(off) (STACK_SIZE + (off))
-
#if __LINUX_ARM_ARCH__ < 7
static u16 imm_offset(u32 k, struct jit_ctx *ctx)
@@ -472,7 +475,7 @@ static bool is_stacked(s8 reg)
static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx)
{
if (is_stacked(reg)) {
- emit(ARM_LDR_I(tmp, ARM_SP, STACK_VAR(reg)), ctx);
+ emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);
reg = tmp;
}
return reg;
@@ -482,8 +485,10 @@ static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp,
struct jit_ctx *ctx)
{
if (is_stacked(reg[1])) {
- emit(ARM_LDR_I(tmp[1], ARM_SP, STACK_VAR(reg[1])), ctx);
- emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(reg[0])), ctx);
+ emit(ARM_LDR_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[1])),
+ ctx);
+ emit(ARM_LDR_I(tmp[0], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[0])),
+ ctx);
reg = tmp;
}
return reg;
@@ -496,7 +501,7 @@ static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp,
static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx)
{
if (is_stacked(reg))
- emit(ARM_STR_I(src, ARM_SP, STACK_VAR(reg)), ctx);
+ emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);
else if (reg != src)
emit(ARM_MOV_R(reg, src), ctx);
}
@@ -505,8 +510,10 @@ static void arm_bpf_put_reg64(const s8 *reg, const s8 *src,
struct jit_ctx *ctx)
{
if (is_stacked(reg[1])) {
- emit(ARM_STR_I(src[1], ARM_SP, STACK_VAR(reg[1])), ctx);
- emit(ARM_STR_I(src[0], ARM_SP, STACK_VAR(reg[0])), ctx);
+ emit(ARM_STR_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[1])),
+ ctx);
+ emit(ARM_STR_I(src[0], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg[0])),
+ ctx);
} else {
if (reg[1] != src[1])
emit(ARM_MOV_R(reg[1], src[1]), ctx);
@@ -1103,16 +1110,15 @@ static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx)
}
// push the scratch stack register on top of the stack
-static inline void emit_push_r64(const s8 src[], const u8 shift,
- struct jit_ctx *ctx)
+static inline void emit_push_r64(const s8 src[], struct jit_ctx *ctx)
{
const s8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *rt;
u16 reg_set = 0;
- emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(src[1]+shift)), ctx);
- emit(ARM_LDR_I(tmp2[0], ARM_SP, STACK_VAR(src[0]+shift)), ctx);
+ rt = arm_bpf_get_reg64(src, tmp2, ctx);
- reg_set = (1 << tmp2[1]) | (1 << tmp2[0]);
+ reg_set = (1 << rt[1]) | (1 << rt[0]);
emit(ARM_PUSH(reg_set), ctx);
}
@@ -1155,8 +1161,8 @@ static void build_prologue(struct jit_ctx *ctx)
emit(ARM_MOV_R(r3, r4), ctx);
emit(ARM_MOV_R(r2, r0), ctx);
/* Initialize Tail Count */
- emit(ARM_STR_I(r4, ARM_SP, STACK_VAR(tcc[0])), ctx);
- emit(ARM_STR_I(r4, ARM_SP, STACK_VAR(tcc[1])), ctx);
+ emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[0])), ctx);
+ emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[1])), ctx);
/* end of prologue */
}
@@ -1606,9 +1612,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit_a32_mov_r64(true, r0, r1, ctx);
emit_a32_mov_r64(true, r1, r2, ctx);
- emit_push_r64(r5, 0, ctx);
- emit_push_r64(r4, 8, ctx);
- emit_push_r64(r3, 16, ctx);
+ emit_push_r64(r5, ctx);
+ emit_push_r64(r4, ctx);
+ emit_push_r64(r3, ctx);
emit_a32_mov_i(tmp[1], func, ctx);
emit_blx_r(tmp[1], ctx);
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 06/13] ARM: net: bpf: 64-bit accessor functions for BPF registers
From: Russell King @ 2018-07-10 12:36 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
In-Reply-To: <20180710123340.GK17271@n2100.armlinux.org.uk>
Provide a couple of 64-bit register accessors, and use them where
appropriate
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
arch/arm/net/bpf_jit_32.c | 235 ++++++++++++++++++++++++----------------------
1 file changed, 122 insertions(+), 113 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index 08fb4eb285a2..45a3599e94a4 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -478,6 +478,17 @@ static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx)
return reg;
}
+static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp,
+ struct jit_ctx *ctx)
+{
+ if (is_stacked(reg[1])) {
+ emit(ARM_LDR_I(tmp[1], ARM_SP, STACK_VAR(reg[1])), ctx);
+ emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(reg[0])), ctx);
+ reg = tmp;
+ }
+ return reg;
+}
+
/* If a BPF register is on the stack (stk is true), save the register
* back to the stack. If the source register is not the same, then
* move it into the correct register.
@@ -490,6 +501,20 @@ static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx)
emit(ARM_MOV_R(reg, src), ctx);
}
+static void arm_bpf_put_reg64(const s8 *reg, const s8 *src,
+ struct jit_ctx *ctx)
+{
+ if (is_stacked(reg[1])) {
+ emit(ARM_STR_I(src[1], ARM_SP, STACK_VAR(reg[1])), ctx);
+ emit(ARM_STR_I(src[0], ARM_SP, STACK_VAR(reg[0])), ctx);
+ } else {
+ if (reg[1] != src[1])
+ emit(ARM_MOV_R(reg[1], src[1]), ctx);
+ if (reg[0] != src[0])
+ emit(ARM_MOV_R(reg[0], src[0]), ctx);
+ }
+}
+
static inline void emit_a32_mov_i(const s8 dst, const u32 val,
struct jit_ctx *ctx)
{
@@ -669,18 +694,16 @@ static inline void emit_a32_alu_i(const s8 dst, const u32 val,
static inline void emit_a32_neg64(const s8 dst[],
struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rd, rm;
+ const s8 *rd;
/* Setup Operand */
- rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
/* Do Negate Operation */
- emit(ARM_RSBS_I(rd, rd, 0), ctx);
- emit(ARM_RSC_I(rm, rm, 0), ctx);
+ emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx);
+ emit(ARM_RSC_I(rd[0], rd[0], 0), ctx);
- arm_bpf_put_reg32(dst_lo, rd, ctx);
- arm_bpf_put_reg32(dst_hi, rm, ctx);
+ arm_bpf_put_reg64(dst, rd, ctx);
}
/* dst = dst << src */
@@ -688,20 +711,20 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[],
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- s8 rt, rd, rm;
+ const s8 *rd;
+ s8 rt;
/* Setup Operands */
rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
- rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
/* Do LSH operation */
emit(ARM_SUB_I(ARM_IP, rt, 32), ctx);
emit(ARM_RSB_I(tmp2[0], rt, 32), ctx);
- emit(ARM_MOV_SR(ARM_LR, rm, SRTYPE_ASL, rt), ctx);
- emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd, SRTYPE_ASL, ARM_IP), ctx);
- emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd, SRTYPE_LSR, tmp2[0]), ctx);
- emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_ASL, rt), ctx);
+ emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx);
+ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx);
+ emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx);
+ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx);
arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
@@ -712,21 +735,21 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[],
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- s8 rt, rd, rm;
+ const s8 *rd;
+ s8 rt;
/* Setup Operands */
rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
- rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
/* Do the ARSH operation */
emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
- emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_LSR, rt), ctx);
- emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx);
+ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);
+ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);
_emit(ARM_COND_MI, ARM_B(0), ctx);
- emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASR, tmp2[0]), ctx);
- emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_ASR, rt), ctx);
+ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASR, tmp2[0]), ctx);
+ emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx);
arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
@@ -737,20 +760,20 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[],
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- s8 rt, rd, rm;
+ const s8 *rd;
+ s8 rt;
/* Setup Operands */
rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
- rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
/* Do RSH operation */
emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
- emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_LSR, rt), ctx);
- emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx);
- emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_LSR, tmp2[0]), ctx);
- emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_LSR, rt), ctx);
+ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);
+ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);
+ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx);
+ emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx);
arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
@@ -761,27 +784,25 @@ static inline void emit_a32_lsh_i64(const s8 dst[],
const u32 val, struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- s8 rd, rm;
+ const s8 *rd;
/* Setup operands */
- rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
/* Do LSH operation */
if (val < 32) {
- emit(ARM_MOV_SI(tmp2[0], rm, SRTYPE_ASL, val), ctx);
- emit(ARM_ORR_SI(rm, tmp2[0], rd, SRTYPE_LSR, 32 - val), ctx);
- emit(ARM_MOV_SI(rd, rd, SRTYPE_ASL, val), ctx);
+ emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx);
+ emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx);
+ emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx);
} else {
if (val == 32)
- emit(ARM_MOV_R(rm, rd), ctx);
+ emit(ARM_MOV_R(rd[0], rd[1]), ctx);
else
- emit(ARM_MOV_SI(rm, rd, SRTYPE_ASL, val - 32), ctx);
- emit(ARM_EOR_R(rd, rd, rd), ctx);
+ emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx);
+ emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx);
}
- arm_bpf_put_reg32(dst_lo, rd, ctx);
- arm_bpf_put_reg32(dst_hi, rm, ctx);
+ arm_bpf_put_reg64(dst, rd, ctx);
}
/* dst = dst >> val */
@@ -789,27 +810,25 @@ static inline void emit_a32_rsh_i64(const s8 dst[],
const u32 val, struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- s8 rd, rm;
+ const s8 *rd;
/* Setup operands */
- rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
/* Do LSR operation */
if (val < 32) {
- emit(ARM_MOV_SI(tmp2[1], rd, SRTYPE_LSR, val), ctx);
- emit(ARM_ORR_SI(rd, tmp2[1], rm, SRTYPE_ASL, 32 - val), ctx);
- emit(ARM_MOV_SI(rm, rm, SRTYPE_LSR, val), ctx);
+ emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);
+ emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);
+ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx);
} else if (val == 32) {
- emit(ARM_MOV_R(rd, rm), ctx);
- emit(ARM_MOV_I(rm, 0), ctx);
+ emit(ARM_MOV_R(rd[1], rd[0]), ctx);
+ emit(ARM_MOV_I(rd[0], 0), ctx);
} else {
- emit(ARM_MOV_SI(rd, rm, SRTYPE_LSR, val - 32), ctx);
- emit(ARM_MOV_I(rm, 0), ctx);
+ emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx);
+ emit(ARM_MOV_I(rd[0], 0), ctx);
}
- arm_bpf_put_reg32(dst_lo, rd, ctx);
- arm_bpf_put_reg32(dst_hi, rm, ctx);
+ arm_bpf_put_reg64(dst, rd, ctx);
}
/* dst = dst >> val (signed) */
@@ -817,51 +836,47 @@ static inline void emit_a32_arsh_i64(const s8 dst[],
const u32 val, struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- s8 rd, rm;
+ const s8 *rd;
/* Setup operands */
- rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
/* Do ARSH operation */
if (val < 32) {
- emit(ARM_MOV_SI(tmp2[1], rd, SRTYPE_LSR, val), ctx);
- emit(ARM_ORR_SI(rd, tmp2[1], rm, SRTYPE_ASL, 32 - val), ctx);
- emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, val), ctx);
+ emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);
+ emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);
+ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx);
} else if (val == 32) {
- emit(ARM_MOV_R(rd, rm), ctx);
- emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx);
+ emit(ARM_MOV_R(rd[1], rd[0]), ctx);
+ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);
} else {
- emit(ARM_MOV_SI(rd, rm, SRTYPE_ASR, val - 32), ctx);
- emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx);
+ emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx);
+ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);
}
- arm_bpf_put_reg32(dst_lo, rd, ctx);
- arm_bpf_put_reg32(dst_hi, rm, ctx);
+ arm_bpf_put_reg64(dst, rd, ctx);
}
static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[],
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- s8 rd, rm, rt, rn;
+ const s8 *rd, *rt;
/* Setup operands for multiplication */
- rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
- rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
- rn = arm_bpf_get_reg32(src_hi, tmp2[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
+ rt = arm_bpf_get_reg64(src, tmp2, ctx);
/* Do Multiplication */
- emit(ARM_MUL(ARM_IP, rd, rn), ctx);
- emit(ARM_MUL(ARM_LR, rm, rt), ctx);
+ emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx);
+ emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx);
emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx);
- emit(ARM_UMULL(ARM_IP, rm, rd, rt), ctx);
- emit(ARM_ADD_R(rm, ARM_LR, rm), ctx);
+ emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx);
+ emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx);
arm_bpf_put_reg32(dst_lo, ARM_IP, ctx);
- arm_bpf_put_reg32(dst_hi, rm, ctx);
+ arm_bpf_put_reg32(dst_hi, rd[0], ctx);
}
/* *(size *)(dst + off) = src */
@@ -918,17 +933,17 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src,
case BPF_B:
/* Load a Byte */
emit(ARM_LDRB_I(rd[1], rm, off), ctx);
- emit_a32_mov_i(dst[0], 0, ctx);
+ emit_a32_mov_i(rd[0], 0, ctx);
break;
case BPF_H:
/* Load a HalfWord */
emit(ARM_LDRH_I(rd[1], rm, off), ctx);
- emit_a32_mov_i(dst[0], 0, ctx);
+ emit_a32_mov_i(rd[0], 0, ctx);
break;
case BPF_W:
/* Load a Word */
emit(ARM_LDR_I(rd[1], rm, off), ctx);
- emit_a32_mov_i(dst[0], 0, ctx);
+ emit_a32_mov_i(rd[0], 0, ctx);
break;
case BPF_DW:
/* Load a Double Word */
@@ -936,9 +951,7 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src,
emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);
break;
}
- arm_bpf_put_reg32(dst[1], rd[1], ctx);
- if (sz == BPF_DW)
- arm_bpf_put_reg32(dst[0], rd[0], ctx);
+ arm_bpf_put_reg64(dst, rd, ctx);
}
/* Arithmatic Operation */
@@ -982,11 +995,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
const s8 *tcc = bpf2a32[TCALL_CNT];
+ const s8 *tc;
const int idx0 = ctx->idx;
#define cur_offset (ctx->idx - idx0)
#define jmp_offset (out_offset - (cur_offset) - 2)
u32 off, lo, hi;
- s8 r_array, r_index, r_tc_lo, r_tc_hi;
+ s8 r_array, r_index;
/* if (index >= array->map.max_entries)
* goto out;
@@ -1008,15 +1022,13 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
*/
lo = (u32)MAX_TAIL_CALL_CNT;
hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32);
- r_tc_lo = arm_bpf_get_reg32(tcc[1], tmp[1], ctx);
- r_tc_hi = arm_bpf_get_reg32(tcc[0], tmp[0], ctx);
- emit(ARM_CMP_I(r_tc_hi, hi), ctx);
- _emit(ARM_COND_EQ, ARM_CMP_I(r_tc_lo, lo), ctx);
+ tc = arm_bpf_get_reg64(tcc, tmp, ctx);
+ emit(ARM_CMP_I(tc[0], hi), ctx);
+ _emit(ARM_COND_EQ, ARM_CMP_I(tc[1], lo), ctx);
_emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
- emit(ARM_ADDS_I(r_tc_lo, r_tc_lo, 1), ctx);
- emit(ARM_ADC_I(r_tc_hi, r_tc_hi, 0), ctx);
- arm_bpf_put_reg32(tcc[1], r_tc_lo, ctx);
- arm_bpf_put_reg32(tcc[0], r_tc_hi, ctx);
+ emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx);
+ emit(ARM_ADC_I(tc[0], tc[0], 0), ctx);
+ arm_bpf_put_reg64(tcc, tmp, ctx);
/* prog = array->ptrs[index]
* if (prog == NULL)
@@ -1183,7 +1195,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
const s32 imm = insn->imm;
const int i = insn - ctx->prog->insnsi;
const bool is64 = BPF_CLASS(code) == BPF_ALU64;
- s8 rd, rt, rm, rn;
+ const s8 *rd, *rs;
+ s8 rd_lo, rt, rm, rn;
s32 jmp_offset;
#define check_imm(bits, imm) do { \
@@ -1270,7 +1283,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU | BPF_MOD | BPF_K:
case BPF_ALU | BPF_MOD | BPF_X:
- rd = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx);
+ rd_lo = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx);
switch (BPF_SRC(code)) {
case BPF_X:
rt = arm_bpf_get_reg32(src_lo, tmp2[0], ctx);
@@ -1283,8 +1296,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
rt = src_lo;
break;
}
- emit_udivmod(rd, rd, rt, ctx, BPF_OP(code));
- arm_bpf_put_reg32(dst_lo, rd, ctx);
+ emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code));
+ arm_bpf_put_reg32(dst_lo, rd_lo, ctx);
emit_a32_mov_i(dst_hi, 0, ctx);
break;
case BPF_ALU64 | BPF_DIV | BPF_K:
@@ -1364,21 +1377,20 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
/* dst = htobe(dst) */
case BPF_ALU | BPF_END | BPF_FROM_LE:
case BPF_ALU | BPF_END | BPF_FROM_BE:
- rt = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rd = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
if (BPF_SRC(code) == BPF_FROM_LE)
goto emit_bswap_uxt;
switch (imm) {
case 16:
- emit_rev16(rt, rt, ctx);
+ emit_rev16(rd[1], rd[1], ctx);
goto emit_bswap_uxt;
case 32:
- emit_rev32(rt, rt, ctx);
+ emit_rev32(rd[1], rd[1], ctx);
goto emit_bswap_uxt;
case 64:
- emit_rev32(ARM_LR, rt, ctx);
- emit_rev32(rt, rd, ctx);
- emit(ARM_MOV_R(rd, ARM_LR), ctx);
+ emit_rev32(ARM_LR, rd[1], ctx);
+ emit_rev32(rd[1], rd[0], ctx);
+ emit(ARM_MOV_R(rd[0], ARM_LR), ctx);
break;
}
goto exit;
@@ -1388,23 +1400,22 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
/* zero-extend 16 bits into 64 bits */
#if __LINUX_ARM_ARCH__ < 6
emit_a32_mov_i(tmp2[1], 0xffff, ctx);
- emit(ARM_AND_R(rt, rt, tmp2[1]), ctx);
+ emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx);
#else /* ARMv6+ */
- emit(ARM_UXTH(rt, rt), ctx);
+ emit(ARM_UXTH(rd[1], rd[1]), ctx);
#endif
- emit(ARM_EOR_R(rd, rd, rd), ctx);
+ emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
break;
case 32:
/* zero-extend 32 bits into 64 bits */
- emit(ARM_EOR_R(rd, rd, rd), ctx);
+ emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
break;
case 64:
/* nop */
break;
}
exit:
- arm_bpf_put_reg32(dst_lo, rt, ctx);
- arm_bpf_put_reg32(dst_hi, rd, ctx);
+ arm_bpf_put_reg64(dst, rd, ctx);
break;
/* dst = imm64 */
case BPF_LD | BPF_IMM | BPF_DW:
@@ -1459,15 +1470,14 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
{
u8 sz = BPF_SIZE(code);
- rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
- rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx);
+ rs = arm_bpf_get_reg64(src, tmp2, ctx);
/* Store the value */
if (BPF_SIZE(code) == BPF_DW) {
- emit_str_r(dst_lo, rn, off, ctx, BPF_W);
- emit_str_r(dst_lo, rm, off+4, ctx, BPF_W);
+ emit_str_r(dst_lo, rs[1], off, ctx, BPF_W);
+ emit_str_r(dst_lo, rs[0], off+4, ctx, BPF_W);
} else {
- emit_str_r(dst_lo, rn, off, ctx, sz);
+ emit_str_r(dst_lo, rs[1], off, ctx, sz);
}
break;
}
@@ -1527,11 +1537,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit_a32_mov_i64(true, tmp2, imm, ctx);
go_jmp:
/* Setup destination register */
- rt = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
- rd = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rd = arm_bpf_get_reg64(dst, tmp, ctx);
/* Check for the condition */
- emit_ar_r(rd, rt, rm, rn, ctx, BPF_OP(code));
+ emit_ar_r(rd[0], rd[1], rm, rn, ctx, BPF_OP(code));
/* Setup JUMP instruction */
jmp_offset = bpf2a32_offset(i+off, i, ctx);
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 05/13] ARM: net: bpf: provide accessor functions for BPF registers
From: Russell King @ 2018-07-10 12:36 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
In-Reply-To: <20180710123340.GK17271@n2100.armlinux.org.uk>
Many of the code paths need to have knowledge about whether a register
is stacked or in a CPU register. Move this decision making to a pair
of helper functions instead of having it scattered throughout the
code.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
arch/arm/net/bpf_jit_32.c | 329 ++++++++++++++++++----------------------------
1 file changed, 128 insertions(+), 201 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index e81401aca2df..08fb4eb285a2 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -465,6 +465,31 @@ static bool is_stacked(s8 reg)
return reg < 0;
}
+/* If a BPF register is on the stack (stk is true), load it to the
+ * supplied temporary register and return the temporary register
+ * for subsequent operations, otherwise just use the CPU register.
+ */
+static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx)
+{
+ if (is_stacked(reg)) {
+ emit(ARM_LDR_I(tmp, ARM_SP, STACK_VAR(reg)), ctx);
+ reg = tmp;
+ }
+ return reg;
+}
+
+/* If a BPF register is on the stack (stk is true), save the register
+ * back to the stack. If the source register is not the same, then
+ * move it into the correct register.
+ */
+static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx)
+{
+ if (is_stacked(reg))
+ emit(ARM_STR_I(src, ARM_SP, STACK_VAR(reg)), ctx);
+ else if (reg != src)
+ emit(ARM_MOV_R(reg, src), ctx);
+}
+
static inline void emit_a32_mov_i(const s8 dst, const u32 val,
struct jit_ctx *ctx)
{
@@ -472,7 +497,7 @@ static inline void emit_a32_mov_i(const s8 dst, const u32 val,
if (is_stacked(dst)) {
emit_mov_i(tmp[1], val, ctx);
- emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(dst)), ctx);
+ arm_bpf_put_reg32(dst, tmp[1], ctx);
} else {
emit_mov_i(dst, val, ctx);
}
@@ -572,19 +597,13 @@ static inline void emit_a32_alu_r(const s8 dst, const s8 src,
struct jit_ctx *ctx, const bool is64,
const bool hi, const u8 op) {
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rn = is_stacked(src) ? tmp[1] : src;
-
- if (is_stacked(src))
- emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src)), ctx);
+ s8 rn, rd;
+ rn = arm_bpf_get_reg32(src, tmp[1], ctx);
+ rd = arm_bpf_get_reg32(dst, tmp[0], ctx);
/* ALU operation */
- if (is_stacked(dst)) {
- emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx);
- emit_alu_r(tmp[0], rn, is64, hi, op, ctx);
- emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx);
- } else {
- emit_alu_r(dst, rn, is64, hi, op, ctx);
- }
+ emit_alu_r(rd, rn, is64, hi, op, ctx);
+ arm_bpf_put_reg32(dst, rd, ctx);
}
/* ALU operation (64 bit) */
@@ -598,18 +617,14 @@ static inline void emit_a32_alu_r64(const bool is64, const s8 dst[],
emit_a32_mov_i(dst_hi, 0, ctx);
}
-/* dst = imm (4 bytes)*/
+/* dst = src (4 bytes)*/
static inline void emit_a32_mov_r(const s8 dst, const s8 src,
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rt = is_stacked(src) ? tmp[0] : src;
+ s8 rt;
- if (is_stacked(src))
- emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(src)), ctx);
- if (is_stacked(dst))
- emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst)), ctx);
- else
- emit(ARM_MOV_R(dst, rt), ctx);
+ rt = arm_bpf_get_reg32(src, tmp[0], ctx);
+ arm_bpf_put_reg32(dst, rt, ctx);
}
/* dst = src */
@@ -630,10 +645,9 @@ static inline void emit_a32_mov_r64(const bool is64, const s8 dst[],
static inline void emit_a32_alu_i(const s8 dst, const u32 val,
struct jit_ctx *ctx, const u8 op) {
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rd = is_stacked(dst) ? tmp[0] : dst;
+ s8 rd;
- if (is_stacked(dst))
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
+ rd = arm_bpf_get_reg32(dst, tmp[0], ctx);
/* Do shift operation */
switch (op) {
@@ -648,31 +662,25 @@ static inline void emit_a32_alu_i(const s8 dst, const u32 val,
break;
}
- if (is_stacked(dst))
- emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
+ arm_bpf_put_reg32(dst, rd, ctx);
}
/* dst = ~dst (64 bit) */
static inline void emit_a32_neg64(const s8 dst[],
struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rd = is_stacked(dst_lo) ? tmp[1] : dst[1];
- s8 rm = is_stacked(dst_lo) ? tmp[0] : dst[0];
+ s8 rd, rm;
/* Setup Operand */
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
/* Do Negate Operation */
emit(ARM_RSBS_I(rd, rd, 0), ctx);
emit(ARM_RSC_I(rm, rm, 0), ctx);
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ arm_bpf_put_reg32(dst_lo, rd, ctx);
+ arm_bpf_put_reg32(dst_hi, rm, ctx);
}
/* dst = dst << src */
@@ -680,18 +688,12 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[],
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
+ s8 rt, rd, rm;
/* Setup Operands */
- s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo;
- s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
-
- if (is_stacked(src_lo))
- emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
+ rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
/* Do LSH operation */
emit(ARM_SUB_I(ARM_IP, rt, 32), ctx);
@@ -701,13 +703,8 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[],
emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd, SRTYPE_LSR, tmp2[0]), ctx);
emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_ASL, rt), ctx);
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
- } else {
- emit(ARM_MOV_R(rd, ARM_LR), ctx);
- emit(ARM_MOV_R(rm, ARM_IP), ctx);
- }
+ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
+ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
}
/* dst = dst >> src (signed)*/
@@ -715,17 +712,12 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[],
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
+ s8 rt, rd, rm;
+
/* Setup Operands */
- s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo;
- s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
-
- if (is_stacked(src_lo))
- emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
+ rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
/* Do the ARSH operation */
emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
@@ -735,13 +727,9 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[],
_emit(ARM_COND_MI, ARM_B(0), ctx);
emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASR, tmp2[0]), ctx);
emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_ASR, rt), ctx);
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
- } else {
- emit(ARM_MOV_R(rd, ARM_LR), ctx);
- emit(ARM_MOV_R(rm, ARM_IP), ctx);
- }
+
+ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
+ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
}
/* dst = dst >> src */
@@ -749,17 +737,12 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[],
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
+ s8 rt, rd, rm;
+
/* Setup Operands */
- s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo;
- s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
-
- if (is_stacked(src_lo))
- emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
+ rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
/* Do RSH operation */
emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
@@ -768,13 +751,9 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[],
emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx);
emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_LSR, tmp2[0]), ctx);
emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_LSR, rt), ctx);
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
- } else {
- emit(ARM_MOV_R(rd, ARM_LR), ctx);
- emit(ARM_MOV_R(rm, ARM_IP), ctx);
- }
+
+ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
+ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
}
/* dst = dst << val */
@@ -782,14 +761,11 @@ static inline void emit_a32_lsh_i64(const s8 dst[],
const u32 val, struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- /* Setup operands */
- s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
+ s8 rd, rm;
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ /* Setup operands */
+ rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
/* Do LSH operation */
if (val < 32) {
@@ -804,10 +780,8 @@ static inline void emit_a32_lsh_i64(const s8 dst[],
emit(ARM_EOR_R(rd, rd, rd), ctx);
}
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ arm_bpf_put_reg32(dst_lo, rd, ctx);
+ arm_bpf_put_reg32(dst_hi, rm, ctx);
}
/* dst = dst >> val */
@@ -815,14 +789,11 @@ static inline void emit_a32_rsh_i64(const s8 dst[],
const u32 val, struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- /* Setup operands */
- s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
+ s8 rd, rm;
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ /* Setup operands */
+ rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
/* Do LSR operation */
if (val < 32) {
@@ -837,10 +808,8 @@ static inline void emit_a32_rsh_i64(const s8 dst[],
emit(ARM_MOV_I(rm, 0), ctx);
}
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ arm_bpf_put_reg32(dst_lo, rd, ctx);
+ arm_bpf_put_reg32(dst_hi, rm, ctx);
}
/* dst = dst >> val (signed) */
@@ -848,14 +817,11 @@ static inline void emit_a32_arsh_i64(const s8 dst[],
const u32 val, struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
- /* Setup operands */
- s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
+ s8 rd, rm;
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ /* Setup operands */
+ rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
/* Do ARSH operation */
if (val < 32) {
@@ -870,30 +836,21 @@ static inline void emit_a32_arsh_i64(const s8 dst[],
emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx);
}
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ arm_bpf_put_reg32(dst_lo, rd, ctx);
+ arm_bpf_put_reg32(dst_hi, rm, ctx);
}
static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[],
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
+ s8 rd, rm, rt, rn;
+
/* Setup operands for multiplication */
- s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo;
- s8 rn = is_stacked(src_lo) ? tmp2[0] : src_hi;
-
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
- if (is_stacked(src_lo)) {
- emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
- emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_hi)), ctx);
- }
+ rd = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rm = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
+ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
+ rn = arm_bpf_get_reg32(src_hi, tmp2[0], ctx);
/* Do Multiplication */
emit(ARM_MUL(ARM_IP, rd, rn), ctx);
@@ -902,22 +859,18 @@ static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[],
emit(ARM_UMULL(ARM_IP, rm, rd, rt), ctx);
emit(ARM_ADD_R(rm, ARM_LR, rm), ctx);
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
- } else {
- emit(ARM_MOV_R(rd, ARM_IP), ctx);
- }
+
+ arm_bpf_put_reg32(dst_lo, ARM_IP, ctx);
+ arm_bpf_put_reg32(dst_hi, rm, ctx);
}
/* *(size *)(dst + off) = src */
static inline void emit_str_r(const s8 dst, const s8 src,
const s32 off, struct jit_ctx *ctx, const u8 sz){
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rd = is_stacked(dst) ? tmp[1] : dst;
+ s8 rd;
- if (is_stacked(dst))
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
+ rd = arm_bpf_get_reg32(dst, tmp[1], ctx);
if (off) {
emit_a32_mov_i(tmp[0], off, ctx);
emit(ARM_ADD_R(tmp[0], rd, tmp[0]), ctx);
@@ -983,10 +936,9 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src,
emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);
break;
}
- if (is_stacked(dst_lo))
- emit(ARM_STR_I(rd[1], ARM_SP, STACK_VAR(dst_lo)), ctx);
- if (is_stacked(dst_lo) && sz == BPF_DW)
- emit(ARM_STR_I(rd[0], ARM_SP, STACK_VAR(dst_hi)), ctx);
+ arm_bpf_put_reg32(dst[1], rd[1], ctx);
+ if (sz == BPF_DW)
+ arm_bpf_put_reg32(dst[0], rd[0], ctx);
}
/* Arithmatic Operation */
@@ -1034,6 +986,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
#define cur_offset (ctx->idx - idx0)
#define jmp_offset (out_offset - (cur_offset) - 2)
u32 off, lo, hi;
+ s8 r_array, r_index, r_tc_lo, r_tc_hi;
/* if (index >= array->map.max_entries)
* goto out;
@@ -1041,12 +994,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
off = offsetof(struct bpf_array, map.max_entries);
/* array->map.max_entries */
emit_a32_mov_i(tmp[1], off, ctx);
- emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx);
- emit(ARM_LDR_R(tmp[1], tmp2[1], tmp[1]), ctx);
+ r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx);
+ emit(ARM_LDR_R(tmp[1], r_array, tmp[1]), ctx);
/* index is 32-bit for arrays */
- emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx);
+ r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx);
/* index >= array->map.max_entries */
- emit(ARM_CMP_R(tmp2[1], tmp[1]), ctx);
+ emit(ARM_CMP_R(r_index, tmp[1]), ctx);
_emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
/* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
@@ -1055,15 +1008,15 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
*/
lo = (u32)MAX_TAIL_CALL_CNT;
hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32);
- emit(ARM_LDR_I(tmp[1], ARM_SP, STACK_VAR(tcc[1])), ctx);
- emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(tcc[0])), ctx);
- emit(ARM_CMP_I(tmp[0], hi), ctx);
- _emit(ARM_COND_EQ, ARM_CMP_I(tmp[1], lo), ctx);
+ r_tc_lo = arm_bpf_get_reg32(tcc[1], tmp[1], ctx);
+ r_tc_hi = arm_bpf_get_reg32(tcc[0], tmp[0], ctx);
+ emit(ARM_CMP_I(r_tc_hi, hi), ctx);
+ _emit(ARM_COND_EQ, ARM_CMP_I(r_tc_lo, lo), ctx);
_emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
- emit(ARM_ADDS_I(tmp[1], tmp[1], 1), ctx);
- emit(ARM_ADC_I(tmp[0], tmp[0], 0), ctx);
- emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(tcc[1])), ctx);
- emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(tcc[0])), ctx);
+ emit(ARM_ADDS_I(r_tc_lo, r_tc_lo, 1), ctx);
+ emit(ARM_ADC_I(r_tc_hi, r_tc_hi, 0), ctx);
+ arm_bpf_put_reg32(tcc[1], r_tc_lo, ctx);
+ arm_bpf_put_reg32(tcc[0], r_tc_hi, ctx);
/* prog = array->ptrs[index]
* if (prog == NULL)
@@ -1071,10 +1024,10 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
*/
off = offsetof(struct bpf_array, ptrs);
emit_a32_mov_i(tmp[1], off, ctx);
- emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx);
- emit(ARM_ADD_R(tmp[1], tmp2[1], tmp[1]), ctx);
- emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx);
- emit(ARM_MOV_SI(tmp[0], tmp2[1], SRTYPE_ASL, 2), ctx);
+ r_array = arm_bpf_get_reg32(r2[1], tmp2[1], ctx);
+ emit(ARM_ADD_R(tmp[1], r_array, tmp[1]), ctx);
+ r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx);
+ emit(ARM_MOV_SI(tmp[0], r_index, SRTYPE_ASL, 2), ctx);
emit(ARM_LDR_R(tmp[1], tmp[1], tmp[0]), ctx);
emit(ARM_CMP_I(tmp[1], 0), ctx);
_emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
@@ -1317,15 +1270,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU | BPF_MOD | BPF_K:
case BPF_ALU | BPF_MOD | BPF_X:
- rd = is_stacked(dst_lo) ? tmp2[1] : dst_lo;
- if (is_stacked(dst_lo))
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
+ rd = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx);
switch (BPF_SRC(code)) {
case BPF_X:
- rt = is_stacked(rt) ? tmp2[0] : src_lo;
- if (is_stacked(src_lo))
- emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)),
- ctx);
+ rt = arm_bpf_get_reg32(src_lo, tmp2[0], ctx);
break;
case BPF_K:
rt = tmp2[0];
@@ -1336,8 +1284,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
break;
}
emit_udivmod(rd, rd, rt, ctx, BPF_OP(code));
- if (is_stacked(dst_lo))
- emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
+ arm_bpf_put_reg32(dst_lo, rd, ctx);
emit_a32_mov_i(dst_hi, 0, ctx);
break;
case BPF_ALU64 | BPF_DIV | BPF_K:
@@ -1417,12 +1364,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
/* dst = htobe(dst) */
case BPF_ALU | BPF_END | BPF_FROM_LE:
case BPF_ALU | BPF_END | BPF_FROM_BE:
- rd = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- rt = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ rt = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rd = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
if (BPF_SRC(code) == BPF_FROM_LE)
goto emit_bswap_uxt;
switch (imm) {
@@ -1460,10 +1403,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
break;
}
exit:
- if (is_stacked(dst_lo)) {
- emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ arm_bpf_put_reg32(dst_lo, rt, ctx);
+ arm_bpf_put_reg32(dst_hi, rd, ctx);
break;
/* dst = imm64 */
case BPF_LD | BPF_IMM | BPF_DW:
@@ -1482,9 +1423,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_LDX | BPF_MEM | BPF_H:
case BPF_LDX | BPF_MEM | BPF_B:
case BPF_LDX | BPF_MEM | BPF_DW:
- rn = is_stacked(src_lo) ? tmp2[1] : src_lo;
- if (is_stacked(src_lo))
- emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
+ rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code));
break;
/* ST: *(size *)(dst + off) = imm */
@@ -1520,12 +1459,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
{
u8 sz = BPF_SIZE(code);
- rn = is_stacked(src_lo) ? tmp2[1] : src_lo;
- rm = is_stacked(src_lo) ? tmp2[0] : src_hi;
- if (is_stacked(src_lo)) {
- emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx);
- }
+ rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
+ rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx);
/* Store the value */
if (BPF_SIZE(code) == BPF_DW) {
@@ -1559,12 +1494,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_JMP | BPF_JSLT | BPF_X:
case BPF_JMP | BPF_JSLE | BPF_X:
/* Setup source registers */
- rm = is_stacked(src_lo) ? tmp2[0] : src_hi;
- rn = is_stacked(src_lo) ? tmp2[1] : src_lo;
- if (is_stacked(src_lo)) {
- emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
- emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx);
- }
+ rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx);
+ rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
goto go_jmp;
/* PC += off if dst == imm */
/* PC += off if dst > imm */
@@ -1596,12 +1527,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit_a32_mov_i64(true, tmp2, imm, ctx);
go_jmp:
/* Setup destination register */
- rd = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- rt = is_stacked(dst_lo) ? tmp[1] : dst_lo;
- if (is_stacked(dst_lo)) {
- emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
- }
+ rt = arm_bpf_get_reg32(dst_lo, tmp[1], ctx);
+ rd = arm_bpf_get_reg32(dst_hi, tmp[0], ctx);
/* Check for the condition */
emit_ar_r(rd, rt, rm, rn, ctx, BPF_OP(code));
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 04/13] ARM: net: bpf: remove is_on_stack() and sstk/dstk
From: Russell King @ 2018-07-10 12:36 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
In-Reply-To: <20180710123340.GK17271@n2100.armlinux.org.uk>
The decision about whether a BPF register is on the stack or in a CPU
register is detected at the top BPF insn processing level, and then
percolated throughout the remainder of the code. Since we now use
negative register values to represent stacked registers, we can detect
where a BPF register is stored without restoring to carrying this
additional metadata through all code paths.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
arch/arm/net/bpf_jit_32.c | 336 ++++++++++++++++++++++------------------------
1 file changed, 160 insertions(+), 176 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index 69bf7ab18bf9..e81401aca2df 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -459,27 +459,18 @@ static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op)
emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx);
}
-/* Checks whether BPF register is on scratch stack space or not. */
-static inline bool is_on_stack(u8 bpf_reg)
+/* Is the translated BPF register on stack? */
+static bool is_stacked(s8 reg)
{
- static u8 stack_regs[] = {BPF_REG_AX, BPF_REG_3, BPF_REG_4, BPF_REG_5,
- BPF_REG_7, BPF_REG_8, BPF_REG_9, TCALL_CNT,
- BPF_REG_2, BPF_REG_FP};
- int i, reg_len = sizeof(stack_regs);
-
- for (i = 0 ; i < reg_len ; i++) {
- if (bpf_reg == stack_regs[i])
- return true;
- }
- return false;
+ return reg < 0;
}
static inline void emit_a32_mov_i(const s8 dst, const u32 val,
- bool dstk, struct jit_ctx *ctx)
+ struct jit_ctx *ctx)
{
const s8 *tmp = bpf2a32[TMP_REG_1];
- if (dstk) {
+ if (is_stacked(dst)) {
emit_mov_i(tmp[1], val, ctx);
emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(dst)), ctx);
} else {
@@ -489,14 +480,13 @@ static inline void emit_a32_mov_i(const s8 dst, const u32 val,
/* Sign extended move */
static inline void emit_a32_mov_i64(const bool is64, const s8 dst[],
- const u32 val, bool dstk,
- struct jit_ctx *ctx) {
+ const u32 val, struct jit_ctx *ctx) {
u32 hi = 0;
if (is64 && (val & (1<<31)))
hi = (u32)~0;
- emit_a32_mov_i(dst_lo, val, dstk, ctx);
- emit_a32_mov_i(dst_hi, hi, dstk, ctx);
+ emit_a32_mov_i(dst_lo, val, ctx);
+ emit_a32_mov_i(dst_hi, hi, ctx);
}
static inline void emit_a32_add_r(const u8 dst, const u8 src,
@@ -579,17 +569,16 @@ static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64,
* dst = dst (op) src
*/
static inline void emit_a32_alu_r(const s8 dst, const s8 src,
- bool dstk, bool sstk,
struct jit_ctx *ctx, const bool is64,
const bool hi, const u8 op) {
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rn = sstk ? tmp[1] : src;
+ s8 rn = is_stacked(src) ? tmp[1] : src;
- if (sstk)
+ if (is_stacked(src))
emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src)), ctx);
/* ALU operation */
- if (dstk) {
+ if (is_stacked(dst)) {
emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx);
emit_alu_r(tmp[0], rn, is64, hi, op, ctx);
emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx);
@@ -600,26 +589,24 @@ static inline void emit_a32_alu_r(const s8 dst, const s8 src,
/* ALU operation (64 bit) */
static inline void emit_a32_alu_r64(const bool is64, const s8 dst[],
- const s8 src[], bool dstk,
- bool sstk, struct jit_ctx *ctx,
+ const s8 src[], struct jit_ctx *ctx,
const u8 op) {
- emit_a32_alu_r(dst_lo, src_lo, dstk, sstk, ctx, is64, false, op);
+ emit_a32_alu_r(dst_lo, src_lo, ctx, is64, false, op);
if (is64)
- emit_a32_alu_r(dst_hi, src_hi, dstk, sstk, ctx, is64, true, op);
+ emit_a32_alu_r(dst_hi, src_hi, ctx, is64, true, op);
else
- emit_a32_mov_i(dst_hi, 0, dstk, ctx);
+ emit_a32_mov_i(dst_hi, 0, ctx);
}
/* dst = imm (4 bytes)*/
static inline void emit_a32_mov_r(const s8 dst, const s8 src,
- bool dstk, bool sstk,
struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rt = sstk ? tmp[0] : src;
+ s8 rt = is_stacked(src) ? tmp[0] : src;
- if (sstk)
+ if (is_stacked(src))
emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(src)), ctx);
- if (dstk)
+ if (is_stacked(dst))
emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst)), ctx);
else
emit(ARM_MOV_R(dst, rt), ctx);
@@ -627,25 +614,25 @@ static inline void emit_a32_mov_r(const s8 dst, const s8 src,
/* dst = src */
static inline void emit_a32_mov_r64(const bool is64, const s8 dst[],
- const s8 src[], bool dstk,
- bool sstk, struct jit_ctx *ctx) {
- emit_a32_mov_r(dst_lo, src_lo, dstk, sstk, ctx);
+ const s8 src[],
+ struct jit_ctx *ctx) {
+ emit_a32_mov_r(dst_lo, src_lo, ctx);
if (is64) {
/* complete 8 byte move */
- emit_a32_mov_r(dst_hi, src_hi, dstk, sstk, ctx);
+ emit_a32_mov_r(dst_hi, src_hi, ctx);
} else {
/* Zero out high 4 bytes */
- emit_a32_mov_i(dst_hi, 0, dstk, ctx);
+ emit_a32_mov_i(dst_hi, 0, ctx);
}
}
/* Shift operations */
-static inline void emit_a32_alu_i(const s8 dst, const u32 val, bool dstk,
+static inline void emit_a32_alu_i(const s8 dst, const u32 val,
struct jit_ctx *ctx, const u8 op) {
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rd = dstk ? tmp[0] : dst;
+ s8 rd = is_stacked(dst) ? tmp[0] : dst;
- if (dstk)
+ if (is_stacked(dst))
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
/* Do shift operation */
@@ -661,19 +648,19 @@ static inline void emit_a32_alu_i(const s8 dst, const u32 val, bool dstk,
break;
}
- if (dstk)
+ if (is_stacked(dst))
emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
}
/* dst = ~dst (64 bit) */
-static inline void emit_a32_neg64(const s8 dst[], bool dstk,
+static inline void emit_a32_neg64(const s8 dst[],
struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rd = dstk ? tmp[1] : dst[1];
- s8 rm = dstk ? tmp[0] : dst[0];
+ s8 rd = is_stacked(dst_lo) ? tmp[1] : dst[1];
+ s8 rm = is_stacked(dst_lo) ? tmp[0] : dst[0];
/* Setup Operand */
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -682,26 +669,26 @@ static inline void emit_a32_neg64(const s8 dst[], bool dstk,
emit(ARM_RSBS_I(rd, rd, 0), ctx);
emit(ARM_RSC_I(rm, rm, 0), ctx);
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
}
/* dst = dst << src */
-static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], bool dstk,
- bool sstk, struct jit_ctx *ctx) {
+static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[],
+ struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup Operands */
- s8 rt = sstk ? tmp2[1] : src_lo;
- s8 rd = dstk ? tmp[1] : dst_lo;
- s8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo;
+ s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- if (sstk)
+ if (is_stacked(src_lo))
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -714,7 +701,7 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], bool dstk,
emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd, SRTYPE_LSR, tmp2[0]), ctx);
emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_ASL, rt), ctx);
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
} else {
@@ -724,18 +711,18 @@ static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], bool dstk,
}
/* dst = dst >> src (signed)*/
-static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], bool dstk,
- bool sstk, struct jit_ctx *ctx) {
+static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[],
+ struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup Operands */
- s8 rt = sstk ? tmp2[1] : src_lo;
- s8 rd = dstk ? tmp[1] : dst_lo;
- s8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo;
+ s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- if (sstk)
+ if (is_stacked(src_lo))
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -748,7 +735,7 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], bool dstk,
_emit(ARM_COND_MI, ARM_B(0), ctx);
emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASR, tmp2[0]), ctx);
emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_ASR, rt), ctx);
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
} else {
@@ -758,18 +745,18 @@ static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], bool dstk,
}
/* dst = dst >> src */
-static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], bool dstk,
- bool sstk, struct jit_ctx *ctx) {
+static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[],
+ struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup Operands */
- s8 rt = sstk ? tmp2[1] : src_lo;
- s8 rd = dstk ? tmp[1] : dst_lo;
- s8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo;
+ s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- if (sstk)
+ if (is_stacked(src_lo))
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -781,7 +768,7 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], bool dstk,
emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx);
emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_LSR, tmp2[0]), ctx);
emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_LSR, rt), ctx);
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
} else {
@@ -791,15 +778,15 @@ static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], bool dstk,
}
/* dst = dst << val */
-static inline void emit_a32_lsh_i64(const s8 dst[], bool dstk,
- const u32 val, struct jit_ctx *ctx){
+static inline void emit_a32_lsh_i64(const s8 dst[],
+ const u32 val, struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup operands */
- s8 rd = dstk ? tmp[1] : dst_lo;
- s8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -817,22 +804,22 @@ static inline void emit_a32_lsh_i64(const s8 dst[], bool dstk,
emit(ARM_EOR_R(rd, rd, rd), ctx);
}
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
}
/* dst = dst >> val */
-static inline void emit_a32_rsh_i64(const s8 dst[], bool dstk,
+static inline void emit_a32_rsh_i64(const s8 dst[],
const u32 val, struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup operands */
- s8 rd = dstk ? tmp[1] : dst_lo;
- s8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -850,22 +837,22 @@ static inline void emit_a32_rsh_i64(const s8 dst[], bool dstk,
emit(ARM_MOV_I(rm, 0), ctx);
}
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
}
/* dst = dst >> val (signed) */
-static inline void emit_a32_arsh_i64(const s8 dst[], bool dstk,
+static inline void emit_a32_arsh_i64(const s8 dst[],
const u32 val, struct jit_ctx *ctx){
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup operands */
- s8 rd = dstk ? tmp[1] : dst_lo;
- s8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -883,27 +870,27 @@ static inline void emit_a32_arsh_i64(const s8 dst[], bool dstk,
emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx);
}
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
}
-static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], bool dstk,
- bool sstk, struct jit_ctx *ctx) {
+static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[],
+ struct jit_ctx *ctx) {
const s8 *tmp = bpf2a32[TMP_REG_1];
const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup operands for multiplication */
- s8 rd = dstk ? tmp[1] : dst_lo;
- s8 rm = dstk ? tmp[0] : dst_hi;
- s8 rt = sstk ? tmp2[1] : src_lo;
- s8 rn = sstk ? tmp2[0] : src_hi;
+ s8 rd = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ s8 rm = is_stacked(dst_lo) ? tmp[0] : dst_hi;
+ s8 rt = is_stacked(src_lo) ? tmp2[1] : src_lo;
+ s8 rn = is_stacked(src_lo) ? tmp2[0] : src_hi;
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
- if (sstk) {
+ if (is_stacked(src_lo)) {
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_hi)), ctx);
}
@@ -915,7 +902,7 @@ static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], bool dstk,
emit(ARM_UMULL(ARM_IP, rm, rd, rt), ctx);
emit(ARM_ADD_R(rm, ARM_LR, rm), ctx);
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
} else {
@@ -924,15 +911,15 @@ static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], bool dstk,
}
/* *(size *)(dst + off) = src */
-static inline void emit_str_r(const s8 dst, const s8 src, bool dstk,
+static inline void emit_str_r(const s8 dst, const s8 src,
const s32 off, struct jit_ctx *ctx, const u8 sz){
const s8 *tmp = bpf2a32[TMP_REG_1];
- s8 rd = dstk ? tmp[1] : dst;
+ s8 rd = is_stacked(dst) ? tmp[1] : dst;
- if (dstk)
+ if (is_stacked(dst))
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
if (off) {
- emit_a32_mov_i(tmp[0], off, false, ctx);
+ emit_a32_mov_i(tmp[0], off, ctx);
emit(ARM_ADD_R(tmp[0], rd, tmp[0]), ctx);
rd = tmp[0];
}
@@ -953,10 +940,10 @@ static inline void emit_str_r(const s8 dst, const s8 src, bool dstk,
}
/* dst = *(size*)(src + off) */
-static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk,
+static inline void emit_ldx_r(const s8 dst[], const s8 src,
s32 off, struct jit_ctx *ctx, const u8 sz){
const s8 *tmp = bpf2a32[TMP_REG_1];
- const s8 *rd = dstk ? tmp : dst;
+ const s8 *rd = is_stacked(dst_lo) ? tmp : dst;
s8 rm = src;
s32 off_max;
@@ -966,7 +953,7 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk,
off_max = 0xfff;
if (off < 0 || off > off_max) {
- emit_a32_mov_i(tmp[0], off, false, ctx);
+ emit_a32_mov_i(tmp[0], off, ctx);
emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);
rm = tmp[0];
off = 0;
@@ -978,17 +965,17 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk,
case BPF_B:
/* Load a Byte */
emit(ARM_LDRB_I(rd[1], rm, off), ctx);
- emit_a32_mov_i(dst[0], 0, dstk, ctx);
+ emit_a32_mov_i(dst[0], 0, ctx);
break;
case BPF_H:
/* Load a HalfWord */
emit(ARM_LDRH_I(rd[1], rm, off), ctx);
- emit_a32_mov_i(dst[0], 0, dstk, ctx);
+ emit_a32_mov_i(dst[0], 0, ctx);
break;
case BPF_W:
/* Load a Word */
emit(ARM_LDR_I(rd[1], rm, off), ctx);
- emit_a32_mov_i(dst[0], 0, dstk, ctx);
+ emit_a32_mov_i(dst[0], 0, ctx);
break;
case BPF_DW:
/* Load a Double Word */
@@ -996,10 +983,10 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk,
emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);
break;
}
- if (dstk)
- emit(ARM_STR_I(rd[1], ARM_SP, STACK_VAR(dst[1])), ctx);
- if (dstk && sz == BPF_DW)
- emit(ARM_STR_I(rd[0], ARM_SP, STACK_VAR(dst[0])), ctx);
+ if (is_stacked(dst_lo))
+ emit(ARM_STR_I(rd[1], ARM_SP, STACK_VAR(dst_lo)), ctx);
+ if (is_stacked(dst_lo) && sz == BPF_DW)
+ emit(ARM_STR_I(rd[0], ARM_SP, STACK_VAR(dst_hi)), ctx);
}
/* Arithmatic Operation */
@@ -1053,7 +1040,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
*/
off = offsetof(struct bpf_array, map.max_entries);
/* array->map.max_entries */
- emit_a32_mov_i(tmp[1], off, false, ctx);
+ emit_a32_mov_i(tmp[1], off, ctx);
emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx);
emit(ARM_LDR_R(tmp[1], tmp2[1], tmp[1]), ctx);
/* index is 32-bit for arrays */
@@ -1083,7 +1070,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
* goto out;
*/
off = offsetof(struct bpf_array, ptrs);
- emit_a32_mov_i(tmp[1], off, false, ctx);
+ emit_a32_mov_i(tmp[1], off, ctx);
emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx);
emit(ARM_ADD_R(tmp[1], tmp2[1], tmp[1]), ctx);
emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx);
@@ -1094,7 +1081,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
/* goto *(prog->bpf_func + prologue_size); */
off = offsetof(struct bpf_prog, bpf_func);
- emit_a32_mov_i(tmp2[1], off, false, ctx);
+ emit_a32_mov_i(tmp2[1], off, ctx);
emit(ARM_LDR_R(tmp[1], tmp[1], tmp2[1]), ctx);
emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx);
emit_bx_r(tmp[1], ctx);
@@ -1193,8 +1180,8 @@ static void build_prologue(struct jit_ctx *ctx)
emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx);
/* Set up BPF prog stack base register */
- emit_a32_mov_r(fplo, ARM_IP, true, false, ctx);
- emit_a32_mov_i(fphi, 0, true, ctx);
+ emit_a32_mov_r(fplo, ARM_IP, ctx);
+ emit_a32_mov_i(fphi, 0, ctx);
/* mov r4, 0 */
emit(ARM_MOV_I(r4, 0), ctx);
@@ -1243,8 +1230,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
const s32 imm = insn->imm;
const int i = insn - ctx->prog->insnsi;
const bool is64 = BPF_CLASS(code) == BPF_ALU64;
- const bool dstk = is_on_stack(insn->dst_reg);
- const bool sstk = is_on_stack(insn->src_reg);
s8 rd, rt, rm, rn;
s32 jmp_offset;
@@ -1268,11 +1253,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_ALU64 | BPF_MOV | BPF_X:
switch (BPF_SRC(code)) {
case BPF_X:
- emit_a32_mov_r64(is64, dst, src, dstk, sstk, ctx);
+ emit_a32_mov_r64(is64, dst, src, ctx);
break;
case BPF_K:
/* Sign-extend immediate value to destination reg */
- emit_a32_mov_i64(is64, dst, imm, dstk, ctx);
+ emit_a32_mov_i64(is64, dst, imm, ctx);
break;
}
break;
@@ -1312,8 +1297,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_ALU64 | BPF_XOR | BPF_X:
switch (BPF_SRC(code)) {
case BPF_X:
- emit_a32_alu_r64(is64, dst, src, dstk, sstk,
- ctx, BPF_OP(code));
+ emit_a32_alu_r64(is64, dst, src, ctx, BPF_OP(code));
break;
case BPF_K:
/* Move immediate value to the temporary register
@@ -1322,9 +1306,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
* value into temporary reg and then it would be
* safe to do the operation on it.
*/
- emit_a32_mov_i64(is64, tmp2, imm, false, ctx);
- emit_a32_alu_r64(is64, dst, tmp2, dstk, false,
- ctx, BPF_OP(code));
+ emit_a32_mov_i64(is64, tmp2, imm, ctx);
+ emit_a32_alu_r64(is64, dst, tmp2, ctx, BPF_OP(code));
break;
}
break;
@@ -1334,26 +1317,28 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU | BPF_MOD | BPF_K:
case BPF_ALU | BPF_MOD | BPF_X:
- rt = src_lo;
- rd = dstk ? tmp2[1] : dst_lo;
- if (dstk)
+ rd = is_stacked(dst_lo) ? tmp2[1] : dst_lo;
+ if (is_stacked(dst_lo))
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
switch (BPF_SRC(code)) {
case BPF_X:
- rt = sstk ? tmp2[0] : rt;
- if (sstk)
+ rt = is_stacked(rt) ? tmp2[0] : src_lo;
+ if (is_stacked(src_lo))
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)),
ctx);
break;
case BPF_K:
rt = tmp2[0];
- emit_a32_mov_i(rt, imm, false, ctx);
+ emit_a32_mov_i(rt, imm, ctx);
+ break;
+ default:
+ rt = src_lo;
break;
}
emit_udivmod(rd, rd, rt, ctx, BPF_OP(code));
- if (dstk)
+ if (is_stacked(dst_lo))
emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
- emit_a32_mov_i(dst_hi, 0, dstk, ctx);
+ emit_a32_mov_i(dst_hi, 0, ctx);
break;
case BPF_ALU64 | BPF_DIV | BPF_K:
case BPF_ALU64 | BPF_DIV | BPF_X:
@@ -1367,54 +1352,54 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
if (unlikely(imm > 31))
return -EINVAL;
if (imm)
- emit_a32_alu_i(dst_lo, imm, dstk, ctx, BPF_OP(code));
- emit_a32_mov_i(dst_hi, 0, dstk, ctx);
+ emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code));
+ emit_a32_mov_i(dst_hi, 0, ctx);
break;
/* dst = dst << imm */
case BPF_ALU64 | BPF_LSH | BPF_K:
if (unlikely(imm > 63))
return -EINVAL;
- emit_a32_lsh_i64(dst, dstk, imm, ctx);
+ emit_a32_lsh_i64(dst, imm, ctx);
break;
/* dst = dst >> imm */
case BPF_ALU64 | BPF_RSH | BPF_K:
if (unlikely(imm > 63))
return -EINVAL;
- emit_a32_rsh_i64(dst, dstk, imm, ctx);
+ emit_a32_rsh_i64(dst, imm, ctx);
break;
/* dst = dst << src */
case BPF_ALU64 | BPF_LSH | BPF_X:
- emit_a32_lsh_r64(dst, src, dstk, sstk, ctx);
+ emit_a32_lsh_r64(dst, src, ctx);
break;
/* dst = dst >> src */
case BPF_ALU64 | BPF_RSH | BPF_X:
- emit_a32_rsh_r64(dst, src, dstk, sstk, ctx);
+ emit_a32_rsh_r64(dst, src, ctx);
break;
/* dst = dst >> src (signed) */
case BPF_ALU64 | BPF_ARSH | BPF_X:
- emit_a32_arsh_r64(dst, src, dstk, sstk, ctx);
+ emit_a32_arsh_r64(dst, src, ctx);
break;
/* dst = dst >> imm (signed) */
case BPF_ALU64 | BPF_ARSH | BPF_K:
if (unlikely(imm > 63))
return -EINVAL;
- emit_a32_arsh_i64(dst, dstk, imm, ctx);
+ emit_a32_arsh_i64(dst, imm, ctx);
break;
/* dst = ~dst */
case BPF_ALU | BPF_NEG:
- emit_a32_alu_i(dst_lo, 0, dstk, ctx, BPF_OP(code));
- emit_a32_mov_i(dst_hi, 0, dstk, ctx);
+ emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code));
+ emit_a32_mov_i(dst_hi, 0, ctx);
break;
/* dst = ~dst (64 bit) */
case BPF_ALU64 | BPF_NEG:
- emit_a32_neg64(dst, dstk, ctx);
+ emit_a32_neg64(dst, ctx);
break;
/* dst = dst * src/imm */
case BPF_ALU64 | BPF_MUL | BPF_X:
case BPF_ALU64 | BPF_MUL | BPF_K:
switch (BPF_SRC(code)) {
case BPF_X:
- emit_a32_mul_r64(dst, src, dstk, sstk, ctx);
+ emit_a32_mul_r64(dst, src, ctx);
break;
case BPF_K:
/* Move immediate value to the temporary register
@@ -1423,8 +1408,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
* reg then it would be safe to do the operation
* on it.
*/
- emit_a32_mov_i64(is64, tmp2, imm, false, ctx);
- emit_a32_mul_r64(dst, tmp2, dstk, false, ctx);
+ emit_a32_mov_i64(is64, tmp2, imm, ctx);
+ emit_a32_mul_r64(dst, tmp2, ctx);
break;
}
break;
@@ -1432,9 +1417,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
/* dst = htobe(dst) */
case BPF_ALU | BPF_END | BPF_FROM_LE:
case BPF_ALU | BPF_END | BPF_FROM_BE:
- rd = dstk ? tmp[0] : dst_hi;
- rt = dstk ? tmp[1] : dst_lo;
- if (dstk) {
+ rd = is_stacked(dst_lo) ? tmp[0] : dst_hi;
+ rt = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -1459,7 +1444,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case 16:
/* zero-extend 16 bits into 64 bits */
#if __LINUX_ARM_ARCH__ < 6
- emit_a32_mov_i(tmp2[1], 0xffff, false, ctx);
+ emit_a32_mov_i(tmp2[1], 0xffff, ctx);
emit(ARM_AND_R(rt, rt, tmp2[1]), ctx);
#else /* ARMv6+ */
emit(ARM_UXTH(rt, rt), ctx);
@@ -1475,7 +1460,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
break;
}
exit:
- if (dstk) {
+ if (is_stacked(dst_lo)) {
emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -1487,8 +1472,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
u32 hi, lo = imm;
hi = insn1.imm;
- emit_a32_mov_i(dst_lo, lo, dstk, ctx);
- emit_a32_mov_i(dst_hi, hi, dstk, ctx);
+ emit_a32_mov_i(dst_lo, lo, ctx);
+ emit_a32_mov_i(dst_hi, hi, ctx);
return 1;
}
@@ -1497,10 +1482,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_LDX | BPF_MEM | BPF_H:
case BPF_LDX | BPF_MEM | BPF_B:
case BPF_LDX | BPF_MEM | BPF_DW:
- rn = sstk ? tmp2[1] : src_lo;
- if (sstk)
+ rn = is_stacked(src_lo) ? tmp2[1] : src_lo;
+ if (is_stacked(src_lo))
emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
- emit_ldx_r(dst, rn, dstk, off, ctx, BPF_SIZE(code));
+ emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code));
break;
/* ST: *(size *)(dst + off) = imm */
case BPF_ST | BPF_MEM | BPF_W:
@@ -1510,16 +1495,15 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
switch (BPF_SIZE(code)) {
case BPF_DW:
/* Sign-extend immediate value into temp reg */
- emit_a32_mov_i64(true, tmp2, imm, false, ctx);
- emit_str_r(dst_lo, tmp2[1], dstk, off, ctx, BPF_W);
- emit_str_r(dst_lo, tmp2[0], dstk, off+4, ctx, BPF_W);
+ emit_a32_mov_i64(true, tmp2, imm, ctx);
+ emit_str_r(dst_lo, tmp2[1], off, ctx, BPF_W);
+ emit_str_r(dst_lo, tmp2[0], off+4, ctx, BPF_W);
break;
case BPF_W:
case BPF_H:
case BPF_B:
- emit_a32_mov_i(tmp2[1], imm, false, ctx);
- emit_str_r(dst_lo, tmp2[1], dstk, off, ctx,
- BPF_SIZE(code));
+ emit_a32_mov_i(tmp2[1], imm, ctx);
+ emit_str_r(dst_lo, tmp2[1], off, ctx, BPF_SIZE(code));
break;
}
break;
@@ -1536,19 +1520,19 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
{
u8 sz = BPF_SIZE(code);
- rn = sstk ? tmp2[1] : src_lo;
- rm = sstk ? tmp2[0] : src_hi;
- if (sstk) {
+ rn = is_stacked(src_lo) ? tmp2[1] : src_lo;
+ rm = is_stacked(src_lo) ? tmp2[0] : src_hi;
+ if (is_stacked(src_lo)) {
emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx);
}
/* Store the value */
if (BPF_SIZE(code) == BPF_DW) {
- emit_str_r(dst_lo, rn, dstk, off, ctx, BPF_W);
- emit_str_r(dst_lo, rm, dstk, off+4, ctx, BPF_W);
+ emit_str_r(dst_lo, rn, off, ctx, BPF_W);
+ emit_str_r(dst_lo, rm, off+4, ctx, BPF_W);
} else {
- emit_str_r(dst_lo, rn, dstk, off, ctx, sz);
+ emit_str_r(dst_lo, rn, off, ctx, sz);
}
break;
}
@@ -1575,9 +1559,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case BPF_JMP | BPF_JSLT | BPF_X:
case BPF_JMP | BPF_JSLE | BPF_X:
/* Setup source registers */
- rm = sstk ? tmp2[0] : src_hi;
- rn = sstk ? tmp2[1] : src_lo;
- if (sstk) {
+ rm = is_stacked(src_lo) ? tmp2[0] : src_hi;
+ rn = is_stacked(src_lo) ? tmp2[1] : src_lo;
+ if (is_stacked(src_lo)) {
emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx);
}
@@ -1609,12 +1593,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
rm = tmp2[0];
rn = tmp2[1];
/* Sign-extend immediate value */
- emit_a32_mov_i64(true, tmp2, imm, false, ctx);
+ emit_a32_mov_i64(true, tmp2, imm, ctx);
go_jmp:
/* Setup destination register */
- rd = dstk ? tmp[0] : dst_hi;
- rt = dstk ? tmp[1] : dst_lo;
- if (dstk) {
+ rd = is_stacked(dst_lo) ? tmp[0] : dst_hi;
+ rt = is_stacked(dst_lo) ? tmp[1] : dst_lo;
+ if (is_stacked(dst_lo)) {
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
}
@@ -1684,13 +1668,13 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
const s8 *r5 = bpf2a32[BPF_REG_5];
const u32 func = (u32)__bpf_call_base + (u32)imm;
- emit_a32_mov_r64(true, r0, r1, false, false, ctx);
- emit_a32_mov_r64(true, r1, r2, false, true, ctx);
+ emit_a32_mov_r64(true, r0, r1, ctx);
+ emit_a32_mov_r64(true, r1, r2, ctx);
emit_push_r64(r5, 0, ctx);
emit_push_r64(r4, 8, ctx);
emit_push_r64(r3, 16, ctx);
- emit_a32_mov_i(tmp[1], func, false, ctx);
+ emit_a32_mov_i(tmp[1], func, ctx);
emit_blx_r(tmp[1], ctx);
emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 03/13] ARM: net: bpf: use negative numbers for stacked registers
From: Russell King @ 2018-07-10 12:36 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
In-Reply-To: <20180710123340.GK17271@n2100.armlinux.org.uk>
Use negative numbers for eBPF registers that live on the stack.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
arch/arm/net/bpf_jit_32.c | 200 +++++++++++++++++++++++-----------------------
1 file changed, 102 insertions(+), 98 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index c81da1a50834..69bf7ab18bf9 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -101,7 +101,11 @@ enum {
BPF_JIT_SCRATCH_REGS,
};
-#define STACK_OFFSET(k) ((k) * 4)
+/*
+ * Negative "register" values indicate the register is stored on the stack
+ * and are the offset from the top of the eBPF JIT scratch space.
+ */
+#define STACK_OFFSET(k) (-4 - (k) * 4)
#define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4)
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */
@@ -125,7 +129,7 @@ enum {
* scratch memory space and we have to build eBPF 64 bit register from those.
*
*/
-static const u8 bpf2a32[][2] = {
+static const s8 bpf2a32[][2] = {
/* return value from in-kernel function, and exit value from eBPF */
[BPF_REG_0] = {ARM_R1, ARM_R0},
/* arguments from eBPF program to in-kernel function */
@@ -291,7 +295,7 @@ static void jit_fill_hole(void *area, unsigned int size)
#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
/* Get the offset of eBPF REGISTERs stored on scratch space. */
-#define STACK_VAR(off) (STACK_SIZE - off)
+#define STACK_VAR(off) (STACK_SIZE + (off))
#if __LINUX_ARM_ARCH__ < 7
@@ -408,7 +412,7 @@ static inline int epilogue_offset(const struct jit_ctx *ctx)
static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op)
{
- const u8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
#if __LINUX_ARM_ARCH__ == 7
if (elf_hwcap & HWCAP_IDIVA) {
@@ -470,10 +474,10 @@ static inline bool is_on_stack(u8 bpf_reg)
return false;
}
-static inline void emit_a32_mov_i(const u8 dst, const u32 val,
+static inline void emit_a32_mov_i(const s8 dst, const u32 val,
bool dstk, struct jit_ctx *ctx)
{
- const u8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
if (dstk) {
emit_mov_i(tmp[1], val, ctx);
@@ -484,7 +488,7 @@ static inline void emit_a32_mov_i(const u8 dst, const u32 val,
}
/* Sign extended move */
-static inline void emit_a32_mov_i64(const bool is64, const u8 dst[],
+static inline void emit_a32_mov_i64(const bool is64, const s8 dst[],
const u32 val, bool dstk,
struct jit_ctx *ctx) {
u32 hi = 0;
@@ -574,12 +578,12 @@ static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64,
/* ALU operation (32 bit)
* dst = dst (op) src
*/
-static inline void emit_a32_alu_r(const u8 dst, const u8 src,
+static inline void emit_a32_alu_r(const s8 dst, const s8 src,
bool dstk, bool sstk,
struct jit_ctx *ctx, const bool is64,
const bool hi, const u8 op) {
- const u8 *tmp = bpf2a32[TMP_REG_1];
- u8 rn = sstk ? tmp[1] : src;
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ s8 rn = sstk ? tmp[1] : src;
if (sstk)
emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src)), ctx);
@@ -595,8 +599,8 @@ static inline void emit_a32_alu_r(const u8 dst, const u8 src,
}
/* ALU operation (64 bit) */
-static inline void emit_a32_alu_r64(const bool is64, const u8 dst[],
- const u8 src[], bool dstk,
+static inline void emit_a32_alu_r64(const bool is64, const s8 dst[],
+ const s8 src[], bool dstk,
bool sstk, struct jit_ctx *ctx,
const u8 op) {
emit_a32_alu_r(dst_lo, src_lo, dstk, sstk, ctx, is64, false, op);
@@ -607,11 +611,11 @@ static inline void emit_a32_alu_r64(const bool is64, const u8 dst[],
}
/* dst = imm (4 bytes)*/
-static inline void emit_a32_mov_r(const u8 dst, const u8 src,
+static inline void emit_a32_mov_r(const s8 dst, const s8 src,
bool dstk, bool sstk,
struct jit_ctx *ctx) {
- const u8 *tmp = bpf2a32[TMP_REG_1];
- u8 rt = sstk ? tmp[0] : src;
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ s8 rt = sstk ? tmp[0] : src;
if (sstk)
emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(src)), ctx);
@@ -622,8 +626,8 @@ static inline void emit_a32_mov_r(const u8 dst, const u8 src,
}
/* dst = src */
-static inline void emit_a32_mov_r64(const bool is64, const u8 dst[],
- const u8 src[], bool dstk,
+static inline void emit_a32_mov_r64(const bool is64, const s8 dst[],
+ const s8 src[], bool dstk,
bool sstk, struct jit_ctx *ctx) {
emit_a32_mov_r(dst_lo, src_lo, dstk, sstk, ctx);
if (is64) {
@@ -636,10 +640,10 @@ static inline void emit_a32_mov_r64(const bool is64, const u8 dst[],
}
/* Shift operations */
-static inline void emit_a32_alu_i(const u8 dst, const u32 val, bool dstk,
+static inline void emit_a32_alu_i(const s8 dst, const u32 val, bool dstk,
struct jit_ctx *ctx, const u8 op) {
- const u8 *tmp = bpf2a32[TMP_REG_1];
- u8 rd = dstk ? tmp[0] : dst;
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ s8 rd = dstk ? tmp[0] : dst;
if (dstk)
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
@@ -662,11 +666,11 @@ static inline void emit_a32_alu_i(const u8 dst, const u32 val, bool dstk,
}
/* dst = ~dst (64 bit) */
-static inline void emit_a32_neg64(const u8 dst[], bool dstk,
+static inline void emit_a32_neg64(const s8 dst[], bool dstk,
struct jit_ctx *ctx){
- const u8 *tmp = bpf2a32[TMP_REG_1];
- u8 rd = dstk ? tmp[1] : dst[1];
- u8 rm = dstk ? tmp[0] : dst[0];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ s8 rd = dstk ? tmp[1] : dst[1];
+ s8 rm = dstk ? tmp[0] : dst[0];
/* Setup Operand */
if (dstk) {
@@ -685,15 +689,15 @@ static inline void emit_a32_neg64(const u8 dst[], bool dstk,
}
/* dst = dst << src */
-static inline void emit_a32_lsh_r64(const u8 dst[], const u8 src[], bool dstk,
+static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], bool dstk,
bool sstk, struct jit_ctx *ctx) {
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup Operands */
- u8 rt = sstk ? tmp2[1] : src_lo;
- u8 rd = dstk ? tmp[1] : dst_lo;
- u8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rt = sstk ? tmp2[1] : src_lo;
+ s8 rd = dstk ? tmp[1] : dst_lo;
+ s8 rm = dstk ? tmp[0] : dst_hi;
if (sstk)
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
@@ -720,14 +724,14 @@ static inline void emit_a32_lsh_r64(const u8 dst[], const u8 src[], bool dstk,
}
/* dst = dst >> src (signed)*/
-static inline void emit_a32_arsh_r64(const u8 dst[], const u8 src[], bool dstk,
+static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], bool dstk,
bool sstk, struct jit_ctx *ctx) {
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup Operands */
- u8 rt = sstk ? tmp2[1] : src_lo;
- u8 rd = dstk ? tmp[1] : dst_lo;
- u8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rt = sstk ? tmp2[1] : src_lo;
+ s8 rd = dstk ? tmp[1] : dst_lo;
+ s8 rm = dstk ? tmp[0] : dst_hi;
if (sstk)
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
@@ -754,14 +758,14 @@ static inline void emit_a32_arsh_r64(const u8 dst[], const u8 src[], bool dstk,
}
/* dst = dst >> src */
-static inline void emit_a32_rsh_r64(const u8 dst[], const u8 src[], bool dstk,
+static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], bool dstk,
bool sstk, struct jit_ctx *ctx) {
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup Operands */
- u8 rt = sstk ? tmp2[1] : src_lo;
- u8 rd = dstk ? tmp[1] : dst_lo;
- u8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rt = sstk ? tmp2[1] : src_lo;
+ s8 rd = dstk ? tmp[1] : dst_lo;
+ s8 rm = dstk ? tmp[0] : dst_hi;
if (sstk)
emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
@@ -787,13 +791,13 @@ static inline void emit_a32_rsh_r64(const u8 dst[], const u8 src[], bool dstk,
}
/* dst = dst << val */
-static inline void emit_a32_lsh_i64(const u8 dst[], bool dstk,
+static inline void emit_a32_lsh_i64(const s8 dst[], bool dstk,
const u32 val, struct jit_ctx *ctx){
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup operands */
- u8 rd = dstk ? tmp[1] : dst_lo;
- u8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rd = dstk ? tmp[1] : dst_lo;
+ s8 rm = dstk ? tmp[0] : dst_hi;
if (dstk) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
@@ -820,13 +824,13 @@ static inline void emit_a32_lsh_i64(const u8 dst[], bool dstk,
}
/* dst = dst >> val */
-static inline void emit_a32_rsh_i64(const u8 dst[], bool dstk,
+static inline void emit_a32_rsh_i64(const s8 dst[], bool dstk,
const u32 val, struct jit_ctx *ctx) {
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup operands */
- u8 rd = dstk ? tmp[1] : dst_lo;
- u8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rd = dstk ? tmp[1] : dst_lo;
+ s8 rm = dstk ? tmp[0] : dst_hi;
if (dstk) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
@@ -853,13 +857,13 @@ static inline void emit_a32_rsh_i64(const u8 dst[], bool dstk,
}
/* dst = dst >> val (signed) */
-static inline void emit_a32_arsh_i64(const u8 dst[], bool dstk,
+static inline void emit_a32_arsh_i64(const s8 dst[], bool dstk,
const u32 val, struct jit_ctx *ctx){
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup operands */
- u8 rd = dstk ? tmp[1] : dst_lo;
- u8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rd = dstk ? tmp[1] : dst_lo;
+ s8 rm = dstk ? tmp[0] : dst_hi;
if (dstk) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
@@ -885,15 +889,15 @@ static inline void emit_a32_arsh_i64(const u8 dst[], bool dstk,
}
}
-static inline void emit_a32_mul_r64(const u8 dst[], const u8 src[], bool dstk,
+static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], bool dstk,
bool sstk, struct jit_ctx *ctx) {
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
/* Setup operands for multiplication */
- u8 rd = dstk ? tmp[1] : dst_lo;
- u8 rm = dstk ? tmp[0] : dst_hi;
- u8 rt = sstk ? tmp2[1] : src_lo;
- u8 rn = sstk ? tmp2[0] : src_hi;
+ s8 rd = dstk ? tmp[1] : dst_lo;
+ s8 rm = dstk ? tmp[0] : dst_hi;
+ s8 rt = sstk ? tmp2[1] : src_lo;
+ s8 rn = sstk ? tmp2[0] : src_hi;
if (dstk) {
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
@@ -920,10 +924,10 @@ static inline void emit_a32_mul_r64(const u8 dst[], const u8 src[], bool dstk,
}
/* *(size *)(dst + off) = src */
-static inline void emit_str_r(const u8 dst, const u8 src, bool dstk,
+static inline void emit_str_r(const s8 dst, const s8 src, bool dstk,
const s32 off, struct jit_ctx *ctx, const u8 sz){
- const u8 *tmp = bpf2a32[TMP_REG_1];
- u8 rd = dstk ? tmp[1] : dst;
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ s8 rd = dstk ? tmp[1] : dst;
if (dstk)
emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
@@ -949,11 +953,11 @@ static inline void emit_str_r(const u8 dst, const u8 src, bool dstk,
}
/* dst = *(size*)(src + off) */
-static inline void emit_ldx_r(const u8 dst[], const u8 src, bool dstk,
+static inline void emit_ldx_r(const s8 dst[], const s8 src, bool dstk,
s32 off, struct jit_ctx *ctx, const u8 sz){
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *rd = dstk ? tmp : dst;
- u8 rm = src;
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *rd = dstk ? tmp : dst;
+ s8 rm = src;
s32 off_max;
if (sz == BPF_H)
@@ -1034,11 +1038,11 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
{
/* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */
- const u8 *r2 = bpf2a32[BPF_REG_2];
- const u8 *r3 = bpf2a32[BPF_REG_3];
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
- const u8 *tcc = bpf2a32[TCALL_CNT];
+ const s8 *r2 = bpf2a32[BPF_REG_2];
+ const s8 *r3 = bpf2a32[BPF_REG_3];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tcc = bpf2a32[TCALL_CNT];
const int idx0 = ctx->idx;
#define cur_offset (ctx->idx - idx0)
#define jmp_offset (out_offset - (cur_offset) - 2)
@@ -1112,7 +1116,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx)
{
#if __LINUX_ARM_ARCH__ < 6
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx);
@@ -1127,7 +1131,7 @@ static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx)
static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx)
{
#if __LINUX_ARM_ARCH__ < 6
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx);
@@ -1147,10 +1151,10 @@ static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx)
}
// push the scratch stack register on top of the stack
-static inline void emit_push_r64(const u8 src[], const u8 shift,
+static inline void emit_push_r64(const s8 src[], const u8 shift,
struct jit_ctx *ctx)
{
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
u16 reg_set = 0;
emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(src[1]+shift)), ctx);
@@ -1162,13 +1166,13 @@ static inline void emit_push_r64(const u8 src[], const u8 shift,
static void build_prologue(struct jit_ctx *ctx)
{
- const u8 r0 = bpf2a32[BPF_REG_0][1];
- const u8 r2 = bpf2a32[BPF_REG_1][1];
- const u8 r3 = bpf2a32[BPF_REG_1][0];
- const u8 r4 = bpf2a32[BPF_REG_6][1];
- const u8 fplo = bpf2a32[BPF_REG_FP][1];
- const u8 fphi = bpf2a32[BPF_REG_FP][0];
- const u8 *tcc = bpf2a32[TCALL_CNT];
+ const s8 r0 = bpf2a32[BPF_REG_0][1];
+ const s8 r2 = bpf2a32[BPF_REG_1][1];
+ const s8 r3 = bpf2a32[BPF_REG_1][0];
+ const s8 r4 = bpf2a32[BPF_REG_6][1];
+ const s8 fplo = bpf2a32[BPF_REG_FP][1];
+ const s8 fphi = bpf2a32[BPF_REG_FP][0];
+ const s8 *tcc = bpf2a32[TCALL_CNT];
/* Save callee saved registers. */
#ifdef CONFIG_FRAME_POINTER
@@ -1231,17 +1235,17 @@ static void build_epilogue(struct jit_ctx *ctx)
static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
{
const u8 code = insn->code;
- const u8 *dst = bpf2a32[insn->dst_reg];
- const u8 *src = bpf2a32[insn->src_reg];
- const u8 *tmp = bpf2a32[TMP_REG_1];
- const u8 *tmp2 = bpf2a32[TMP_REG_2];
+ const s8 *dst = bpf2a32[insn->dst_reg];
+ const s8 *src = bpf2a32[insn->src_reg];
+ const s8 *tmp = bpf2a32[TMP_REG_1];
+ const s8 *tmp2 = bpf2a32[TMP_REG_2];
const s16 off = insn->off;
const s32 imm = insn->imm;
const int i = insn - ctx->prog->insnsi;
const bool is64 = BPF_CLASS(code) == BPF_ALU64;
const bool dstk = is_on_stack(insn->dst_reg);
const bool sstk = is_on_stack(insn->src_reg);
- u8 rd, rt, rm, rn;
+ s8 rd, rt, rm, rn;
s32 jmp_offset;
#define check_imm(bits, imm) do { \
@@ -1672,12 +1676,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
/* function call */
case BPF_JMP | BPF_CALL:
{
- const u8 *r0 = bpf2a32[BPF_REG_0];
- const u8 *r1 = bpf2a32[BPF_REG_1];
- const u8 *r2 = bpf2a32[BPF_REG_2];
- const u8 *r3 = bpf2a32[BPF_REG_3];
- const u8 *r4 = bpf2a32[BPF_REG_4];
- const u8 *r5 = bpf2a32[BPF_REG_5];
+ const s8 *r0 = bpf2a32[BPF_REG_0];
+ const s8 *r1 = bpf2a32[BPF_REG_1];
+ const s8 *r2 = bpf2a32[BPF_REG_2];
+ const s8 *r3 = bpf2a32[BPF_REG_3];
+ const s8 *r4 = bpf2a32[BPF_REG_4];
+ const s8 *r5 = bpf2a32[BPF_REG_5];
const u32 func = (u32)__bpf_call_base + (u32)imm;
emit_a32_mov_r64(true, r0, r1, false, false, ctx);
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 02/13] ARM: net: bpf: provide load/store ops with negative immediates
From: Russell King @ 2018-07-10 12:36 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
In-Reply-To: <20180710123340.GK17271@n2100.armlinux.org.uk>
Provide a set of load/store opcode generators that work with negative
immediates as well as positive ones.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
arch/arm/net/bpf_jit_32.c | 28 ++++++++++++++++++++++++++++
arch/arm/net/bpf_jit_32.h | 35 +++++++++++++----------------------
2 files changed, 41 insertions(+), 22 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index f2e6ffe57788..c81da1a50834 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -239,6 +239,34 @@ static int16_t imm8m(u32 x)
return -1;
}
+static u32 arm_bpf_ldst_imm12(u32 op, u8 rt, u8 rn, s16 imm12)
+{
+ op |= rt << 12 | rn << 16;
+ if (imm12 >= 0)
+ op |= ARM_INST_LDST__U;
+ else
+ imm12 = -imm12;
+ return op | (imm12 & 0xfff);
+}
+
+static u32 arm_bpf_ldst_imm8(u32 op, u8 rt, u8 rn, s16 imm8)
+{
+ op |= rt << 12 | rn << 16;
+ if (imm8 >= 0)
+ op |= ARM_INST_LDST__U;
+ else
+ imm8 = -imm8;
+ return op | (imm8 & 0xf0) << 4 | (imm8 & 0x0f);
+}
+
+#define ARM_LDR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDR_I, rt, rn, off)
+#define ARM_LDRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDRB_I, rt, rn, off)
+#define ARM_LDRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRH_I, rt, rn, off)
+
+#define ARM_STR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STR_I, rt, rn, off)
+#define ARM_STRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STRB_I, rt, rn, off)
+#define ARM_STRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRH_I, rt, rn, off)
+
/*
* Initializes the JIT space with undefined instructions.
*/
diff --git a/arch/arm/net/bpf_jit_32.h b/arch/arm/net/bpf_jit_32.h
index d5cf5f6208aa..c55bc39d3e22 100644
--- a/arch/arm/net/bpf_jit_32.h
+++ b/arch/arm/net/bpf_jit_32.h
@@ -77,11 +77,12 @@
#define ARM_INST_EOR_R 0x00200000
#define ARM_INST_EOR_I 0x02200000
-#define ARM_INST_LDRB_I 0x05d00000
+#define ARM_INST_LDST__U 0x00800000
+#define ARM_INST_LDRB_I 0x05500000
#define ARM_INST_LDRB_R 0x07d00000
-#define ARM_INST_LDRH_I 0x01d000b0
+#define ARM_INST_LDRH_I 0x015000b0
#define ARM_INST_LDRH_R 0x019000b0
-#define ARM_INST_LDR_I 0x05900000
+#define ARM_INST_LDR_I 0x05100000
#define ARM_INST_LDR_R 0x07900000
#define ARM_INST_LDM 0x08900000
@@ -124,9 +125,9 @@
#define ARM_INST_SBC_R 0x00c00000
#define ARM_INST_SBCS_R 0x00d00000
-#define ARM_INST_STR_I 0x05800000
-#define ARM_INST_STRB_I 0x05c00000
-#define ARM_INST_STRH_I 0x01c000b0
+#define ARM_INST_STR_I 0x05000000
+#define ARM_INST_STRB_I 0x05400000
+#define ARM_INST_STRH_I 0x014000b0
#define ARM_INST_TST_R 0x01100000
#define ARM_INST_TST_I 0x03100000
@@ -183,17 +184,14 @@
#define ARM_EOR_R(rd, rn, rm) _AL3_R(ARM_INST_EOR, rd, rn, rm)
#define ARM_EOR_I(rd, rn, imm) _AL3_I(ARM_INST_EOR, rd, rn, imm)
-#define ARM_LDR_I(rt, rn, off) (ARM_INST_LDR_I | (rt) << 12 | (rn) << 16 \
- | ((off) & 0xfff))
-#define ARM_LDR_R(rt, rn, rm) (ARM_INST_LDR_R | (rt) << 12 | (rn) << 16 \
+#define ARM_LDR_R(rt, rn, rm) (ARM_INST_LDR_R | ARM_INST_LDST__U \
+ | (rt) << 12 | (rn) << 16 \
| (rm))
-#define ARM_LDRB_I(rt, rn, off) (ARM_INST_LDRB_I | (rt) << 12 | (rn) << 16 \
- | (off))
-#define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | (rt) << 12 | (rn) << 16 \
+#define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | ARM_INST_LDST__U \
+ | (rt) << 12 | (rn) << 16 \
| (rm))
-#define ARM_LDRH_I(rt, rn, off) (ARM_INST_LDRH_I | (rt) << 12 | (rn) << 16 \
- | (((off) & 0xf0) << 4) | ((off) & 0xf))
-#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | (rt) << 12 | (rn) << 16 \
+#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | ARM_INST_LDST__U \
+ | (rt) << 12 | (rn) << 16 \
| (rm))
#define ARM_LDM(rn, regs) (ARM_INST_LDM | (rn) << 16 | (regs))
@@ -254,13 +252,6 @@
#define ARM_SUBS_I(rd, rn, imm) _AL3_I(ARM_INST_SUBS, rd, rn, imm)
#define ARM_SBC_I(rd, rn, imm) _AL3_I(ARM_INST_SBC, rd, rn, imm)
-#define ARM_STR_I(rt, rn, off) (ARM_INST_STR_I | (rt) << 12 | (rn) << 16 \
- | ((off) & 0xfff))
-#define ARM_STRH_I(rt, rn, off) (ARM_INST_STRH_I | (rt) << 12 | (rn) << 16 \
- | (((off) & 0xf0) << 4) | ((off) & 0xf))
-#define ARM_STRB_I(rt, rn, off) (ARM_INST_STRB_I | (rt) << 12 | (rn) << 16 \
- | (((off) & 0xf0) << 4) | ((off) & 0xf))
-
#define ARM_TST_R(rn, rm) _AL3_R(ARM_INST_TST, 0, rn, rm)
#define ARM_TST_I(rn, imm) _AL3_I(ARM_INST_TST, 0, rn, imm)
--
2.7.4
^ permalink raw reply related
* [PATCH net-next 01/13] ARM: net: bpf: enumerate the JIT scratch stack layout
From: Russell King @ 2018-07-10 12:36 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
In-Reply-To: <20180710123340.GK17271@n2100.armlinux.org.uk>
Enumerate the contents of the JIT scratch stack layout used for storing
some of the JITs 64-bit registers, tail call counter and AX register.
XXX: what about the skb_copy_bits buffer - this appears to overlap with
the first word of the JITs accessible stack.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
arch/arm/net/bpf_jit_32.c | 59 +++++++++++++++++++++++++++++++++--------------
1 file changed, 42 insertions(+), 17 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index f6a62ae44a65..f2e6ffe57788 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -72,7 +72,38 @@
#define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR)
#define CALLEE_POP_MASK (CALLEE_MASK | 1 << ARM_PC)
-#define STACK_OFFSET(k) (k)
+enum {
+ /* Stack layout - these are offsets from (top of stack - 4) */
+ BPF_R2_HI,
+ BPF_R2_LO,
+ BPF_R3_HI,
+ BPF_R3_LO,
+ BPF_R4_HI,
+ BPF_R4_LO,
+ BPF_R5_HI,
+ BPF_R5_LO,
+ BPF_R7_HI,
+ BPF_R7_LO,
+ BPF_R8_HI,
+ BPF_R8_LO,
+ BPF_R9_HI,
+ BPF_R9_LO,
+ BPF_FP_HI,
+ BPF_FP_LO,
+ BPF_TC_HI,
+ BPF_TC_LO,
+ BPF_AX_HI,
+ BPF_AX_LO,
+ /* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4,
+ * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9,
+ * BPF_REG_FP and Tail call counts.
+ */
+ BPF_JIT_SCRATCH_REGS,
+};
+
+#define STACK_OFFSET(k) ((k) * 4)
+#define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4)
+
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */
#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */
#define TCALL_CNT (MAX_BPF_JIT_REG + 2) /* Tail Call Count */
@@ -100,29 +131,29 @@ static const u8 bpf2a32[][2] = {
/* arguments from eBPF program to in-kernel function */
[BPF_REG_1] = {ARM_R3, ARM_R2},
/* Stored on stack scratch space */
- [BPF_REG_2] = {STACK_OFFSET(0), STACK_OFFSET(4)},
- [BPF_REG_3] = {STACK_OFFSET(8), STACK_OFFSET(12)},
- [BPF_REG_4] = {STACK_OFFSET(16), STACK_OFFSET(20)},
- [BPF_REG_5] = {STACK_OFFSET(24), STACK_OFFSET(28)},
+ [BPF_REG_2] = {STACK_OFFSET(BPF_R2_HI), STACK_OFFSET(BPF_R2_LO)},
+ [BPF_REG_3] = {STACK_OFFSET(BPF_R3_HI), STACK_OFFSET(BPF_R3_LO)},
+ [BPF_REG_4] = {STACK_OFFSET(BPF_R4_HI), STACK_OFFSET(BPF_R4_LO)},
+ [BPF_REG_5] = {STACK_OFFSET(BPF_R5_HI), STACK_OFFSET(BPF_R5_LO)},
/* callee saved registers that in-kernel function will preserve */
[BPF_REG_6] = {ARM_R5, ARM_R4},
/* Stored on stack scratch space */
- [BPF_REG_7] = {STACK_OFFSET(32), STACK_OFFSET(36)},
- [BPF_REG_8] = {STACK_OFFSET(40), STACK_OFFSET(44)},
- [BPF_REG_9] = {STACK_OFFSET(48), STACK_OFFSET(52)},
+ [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)},
+ [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)},
+ [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)},
/* Read only Frame Pointer to access Stack */
- [BPF_REG_FP] = {STACK_OFFSET(56), STACK_OFFSET(60)},
+ [BPF_REG_FP] = {STACK_OFFSET(BPF_FP_HI), STACK_OFFSET(BPF_FP_LO)},
/* Temporary Register for internal BPF JIT, can be used
* for constant blindings and others.
*/
[TMP_REG_1] = {ARM_R7, ARM_R6},
[TMP_REG_2] = {ARM_R10, ARM_R8},
/* Tail call count. Stored on stack scratch space. */
- [TCALL_CNT] = {STACK_OFFSET(64), STACK_OFFSET(68)},
+ [TCALL_CNT] = {STACK_OFFSET(BPF_TC_HI), STACK_OFFSET(BPF_TC_LO)},
/* temporary register for blinding constants.
* Stored on stack scratch space.
*/
- [BPF_REG_AX] = {STACK_OFFSET(72), STACK_OFFSET(76)},
+ [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)},
};
#define dst_lo dst[1]
@@ -227,12 +258,6 @@ static void jit_fill_hole(void *area, unsigned int size)
#define STACK_ALIGNMENT 4
#endif
-/* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4,
- * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9,
- * BPF_REG_FP and Tail call counts.
- */
-#define SCRATCH_SIZE 80
-
/* total stack size used in JITed code */
#define _STACK_SIZE (ctx->prog->aux->stack_depth + SCRATCH_SIZE)
#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
--
2.7.4
^ permalink raw reply related
* [PATCH 00/13] ARM BPF jit compiler improvements
From: Russell King - ARM Linux @ 2018-07-10 12:33 UTC (permalink / raw)
To: netdev, linux-arm-kernel; +Cc: Daniel Borkmann
Hi,
This series improves the ARM BPF JIT compiler by:
- enumerating the stack layout rather than using constants that happen
to be multiples of four
- rejig the BPF "register" accesses to use negative numbers instead of
positive, which could be confused with register numbers in the bpf2a32
array.
- since we maintain the ARM FP register as a pointer to the top of our
scratch space (or, with frame pointers enabled, a valid ARM frame
pointer register), we can access our scratch space using FP, which is
constant across all BPF programs, including tail-called programs.
- use immediate forms of ARM instructions where possible, rather than
first loading the immediate into an ARM register.
- use load-with-shift instruction rather than seperate shift instruction
followed by load
- avoid reloading index and array in the tail-call code
- use double-word load/store instructions where available
arch/arm/net/bpf_jit_32.c | 927 +++++++++++++++++++++++-----------------------
arch/arm/net/bpf_jit_32.h | 44 +--
2 files changed, 493 insertions(+), 478 deletions(-)
--
RMK's Patch system: http://www.armlinux.org.uk/developer/patches/
FTTC broadband for 0.8mile line in suburbia: sync at 13.8Mbps down 630kbps up
According to speedtest.net: 13Mbps down 490kbps up
^ permalink raw reply
* Re: [PATCH bpf] bpf: fix ldx in ld_abs rewrite for large offsets
From: Mark Rutland @ 2018-07-10 12:27 UTC (permalink / raw)
To: Daniel Borkmann; +Cc: alexei.starovoitov, netdev
In-Reply-To: <ae1f7041-8b81-ae99-c51c-d40925440dd0@iogearbox.net>
On Tue, Jul 10, 2018 at 02:13:42PM +0200, Daniel Borkmann wrote:
> On 07/10/2018 12:14 PM, Mark Rutland wrote:
> > On Tue, Jul 10, 2018 at 12:43:22AM +0200, Daniel Borkmann wrote:
> >> Mark reported that syzkaller triggered a KASAN detected slab-out-of-bounds
> >> bug in ___bpf_prog_run() with a BPF_LD | BPF_ABS word load at offset 0x8001.
> >> After further investigation it became clear that the issue was the
> >> BPF_LDX_MEM() which takes offset as an argument whereas it cannot encode
> >> larger than S16_MAX offsets into it. For this synthetical case we need to
> >> move the full address into tmp register instead and do the LDX without
> >> immediate value.
> >>
> >> Fixes: e0cea7ce988c ("bpf: implement ld_abs/ld_ind in native bpf")
> >> Reported-by: syzbot <syzkaller@googlegroups.com>
> >> Reported-by: Mark Rutland <mark.rutland@arm.com>
> >> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
> >> ---
> >> net/core/filter.c | 16 +++++++++++++---
> >> 1 file changed, 13 insertions(+), 3 deletions(-)
> >>
> >> diff --git a/net/core/filter.c b/net/core/filter.c
> >> index 5fa66a3..a13f5b1 100644
> >> --- a/net/core/filter.c
> >> +++ b/net/core/filter.c
> >> @@ -459,11 +459,21 @@ static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp)
> >> (!unaligned_ok && offset >= 0 &&
> >> offset + ip_align >= 0 &&
> >> offset + ip_align % size == 0))) {
> >> + bool ldx_off_ok = offset <= S16_MAX;
> >> +
> >
> > Given offset is a (signed) int, is it possible for that to be a negative
> > value less than S16_MIN? ... or is that ruled out elsewhere?
>
> This branch here handles only positive offset.
Ah, sorry. I missed the "offset >= 0" check in the context above.
> offset can be negative,
> but in that case these insns won't be emitted and handling will be done
> in 'slow path' via bpf_skb_load_helper_*().
Ok; looks good to me, then.
Thanks,
Mark.
^ permalink raw reply
* Re: [PATCH net v2 4/5] net/ipv6: propagate net.ipv6.conf.all.addr_gen_mode to devices
From: David Ahern @ 2018-07-10 12:19 UTC (permalink / raw)
To: Sabrina Dubroca; +Cc: netdev, Jiri Pirko, Felix Jia
In-Reply-To: <20180710101352.GA31570@bistromath.localdomain>
On 7/10/18 4:13 AM, Sabrina Dubroca wrote:
> 2018-07-09, 11:24:49 -0600, David Ahern wrote:
>> On 7/9/18 4:25 AM, Sabrina Dubroca wrote:
>>> This aligns the addr_gen_mode sysctl with the expected behavior of the
>>> "all" variant.
>>>
>>> Fixes: d35a00b8e33d ("net/ipv6: allow sysctl to change link-local address generation mode")
>>> Suggested-by: David Ahern <dsahern@gmail.com>
>>> Signed-off-by: Sabrina Dubroca <sd@queasysnail.net>
>>> ---
>>> net/ipv6/addrconf.c | 12 ++++++++++++
>>> 1 file changed, 12 insertions(+)
>>>
>>> diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
>>> index e89bca83e0e4..1659a6b3cf42 100644
>>> --- a/net/ipv6/addrconf.c
>>> +++ b/net/ipv6/addrconf.c
>>> @@ -5926,6 +5926,18 @@ static int addrconf_sysctl_addr_gen_mode(struct ctl_table *ctl, int write,
>>> idev->cnf.addr_gen_mode = new_val;
>>> addrconf_dev_config(idev->dev);
>>> }
>>> + } else if (&net->ipv6.devconf_all->addr_gen_mode == ctl->data) {
>>> + struct net_device *dev;
>>> +
>>> + net->ipv6.devconf_dflt->addr_gen_mode = new_val;
>>> + for_each_netdev(net, dev) {
>>> + idev = __in6_dev_get(dev);
>>> + if (idev &&
>>> + idev->cnf.addr_gen_mode != new_val) {
>>> + idev->cnf.addr_gen_mode = new_val;
>>> + addrconf_dev_config(idev->dev);
>>
>> This call is adding a new LL address without removing the previous one:
>>
>> # ip -6 addr sh dev eth2
>> 4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
>> inet6 2001:db8:2::4/64 scope global
>> valid_lft forever preferred_lft forever
>> inet6 fe80::e0:f9ff:fe45:6480/64 scope link
>> valid_lft forever preferred_lft forever
>>
>> # sysctl -w net.ipv6.conf.eth2.addr_gen_mode=3
>> net.ipv6.conf.eth2.addr_gen_mode = 3
>>
>> # ip -6 addr sh dev eth2
>> 4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
>> inet6 2001:db8:2::4/64 scope global
>> valid_lft forever preferred_lft forever
>> inet6 fe80::bc31:8009:270d:e019/64 scope link stable-privacy
>> valid_lft forever preferred_lft forever
>> inet6 fe80::e0:f9ff:fe45:6480/64 scope link
>> valid_lft forever preferred_lft forever
>
> Yes. That's also what will happen with global addresses, once the next
> RA is received: a new address corresponding to the new generation mode
> will be added, and the old one isn't removed.
Interesting.
>
> I think that was the expected behavior of d35a00b8e33d, but since it
> never actually worked... OTOH, the netlink attribute only sets
> idev->cnf.addr_gen_mode and doesn't add the new LL address (not until
> a DOWN/UP cycle), which I personally find surprising. If I set the
> mode to random or stable_secret, I would expect the privacy address to
> show up without having to take the device down and then up.
>
> I think removing the previous address immediately would break things
> (and the user wouldn't expect an address to disappear that way, since
> they're not explicitly asking for it to be removed), but I guess we
> could play games with the lifetimes (reduce the lifetime of the old
> address from forever to some limit). That limit would need to be
> configurable I think, and I would rather target that change for
> net-next.
>
The setting is address generation mode. Addresses are generated on admin
up only - I believe this is well established behavior. If the
documentation contains a note that changing the mode requires an admin
down/up, then it should not be a surprise to users.
I can't imagine this setting is changed on the fly and expected to work
immediately. Rather I would expect this is set at boot via the sysctl
files or by an interface manager before interfaces are brought up.
I'll ask around, but I am traveling today so will be a delay getting
back on this.
^ permalink raw reply
* Re: [PATCH] netfilter: NFT_SOCKET don't use NF_SOCKET_IPV6 without NF_TABLES_IPV6
From: Pablo Neira Ayuso @ 2018-07-10 12:14 UTC (permalink / raw)
To: Máté Eckl
Cc: Arnd Bergmann, Jozsef Kadlecsik, Florian Westphal,
David S. Miller, Flavio Leitner, netfilter-devel, coreteam,
Networking, Linux Kernel Mailing List
In-Reply-To: <20180710114517.nxnkav6mimyivh4m@sch.bme.hu>
On Tue, Jul 10, 2018 at 01:45:17PM +0200, Máté Eckl wrote:
> On Tue, Jul 10, 2018 at 01:24:59PM +0200, Pablo Neira Ayuso wrote:
> > On Tue, Jul 10, 2018 at 01:15:36PM +0200, Máté Eckl wrote:
> > > On Tue, Jul 10, 2018 at 12:56:05PM +0200, Pablo Neira Ayuso wrote:
> > [...]
> > > This patch only solves the nf_socket and nft_socket modules problem so I can
> > > only submit a v2 for 'netfilter: Kconfig: Change IPv6 select dependencies' but
> > > you already applied it so it would meen a force push. Should I do this?
> > >
> > > I think Arnd's patch solves these problems in case we don't want to force-push
> > > or rebase.
> >
> > You are refering to these two patches, right?
> >
> > https://patchwork.ozlabs.org/patch/941374/
> > https://patchwork.ozlabs.org/patch/941696/
>
> Oh, I missed the first one. So basically I should just squash them and resubmit
> right? Or even replace ("netfilter: Kconfig: Change IPv6 select
> dependencies") with these changes?
That's one possibility, yes.
^ permalink raw reply
* Re: [PATCH bpf] bpf: fix ldx in ld_abs rewrite for large offsets
From: Daniel Borkmann @ 2018-07-10 12:13 UTC (permalink / raw)
To: Mark Rutland; +Cc: alexei.starovoitov, netdev
In-Reply-To: <20180710101432.u5rdnmofd3imf5vu@lakrids.cambridge.arm.com>
On 07/10/2018 12:14 PM, Mark Rutland wrote:
> On Tue, Jul 10, 2018 at 12:43:22AM +0200, Daniel Borkmann wrote:
>> Mark reported that syzkaller triggered a KASAN detected slab-out-of-bounds
>> bug in ___bpf_prog_run() with a BPF_LD | BPF_ABS word load at offset 0x8001.
>> After further investigation it became clear that the issue was the
>> BPF_LDX_MEM() which takes offset as an argument whereas it cannot encode
>> larger than S16_MAX offsets into it. For this synthetical case we need to
>> move the full address into tmp register instead and do the LDX without
>> immediate value.
>>
>> Fixes: e0cea7ce988c ("bpf: implement ld_abs/ld_ind in native bpf")
>> Reported-by: syzbot <syzkaller@googlegroups.com>
>> Reported-by: Mark Rutland <mark.rutland@arm.com>
>> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
>> ---
>> net/core/filter.c | 16 +++++++++++++---
>> 1 file changed, 13 insertions(+), 3 deletions(-)
>>
>> diff --git a/net/core/filter.c b/net/core/filter.c
>> index 5fa66a3..a13f5b1 100644
>> --- a/net/core/filter.c
>> +++ b/net/core/filter.c
>> @@ -459,11 +459,21 @@ static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp)
>> (!unaligned_ok && offset >= 0 &&
>> offset + ip_align >= 0 &&
>> offset + ip_align % size == 0))) {
>> + bool ldx_off_ok = offset <= S16_MAX;
>> +
>
> Given offset is a (signed) int, is it possible for that to be a negative
> value less than S16_MIN? ... or is that ruled out elsewhere?
This branch here handles only positive offset. offset can be negative,
but in that case these insns won't be emitted and handling will be done
in 'slow path' via bpf_skb_load_helper_*().
> Thanks,
> Mark.
>
>> *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
>> *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
>> - *insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP, size, 2 + endian);
>> - *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A, BPF_REG_D,
>> - offset);
>> + *insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP,
>> + size, 2 + endian + (!ldx_off_ok * 2));
>> + if (ldx_off_ok) {
>> + *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
>> + BPF_REG_D, offset);
>> + } else {
>> + *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_D);
>> + *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_TMP, offset);
>> + *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
>> + BPF_REG_TMP, 0);
>> + }
>> if (endian)
>> *insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8);
>> *insn++ = BPF_JMP_A(8);
>> --
>> 2.9.5
>>
^ permalink raw reply
* Re: [PATCH] netfilter: NFT_SOCKET don't use NF_SOCKET_IPV6 without NF_TABLES_IPV6
From: Máté Eckl @ 2018-07-10 11:45 UTC (permalink / raw)
To: Pablo Neira Ayuso
Cc: Arnd Bergmann, Jozsef Kadlecsik, Florian Westphal,
David S. Miller, Flavio Leitner, netfilter-devel, coreteam,
Networking, Linux Kernel Mailing List
In-Reply-To: <20180710112459.ielgaydrgd4kxizo@salvia>
On Tue, Jul 10, 2018 at 01:24:59PM +0200, Pablo Neira Ayuso wrote:
> On Tue, Jul 10, 2018 at 01:15:36PM +0200, Máté Eckl wrote:
> > On Tue, Jul 10, 2018 at 12:56:05PM +0200, Pablo Neira Ayuso wrote:
> [...]
> > This patch only solves the nf_socket and nft_socket modules problem so I can
> > only submit a v2 for 'netfilter: Kconfig: Change IPv6 select dependencies' but
> > you already applied it so it would meen a force push. Should I do this?
> >
> > I think Arnd's patch solves these problems in case we don't want to force-push
> > or rebase.
>
> You are refering to these two patches, right?
>
> https://patchwork.ozlabs.org/patch/941374/
> https://patchwork.ozlabs.org/patch/941696/
Oh, I missed the first one. So basically I should just squash them and resubmit
right? Or even replace ("netfilter: Kconfig: Change IPv6 select
dependencies") with these changes?
> > > I think it's better if we toss your original patch in the tree and
> > > rebase, ie. take the new one that fixes all issues that Arnd is
> > > reporting. It would be good if we can sort out this before I send the
> > > next pull request for net-next stuff.
> > >
> > > I was afraid of fallout like this when I saw your original patch,
> > > kbuild is always tricky.
> >
> > This patch is not related to the nft_tproxy module (it seems that you refer to
> > that) as Arnd didn't have that in the tree when doing this. I'll send a v4 fot
> > the tproxy module, but that cannot be related to this one as it is not in tree
> > yet.
>
> No, I'm refering to 35bf1ccecaaa ("netfilter: Kconfig: Change IPv6
> select dependencies"), that is causing the issues.
>
> Thanks.
^ permalink raw reply
* Re: [PATCH v2] can: m_can: Move accessing of message ram to after clocks are enabled
From: Faiz Abbas @ 2018-07-10 11:36 UTC (permalink / raw)
To: linux-kernel, netdev, linux-can, linux-omap; +Cc: wg, mkl, davem
In-Reply-To: <20180703111710.27084-1-faiz_abbas@ti.com>
Hi,
On Tuesday 03 July 2018 04:47 PM, Faiz Abbas wrote:
> MCAN message ram should only be accessed once clocks are enabled.
> Therefore, move the call to parse/init the message ram to after
> clocks are enabled.
>
> Signed-off-by: Faiz Abbas <faiz_abbas@ti.com>
> ---
> Changes in v2:
>
> rebased to latest mainline
>
> drivers/net/can/m_can/m_can.c | 7 +++----
> 1 file changed, 3 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c
> index ac4c6dc2f8c8..04c48371ab2a 100644
> --- a/drivers/net/can/m_can/m_can.c
> +++ b/drivers/net/can/m_can/m_can.c
> @@ -1644,8 +1644,6 @@ static int m_can_plat_probe(struct platform_device *pdev)
> priv->can.clock.freq = clk_get_rate(cclk);
> priv->mram_base = mram_addr;
>
> - m_can_of_parse_mram(priv, mram_config_vals);
> -
> platform_set_drvdata(pdev, dev);
> SET_NETDEV_DEV(dev, &pdev->dev);
>
> @@ -1668,6 +1666,8 @@ static int m_can_plat_probe(struct platform_device *pdev)
> goto clk_disable;
> }
>
> + m_can_of_parse_mram(priv, mram_config_vals);
> +
> devm_can_led_init(dev);
>
> of_can_transceiver(dev);
> @@ -1715,8 +1715,6 @@ static __maybe_unused int m_can_resume(struct device *dev)
>
> pinctrl_pm_select_default_state(dev);
>
> - m_can_init_ram(priv);
> -
> priv->can.state = CAN_STATE_ERROR_ACTIVE;
>
> if (netif_running(ndev)) {
> @@ -1726,6 +1724,7 @@ static __maybe_unused int m_can_resume(struct device *dev)
> if (ret)
> return ret;
>
> + m_can_init_ram(priv);
> m_can_start(ndev);
> netif_device_attach(ndev);
> netif_start_queue(ndev);
>
Gentle ping.
Thanks,
Faiz
^ permalink raw reply
* Re: [PATCH v2] can: m_can: Fix runtime resume call
From: Faiz Abbas @ 2018-07-10 11:36 UTC (permalink / raw)
To: linux-kernel, netdev, linux-can, linux-omap; +Cc: wg, mkl, davem
In-Reply-To: <20180703111102.26502-1-faiz_abbas@ti.com>
Hi,
On Tuesday 03 July 2018 04:41 PM, Faiz Abbas wrote:
> pm_runtime_get_sync() returns a 1 if the state of the device is already
> 'active'. This is not a failure case and should return a success.
>
> Therefore fix error handling for pm_runtime_get_sync() call such that
> it returns success when the value is 1.
>
> Also cleanup the TODO for using runtime PM for sleep mode as that is
> implemented.
>
> Signed-off-by: Faiz Abbas <faiz_abbas@ti.com>
> ---
>
> Changes in v2:
>
> rebased to latest mainline
>
> drivers/net/can/m_can/m_can.c | 8 ++++----
> 1 file changed, 4 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c
> index b397a33f3d32..ac4c6dc2f8c8 100644
> --- a/drivers/net/can/m_can/m_can.c
> +++ b/drivers/net/can/m_can/m_can.c
> @@ -634,10 +634,12 @@ static int m_can_clk_start(struct m_can_priv *priv)
> int err;
>
> err = pm_runtime_get_sync(priv->device);
> - if (err)
> + if (err < 0) {
> pm_runtime_put_noidle(priv->device);
> + return err;
> + }
>
> - return err;
> + return 0;
> }
>
> static void m_can_clk_stop(struct m_can_priv *priv)
> @@ -1687,8 +1689,6 @@ static int m_can_plat_probe(struct platform_device *pdev)
> return ret;
> }
>
> -/* TODO: runtime PM with power down or sleep mode */
> -
> static __maybe_unused int m_can_suspend(struct device *dev)
> {
> struct net_device *ndev = dev_get_drvdata(dev);
>
Gentle ping.
Thanks,
Faiz
^ permalink raw reply
* Re: [PATCH] netfilter: NFT_SOCKET don't use NF_SOCKET_IPV6 without NF_TABLES_IPV6
From: Pablo Neira Ayuso @ 2018-07-10 11:24 UTC (permalink / raw)
To: Máté Eckl
Cc: Arnd Bergmann, Jozsef Kadlecsik, Florian Westphal,
David S. Miller, Flavio Leitner, netfilter-devel, coreteam,
Networking, Linux Kernel Mailing List
In-Reply-To: <20180710111536.hio2xalz7pyxucc6@sch.bme.hu>
On Tue, Jul 10, 2018 at 01:15:36PM +0200, Máté Eckl wrote:
> On Tue, Jul 10, 2018 at 12:56:05PM +0200, Pablo Neira Ayuso wrote:
[...]
> This patch only solves the nf_socket and nft_socket modules problem so I can
> only submit a v2 for 'netfilter: Kconfig: Change IPv6 select dependencies' but
> you already applied it so it would meen a force push. Should I do this?
>
> I think Arnd's patch solves these problems in case we don't want to force-push
> or rebase.
You are refering to these two patches, right?
https://patchwork.ozlabs.org/patch/941374/
https://patchwork.ozlabs.org/patch/941696/
> > I think it's better if we toss your original patch in the tree and
> > rebase, ie. take the new one that fixes all issues that Arnd is
> > reporting. It would be good if we can sort out this before I send the
> > next pull request for net-next stuff.
> >
> > I was afraid of fallout like this when I saw your original patch,
> > kbuild is always tricky.
>
> This patch is not related to the nft_tproxy module (it seems that you refer to
> that) as Arnd didn't have that in the tree when doing this. I'll send a v4 fot
> the tproxy module, but that cannot be related to this one as it is not in tree
> yet.
No, I'm refering to 35bf1ccecaaa ("netfilter: Kconfig: Change IPv6
select dependencies"), that is causing the issues.
Thanks.
^ permalink raw reply
* Issue with driver i40e stat strings count mismatch
From: Jesper Dangaard Brouer @ 2018-07-10 11:17 UTC (permalink / raw)
To: Jeff Kirsher
Cc: brouer, Björn Töpel, alexander.h.duyck@intel.com,
intel-wired-lan, netdev@vger.kernel.org
Hi Intel-fokes,
Your i40e driver have issues with it's ethtool stats. A warning
triggers at drivers/net/ethernet/intel/i40e/i40e_ethtool.c line 1907
(see splash below) in func i40e_get_stat_strings().
[ 5077.779518] ------------[ cut here ]------------
[ 5077.784493] stat strings count mismatch!
[ 5077.784529] WARNING: CPU: 0 PID: 2293 at drivers/net/ethernet/intel/i40e/i40e_ethtool.c:1907 i40e_get_strings+0x477/0x4b0 [i40e]
[ 5077.800941] Modules linked in: act_gact cls_u32 sch_ingress xt_tcpudp iptable_raw ip_tables x_tables tun nfnetlink bridge stp llc bpfilter sunrpc coretemp kvm_intel kvm irqbypass intel_cstate intel_uncore intel_rapl_perf pcspkr i2c_i801 wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_pad sch_fq_codel ixgbe mlx5_core mlxfw i40e devlink hid_generic igb mdio i2c_algo_bit ptp sd_mod i2c_core pps_core [last unloaded: x_tables]
[ 5077.839833] CPU: 0 PID: 2293 Comm: ethtool Not tainted 4.18.0-rc3-net-next-EdwardCree01+ #484
[ 5077.848962] Hardware name: Supermicro Super Server/X10SRi-F, BIOS 2.0a 08/01/2016
[ 5077.857049] RIP: 0010:i40e_get_strings+0x477/0x4b0 [i40e]
[ 5077.862776] Code: 98 49 39 c4 0f 84 2e fc ff ff 80 3d b3 da 03 00 00 0f 85 21 fc ff ff 48 c7 c7 24 42 19 a0 c6 05 9f da 03 00 01 e8 e9 9c ee e0 <0f> 0b e9 07 fc ff ff 48 83 c4 10 48 c7 c1 80 01 19 a0 be 20 00 00
[ 5077.882506] RSP: 0018:ffffc90003af3c18 EFLAGS: 00010296
[ 5077.888063] RAX: 000000000000001c RBX: ffffc90003ce1440 RCX: 0000000000000006
[ 5077.895528] RDX: 0000000000000007 RSI: 0000000000000096 RDI: ffff88087ca15530
[ 5077.902991] RBP: ffffc90003ce1440 R08: 000000000000001c R09: 0000000000000411
[ 5077.910453] R10: 000fffffffe00000 R11: ffffffff82a4e66d R12: ffffffffffffcbc0
[ 5077.917913] R13: ffff88087c50f000 R14: 0000000000000008 R15: ffffffffa0199620
[ 5077.925376] FS: 00007f7a84bcb740(0000) GS:ffff88087ca00000(0000) knlGS:0000000000000000
[ 5077.934061] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 5077.940133] CR2: 000055c7d709b000 CR3: 000000081acd0004 CR4: 00000000003606f0
[ 5077.947609] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 5077.955070] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 5077.962531] Call Trace:
[ 5077.965309] dev_ethtool+0xf4e/0x2430
[ 5077.969305] ? get_page_from_freelist+0x2bb/0x1240
[ 5077.974428] ? dev_ioctl+0x1e9/0x3c0
[ 5077.978332] dev_ioctl+0x1e9/0x3c0
[ 5077.982062] sock_do_ioctl+0xa8/0x140
[ 5077.986057] ? sock_ioctl+0x1c0/0x300
[ 5077.990051] sock_ioctl+0x1c0/0x300
[ 5077.993864] ? __handle_mm_fault+0xa82/0xfd0
[ 5077.998462] ? do_vfs_ioctl+0x8d/0x5e0
[ 5078.002550] do_vfs_ioctl+0x8d/0x5e0
[ 5078.006456] ? handle_mm_fault+0xd0/0x210
[ 5078.010790] ksys_ioctl+0x70/0x80
[ 5078.014429] __x64_sys_ioctl+0x16/0x20
[ 5078.018505] do_syscall_64+0x42/0xf0
[ 5078.022411] entry_SYSCALL_64_after_hwframe+0x44/0xa9
[ 5078.027789] RIP: 0033:0x7f7a84394dc7
[ 5078.031703] Code: b3 66 90 48 8b 05 d9 00 2d 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 2e 0f 1f 84 00 00 00 00 00 b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d a9 00 2d 00 f7 d8 64 89 01 48
[ 5078.051437] RSP: 002b:00007ffd64d68338 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
[ 5078.059596] RAX: ffffffffffffffda RBX: 000055c7d7099260 RCX: 00007f7a84394dc7
[ 5078.067062] RDX: 00007ffd64d684e0 RSI: 0000000000008946 RDI: 0000000000000003
[ 5078.074524] RBP: 00007ffd64d684d0 R08: ffffffffffffffb0 R09: 000055c7d7099260
[ 5078.081986] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000000001a2
[ 5078.089447] R13: 0000000000000001 R14: 0000000000000000 R15: 00007ffd64d684e0
[ 5078.096911] ---[ end trace faf82a00c0d8b6b1 ]---
$ gdb ./drivers/net/ethernet/intel/i40e/i40e.ko
[...]
Reading symbols from ./drivers/net/ethernet/intel/i40e/i40e.ko...done.
(gdb) list *(i40e_get_strings)+0x477
0x17787 is in i40e_get_strings (drivers/net/ethernet/intel/i40e/i40e_ethtool.c:1906).
1901 snprintf(data, ETH_GSTRING_LEN,
1902 "port.rx_priority_%u_xon_2_xoff", i);
1903 data += ETH_GSTRING_LEN;
1904 }
1905
1906 WARN_ONCE(p - data != i40e_get_stats_count(netdev) * ETH_GSTRING_LEN,
1907 "stat strings count mismatch!");
1908 }
1909
1910 static void i40e_get_priv_flag_strings(struct net_device *netdev, u8 *data)
--
Best regards,
Jesper Dangaard Brouer
MSc.CS, Principal Kernel Engineer at Red Hat
LinkedIn: http://www.linkedin.com/in/brouer
^ permalink raw reply
* Re: [V9fs-developer] [PATCH] Integer underflow in pdu_read()
From: piaojun @ 2018-07-10 11:16 UTC (permalink / raw)
To: Tomas Bortoli, ericvh, rminnich, lucho
Cc: netdev, linux-kernel, syzkaller, v9fs-developer, davem
In-Reply-To: <20180709192651.28095-1-tomasbortoli@gmail.com>
Hi Tomas,
I found some other subtraction between pdu->size and pdu->offset such as:
p9pdu_vreadf()
--min_t(uint32_t, *count, pdu->size - pdu->offset);
p9_check_zc_errors()
--len = req->rc->size - req->rc->offset;
I wonder if there are underflow risks?
Thanks,
Jun
On 2018/7/10 3:26, Tomas Bortoli wrote:
> The pdu_read() function suffers from an integer underflow.
> When pdu->offset is greater than pdu->size, the length calculation will have
> a wrong result, resulting in an out-of-bound read.
> This patch modifies also pdu_write() in the same way to prevent the same
> issue from happening there and for consistency.
>
> Signed-off-by: Tomas Bortoli <tomasbortoli@gmail.com>
> Reported-by: syzbot+65c6b72f284a39d416b4@syzkaller.appspotmail.com
> ---
> net/9p/protocol.c | 12 ++++++++----
> 1 file changed, 8 insertions(+), 4 deletions(-)
>
> diff --git a/net/9p/protocol.c b/net/9p/protocol.c
> index 931ea00c4fed..f1e2425f920b 100644
> --- a/net/9p/protocol.c
> +++ b/net/9p/protocol.c
> @@ -55,16 +55,20 @@ EXPORT_SYMBOL(p9stat_free);
>
> size_t pdu_read(struct p9_fcall *pdu, void *data, size_t size)
> {
> - size_t len = min(pdu->size - pdu->offset, size);
> - memcpy(data, &pdu->sdata[pdu->offset], len);
> + size_t len = pdu->offset > pdu->size ? 0 :
> + min(pdu->size - pdu->offset, size);
> + if (len != 0)
> + memcpy(data, &pdu->sdata[pdu->offset], len);
> pdu->offset += len;
> return size - len;
> }
>
> static size_t pdu_write(struct p9_fcall *pdu, const void *data, size_t size)
> {
> - size_t len = min(pdu->capacity - pdu->size, size);
> - memcpy(&pdu->sdata[pdu->size], data, len);
> + size_t len = pdu->size > pdu->capacity ? 0 :
> + min(pdu->capacity - pdu->size, size);
> + if (len != 0)
> + memcpy(&pdu->sdata[pdu->size], data, len);
> pdu->size += len;
> return size - len;
> }
>
^ permalink raw reply
* Re: [PATCH] netfilter: NFT_SOCKET don't use NF_SOCKET_IPV6 without NF_TABLES_IPV6
From: Máté Eckl @ 2018-07-10 11:15 UTC (permalink / raw)
To: Pablo Neira Ayuso
Cc: Arnd Bergmann, Jozsef Kadlecsik, Florian Westphal,
David S. Miller, Flavio Leitner, netfilter-devel, coreteam,
Networking, Linux Kernel Mailing List
In-Reply-To: <20180710105605.evgtmjiq5z46zld6@salvia>
On Tue, Jul 10, 2018 at 12:56:05PM +0200, Pablo Neira Ayuso wrote:
> On Tue, Jul 10, 2018 at 11:10:40AM +0200, Arnd Bergmann wrote:
> > On Tue, Jul 10, 2018 at 10:05 AM, Máté Eckl <ecklm94@gmail.com> wrote:
> > > On Tue, Jul 10, 2018 at 10:02:27AM +0200, Máté Eckl wrote:
> > >> On Mon, Jul 09, 2018 at 11:35:09PM +0200, Arnd Bergmann wrote:
> > >> > It is now possible to build the nft_socket module as built-in when
> > >> > NF_TABLES_IPV6 is disabled, and have NF_SOCKET_IPV6=m set manually.
> > >> >
> > >> > In this case, the NF_SOCKET_IPV6 functionality will be useless according
> > >> > to the explanation in commit 35bf1ccecaaa ("netfilter: Kconfig: Change
> > >> > IPv6 select dependencies"), but on top of that it also causes a link
> > >> > error:
> > >> >
> > >> > net/netfilter/nft_socket.o: In function `nft_socket_eval':
> > >> > nft_socket.c:(.text+0x162): undefined reference to `nf_sk_lookup_slow_v6'
> > >> >
> > >> > This changes the compile-time check so we don't attempt to use
> > >> > the NF_SOCKET_IPV6 code when it cannot be used, and make it all
> > >> > compile again. That may lead to unexpected behavior when a user
> > >> > enables NF_SOCKET_IPV6 but cannot use it, but seems to be the
> > >> > logical conclusion of the 35bf1ccecaaa change.
> > >> >
> > >> > Fixes: 35bf1ccecaaa ("netfilter: Kconfig: Change IPv6 select dependencies")
> > >> > Signed-off-by: Arnd Bergmann <arnd@arndb.de>
> > >>
> > >> I think this should be fixed in the Kconfig rather than inside the module(s).
> >
> > Should we revert your patch then, or do you have a better idea?
>
> Máté, would you resubmit a new patch that addresses all the problems
> that Arnd is reporting in one go?
This patch only solves the nf_socket and nft_socket modules problem so I can
only submit a v2 for 'netfilter: Kconfig: Change IPv6 select dependencies' but
you already applied it so it would meen a force push. Should I do this?
I think Arnd's patch solves these problems in case we don't want to force-push
or rebase.
> I think it's better if we toss your original patch in the tree and
> rebase, ie. take the new one that fixes all issues that Arnd is
> reporting. It would be good if we can sort out this before I send the
> next pull request for net-next stuff.
>
> I was afraid of fallout like this when I saw your original patch,
> kbuild is always tricky.
This patch is not related to the nft_tproxy module (it seems that you refer to
that) as Arnd didn't have that in the tree when doing this. I'll send a v4 fot
the tproxy module, but that cannot be related to this one as it is not in tree
yet.
> Please Cc Arnd, Florian and me for review.
>
> Thanks!
^ permalink raw reply
* Re: [V9fs-developer] [PATCH] Integer underflow in pdu_read()
From: piaojun @ 2018-07-10 11:06 UTC (permalink / raw)
To: Tomas Bortoli, ericvh, rminnich, lucho
Cc: netdev, linux-kernel, syzkaller, v9fs-developer, davem
In-Reply-To: <36523cc7-adec-9e61-d34c-dc00806c403a@gmail.com>
Hi Tomas,
Thanks for your explaination, and I get your point.
On 2018/7/10 16:27, Tomas Bortoli wrote:
> Hi Jun,
>
> Intuitively, if you have a packet of size x and you read at an offset y,
> when y>x you are off the packet. That's an out out bound read.
>
> In this specific code when offset > size, the available length
> estimation will fail as there will be an underflow resulting from
> offset-size (it'll give a big big number) that breaks the out-of-bound
> control put in place (if offset-size is a big big number, the asked size
> to read will be probably smaller and therefore allowed).
>
> These definitions might help:
> https://cwe.mitre.org/data/definitions/787.html
> https://cwe.mitre.org/data/definitions/125.html
>
> Tomas
>> Hi Tomas,
>>
>> It looks like pdu->size should always be greater than pdu->offset, right?
>> My question may be very easy for you, please help explaining.
>>
>> Thanks,
>> Jun
>>
>> On 2018/7/10 3:26, Tomas Bortoli wrote:
>>> The pdu_read() function suffers from an integer underflow.
>>> When pdu->offset is greater than pdu->size, the length calculation will have
>>> a wrong result, resulting in an out-of-bound read.
>>> This patch modifies also pdu_write() in the same way to prevent the same
>>> issue from happening there and for consistency.
>>>
>>> Signed-off-by: Tomas Bortoli <tomasbortoli@gmail.com>
>>> Reported-by: syzbot+65c6b72f284a39d416b4@syzkaller.appspotmail.com
>>> ---
>>> net/9p/protocol.c | 12 ++++++++----
>>> 1 file changed, 8 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/net/9p/protocol.c b/net/9p/protocol.c
>>> index 931ea00c4fed..f1e2425f920b 100644
>>> --- a/net/9p/protocol.c
>>> +++ b/net/9p/protocol.c
>>> @@ -55,16 +55,20 @@ EXPORT_SYMBOL(p9stat_free);
>>>
>>> size_t pdu_read(struct p9_fcall *pdu, void *data, size_t size)
>>> {
>>> - size_t len = min(pdu->size - pdu->offset, size);
>>> - memcpy(data, &pdu->sdata[pdu->offset], len);
>>> + size_t len = pdu->offset > pdu->size ? 0 :
>>> + min(pdu->size - pdu->offset, size);
>>> + if (len != 0)
>>> + memcpy(data, &pdu->sdata[pdu->offset], len);
>>> pdu->offset += len;
>>> return size - len;
>>> }
>>>
>>> static size_t pdu_write(struct p9_fcall *pdu, const void *data, size_t size)
>>> {
>>> - size_t len = min(pdu->capacity - pdu->size, size);
>>> - memcpy(&pdu->sdata[pdu->size], data, len);
>>> + size_t len = pdu->size > pdu->capacity ? 0 :
>>> + min(pdu->capacity - pdu->size, size);
>>> + if (len != 0)
>>> + memcpy(&pdu->sdata[pdu->size], data, len);
>>> pdu->size += len;
>>> return size - len;
>>> }
>>>
>
>
>
^ permalink raw reply
* [PATCH net 2/2] sfp: fix module initialisation with netdev already up
From: Russell King @ 2018-07-10 11:05 UTC (permalink / raw)
To: Andrew Lunn, Florian Fainelli; +Cc: David S. Miller, netdev
It was been observed that with a particular order of initialisation,
the netdev can be up, but the SFP module still has its TX_DISABLE
signal asserted. This occurs when the network device brought up before
the SFP kernel module has been inserted by userspace.
This occurs because sfp-bus layer does not hear about the change in
network device state, and so assumes that it is still down. Set
netdev->sfp when the upstream is registered to work around this problem.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
drivers/net/phy/sfp-bus.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
index e355e7db54a7..5e5fcc33421e 100644
--- a/drivers/net/phy/sfp-bus.c
+++ b/drivers/net/phy/sfp-bus.c
@@ -342,7 +342,6 @@ static int sfp_register_bus(struct sfp_bus *bus)
}
if (bus->started)
bus->socket_ops->start(bus->sfp);
- bus->netdev->sfp_bus = bus;
bus->registered = true;
return 0;
}
@@ -357,7 +356,6 @@ static void sfp_unregister_bus(struct sfp_bus *bus)
if (bus->phydev && ops && ops->disconnect_phy)
ops->disconnect_phy(bus->upstream);
}
- bus->netdev->sfp_bus = NULL;
bus->registered = false;
}
@@ -433,6 +431,7 @@ static void sfp_upstream_clear(struct sfp_bus *bus)
{
bus->upstream_ops = NULL;
bus->upstream = NULL;
+ bus->netdev->sfp_bus = NULL;
bus->netdev = NULL;
}
@@ -461,6 +460,7 @@ struct sfp_bus *sfp_register_upstream(struct fwnode_handle *fwnode,
bus->upstream_ops = ops;
bus->upstream = upstream;
bus->netdev = ndev;
+ ndev->sfp_bus = bus;
if (bus->sfp) {
ret = sfp_register_bus(bus);
--
2.7.4
^ permalink raw reply related
* [PATCH net 1/2] sfp: ensure we clean up properly on bus registration failure
From: Russell King @ 2018-07-10 11:05 UTC (permalink / raw)
To: Andrew Lunn, Florian Fainelli; +Cc: David S. Miller, netdev
We fail to correctly clean up after a bus registration failure, which
can lead to an incorrect assumption about the registration state of
the upstream or sfp cage.
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
---
drivers/net/phy/sfp-bus.c | 31 ++++++++++++++++++++++++-------
1 file changed, 24 insertions(+), 7 deletions(-)
diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
index fd6c23f69c2f..e355e7db54a7 100644
--- a/drivers/net/phy/sfp-bus.c
+++ b/drivers/net/phy/sfp-bus.c
@@ -429,6 +429,13 @@ void sfp_upstream_stop(struct sfp_bus *bus)
}
EXPORT_SYMBOL_GPL(sfp_upstream_stop);
+static void sfp_upstream_clear(struct sfp_bus *bus)
+{
+ bus->upstream_ops = NULL;
+ bus->upstream = NULL;
+ bus->netdev = NULL;
+}
+
/**
* sfp_register_upstream() - Register the neighbouring device
* @fwnode: firmware node for the SFP bus
@@ -455,8 +462,11 @@ struct sfp_bus *sfp_register_upstream(struct fwnode_handle *fwnode,
bus->upstream = upstream;
bus->netdev = ndev;
- if (bus->sfp)
+ if (bus->sfp) {
ret = sfp_register_bus(bus);
+ if (ret)
+ sfp_upstream_clear(bus);
+ }
rtnl_unlock();
}
@@ -481,8 +491,7 @@ void sfp_unregister_upstream(struct sfp_bus *bus)
rtnl_lock();
if (bus->sfp)
sfp_unregister_bus(bus);
- bus->upstream = NULL;
- bus->netdev = NULL;
+ sfp_upstream_clear(bus);
rtnl_unlock();
sfp_bus_put(bus);
@@ -554,6 +563,13 @@ void sfp_module_remove(struct sfp_bus *bus)
}
EXPORT_SYMBOL_GPL(sfp_module_remove);
+static void sfp_socket_clear(struct sfp_bus *bus)
+{
+ bus->sfp_dev = NULL;
+ bus->sfp = NULL;
+ bus->socket_ops = NULL;
+}
+
struct sfp_bus *sfp_register_socket(struct device *dev, struct sfp *sfp,
const struct sfp_socket_ops *ops)
{
@@ -566,8 +582,11 @@ struct sfp_bus *sfp_register_socket(struct device *dev, struct sfp *sfp,
bus->sfp = sfp;
bus->socket_ops = ops;
- if (bus->netdev)
+ if (bus->netdev) {
ret = sfp_register_bus(bus);
+ if (ret)
+ sfp_socket_clear(bus);
+ }
rtnl_unlock();
}
@@ -585,9 +604,7 @@ void sfp_unregister_socket(struct sfp_bus *bus)
rtnl_lock();
if (bus->netdev)
sfp_unregister_bus(bus);
- bus->sfp_dev = NULL;
- bus->sfp = NULL;
- bus->socket_ops = NULL;
+ sfp_socket_clear(bus);
rtnl_unlock();
sfp_bus_put(bus);
--
2.7.4
^ permalink raw reply related
* 答复: [PATCH][net-next][v2] net: limit each hash list length to MAX_GRO_SKBS
From: Li,Rongqing @ 2018-07-10 10:59 UTC (permalink / raw)
To: Eric Dumazet, David Miller; +Cc: netdev@vger.kernel.org
In-Reply-To: <4df439f1-d80c-a2ff-d1ac-d5475694a59e@gmail.com>
> -----邮件原件-----
> 发件人: Eric Dumazet [mailto:eric.dumazet@gmail.com]
> 发送时间: 2018年7月8日 8:22
> 收件人: David Miller <davem@davemloft.net>; Li,Rongqing
> <lirongqing@baidu.com>
> 抄送: netdev@vger.kernel.org
> 主题: Re: [PATCH][net-next][v2] net: limit each hash list length to
> MAX_GRO_SKBS
>
>
>
> On 07/05/2018 03:20 AM, David Miller wrote:
> > From: Li RongQing <lirongqing@baidu.com>
> > Date: Thu, 5 Jul 2018 14:34:32 +0800
> >
> >> After commit 07d78363dcff ("net: Convert NAPI gro list into a small
> >> hash table.")' there is 8 hash buckets, which allows more flows to be
> >> held for merging. but MAX_GRO_SKBS, the total held skb for merging,
> >> is 8 skb still, limit the hash table performance.
> >>
> >> keep MAX_GRO_SKBS as 8 skb, but limit each hash list length to 8 skb,
> >> not the total 8 skb
> >>
> >> Signed-off-by: Li RongQing <lirongqing@baidu.com>
> >
> > Applied, thanks.
> >
>
> Maybe gro_count should be replaced by a bitmask, so that we can speed up
> napi_gro_flush(), since it now has to use 3 cache lines (gro_hash[] size is 192
> bytes)
Do you means that?
Subject: [PATCH][RFC][net-next] net: convert gro_count to bitmask
convert gro_count to a bitmask, and rename it as gro_bitmask to speed
up napi_gro_flush(), since gro_hash now has to use 3 cache lines
---
include/linux/netdevice.h | 2 +-
net/core/dev.c | 36 ++++++++++++++++++++++++------------
2 files changed, 25 insertions(+), 13 deletions(-)
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index b683971e500d..df49b36ef378 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -322,7 +322,7 @@ struct napi_struct {
unsigned long state;
int weight;
- unsigned int gro_count;
+ unsigned long gro_bitmask;
int (*poll)(struct napi_struct *, int);
#ifdef CONFIG_NETPOLL
int poll_owner;
diff --git a/net/core/dev.c b/net/core/dev.c
index 89825c1eccdc..da2d1185eb82 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5161,9 +5161,11 @@ static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
return;
list_del_init(&skb->list);
napi_gro_complete(skb);
- napi->gro_count--;
napi->gro_hash[index].count--;
}
+
+ if (!napi->gro_hash[index].count)
+ clear_bit(index, &napi->gro_bitmask);
}
/* napi->gro_hash[].list contains packets ordered by age.
@@ -5174,8 +5176,10 @@ void napi_gro_flush(struct napi_struct *napi, bool flush_old)
{
u32 i;
- for (i = 0; i < GRO_HASH_BUCKETS; i++)
- __napi_gro_flush_chain(napi, i, flush_old);
+ for (i = 0; i < GRO_HASH_BUCKETS; i++) {
+ if (test_bit(i, &napi->gro_bitmask))
+ __napi_gro_flush_chain(napi, i, flush_old);
+ }
}
EXPORT_SYMBOL(napi_gro_flush);
@@ -5267,8 +5271,8 @@ static void gro_flush_oldest(struct list_head *head)
if (WARN_ON_ONCE(!oldest))
return;
- /* Do not adjust napi->gro_count, caller is adding a new SKB to
- * the chain.
+ /* Do not adjust napi->gro_hash[].count, caller is adding a new
+ * SKB to the chain.
*/
list_del(&oldest->list);
napi_gro_complete(oldest);
@@ -5342,7 +5346,6 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
if (pp) {
list_del_init(&pp->list);
napi_gro_complete(pp);
- napi->gro_count--;
napi->gro_hash[hash].count--;
}
@@ -5355,7 +5358,6 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
gro_flush_oldest(gro_head);
} else {
- napi->gro_count++;
napi->gro_hash[hash].count++;
}
NAPI_GRO_CB(skb)->count = 1;
@@ -5370,6 +5372,13 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
if (grow > 0)
gro_pull_from_frag0(skb, grow);
ok:
+
+ if (napi->gro_hash[hash].count)
+ if (!test_bit(hash, &napi->gro_bitmask))
+ set_bit(hash, &napi->gro_bitmask);
+ else if (test_bit(hash, &napi->gro_bitmask))
+ clear_bit(hash, &napi->gro_bitmask);
+
return ret;
normal:
@@ -5768,7 +5777,7 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
NAPIF_STATE_IN_BUSY_POLL)))
return false;
- if (n->gro_count) {
+ if (n->gro_bitmask) {
unsigned long timeout = 0;
if (work_done)
@@ -5977,7 +5986,7 @@ static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
/* Note : we use a relaxed variant of napi_schedule_prep() not setting
* NAPI_STATE_MISSED, since we do not react to a device IRQ.
*/
- if (napi->gro_count && !napi_disable_pending(napi) &&
+ if (napi->gro_bitmask && !napi_disable_pending(napi) &&
!test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
__napi_schedule_irqoff(napi);
@@ -5992,7 +6001,7 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
INIT_LIST_HEAD(&napi->poll_list);
hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
napi->timer.function = napi_watchdog;
- napi->gro_count = 0;
+ napi->gro_bitmask = 0;
for (i = 0; i < GRO_HASH_BUCKETS; i++) {
INIT_LIST_HEAD(&napi->gro_hash[i].list);
napi->gro_hash[i].count = 0;
@@ -6052,7 +6061,7 @@ void netif_napi_del(struct napi_struct *napi)
napi_free_frags(napi);
flush_gro_hash(napi);
- napi->gro_count = 0;
+ napi->gro_bitmask = 0;
}
EXPORT_SYMBOL(netif_napi_del);
@@ -6094,7 +6103,7 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll)
goto out_unlock;
}
- if (n->gro_count) {
+ if (n->gro_bitmask) {
/* flush too old packets
* If HZ < 1000, flush all packets.
*/
@@ -9141,6 +9150,9 @@ static struct hlist_head * __net_init netdev_create_hash(void)
/* Initialize per network namespace state */
static int __net_init netdev_init(struct net *net)
{
+ BUILD_BUG_ON(GRO_HASH_BUCKETS >
+ FIELD_SIZEOF(struct napi_struct, gro_bitmask));
+
if (net != &init_net)
INIT_LIST_HEAD(&net->dev_base_head);
--
2.16.2
^ permalink raw reply related
* Re: [PATCH] ib/mlx4: adjust gfp flags for DMA allocations
From: Leon Romanovsky @ 2018-07-10 10:56 UTC (permalink / raw)
To: Jan Dakinevich
Cc: Yishai Hadas, Doug Ledford, Jason Gunthorpe, Tariq Toukan,
David S. Miller, linux-rdma, linux-kernel, netdev, Denis Lunev,
Konstantin Khorenko
In-Reply-To: <1531144273-25959-1-git-send-email-jan.dakinevich@virtuozzo.com>
[-- Attachment #1: Type: text/plain, Size: 1049 bytes --]
On Mon, Jul 09, 2018 at 04:51:13PM +0300, Jan Dakinevich wrote:
> For most cases it is enough to use common GFP_KERNEL flag. However,
> there is one place with defined fallback path on unsuccessful allocation.
> In this case it would be preferred to append on first allocation attempt
> __GFP_NOWARN and __GFP_NORETRY flags to suppress possible warnings and
> to shorten the path to the fallback in a case of memory pressure.
>
> Signed-off-by: Jan Dakinevich <jan.dakinevich@virtuozzo.com>
> ---
> drivers/infiniband/hw/mlx4/cq.c | 2 +-
> drivers/infiniband/hw/mlx4/qp.c | 6 ++++--
> drivers/infiniband/hw/mlx4/srq.c | 2 +-
> drivers/net/ethernet/mellanox/mlx4/alloc.c | 12 ++++++------
> include/linux/mlx4/device.h | 2 +-
> 5 files changed, 13 insertions(+), 11 deletions(-)
>
Thanks for the patch, however we are working on the patch to avoid
second call to mlx4_buf_alloc() and it will eliminate the annoying
WARN splat.
Because it is in latest stages of internal review, NAK.
Thanks
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 801 bytes --]
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox