[PATCH bpf-next 04/11] bpf: Implement hidden BPF_PUSH64 and BPF_POP64 instructions

[Stanislav Fomichev <sdf@google.com>]

Implemented for:
- x86_64 jit (tested)
- arm64 jit (untested)

Interpreter is not implemented because push/pop are currently
used only with xdp kfunc and jit is required to use kfuncs.

Fundamentally:
  BPF_ST | BPF_STACK + src_reg == store into the stack
  BPF_LD | BPF_STACK + dst_reg == load from the stack
  off/imm are unused

Updated disasm code to properly dump these new instructions:

  31: (e2) push r1
  32: (79) r5 = *(u64 *)(r1 +56)
  33: (55) if r5 != 0x0 goto pc+2
  34: (b7) r0 = 0
  35: (05) goto pc+1
  36: (79) r0 = *(u64 *)(r5 +32)
  37: (e0) pop r1

Cc: Zi Shen Lim <zlim.lnx@gmail.com>
Suggested-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
---
 arch/arm64/net/bpf_jit_comp.c |  8 ++++++++
 arch/x86/net/bpf_jit_comp.c   |  8 ++++++++
 include/linux/filter.h        | 23 +++++++++++++++++++++++
 kernel/bpf/disasm.c           |  6 ++++++
 4 files changed, 45 insertions(+)

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 62f805f427b7..4c0e70e6572a 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -1185,6 +1185,14 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 		 */
 		break;
 
+		/* kernel hidden stack operations */
+	case BPF_ST | BPF_STACK:
+		emit(A64_PUSH(src, src, A64_SP), ctx);
+		break;
+	case BPF_LD | BPF_STACK:
+		emit(A64_POP(dst, dst, A64_SP), ctx);
+		break;
+
 	/* ST: *(size *)(dst + off) = imm */
 	case BPF_ST | BPF_MEM | BPF_W:
 	case BPF_ST | BPF_MEM | BPF_H:
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index cec5195602bc..528bece87ca4 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1324,6 +1324,14 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
 			EMIT_LFENCE();
 			break;
 
+			/* kernel hidden stack operations */
+		case BPF_ST | BPF_STACK:
+			EMIT1(add_1reg(0x50, src_reg)); /* pushq  */
+			break;
+		case BPF_LD | BPF_STACK:
+			EMIT1(add_1reg(0x58, dst_reg)); /* popq */
+			break;
+
 			/* ST: *(u8*)(dst_reg + off) = imm */
 		case BPF_ST | BPF_MEM | BPF_B:
 			if (is_ereg(dst_reg))
diff --git a/include/linux/filter.h b/include/linux/filter.h
index efc42a6e3aed..42c61ec8f895 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -76,6 +76,9 @@ struct ctl_table_header;
  */
 #define BPF_NOSPEC	0xc0
 
+/* unused opcode for kernel hidden stack operations */
+#define BPF_STACK	0xe0
+
 /* As per nm, we expose JITed images as text (code) section for
  * kallsyms. That way, tools like perf can find it to match
  * addresses.
@@ -402,6 +405,26 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
 		.off   = 0,					\
 		.imm   = 0 })
 
+/* Push SRC register value onto the stack */
+
+#define BPF_PUSH64(SRC)						\
+	((struct bpf_insn) {					\
+		.code  = BPF_ST | BPF_STACK,			\
+		.dst_reg = 0,					\
+		.src_reg = SRC,					\
+		.off   = 0,					\
+		.imm   = 0 })
+
+/* Pop stack value into DST register */
+
+#define BPF_POP64(DST)						\
+	((struct bpf_insn) {					\
+		.code  = BPF_LD | BPF_STACK,			\
+		.dst_reg = DST,					\
+		.src_reg = 0,					\
+		.off   = 0,					\
+		.imm   = 0 })
+
 /* Internal classic blocks for direct assignment */
 
 #define __BPF_STMT(CODE, K)					\
diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c
index 7b4afb7d96db..9cd22f3591de 100644
--- a/kernel/bpf/disasm.c
+++ b/kernel/bpf/disasm.c
@@ -214,6 +214,9 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
 				insn->off, insn->imm);
 		} else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) {
 			verbose(cbs->private_data, "(%02x) nospec\n", insn->code);
+		} else if (BPF_MODE(insn->code) == 0xe0 /* BPF_STACK, no UAPI */) {
+			verbose(cbs->private_data, "(%02x) push r%d\n",
+				insn->code, insn->src_reg);
 		} else {
 			verbose(cbs->private_data, "BUG_st_%02x\n", insn->code);
 		}
@@ -254,6 +257,9 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
 				insn->code, insn->dst_reg,
 				__func_imm_name(cbs, insn, imm,
 						tmp, sizeof(tmp)));
+		} else if (BPF_MODE(insn->code) == 0xe0 /* BPF_STACK, no UAPI */) {
+			verbose(cbs->private_data, "(%02x) pop r%d\n",
+				insn->code, insn->dst_reg);
 		} else {
 			verbose(cbs->private_data, "BUG_ld_%02x\n", insn->code);
 			return;
-- 
2.38.1.431.g37b22c650d-goog