[PATCH bpf-next] s390/bpf: inline smp_processor_id and current_task

[PATCH bpf-next] s390/bpf: inline smp_processor_id and current_task

[Maxim Khmelevskii]

Inline these calls in bpf jit:
 - bpf_get_smp_processor_id()
 - bpf_get_current_task()
 - bpf_get_current_task_btf()

s390 has a 8 KiB per-CPU prefix area in the CPU's
virtual address space, called the lowcore. It is a
struct that contains the cpu number and a pointer
to the current task. These are exactly the values
returned by the BPF helpers.

Emit a load from the lowcore instead of a helper
function call.

JIT output for `bpf_get_smp_processor_id`:

 Before:                       After:
---------------               ----------------
brasl   %r14,0x3ffe0385460    ly      %r14,928
lgr     %r14,%r2

JIT output for `bpf_get_current_task`:

 Before:                        After:
---------------                ----------------
brasl   %r14,0x3ffe0362a90     lg      %r14,832
lgr     %r14,%r2

Benchmark using [1] on KVM(virtme-ng).

./benchs/run_bench_trigger.sh glob-arr-inc arr-inc hash-inc

+---------------+--------------------+--------------------+--------------+
|     Name      |       Before       |       After        |   % change   |
|---------------+--------------------+--------------------+--------------|
| glob-arr-inc  | 244.954 ± 0.654M/s | 278.501 ± 0.834M/s |   + 13.70%   |
| arr-inc       | 311.597 ± 1.016M/s | 313.610 ± 0.331M/s |   + 0.65%    |
| hash-inc      | 47.421 ± 0.017M/s  | 47.600 ± 0.004M/s  |   + 0.38%    |
+---------------+--------------------+--------------------+--------------+

[1] https://github.com/anakryiko/linux/commit/8dec900975ef

Signed-off-by: Maxim Khmelevskii <max@linux.ibm.com>
Reviewed-by: Ilya Leoshkevich <iii@linux.ibm.com>
---
 arch/s390/net/bpf_jit_comp.c | 37 ++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index d08d159b6319..93ab2fb16b99 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -27,6 +27,7 @@
 #include <asm/extable.h>
 #include <asm/dis.h>
 #include <asm/facility.h>
+#include <asm/lowcore.h>
 #include <asm/nospec-branch.h>
 #include <asm/set_memory.h>
 #include <asm/text-patching.h>
@@ -1777,6 +1778,30 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
 		int j, ret;
 		u64 func;
 
+		/* Implement helper call to bpf_get_smp_processor_id() inline */
+		if (insn->src_reg == 0 &&
+		    insn->imm == BPF_FUNC_get_smp_processor_id) {
+			const u32 *cpu_nr = &get_lowcore()->cpu_nr;
+
+			/* ly %b0, cpu_nr */
+			EMIT6_DISP_LH(0xe3000000, 0x0058, BPF_REG_0, REG_0, REG_0,
+				      (unsigned long)cpu_nr);
+			break;
+		}
+
+		/* Implement helper call to bpf_get_current_task/_btf() inline */
+		if (insn->src_reg == 0 &&
+		    (insn->imm == BPF_FUNC_get_current_task ||
+		     insn->imm == BPF_FUNC_get_current_task_btf)) {
+			const u64 *current_task =
+				&get_lowcore()->current_task;
+
+			/* lg %b0, current_task */
+			EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_0, REG_0, REG_0,
+				      (unsigned long)current_task);
+			break;
+		}
+
 		ret = bpf_jit_get_func_addr(fp, insn, extra_pass,
 					    &func, &func_addr_fixed);
 		if (ret < 0)
@@ -3076,3 +3101,15 @@ bool bpf_jit_supports_timed_may_goto(void)
 {
 	return true;
 }
+
+bool bpf_jit_inlines_helper_call(s32 imm)
+{
+	switch (imm) {
+	case BPF_FUNC_get_smp_processor_id:
+	case BPF_FUNC_get_current_task:
+	case BPF_FUNC_get_current_task_btf:
+		return true;
+	default:
+		return false;
+	}
+}
-- 
2.53.0