[PATCH bpf-next v6 3/9] bpf: Support private stack for struct ops programs

[Yonghong Song <yonghong.song@linux.dev>]

To identify whether a st_ops program requests private stack or not,
the st_ops stub function is checked. If the stub function has the
following name
   <st_ops_name>__<member_name>__priv_stack
then the corresponding st_ops member func requests to use private
stack. The information that the private stack is requested or not
is encoded in struct bpf_struct_ops_func_info which will later be
used by verifier.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 include/linux/bpf.h         |  2 ++
 kernel/bpf/bpf_struct_ops.c | 35 +++++++++++++++++++++++++----------
 kernel/bpf/verifier.c       |  8 +++++++-
 3 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index f3884ce2603d..376e43fc72b9 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1491,6 +1491,7 @@ struct bpf_prog_aux {
 	bool exception_boundary;
 	bool is_extended; /* true if extended by freplace program */
 	bool priv_stack_eligible;
+	bool priv_stack_always;
 	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
 	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
 	struct bpf_arena *arena;
@@ -1776,6 +1777,7 @@ struct bpf_struct_ops {
 struct bpf_struct_ops_func_info {
 	struct bpf_ctx_arg_aux *info;
 	u32 cnt;
+	bool priv_stack_always;
 };
 
 struct bpf_struct_ops_desc {
diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c
index 8279b5a57798..2cd4bd086c7a 100644
--- a/kernel/bpf/bpf_struct_ops.c
+++ b/kernel/bpf/bpf_struct_ops.c
@@ -145,33 +145,44 @@ void bpf_struct_ops_image_free(void *image)
 }
 
 #define MAYBE_NULL_SUFFIX "__nullable"
-#define MAX_STUB_NAME 128
+#define MAX_STUB_NAME 140
 
 /* Return the type info of a stub function, if it exists.
  *
- * The name of a stub function is made up of the name of the struct_ops and
- * the name of the function pointer member, separated by "__". For example,
- * if the struct_ops type is named "foo_ops" and the function pointer
- * member is named "bar", the stub function name would be "foo_ops__bar".
+ * The name of a stub function is made up of the name of the struct_ops,
+ * the name of the function pointer member and optionally "priv_stack"
+ * suffix, separated by "__". For example, if the struct_ops type is named
+ * "foo_ops" and the function pointer  member is named "bar", the stub
+ * function name would be "foo_ops__bar". If a suffix "priv_stack" exists,
+ * the stub function name would be "foo_ops__bar__priv_stack".
  */
 static const struct btf_type *
 find_stub_func_proto(const struct btf *btf, const char *st_op_name,
-		     const char *member_name)
+		     const char *member_name, bool *priv_stack_always)
 {
 	char stub_func_name[MAX_STUB_NAME];
 	const struct btf_type *func_type;
 	s32 btf_id;
 	int cp;
 
-	cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s",
+	cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s__priv_stack",
 		      st_op_name, member_name);
 	if (cp >= MAX_STUB_NAME) {
 		pr_warn("Stub function name too long\n");
 		return NULL;
 	}
+
 	btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC);
-	if (btf_id < 0)
-		return NULL;
+	if (btf_id >= 0) {
+		*priv_stack_always = true;
+	} else {
+		cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s",
+			      st_op_name, member_name);
+		btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC);
+		if (btf_id < 0)
+			return NULL;
+	}
+
 	func_type = btf_type_by_id(btf, btf_id);
 	if (!func_type)
 		return NULL;
@@ -209,10 +220,12 @@ static int prepare_func_info(struct btf *btf,
 	const struct btf_param *stub_args, *args;
 	struct bpf_ctx_arg_aux *info, *info_buf;
 	u32 nargs, arg_no, info_cnt = 0;
+	bool priv_stack_always = false;
 	u32 arg_btf_id;
 	int offset;
 
-	stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name);
+	stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name,
+					       &priv_stack_always);
 	if (!stub_func_proto)
 		return 0;
 
@@ -226,6 +239,8 @@ static int prepare_func_info(struct btf *btf,
 		return -EINVAL;
 	}
 
+	func_info->priv_stack_always = priv_stack_always;
+
 	if (!nargs)
 		return 0;
 
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index ccfe159cfbde..25283ee6f86f 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5995,6 +5995,8 @@ static bool bpf_enable_private_stack(struct bpf_verifier_env *env)
 	case BPF_PROG_TYPE_PERF_EVENT:
 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
 		return true;
+	case BPF_PROG_TYPE_STRUCT_OPS:
+		return env->prog->aux->priv_stack_always;
 	case BPF_PROG_TYPE_TRACING:
 		if (env->prog->expected_attach_type != BPF_TRACE_ITER)
 			return true;
@@ -6092,7 +6094,9 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 			return -EACCES;
 		}
 
-		if (!priv_stack_eligible && depth >= BPF_PRIV_STACK_MIN_SUBTREE_SIZE) {
+		if (!priv_stack_eligible &&
+		    (env->prog->aux->priv_stack_always ||
+		     depth >= BPF_PRIV_STACK_MIN_SUBTREE_SIZE)) {
 			subprog[orig_idx].priv_stack_eligible = true;
 			env->prog->aux->priv_stack_eligible = priv_stack_eligible = true;
 		}
@@ -21883,6 +21887,8 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
 		st_ops_desc->func_info[member_idx].info;
 	prog->aux->ctx_arg_info_size =
 		st_ops_desc->func_info[member_idx].cnt;
+	prog->aux->priv_stack_always =
+		st_ops_desc->func_info[member_idx].priv_stack_always;
 
 	prog->aux->attach_func_proto = func_proto;
 	prog->aux->attach_func_name = mname;
-- 
2.43.5