[PATCH bpf-next v10 18/24] libbpf: Collect .bpf_cleanup records and pass them to the kernel

[Yonghong Song <yonghong.song@linux.dev>]

Parse the compiler-emitted .bpf_cleanup section and hand the table to
BPF_PROG_LOAD:

 - At open, each record field, a byte offset into a code section, is
   resolved through its .rel.bpf_cleanup relocation: R_BPF_64_NODYLD32
   from LLVM or R_BPF_64_ABS32 from GNU as.
 - At relocation, once a main program's subprogs are appended, the
   records in the main program and in those subprogs get final
   instruction indices and are sorted by begin_off, as the kernel wants.
   A record in code no function claims belongs to a weak function the
   static linker kept after overriding it, and is skipped, as that
   code's relocations are.
 - At load, the table goes through the bpf_prog_load() options, from
   bpf_object_load_prog() and from bpf_program__clone(), the path
   veristat uses.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 tools/lib/bpf/libbpf.c          | 285 +++++++++++++++++++++++++++++++-
 tools/lib/bpf/libbpf_internal.h |   3 +
 2 files changed, 286 insertions(+), 2 deletions(-)

diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 5f41d3ba2822..a0e307a4aa5a 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -516,6 +516,11 @@ struct bpf_program {
 	void *line_info;
 	__u32 line_info_rec_size;
 	__u32 line_info_cnt;
+
+	struct bpf_cleanup_info *cleanup_info;
+	__u32 cleanup_info_rec_size;
+	__u32 cleanup_info_cnt;
+
 	__u32 prog_flags;
 	__u8  hash[SHA256_DIGEST_LENGTH];
 
@@ -572,6 +577,7 @@ struct bpf_struct_ops {
  * are loaded with BPF_F_ARENA_SCALAR.
  */
 #define ARENA_DATA_SEC ".arena.data"
+#define CLEANUP_SEC ".bpf_cleanup"
 
 enum libbpf_map_type {
 	LIBBPF_MAP_UNSPEC,
@@ -710,6 +716,25 @@ struct elf_sec_desc {
 	Elf_Data *data;
 };
 
+#define CLEANUP_REC_FIELDS	(sizeof(struct bpf_cleanup_info) / sizeof(__u32))
+
+/* Index of each field of struct bpf_cleanup_info, read as an array of __u32. */
+enum {
+	CLEANUP_REC_BEGIN,
+	CLEANUP_REC_END,
+	CLEANUP_REC_PAD,
+};
+
+/*
+ * One (begin, end, landing_pad) triple from .bpf_cleanup, each field resolved
+ * from its relocation to a section and an instruction index within it. Final
+ * indices wait for subprogram placement, which differs per main program.
+ */
+struct cleanup_raw_rec {
+	int sec_idx[CLEANUP_REC_FIELDS];
+	size_t insn_idx[CLEANUP_REC_FIELDS];
+};
+
 struct elf_state {
 	int fd;
 	const void *obj_buf;
@@ -729,6 +754,8 @@ struct elf_state {
 	bool has_st_ops;
 	int arena_data_shndx;
 	int jumptables_data_shndx;
+	Elf_Data *cleanup_data;
+	int cleanup_shndx;
 };
 
 struct usdt_manager;
@@ -817,6 +844,9 @@ struct bpf_object {
 	void *jumptables_data;
 	size_t jumptables_data_sz;
 
+	struct cleanup_raw_rec *cleanup_recs;
+	size_t cleanup_rec_cnt;
+
 	struct {
 		struct bpf_program *prog;
 		unsigned int sym_off;
@@ -874,6 +904,7 @@ void bpf_program__unload(struct bpf_program *prog)
 	zfree(&prog->func_info);
 	zfree(&prog->line_info);
 	zfree(&prog->subprogs);
+	zfree(&prog->cleanup_info);
 }
 
 static void bpf_program__exit(struct bpf_program *prog)
@@ -1639,6 +1670,7 @@ static struct bpf_object *bpf_object__new(const char *path,
 	obj->efile.obj_buf_sz = obj_buf_sz;
 	obj->efile.btf_maps_shndx = -1;
 	obj->efile.arena_data_shndx = -1;
+	obj->efile.cleanup_shndx = -1;
 	obj->kconfig_map_idx = -1;
 	obj->arena_map_idx = -1;
 
@@ -1658,6 +1690,7 @@ static void bpf_object__elf_finish(struct bpf_object *obj)
 	obj->efile.ehdr = NULL;
 	obj->efile.symbols = NULL;
 	obj->efile.arena_data = NULL;
+	obj->efile.cleanup_data = NULL;
 
 	zfree(&obj->efile.secs);
 	obj->efile.sec_cnt = 0;
@@ -4275,6 +4308,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
 				sec_desc->shdr = sh;
 				sec_desc->data = data;
 				obj->efile.has_st_ops = true;
+			} else if (strcmp(name, CLEANUP_SEC) == 0) {
+				obj->efile.cleanup_data = data;
+				obj->efile.cleanup_shndx = idx;
 			} else if (strcmp(name, ARENA_SEC) == 0) {
 				obj->efile.arena_data = data;
 				obj->efile.arena_data_shndx = idx;
@@ -4300,8 +4336,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
 
 			/*
 			 * Only do relo for section with exec instructions,
-			 * struct_ops, maps, and read-only data that might
-			 * have pointers to functions.
+			 * struct_ops, maps, read-only data that might have
+			 * pointers to functions, and the exception cleanup
+			 * table.
 			 */
 			if (!section_have_execinstr(obj, targ_sec_idx) &&
 			    !(obj->data_in_arena &&
@@ -4315,6 +4352,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
 			    strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) &&
 			    strcmp(name, ".rel?" STRUCT_OPS_SEC) &&
 			    strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) &&
+			    strcmp(name, ".rel" CLEANUP_SEC) &&
 			    strcmp(name, ".rel" MAPS_ELF_SEC)) {
 				pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
 					idx, name, targ_sec_idx,
@@ -5099,6 +5137,217 @@ static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
 	return NULL;
 }
 
+static int bpf_object_init_cleanup_info(struct bpf_object *obj)
+{
+	Elf_Data *data = obj->efile.cleanup_data;
+	size_t i, nrels, nslots, nrecs;
+	struct cleanup_raw_rec *recs;
+	Elf_Data *relo = NULL;
+	const __u32 *vals;
+	int ret = 0;
+	bool native;
+
+	if (!data || obj->efile.cleanup_shndx < 0 || !data->d_size)
+		return 0;
+
+	native = is_native_endianness(obj);
+
+	for (i = 0; i < obj->efile.sec_cnt; i++) {
+		struct elf_sec_desc *sd = &obj->efile.secs[i];
+
+		if (sd->sec_type == SEC_RELO && sd->shdr &&
+		    sd->shdr->sh_info == (Elf64_Word)obj->efile.cleanup_shndx) {
+			relo = sd->data;
+			break;
+		}
+	}
+	if (!relo) {
+		pr_warn("%s present without relocations\n", CLEANUP_SEC);
+		return -LIBBPF_ERRNO__FORMAT;
+	}
+	if (data->d_size % sizeof(struct bpf_cleanup_info)) {
+		pr_warn("%s size %zu is not a multiple of the record size %zu\n",
+			CLEANUP_SEC, data->d_size, sizeof(struct bpf_cleanup_info));
+		return -LIBBPF_ERRNO__FORMAT;
+	}
+
+	vals = data->d_buf;
+	nslots = data->d_size / sizeof(__u32);
+	nrecs = data->d_size / sizeof(struct bpf_cleanup_info);
+
+	recs = calloc(nrecs, sizeof(*recs));
+	if (!recs)
+		return -ENOMEM;
+	for (i = 0; i < nslots; i++)
+		recs[i / CLEANUP_REC_FIELDS].sec_idx[i % CLEANUP_REC_FIELDS] = -1;
+
+	/* One relocation per 4-byte field, naming the section it points into. */
+	nrels = relo->d_size / sizeof(Elf64_Rel);
+	for (i = 0; i < nrels; i++) {
+		Elf64_Rel *rel = elf_rel_by_idx(relo, i);
+		Elf64_Sym *sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
+		size_t type = ELF64_R_TYPE(rel->r_info);
+		size_t slot = rel->r_offset / sizeof(__u32);
+		struct cleanup_raw_rec *rec;
+		size_t off;
+
+		if (type != R_BPF_64_NODYLD32 && type != R_BPF_64_ABS32) {
+			pr_warn("%s: relocation %zu has unexpected type %zu\n",
+				CLEANUP_SEC, i, type);
+			ret = -LIBBPF_ERRNO__FORMAT;
+			goto out;
+		}
+		if (!sym || slot >= nslots || rel->r_offset % sizeof(__u32)) {
+			pr_warn("%s: bad relocation %zu\n", CLEANUP_SEC, i);
+			ret = -LIBBPF_ERRNO__FORMAT;
+			goto out;
+		}
+		/*
+		 * The addend lives in the section data, which libelf leaves in
+		 * the object's byte order; a non-section symbol additionally
+		 * contributes its own value.
+		 */
+		off = (native ? vals[slot] : bswap_32(vals[slot])) + sym->st_value;
+		if (off % BPF_INSN_SZ) {
+			pr_warn("%s: field %zu offset %zu is not instruction aligned\n",
+				CLEANUP_SEC, slot, off);
+			ret = -LIBBPF_ERRNO__FORMAT;
+			goto out;
+		}
+		rec = &recs[slot / CLEANUP_REC_FIELDS];
+		if (rec->sec_idx[slot % CLEANUP_REC_FIELDS] >= 0) {
+			pr_warn("%s: field %zu has more than one relocation\n",
+				CLEANUP_SEC, slot);
+			ret = -LIBBPF_ERRNO__FORMAT;
+			goto out;
+		}
+		rec->sec_idx[slot % CLEANUP_REC_FIELDS] = sym->st_shndx;
+		rec->insn_idx[slot % CLEANUP_REC_FIELDS] = off / BPF_INSN_SZ;
+	}
+
+	for (i = 0; i < nslots; i++) {
+		if (recs[i / CLEANUP_REC_FIELDS].sec_idx[i % CLEANUP_REC_FIELDS] < 0) {
+			pr_warn("%s: field %zu has no relocation\n", CLEANUP_SEC, i);
+			ret = -LIBBPF_ERRNO__FORMAT;
+			goto out;
+		}
+	}
+
+	obj->cleanup_recs = recs;
+	obj->cleanup_rec_cnt = nrecs;
+	return 0;
+out:
+	free(recs);
+	return ret;
+}
+
+static int cmp_cleanup_info(const void *a, const void *b)
+{
+	const struct bpf_cleanup_info *x = a, *y = b;
+
+	if (x->begin_off == y->begin_off)
+		return 0;
+	return x->begin_off < y->begin_off ? -1 : 1;
+}
+
+static int bpf_prog_collect_cleanup_info(struct bpf_object *obj,
+					 struct bpf_program *prog)
+{
+	size_t i;
+	int j;
+
+	for (i = 0; i < obj->cleanup_rec_cnt; i++) {
+		struct cleanup_raw_rec *raw = &obj->cleanup_recs[i];
+		struct bpf_program *owner = NULL;
+		struct bpf_cleanup_info ci = {};
+		__u32 fields[CLEANUP_REC_FIELDS];
+		void *tmp;
+
+		for (j = 0; j < CLEANUP_REC_FIELDS; j++) {
+			size_t idx = raw->insn_idx[j], final;
+			struct bpf_program *p;
+
+			/*
+			 * An exclusive end may name the instruction past the
+			 * last of a function, so ask about the last one the
+			 * range covers, the way the kernel does.
+			 */
+			if (j == CLEANUP_REC_END) {
+				if (!idx) {
+					pr_warn("%s: record %zu ends at instruction 0\n",
+						CLEANUP_SEC, i);
+					return -LIBBPF_ERRNO__FORMAT;
+				}
+				idx--;
+			}
+
+			p = find_prog_by_sec_insn(obj, raw->sec_idx[j], idx);
+			if (!p && j == CLEANUP_REC_BEGIN) {
+				/*
+				 * Code no function claims is an overridden weak
+				 * function the linker kept: skip its records,
+				 * as its relocations are.
+				 */
+				pr_debug("%s: record %zu is in no function, probably an overridden weak function, skipping\n",
+					 CLEANUP_SEC, i);
+				break;
+			}
+			if (!p) {
+				pr_warn("%s: record %zu field %d is not inside a function\n",
+					CLEANUP_SEC, i, j);
+				return -LIBBPF_ERRNO__FORMAT;
+			}
+			if (!owner) {
+				owner = p;
+			} else if (owner != p) {
+				pr_warn("%s: record %zu spans functions '%s' and '%s'\n",
+					CLEANUP_SEC, i, owner->name, p->name);
+				return -LIBBPF_ERRNO__FORMAT;
+			}
+
+			if (owner == prog) {
+				final = raw->insn_idx[j] - prog->sec_insn_off;
+			} else if (prog_is_subprog(obj, owner) && owner->sub_insn_off) {
+				/*
+				 * sub_insn_off is where this subprogram was
+				 * appended to the main program being relocated;
+				 * zero means it is not part of it.
+				 */
+				final = owner->sub_insn_off +
+					raw->insn_idx[j] - owner->sec_insn_off;
+			} else {
+				owner = NULL;
+				break;
+			}
+			fields[j] = final;
+		}
+		if (!owner)
+			continue;
+
+		ci.begin_off = fields[CLEANUP_REC_BEGIN];
+		ci.end_off = fields[CLEANUP_REC_END];
+		ci.landing_pad_off = fields[CLEANUP_REC_PAD];
+
+		tmp = libbpf_reallocarray(prog->cleanup_info, prog->cleanup_info_cnt + 1,
+					  sizeof(*prog->cleanup_info));
+		if (!tmp)
+			return -ENOMEM;
+		prog->cleanup_info = tmp;
+		prog->cleanup_info_rec_size = sizeof(struct bpf_cleanup_info);
+		prog->cleanup_info[prog->cleanup_info_cnt++] = ci;
+
+		pr_debug("prog '%s': cleanup region [%u,%u) -> landing pad %u\n",
+			 prog->name, ci.begin_off, ci.end_off, ci.landing_pad_off);
+	}
+
+	if (!prog->cleanup_info_cnt)
+		return 0;
+
+	qsort(prog->cleanup_info, prog->cleanup_info_cnt,
+	      sizeof(*prog->cleanup_info), cmp_cleanup_info);
+	return 0;
+}
+
 static int
 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
 {
@@ -8197,6 +8446,13 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat
 					return err;
 			}
 		}
+
+		err = bpf_prog_collect_cleanup_info(obj, prog);
+		if (err) {
+			pr_warn("prog '%s': failed to collect cleanup info: %s\n",
+				prog->name, errstr(err));
+			return err;
+		}
 	}
 	for (i = 0; i < obj->nr_programs; i++) {
 		prog = &obj->programs[i];
@@ -8561,6 +8817,9 @@ static int bpf_object__collect_relos(struct bpf_object *obj)
 			return -LIBBPF_ERRNO__INTERNAL;
 		}
 
+		if (idx == obj->efile.cleanup_shndx)
+			continue;
+
 		if (obj->efile.secs[idx].sec_type == SEC_RODATA)
 			err = bpf_object__collect_rodata_relos(obj, shdr, data);
 		else if (obj->efile.secs[idx].sec_type == SEC_DATA)
@@ -8856,6 +9115,11 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
 		load_attr.line_info_rec_size = prog->line_info_rec_size;
 		load_attr.line_info_cnt = prog->line_info_cnt;
 	}
+	if (prog->cleanup_info_cnt) {
+		load_attr.cleanup_info = prog->cleanup_info;
+		load_attr.cleanup_info_cnt = prog->cleanup_info_cnt;
+		load_attr.cleanup_info_rec_size = prog->cleanup_info_rec_size;
+	}
 	load_attr.log_level = log_level;
 	load_attr.prog_flags = prog->prog_flags;
 	/* the program accesses its data in arena through plain numbers */
@@ -9460,6 +9724,7 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf,
 	err = err ? : bpf_object__init_maps(obj, opts);
 	err = err ? : bpf_object_init_progs(obj, opts);
 	err = err ? : bpf_object__collect_relos(obj);
+	err = err ? : bpf_object_init_cleanup_info(obj);
 	if (err)
 		goto out;
 
@@ -10597,6 +10862,9 @@ void bpf_object__close(struct bpf_object *obj)
 	zfree(&obj->jumptables_data);
 	obj->jumptables_data_sz = 0;
 
+	zfree(&obj->cleanup_recs);
+	obj->cleanup_rec_cnt = 0;
+
 	for (i = 0; i < obj->jumptable_map_cnt; i++)
 		close(obj->jumptable_maps[i].fd);
 	zfree(&obj->jumptable_maps);
@@ -11006,6 +11274,19 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts
 		attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size;
 	}
 
+	/* exception cleanup table */
+	info = OPTS_GET(opts, cleanup_info, NULL);
+	info_cnt = OPTS_GET(opts, cleanup_info_cnt, 0);
+	info_rec_size = OPTS_GET(opts, cleanup_info_rec_size, 0);
+	if (!!info != !!info_cnt || !!info != !!info_rec_size) {
+		pr_warn("prog '%s': cleanup_info, cleanup_info_cnt, and cleanup_info_rec_size must all be specified or all omitted\n",
+			prog->name);
+		return libbpf_err(-EINVAL);
+	}
+	attr.cleanup_info = info ?: prog->cleanup_info;
+	attr.cleanup_info_cnt = info ? info_cnt : prog->cleanup_info_cnt;
+	attr.cleanup_info_rec_size = info ? info_rec_size : prog->cleanup_info_rec_size;
+
 	/* Logging is caller-controlled; no fallback to prog/obj log settings */
 	attr.log_buf = OPTS_GET(opts, log_buf, NULL);
 	attr.log_size = OPTS_GET(opts, log_size, 0);
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index 546f65b95cf4..f1630f03d5f5 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -56,6 +56,9 @@
 #ifndef R_BPF_64_ABS32
 #define R_BPF_64_ABS32 3
 #endif
+#ifndef R_BPF_64_NODYLD32
+#define R_BPF_64_NODYLD32 4
+#endif
 #ifndef R_BPF_64_32
 #define R_BPF_64_32 10
 #endif
-- 
2.53.0-Meta