Re: [PATCH v4][makedumpfile 3/7] Implement kernel btf resolving

[Stephen Brennan <stephen.s.brennan@oracle.com>]

Tao Liu <ltao@redhat.com> writes:
> This patch will parse kernel's btf data using libbpf. The kernel's
> btf data is located between __start_BTF and __stop_BTF symbols which
> are resolved by kallsyms of the previous patch. Same as the previous
> one, the .init_ktypes section of makedumpfile and the extensions will
> be iterated, and any types which belongs to vmlinux can be resolved
> at this time.
>
> Another primary function implemented in this patch, is recursively
> diving into anonymous struct/union when encountered any, to find a
> member by given its name.
>
> Suggested-by: Stephen Brennan <stephen.s.brennan@oracle.com>
> Signed-off-by: Tao Liu <ltao@redhat.com>

Reviewed-by: Stephen Brennan <stephen.s.brennan@oracle.com>

> ---
>  Makefile   |   4 +-
>  btf_info.c | 233 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  btf_info.h |  90 +++++++++++++++++++++
>  3 files changed, 325 insertions(+), 2 deletions(-)
>  create mode 100644 btf_info.c
>  create mode 100644 btf_info.h
>
> diff --git a/Makefile b/Makefile
> index a57185e..320677d 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -45,12 +45,12 @@ CFLAGS_ARCH += -m32
>  endif
>  
>  SRC_BASE = makedumpfile.c makedumpfile.h diskdump_mod.h sadump_mod.h sadump_info.h
> -SRC_PART = print_info.c dwarf_info.c elf_info.c erase_info.c sadump_info.c cache.c tools.c printk.c detect_cycle.c kallsyms.c
> +SRC_PART = print_info.c dwarf_info.c elf_info.c erase_info.c sadump_info.c cache.c tools.c printk.c detect_cycle.c kallsyms.c btf_info.c
>  OBJ_PART=$(patsubst %.c,%.o,$(SRC_PART))
>  SRC_ARCH = arch/arm.c arch/arm64.c arch/x86.c arch/x86_64.c arch/ia64.c arch/ppc64.c arch/s390x.c arch/ppc.c arch/sparc64.c arch/mips64.c arch/loongarch64.c arch/riscv64.c
>  OBJ_ARCH=$(patsubst %.c,%.o,$(SRC_ARCH))
>  
> -LIBS = -ldw -lbz2 -ldl -lelf -lz
> +LIBS = -ldw -lbz2 -ldl -lelf -lz -lbpf
>  ifneq ($(LINKTYPE), dynamic)
>  LIBS := -static $(LIBS) -llzma
>  endif
> diff --git a/btf_info.c b/btf_info.c
> new file mode 100644
> index 0000000..1cb66e2
> --- /dev/null
> +++ b/btf_info.c
> @@ -0,0 +1,233 @@
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <bpf/btf.h>
> +#include <bpf/libbpf_legacy.h>
> +#include "makedumpfile.h"
> +#include "kallsyms.h"
> +#include "btf_info.h"
> +
> +struct btf_arr_elem {
> +	struct btf *btf;
> +	char *module;
> +};
> +
> +static struct btf_arr_elem **btf_arr = NULL;
> +static int btf_arr_len = 0;
> +static int btf_arr_cap = 0;
> +
> +/* makedumpfile & extensions' .init_ktypes section range array */
> +static struct section_range **sr = NULL;
> +static int sr_len = 0;
> +static int sr_cap = 0;
> +
> +/* Which mod's btf should be inited? */
> +static char **mods = NULL;
> +static int mods_len = 0;
> +static int mods_cap = 0;
> +
> +static bool add_ktype_modname(char *modname)
> +{
> +	return push_uniq_str((void ***)&mods, &mods_len, &mods_cap, modname);
> +}
> +
> +bool check_ktypes_require_modname(char *modname, int *total)
> +{
> +	if (total)
> +		*total = mods_len;
> +	for (int i = 0; i < mods_len; i++) {
> +		if (!strcmp(modname, mods[i]))
> +			return true;
> +	}
> +	return false;
> +}
> +
> +static void cleanup_ktypes_modname(void)
> +{
> +	if (mods) {
> +		free(mods);
> +		mods = NULL;
> +	}
> +	mods_len = 0;
> +	mods_cap = 0;
> +}
> +
> +/*
> + * Used by makedumpfile and extensions, to register their .init_ktypes section,
> + * so btf_info can know which module/type should be inited.
> +*/
> +REGISTER_SECTION(ktype)
> +
> +static void cleanup_ktypes_section_range(void)
> +{
> +	for (int i = 0; i < sr_len; i++) {
> +		free(sr[i]);
> +	}
> +	if (sr) {
> +		free(sr);
> +		sr = NULL;
> +	}
> +	sr_len = 0;
> +	sr_cap = 0;
> +}
> +
> +static void find_member_recursive(struct btf *btf, int struct_typeid,
> +				  int base_offset, struct ktype_info *ki)
> +{
> +	const struct btf_type *st;
> +	struct btf_member *bm;
> +	int i, vlen;
> +
> +	struct_typeid = btf__resolve_type(btf, struct_typeid);
> +	st = btf__type_by_id(btf, struct_typeid);
> +
> +	if (!st)
> +		return;
> +
> +	if (BTF_INFO_KIND(st->info) != BTF_KIND_STRUCT &&
> +	    BTF_INFO_KIND(st->info) != BTF_KIND_UNION)
> +		return;
> +
> +	vlen = BTF_INFO_VLEN(st->info);
> +	bm = btf_members(st);
> +
> +	for (i = 0; i < vlen; i++, bm++) {
> +		const char *name = btf__name_by_offset(btf, bm->name_off);
> +		int member_bit_offset = btf_member_bit_offset(st, i) + base_offset;
> +		int member_typeid = btf__resolve_type(btf, bm->type);
> +		const struct btf_type *mt = btf__type_by_id(btf, member_typeid);
> +
> +		if (name && strcmp(name, ki->member_name) == 0) {
> +			ki->member_bit_offset = member_bit_offset;
> +			ki->member_bit_sz = btf_member_bitfield_size(st, i);
> +			ki->member_size = btf__resolve_size(btf, member_typeid);
> +			ki->index = i;
> +			return;
> +		}
> +
> +		if (!name || !name[0]) {
> +			if (BTF_INFO_KIND(mt->info) == BTF_KIND_STRUCT ||
> +			    BTF_INFO_KIND(mt->info) == BTF_KIND_UNION) {
> +				find_member_recursive(btf, member_typeid,
> +						      member_bit_offset, ki);
> +			}
> +		}
> +	}
> +}
> +
> +static void get_ktype_info(struct ktype_info *ki, char *mod_to_resolve)
> +{
> +	int i, j, start_id;
> +
> +	if (mod_to_resolve != NULL) {
> +		if (strcmp(ki->modname, mod_to_resolve) != 0)
> +			/* Exit safely */
> +			return;
> +	}
> +
> +	for (i = 0; i < btf_arr_len; i++) {
> +		if (strcmp(btf_arr[i]->module, ki->modname) != 0)
> +			continue;
> +		/*
> +		 * vmlinux(btf_arr[0])'s typeid is 1~vmlinux_type_cnt,
> +		 * modules(btf_arr[1...])'s typeid is vmlinux_type_cnt~btf__type_cnt
> +		 */
> +		start_id = (i == 0 ? 1 : btf__type_cnt(btf_arr[0]->btf));
> +
> +		for (j = start_id; j < btf__type_cnt(btf_arr[i]->btf); j++) {
> +			const struct btf_type *bt =
> +				btf__type_by_id(btf_arr[i]->btf, j);
> +			const char *name =
> +				btf__name_by_offset(btf_arr[i]->btf, bt->name_off);
> +
> +			if (name && strcmp(ki->struct_name, name) == 0) {
> +				if (ki->member_name != NULL) {
> +					/* Retrieve member info */
> +					find_member_recursive(btf_arr[i]->btf, j, 0, ki);
> +				} else {
> +					ki->index = j;
> +				}
> +				ki->struct_size = btf__resolve_size(btf_arr[i]->btf, j);
> +				return;
> +			}
> +		}
> +	}
> +}
> +
> +static bool add_to_btf_arr(struct btf *btf, char *module_name)
> +{
> +	struct btf_arr_elem *new_p;
> +
> +	new_p = malloc(sizeof(struct btf_arr_elem));
> +	if (!new_p)
> +		goto no_mem;
> +
> +	new_p->btf = btf;
> +	new_p->module = module_name;
> +
> +	return add_to_arr((void ***)&btf_arr, &btf_arr_len, &btf_arr_cap, new_p);
> +
> +no_mem:
> +	fprintf(stderr, "%s: Not enough memory!\n", __func__);
> +	return false;
> +}
> +
> +INIT_KERN_SYM(__start_BTF);
> +INIT_KERN_SYM(__stop_BTF);
> +
> +/*
> + * Makedumpfile's .init_ktypes section
> +*/
> +extern struct ktype_info *__start_init_ktypes[];
> +extern struct ktype_info *__stop_init_ktypes[];
> +
> +bool init_kernel_btf(void)
> +{
> +	uint64_t size;
> +	struct btf *btf;
> +	int i;
> +	struct ktype_info **p;
> +	char *buf = NULL;
> +	bool ret = false;
> +
> +	uint64_t start_btf = GET_KERN_SYM(__start_BTF);
> +	uint64_t stop_btf = GET_KERN_SYM(__stop_BTF);
> +	if (!KERN_SYM_EXIST(__start_BTF) ||
> +	    !KERN_SYM_EXIST(__stop_BTF)) {
> +		fprintf(stderr, "%s: symbol __start/stop_BTF not found!\n", __func__);
> +		goto out;
> +	}
> +
> +	if (!register_ktype_section((char *)__start_init_ktypes,
> +				    (char *)__stop_init_ktypes))
> +		return ret;
> +
> +	size = stop_btf - start_btf;
> +	buf = (char *)malloc(size);
> +	if (!buf) {
> +		fprintf(stderr, "%s: Not enough memory!\n", __func__);
> +		goto out;
> +	}
> +	readmem(VADDR, start_btf, buf, size);
> +	btf = btf__new(buf, size);
> +
> +	if (libbpf_get_error(btf) != 0 ||
> +	    add_to_btf_arr(btf, strdup("vmlinux")) == false) {	
> +		fprintf(stderr, "%s: init vmlinux btf fail\n", __func__);
> +		goto out;
> +	}
> +
> +	for (i = 0; i < sr_len; i++) {
> +		for (p = (struct ktype_info **)(sr[i]->start);
> +		     p < (struct ktype_info **)(sr[i]->stop);
> +		     p++) {
> +			get_ktype_info(*p, "vmlinux");
> +		}
> +	}
> +
> +	ret = true;
> +out:
> +	if (buf)
> +		free(buf);
> +	return ret;
> +}
> \ No newline at end of file
> diff --git a/btf_info.h b/btf_info.h
> new file mode 100644
> index 0000000..2cf6b07
> --- /dev/null
> +++ b/btf_info.h
> @@ -0,0 +1,90 @@
> +#ifndef _BTF_INFO_H
> +#define _BTF_INFO_H
> +#include <stdint.h>
> +#include <stdbool.h>
> +
> +struct ktype_info {
> +	/********in******/
> +	char *modname;		// Set to search within the module, in case
> +				// name conflict of different modules
> +	char *struct_name;	// Search by struct name
> +	char *member_name;	// Search by member name
> +	bool struct_required : 1;
> +	bool member_required : 1;
> +	/********out*****/
> +	uint32_t member_bit_offset;	// member offset in bits
> +	uint32_t member_bit_sz;	// member width in bits
> +	uint32_t member_size;	// member size in bytes
> +	uint32_t struct_size;	// struct size in bytes
> +	int index;		// -1 if type not found
> +};
> +
> +bool check_ktypes_require_modname(char *modname, int *total);
> +bool register_ktype_section(char *start, char *stop);
> +bool init_kernel_btf(void);
> +
> +#define QUATE(x) #x
> +#define INIT_MOD_STRUCT_MEMBER_RQD(MOD, S, M, R)			\
> +	struct ktype_info _##MOD##_##S##_##M = {			\
> +		QUATE(MOD), QUATE(S), QUATE(M), R, R, 0, 0, 0, 0, -1	\
> +	};								\
> +	__attribute__((section(".init_ktypes"), used))			\
> +	struct ktype_info * _ptr_##MOD##_##S##_##M = &_##MOD##_##S##_##M
> +
> +/*
> + * Required types will be checked automatically before extension running.
> + * Optinal types should be checked manually at extension runtime.
> + */
> +#define INIT_MOD_STRUCT_MEMBER(MOD, S, M) \
> +	INIT_MOD_STRUCT_MEMBER_RQD(MOD, S, M, 1)
> +#define INIT_OPT_MOD_STRUCT_MEMBER(MOD, S, M) \
> +	INIT_MOD_STRUCT_MEMBER_RQD(MOD, S, M, 0)
> +
> +#define DECLARE_MOD_STRUCT_MEMBER(MOD, S, M) \
> +	extern struct ktype_info _##MOD##_##S##_##M
> +
> +#define GET_MOD_STRUCT_MEMBER_MOFF(MOD, S, M)  (_##MOD##_##S##_##M.member_bit_offset)
> +#define GET_MOD_STRUCT_MEMBER_MSIZE(MOD, S, M) (_##MOD##_##S##_##M.member_size)
> +#define GET_MOD_STRUCT_MEMBER_SSIZE(MOD, S, M) (_##MOD##_##S##_##M.struct_size)
> +#define MOD_STRUCT_MEMBER_EXIST(MOD, S, M) (_##MOD##_##S##_##M.index >= 0)
> +#define TYPE_EXIST(p) ((p)->index >= 0)
> +
> +#define INIT_KERN_STRUCT_MEMBER(S, M) \
> +	INIT_MOD_STRUCT_MEMBER(vmlinux, S, M)
> +#define INIT_OPT_KERN_STRUCT_MEMBER(S, M) \
> +	INIT_OPT_MOD_STRUCT_MEMBER(vmlinux, S, M)
> +
> +#define DECLARE_KERN_STRUCT_MEMBER(S, M) \
> +	DECLARE_MOD_STRUCT_MEMBER(vmlinux, S, M)
> +
> +#define GET_KERN_STRUCT_MEMBER_MOFF(S, M)  GET_MOD_STRUCT_MEMBER_MOFF(vmlinux, S, M)
> +#define GET_KERN_STRUCT_MEMBER_MSIZE(S, M) GET_MOD_STRUCT_MEMBER_MSIZE(vmlinux, S, M)
> +#define GET_KERN_STRUCT_MEMBER_SSIZE(S, M) GET_MOD_STRUCT_MEMBER_SSIZE(vmlinux, S, M)
> +#define KERN_STRUCT_MEMBER_EXIST(S, M) MOD_STRUCT_MEMBER_EXIST(vmlinux, S, M)
> +
> +#define INIT_MOD_STRUCT_RQD(MOD, S, R)				\
> +	struct ktype_info _##MOD##_##S = {			\
> +		QUATE(MOD), QUATE(S), 0, R, 0, 0, 0, 0, 0, -1	\
> +	};							\
> +	__attribute__((section(".init_ktypes"), used))		\
> +	struct ktype_info * _ptr_##MOD##_##S = &_##MOD##_##S
> +
> +#define INIT_MOD_STRUCT(MOD, S) INIT_MOD_STRUCT_RQD(MOD, S, 1)
> +#define INIT_OPT_MOD_STRUCT(MOD, S) INIT_MOD_STRUCT_RQD(MOD, S, 0)
> +
> +#define DECLARE_MOD_STRUCT(MOD, S) \
> +	extern struct ktype_info _##MOD##_##S;
> +
> +#define GET_MOD_STRUCT_SSIZE(MOD, S) (_##MOD##_##S.struct_size)
> +#define MOD_STRUCT_EXIST(MOD, S) (_##MOD##_##S.index >= 0)
> +
> +#define INIT_KERN_STRUCT(S) INIT_MOD_STRUCT(vmlinux, S)
> +#define INIT_OPT_KERN_STRUCT(S) INIT_OPT_MOD_STRUCT(vmlinux, S)
> +
> +#define DECLARE_KERN_STRUCT(S) \
> +	DECLARE_MOD_STRUCT(vmlinux, S)
> +
> +#define GET_KERN_STRUCT_SSIZE(S) GET_MOD_STRUCT_SSIZE(vmlinux, S)
> +#define KERN_STRUCT_EXIST(S) MOD_STRUCT_EXIST(vmlinux, S)
> +
> +#endif /* _BTF_INFO_H */
> \ No newline at end of file
> -- 
> 2.47.0