[PATCH] bpf, btf: store BTF dict id for kernel modules

[PATCH] bpf, btf: store BTF dict id for kernel modules

[Kris Van Hees]

BPF requires a file descriptor (provided by the kernel) to be associated
with BTF type ids for loadble kernel modules.  This file descriptor can
only be obtained from the kernel based on a BTF dict id for the module,
and these ids are assigned dynamically at runtime when modules are
loaded.

After kernel modules are initialized for DTrace, the corresponding BTF
dict ids are associated with their modules.

Signed-off-by: Kris Van Hees <kris.van.hees@oracle.com>
---
 libdtrace/dt_bpf.c  | 58 +++++++++++++++++++++++++++++++++++++++++++--
 libdtrace/dt_bpf.h  |  4 ++++
 libdtrace/dt_btf.c  | 55 +++++++++++++++++++++++++++++++++++++++++-
 libdtrace/dt_btf.h  |  5 +++-
 libdtrace/dt_impl.h |  3 +++
 libdtrace/dt_open.c |  2 +-
 6 files changed, 122 insertions(+), 5 deletions(-)

diff --git a/libdtrace/dt_bpf.c b/libdtrace/dt_bpf.c
index 2e43e545..b680026a 100644
--- a/libdtrace/dt_bpf.c
+++ b/libdtrace/dt_bpf.c
@@ -121,6 +121,60 @@ dt_bpf_prog_load(const dt_probe_t *prp, const dtrace_difo_t *dp,
 				  lvl, buf, sz);
 }
 
+/*
+ * Get BTF dict information based on its fd.
+ */
+int
+dt_bpf_btf_get_info_by_fd(int fd, btf_info_t *info, uint32_t *size)
+{
+	union bpf_attr	attr;
+	int		rc;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.info.bpf_fd = fd;
+	attr.info.info = (uint64_t)info;
+	attr.info.info_len = *size;
+
+	rc = dt_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr);
+	if (rc == 0)
+		*size = attr.info.info_len;
+
+	return rc;
+}
+
+/*
+ * Get a file descriptor for a BTF dict based on its id.
+ */
+int
+dt_bpf_btf_get_fd_by_id(uint32_t id)
+{
+	union bpf_attr	attr;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.btf_id = id;
+
+	return dt_bpf(BPF_BTF_GET_FD_BY_ID, &attr);
+}
+
+/*
+ * Get the id for the "next" BTF dict based on a given id (0 to get the first).
+ */
+int
+dt_bpf_btf_get_next_id(uint32_t curr, uint32_t *next)
+{
+	union bpf_attr	attr;
+	int		rc;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.start_id = curr;
+
+	rc = dt_bpf(BPF_BTF_GET_NEXT_ID, &attr);
+	if (rc == 0)
+		*next = attr.next_id;
+
+	return rc;
+}
+
 /*
  * Create a named BPF map.
  */
diff --git a/libdtrace/dt_bpf.h b/libdtrace/dt_bpf.h
index f9ff7756..5f8e315a 100644
--- a/libdtrace/dt_bpf.h
+++ b/libdtrace/dt_bpf.h
@@ -12,6 +12,7 @@
 #include <linux/bpf.h>
 #include <linux/perf_event.h>
 #include <dtrace/difo.h>
+#include <dt_btf.h>
 #include <dt_impl.h>
 
 struct dtrace_hdl;
@@ -65,6 +66,9 @@ extern int dt_bpf(enum bpf_cmd cmd, union bpf_attr *attr);
 extern int dt_bpf_gmap_create(struct dtrace_hdl *);
 extern int dt_bpf_lockmem_error(struct dtrace_hdl *dtp, const char *msg);
 
+extern int dt_bpf_btf_get_info_by_fd(int fd, btf_info_t *info, uint32_t *size);
+extern int dt_bpf_btf_get_fd_by_id(uint32_t id);
+extern int dt_bpf_btf_get_next_id(uint32_t curr, uint32_t *next);
 extern int dt_bpf_map_lookup(int fd, const void *key, void *val);
 extern int dt_bpf_map_next_key(int fd, const void *key, void *nxt);
 extern int dt_bpf_map_update(int fd, const void *key, const void *val);
diff --git a/libdtrace/dt_btf.c b/libdtrace/dt_btf.c
index 286e9e10..c315f086 100644
--- a/libdtrace/dt_btf.c
+++ b/libdtrace/dt_btf.c
@@ -13,7 +13,8 @@
 #include <linux/btf.h>
 
 #include <dt_impl.h>
-#include <dt_btf.h>
+#include <dt_bpf.h>
+#include <dt_module.h>
 
 typedef struct btf_header	btf_header_t;
 typedef struct btf_type		btf_type_t;
@@ -891,3 +892,55 @@ dt_btf_lookup_name_kind(dtrace_hdl_t *dtp, dt_btf_t *btf, const char *name,
 
 	return -ENOENT;
 }
+
+int
+dt_btf_get_module_ids(dtrace_hdl_t *dtp)
+{
+	uint32_t	id = 0;
+
+	for (;;) {
+		int		rc, fd;
+		btf_info_t	info;
+		char		name[PATH_MAX];
+		uint32_t	size = sizeof(info);
+		dt_module_t	*dmp;
+
+		rc = dt_bpf_btf_get_next_id(id, &id);
+		if (rc) {
+			if (errno == EPERM)
+				return 0;	/* Not supported. */
+			if (errno == ENOENT)
+				return 0;	/* Done. */
+
+			return dt_set_errno(dtp, errno);
+		}
+
+		fd = dt_bpf_btf_get_fd_by_id(id);
+		if (fd < 0) {
+			if (errno == ENOENT)
+				continue;	/* Module got unloaded? */
+
+			return dt_set_errno(dtp, errno);
+		}
+
+		memset(&info, 0, size);
+		info.name = (uint64_t)name;
+		info.name_len = sizeof(name);
+		rc = dt_bpf_btf_get_info_by_fd(fd, &info, &size);
+		close(fd);
+		if (rc)
+			return dt_set_errno(dtp, errno);
+
+		dmp = dt_module_lookup_by_name(dtp, name);
+		if (dmp == NULL)
+			continue;
+
+		dmp->dm_btf_id = id;
+	}
+}
+
+int
+dt_btf_module_fd(const dt_module_t *dmp)
+{
+	return dmp->dm_btf_id ? dt_bpf_btf_get_fd_by_id(dmp->dm_btf_id) : 0;
+}
diff --git a/libdtrace/dt_btf.h b/libdtrace/dt_btf.h
index 0146386b..60256f12 100644
--- a/libdtrace/dt_btf.h
+++ b/libdtrace/dt_btf.h
@@ -15,7 +15,8 @@
 extern "C" {
 #endif
 
-typedef struct dt_btf	dt_btf_t;
+typedef struct dt_btf		dt_btf_t;
+typedef struct bpf_btf_info	btf_info_t;
 
 extern void dt_btf_destroy(dtrace_hdl_t *, dt_btf_t *);
 extern dt_btf_t *dt_btf_load_module(dtrace_hdl_t *, dt_module_t *);
@@ -23,6 +24,8 @@ extern ctf_dict_t *dt_btf_module_ctf(dtrace_hdl_t *, dt_module_t *);
 extern const char *dt_btf_get_string(dtrace_hdl_t *, dt_btf_t *, uint32_t);
 extern int32_t dt_btf_lookup_name_kind(dtrace_hdl_t *, dt_btf_t *,
 				       const char *, uint32_t);
+extern int dt_btf_get_module_ids(dtrace_hdl_t *);
+extern int dt_btf_module_fd(const dt_module_t *);
 
 #ifdef	__cplusplus
 }
diff --git a/libdtrace/dt_impl.h b/libdtrace/dt_impl.h
index 7cf71060..0c154875 100644
--- a/libdtrace/dt_impl.h
+++ b/libdtrace/dt_impl.h
@@ -159,6 +159,7 @@ typedef struct dt_module {
 	ctf_sect_t dm_ctdata;	/* CTF data for module */
 	ctf_sect_t dm_symtab;	/* symbol table */
 	ctf_sect_t dm_strtab;	/* string table */
+	uint32_t dm_btf_id;	/* BTF dict id */
 	struct dt_btf *dm_btf;	/* BTF data for module */
 
 	/*
@@ -339,6 +340,8 @@ struct dtrace_hdl {
 	char *dt_ctfa_path;	/* path to vmlinux.ctfa */
 	ctf_archive_t *dt_ctfa; /* ctf archive for the entire kernel tree */
 	struct dt_btf *dt_shared_btf; /* BTF data for the kernel (shared) */
+	int dt_btf_cnt;		/* BTF id count */
+	int *dt_btf_fds;	/* BTF id-to-fd mapping */
 	ctf_file_t *dt_shared_ctf; /* Handle to the shared CTF */
 	dt_htab_t *dt_kernpaths; /* hash table of dt_kern_path_t's */
 	dt_module_t *dt_exec;	/* pointer to executable module */
diff --git a/libdtrace/dt_open.c b/libdtrace/dt_open.c
index 8c3d3133..225bec50 100644
--- a/libdtrace/dt_open.c
+++ b/libdtrace/dt_open.c
@@ -29,7 +29,6 @@
 
 #include <dt_impl.h>
 #include <dt_bpf.h>
-#include <dt_btf.h>
 #include <dt_pcap.h>
 #include <dt_program.h>
 #include <dt_module.h>
@@ -1155,6 +1154,7 @@ dt_vopen(int version, int flags, int *errp,
 		return set_open_errno(dtp, errp, dtp->dt_errno);
 
 	dt_bpf_init(dtp);
+	dt_btf_get_module_ids(dtp);
 
 	return dtp;
 }
-- 
2.42.0

Re: [PATCH] bpf, btf: store BTF dict id for kernel modules

[Nick Alcock]

On 25 Apr 2024, Kris Van Hees said:

> BPF requires a file descriptor (provided by the kernel) to be associated
> with BTF type ids for loadble kernel modules.  This file descriptor can

s/loadble/loadable/

> only be obtained from the kernel based on a BTF dict id for the module,
> and these ids are assigned dynamically at runtime when modules are
> loaded.

... were they *trying* to make this unnecessarily painful to use?

> Signed-off-by: Kris Van Hees <kris.van.hees@oracle.com>

Reviewed-by: Nick Alcock <nick.alcock@oracle.com>

with a few niggles

> diff --git a/libdtrace/dt_btf.h b/libdtrace/dt_btf.h
> index 0146386b..60256f12 100644
> --- a/libdtrace/dt_btf.h
> +++ b/libdtrace/dt_btf.h
> @@ -15,7 +15,8 @@
>  extern "C" {
>  #endif
>  
> -typedef struct dt_btf	dt_btf_t;
> +typedef struct dt_btf		dt_btf_t;
> +typedef struct bpf_btf_info	btf_info_t;

This sort of thing will start throwing compiler errors whenever any user
#includes the header that gives the concrete definition of any of those
types, because we're probably typedeffing them there too, and unlike
struct definitions you cannot repeat typedefs.

(Not a problem now, but just noting that this sort of pattern tends to
store up trouble for the future.)

> @@ -339,6 +340,8 @@ struct dtrace_hdl {
>  	char *dt_ctfa_path;	/* path to vmlinux.ctfa */
>  	ctf_archive_t *dt_ctfa; /* ctf archive for the entire kernel tree */
>  	struct dt_btf *dt_shared_btf; /* BTF data for the kernel (shared) */
> +	int dt_btf_cnt;		/* BTF id count */
> +	int *dt_btf_fds;	/* BTF id-to-fd mapping */
>  	ctf_file_t *dt_shared_ctf; /* Handle to the shared CTF */
>  	dt_htab_t *dt_kernpaths; /* hash table of dt_kern_path_t's */
>  	dt_module_t *dt_exec;	/* pointer to executable module */

These don't seem to be used anywhere in this patch: did you swap between
storing globally and storing per-module?

-- 
NULL && (void)