Re: [PATCH v6 bpf-next 02/10] libbpf: Support kind layout section handling in BTF

[Mykyta Yatsenko <mykyta.yatsenko5@gmail.com>]

On 12/10/25 20:32, Alan Maguire wrote:
> Support reading in kind layout fixing endian issues on reading;
> also support writing kind layout section to raw BTF object.
> There is not yet an API to populate the kind layout with meaningful
> information.
>
> As part of this, we need to consider multiple valid BTF header
> sizes; the original or the kind layout-extended headers.
> So to support this, the "struct btf" representation is modified
> to always allocate a "struct btf_header" and copy the valid
> portion from the raw data to it; this means we can always safely
> check fields like btf->hdr->kind_layout_len.
>
> Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
> ---
>   tools/lib/bpf/btf.c | 200 ++++++++++++++++++++++++++++++--------------
>   1 file changed, 139 insertions(+), 61 deletions(-)
>
> diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
> index b136572e889a..737adc560818 100644
> --- a/tools/lib/bpf/btf.c
> +++ b/tools/lib/bpf/btf.c
> @@ -40,42 +40,53 @@ struct btf {
>   
>   	/*
>   	 * When BTF is loaded from an ELF or raw memory it is stored
> -	 * in a contiguous memory block. The hdr, type_data, and, strs_data
> +	 * in a contiguous memory block. The  type_data, and, strs_data
>   	 * point inside that memory region to their respective parts of BTF
>   	 * representation:
>   	 *
> -	 * +--------------------------------+
> -	 * |  Header  |  Types  |  Strings  |
> -	 * +--------------------------------+
> -	 * ^          ^         ^
> -	 * |          |         |
> -	 * hdr        |         |
> -	 * types_data-+         |
> -	 * strs_data------------+
> +	 * +--------------------------------+---------------------+
> +	 * |  Header  |  Types  |  Strings  |Optional kind layout |
> +	 * +--------------------------------+---------------------+
> +	 * ^          ^         ^           ^
> +	 * |          |         |           |
> +	 * raw_data   |         |           |
> +	 * types_data-+         |           |
> +	 * strs_data------------+           |
> +	 * kind_layout----------------------+
> +	 *
> +	 * A separate struct btf_header is allocated for btf->hdr,
> +	 * and header information is copied into it.  This allows us
> +	 * to handle header data for various header formats; the original,
> +	 * the extended header with kind layout, etc.
>   	 *
>   	 * If BTF data is later modified, e.g., due to types added or
>   	 * removed, BTF deduplication performed, etc, this contiguous
> -	 * representation is broken up into three independently allocated
> -	 * memory regions to be able to modify them independently.
> +	 * representation is broken up into four independent memory
> +	 * regions.
> +	 *
>   	 * raw_data is nulled out at that point, but can be later allocated
>   	 * and cached again if user calls btf__raw_data(), at which point
> -	 * raw_data will contain a contiguous copy of header, types, and
> -	 * strings:
> +	 * raw_data will contain a contiguous copy of header, types, strings
> +	 * and optionally kind_layout.  kind_layout optionally points to a
> +	 * kind_layout array - this allows us to encode information about
> +	 * the kinds known at encoding time.  If kind_layout is NULL no
> +	 * kind information is encoded.
>   	 *
> -	 * +----------+  +---------+  +-----------+
> -	 * |  Header  |  |  Types  |  |  Strings  |
> -	 * +----------+  +---------+  +-----------+
> -	 * ^             ^            ^
> -	 * |             |            |
> -	 * hdr           |            |
> -	 * types_data----+            |
> -	 * strset__data(strs_set)-----+
> +	 * +----------+  +---------+  +-----------+   +-----------+
> +	 * |  Header  |  |  Types  |  |  Strings  |   |kind_layout|
> +	 * +----------+  +---------+  +-----------+   +-----------+
> +	 * ^             ^            ^               ^
> +	 * |             |            |               |
> +	 * hdr           |            |               |
> +	 * types_data----+            |               |
> +	 * strset__data(strs_set)-----+               |
> +	 * kind_layout--------------------------------+
>   	 *
> -	 *               +----------+---------+-----------+
> -	 *               |  Header  |  Types  |  Strings  |
> -	 * raw_data----->+----------+---------+-----------+
> +	 *               +----------+---------+-----------+---------------------+
> +	 *               |  Header  |  Types  |  Strings  | Optional kind layout|
> +	 * raw_data----->+----------+---------+-----------+---------------------+
>   	 */
> -	struct btf_header *hdr;
> +	struct btf_header *hdr; /* separately-allocated header data */
>   
>   	void *types_data;
>   	size_t types_data_cap; /* used size stored in hdr->type_len */
> @@ -123,6 +134,13 @@ struct btf {
>   	/* whether raw_data is a (read-only) mmap */
>   	bool raw_data_is_mmap;
>   
> +	/* is BTF modifiable? i.e. is it split into separate sections as described above? */
> +	bool modifiable;
> +	/* Points either at raw kind layout data in parsed BTF (if present), or
> +	 * at an allocated kind layout array when BTF is modifiable.
> +	 */
> +	void *kind_layout;
> +
>   	/* BTF object FD, if loaded into kernel */
>   	int fd;
>   
> @@ -214,7 +232,7 @@ static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
>   	return 0;
>   }
>   
> -static void btf_bswap_hdr(struct btf_header *h)
> +static void btf_bswap_hdr(struct btf_header *h, __u32 hdr_len)
>   {
>   	h->magic = bswap_16(h->magic);
>   	h->hdr_len = bswap_32(h->hdr_len);
> @@ -222,50 +240,68 @@ static void btf_bswap_hdr(struct btf_header *h)
>   	h->type_len = bswap_32(h->type_len);
>   	h->str_off = bswap_32(h->str_off);
>   	h->str_len = bswap_32(h->str_len);
> +	/* May be operating on raw data with hdr_len that does not include below fields */
> +	if (hdr_len >= sizeof(struct btf_header)) {
> +		h->kind_layout_off = bswap_32(h->kind_layout_off);
> +		h->kind_layout_len = bswap_32(h->kind_layout_len);
> +	}
>   }
>   
>   static int btf_parse_hdr(struct btf *btf)
>   {
> -	struct btf_header *hdr = btf->hdr;
> +	struct btf_header *hdr = btf->raw_data;
> +	__u32 hdr_len = hdr->hdr_len;
>   	__u32 meta_left;
>   
> -	if (btf->raw_size < sizeof(struct btf_header)) {
> +	if (btf->raw_size < offsetofend(struct btf_header, str_len)) {
>   		pr_debug("BTF header not found\n");
>   		return -EINVAL;
>   	}
>   
>   	if (hdr->magic == bswap_16(BTF_MAGIC)) {
>   		btf->swapped_endian = true;
> -		if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) {
> +		hdr_len = bswap_32(hdr->hdr_len);
> +		if (hdr_len < offsetofend(struct btf_header, str_len)) {
>   			pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n",
> -				bswap_32(hdr->hdr_len));
> +				hdr_len);
>   			return -ENOTSUP;
>   		}
> -		btf_bswap_hdr(hdr);
>   	} else if (hdr->magic != BTF_MAGIC) {
>   		pr_debug("Invalid BTF magic: %x\n", hdr->magic);
>   		return -EINVAL;
>   	}
>   
> -	if (btf->raw_size < hdr->hdr_len) {
> +	if (btf->raw_size < hdr_len) {
>   		pr_debug("BTF header len %u larger than data size %u\n",
> -			 hdr->hdr_len, btf->raw_size);
> +			 hdr_len, btf->raw_size);
>   		return -EINVAL;
>   	}
>   
> -	meta_left = btf->raw_size - hdr->hdr_len;
> -	if (meta_left < (long long)hdr->str_off + hdr->str_len) {
> +	/* At this point, we have basic header information, so allocate btf->hdr */
> +	btf->hdr = calloc(1, sizeof(struct btf_header));
> +	if (!btf->hdr) {
> +		pr_debug("BTF header allocation failed\n");
> +		return -ENOMEM;
> +	}
> +	if (btf->swapped_endian)
> +		btf_bswap_hdr(hdr, hdr_len);
> +	memcpy(btf->hdr, hdr, hdr_len < sizeof(struct btf_header) ? hdr_len :
> +								    sizeof(struct btf_header));
can we simplify this a little bit: memcpy(btf->hdr, hdr, min(hdr_len, 
sizeof(struct btf_header));?
min macro is available in libbpf_internal.h
> +
> +	meta_left = btf->raw_size - hdr_len;
> +	if (meta_left < (long long)btf->hdr->str_off + btf->hdr->str_len) {
>   		pr_debug("Invalid BTF total size: %u\n", btf->raw_size);
>   		return -EINVAL;
>   	}
>   
> -	if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) {
> +	if ((long long)btf->hdr->type_off + btf->hdr->type_len > btf->hdr->str_off) {
>   		pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n",
> -			 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);
> +			 btf->hdr->type_off, btf->hdr->type_len, btf->hdr->str_off,
> +			 btf->hdr->str_len);
>   		return -EINVAL;
>   	}
>   
> -	if (hdr->type_off % 4) {
> +	if (btf->hdr->type_off % 4) {
>   		pr_debug("BTF type section is not aligned to 4 bytes\n");
>   		return -EINVAL;
>   	}
> @@ -292,6 +328,22 @@ static int btf_parse_str_sec(struct btf *btf)
>   	return 0;
>   }
>   
> +static int btf_parse_kind_layout_sec(struct btf *btf)
> +{
> +	const struct btf_header *hdr = btf->hdr;
> +
> +	if (!hdr->kind_layout_off || !hdr->kind_layout_len)
> +		return 0;
> +
> +	if (hdr->kind_layout_len % sizeof(struct btf_kind_layout) != 0) {
> +		pr_debug("Invalid BTF kind layout section\n");
> +		return -EINVAL;
> +	}
> +	btf->kind_layout = btf->raw_data + btf->hdr->hdr_len + btf->hdr->kind_layout_off;
> +
> +	return 0;
> +}
> +
>   static int btf_type_size(const struct btf_type *t)
>   {
>   	const int base_size = sizeof(struct btf_type);
> @@ -951,7 +1003,8 @@ __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
>   
>   static bool btf_is_modifiable(const struct btf *btf)
>   {
> -	return (void *)btf->hdr != btf->raw_data;
> +	/* BTF is modifiable if split into multiple sections */
> +	return btf->modifiable;
>   }
>   
>   static void btf_free_raw_data(struct btf *btf)
> @@ -980,10 +1033,11 @@ void btf__free(struct btf *btf)
>   		 * might still have a cached contiguous raw data present,
>   		 * which will be unconditionally freed below.
>   		 */
> -		free(btf->hdr);
>   		free(btf->types_data);
>   		strset__free(btf->strs_set);
> +		free(btf->kind_layout);
>   	}
> +	free(btf->hdr);
>   	btf_free_raw_data(btf);
>   	free(btf->raw_data_swapped);
>   	free(btf->type_offs);
> @@ -994,6 +1048,7 @@ void btf__free(struct btf *btf)
>   
>   static struct btf *btf_new_empty(struct btf *base_btf)
>   {
> +	struct btf_header *hdr;
>   	struct btf *btf;
>   
>   	btf = calloc(1, sizeof(*btf));
> @@ -1022,14 +1077,20 @@ static struct btf *btf_new_empty(struct btf *base_btf)
>   		return ERR_PTR(-ENOMEM);
>   	}
>   
> -	btf->hdr = btf->raw_data;
> -	btf->hdr->hdr_len = sizeof(struct btf_header);
> -	btf->hdr->magic = BTF_MAGIC;
> -	btf->hdr->version = BTF_VERSION;
> +	hdr = btf->raw_data;
> +	hdr->hdr_len = sizeof(struct btf_header);
> +	hdr->magic = BTF_MAGIC;
> +	hdr->version = BTF_VERSION;
>   
> -	btf->types_data = btf->raw_data + btf->hdr->hdr_len;
> -	btf->strs_data = btf->raw_data + btf->hdr->hdr_len;
> -	btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */
> +	btf->types_data = btf->raw_data + hdr->hdr_len;
> +	btf->strs_data = btf->raw_data + hdr->hdr_len;
> +	hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */
> +	btf->hdr = calloc(1, sizeof(struct btf_header));
> +	if (!btf->hdr) {
> +		free(btf);
> +		return ERR_PTR(-ENOMEM);
> +	}
> +	memcpy(btf->hdr, hdr, sizeof(*hdr));
>   
>   	return btf;
>   }
> @@ -1078,7 +1139,6 @@ static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, b
>   
>   	btf->raw_size = size;
>   
> -	btf->hdr = btf->raw_data;
>   	err = btf_parse_hdr(btf);
>   	if (err)
>   		goto done;
> @@ -1087,6 +1147,7 @@ static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, b
>   	btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off;
>   
>   	err = btf_parse_str_sec(btf);
> +	err = err ?: btf_parse_kind_layout_sec(btf);
>   	err = err ?: btf_parse_type_sec(btf);
>   	err = err ?: btf_sanity_check(btf);
>   	if (err)
> @@ -1550,6 +1611,11 @@ static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endi
>   	}
>   
>   	data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len;
> +	if (btf->kind_layout) {
> +		data_sz = roundup(data_sz, 4);
> +		data_sz += hdr->kind_layout_len;
> +		hdr->kind_layout_off = roundup(hdr->type_len + hdr->str_len, 4);
> +	}
>   	data = calloc(1, data_sz);
>   	if (!data)
>   		return NULL;
> @@ -1557,7 +1623,7 @@ static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endi
>   
>   	memcpy(p, hdr, hdr->hdr_len);
>   	if (swap_endian)
> -		btf_bswap_hdr(p);
> +		btf_bswap_hdr(p, hdr->hdr_len);
>   	p += hdr->hdr_len;
>   
>   	memcpy(p, btf->types_data, hdr->type_len);
> @@ -1576,7 +1642,11 @@ static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endi
>   	p += hdr->type_len;
>   
>   	memcpy(p, btf_strs_data(btf), hdr->str_len);
> -	p += hdr->str_len;
> +
> +	if (btf->kind_layout) {
> +		p = data + hdr->hdr_len + hdr->kind_layout_off;
> +		memcpy(p, btf->kind_layout, hdr->kind_layout_len);
> +	}
>   
>   	*size = data_sz;
>   	return data;
> @@ -1717,13 +1787,13 @@ static void btf_invalidate_raw_data(struct btf *btf)
>   	}
>   }
>   
> -/* Ensure BTF is ready to be modified (by splitting into a three memory
> - * regions for header, types, and strings). Also invalidate cached
> - * raw_data, if any.
> +/* Ensure BTF is ready to be modified (by splitting into memory regions
> + * for types and strings, with kind layout section if needed (btf->hdr
> + * is already a separate region).  Also invalidate cached raw_data, if any.
>    */
>   static int btf_ensure_modifiable(struct btf *btf)
>   {
> -	void *hdr, *types;
> +	void *types, *kind_layout = NULL;
>   	struct strset *set = NULL;
>   	int err = -ENOMEM;
>   
> @@ -1733,15 +1803,20 @@ static int btf_ensure_modifiable(struct btf *btf)
>   		return 0;
>   	}
>   
> -	/* split raw data into three memory regions */
> -	hdr = malloc(btf->hdr->hdr_len);
> +	/* split raw data into memory regions; btf->hdr is done already. */
>   	types = malloc(btf->hdr->type_len);
> -	if (!hdr || !types)
> +	if (!types)
>   		goto err_out;
> -
> -	memcpy(hdr, btf->hdr, btf->hdr->hdr_len);
>   	memcpy(types, btf->types_data, btf->hdr->type_len);
>   
> +	if (btf->hdr->kind_layout_len && btf->hdr->kind_layout_off) {
> +		kind_layout = malloc(btf->hdr->kind_layout_len);
> +		if (!kind_layout)
> +			goto err_out;
> +		memcpy(kind_layout, btf->raw_data + btf->hdr->hdr_len + btf->hdr->kind_layout_off,
> +		       btf->hdr->kind_layout_len);
> +	}
> +
>   	/* build lookup index for all strings */
>   	set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len);
>   	if (IS_ERR(set)) {
> @@ -1750,11 +1825,12 @@ static int btf_ensure_modifiable(struct btf *btf)
>   	}
>   
>   	/* only when everything was successful, update internal state */
> -	btf->hdr = hdr;
>   	btf->types_data = types;
>   	btf->types_data_cap = btf->hdr->type_len;
>   	btf->strs_data = NULL;
>   	btf->strs_set = set;
> +	if (kind_layout)
> +		btf->kind_layout = kind_layout;
>   	/* if BTF was created from scratch, all strings are guaranteed to be
>   	 * unique and deduplicated
>   	 */
> @@ -1766,12 +1842,14 @@ static int btf_ensure_modifiable(struct btf *btf)
>   	/* invalidate raw_data representation */
>   	btf_invalidate_raw_data(btf);
>   
> +	btf->modifiable = true;
> +
>   	return 0;
>   
>   err_out:
>   	strset__free(set);
> -	free(hdr);
>   	free(types);
> +	free(kind_layout);
>   	return err;
>   }
>