* [PATCH bpf-next v7 1/2] bpftool: Use batch lookups for hash map dumps
2026-11-03 19:01 [PATCH bpf-next v7 0/2] bpftool: Batch hash map dumps with bounded buffers Tianyi Chen
@ 2026-11-03 19:01 ` Tianyi Chen
2026-11-04 2:26 ` Alexei Starovoitov
2026-11-03 19:01 ` [PATCH bpf-next v7 2/2] selftests/bpf: Check bpftool batch map dump contents Tianyi Chen
1 sibling, 1 reply; 4+ messages in thread
From: Tianyi Chen @ 2026-11-03 19:01 UTC (permalink / raw)
To: bpf; +Cc: qmo, andrii, eddyz87, ihor.solodrai, linux-kselftest, daniel, ast,
hi
Use BPF_MAP_LOOKUP_BATCH for ordinary hash maps to reduce BPF syscalls
while preserving plain, JSON and BTF output.
Bound the combined key/value buffers to 4 MiB. Start with up to 256
entries and grow on ENOSPC using the same input cursor. At the buffer
limit, continue individual lookups from the last emitted key without
restarting output. Also fall back if the initial batch is unsupported.
Process the final partial batch on ENOENT, but do not read output buffers
on other errors. Keep fatal diagnostics on stderr so JSON arrays contain
only map entries. Traversal remains non-atomic.
For a 100,000-entry map with 4-byte keys and values, median wall times
with stdout redirected to /dev/null were:
baseline batch
plain 87.460 ms 55.677 ms
JSON 78.495 ms 47.157 ms
Both builds used GCC 16.2.1 in an x86-64 KVM guest running
7.3.0-4.cachyos1.fc44.x86_64, pinned to one guest CPU. After two warm-ups,
15 untraced samples per binary and mode alternated baseline and patched
runs. Separate strace runs counted 200,005 versus 396 BPF syscalls for a
plain dump. These results are specific to this fixture and guest.
Link: https://github.com/libbpf/bpftool/issues/63
Acked-by: Quentin Monnet <qmo@kernel.org>
Assisted-by: LLM
Signed-off-by: Tianyi Chen <hi@tychen.cc>
---
tools/bpf/bpftool/map.c | 123 +++++++++++++++++++++++++++++++++++++---
1 file changed, 115 insertions(+), 8 deletions(-)
diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c
index 4b65edf74859b..fdd32cd40709b 100644
--- a/tools/bpf/bpftool/map.c
+++ b/tools/bpf/bpftool/map.c
@@ -741,15 +741,10 @@ static int do_show(int argc, char **argv)
return errno == ENOENT ? 0 : -1;
}
-static int dump_map_elem(int fd, void *key, void *value,
- struct bpf_map_info *map_info, struct btf *btf,
- json_writer_t *btf_wtr, const int *cpu_ids, int cpu_cnt)
+static void print_map_elem(void *key, void *value,
+ struct bpf_map_info *map_info, struct btf *btf,
+ json_writer_t *btf_wtr, const int *cpu_ids, int cpu_cnt)
{
- if (bpf_map_lookup_elem(fd, key, value)) {
- print_entry_error(map_info, key, errno);
- return -1;
- }
-
if (json_output) {
print_entry_json(map_info, key, value, btf, cpu_ids, cpu_cnt);
} else if (btf) {
@@ -763,10 +758,113 @@ static int dump_map_elem(int fd, void *key, void *value,
} else {
print_entry_plain(map_info, key, value, cpu_ids, cpu_cnt);
}
+}
+
+static int dump_map_elem(int fd, void *key, void *value,
+ struct bpf_map_info *map_info, struct btf *btf,
+ json_writer_t *btf_wtr, const int *cpu_ids, int cpu_cnt)
+{
+ if (bpf_map_lookup_elem(fd, key, value)) {
+ print_entry_error(map_info, key, errno);
+ return -1;
+ }
+ print_map_elem(key, value, map_info, btf, btf_wtr, cpu_ids, cpu_cnt);
return 0;
}
+#define MAP_DUMP_BATCH_FALLBACK 1
+#define MAP_DUMP_BATCH_SIZE 256U
+#define MAP_DUMP_BATCH_MAX_BYTES (4 * 1024 * 1024)
+
+static bool map_dump_can_batch(const struct bpf_map_info *info)
+{
+ return info->type == BPF_MAP_TYPE_HASH && info->max_entries &&
+ (__u64)info->key_size + info->value_size <= MAP_DUMP_BATCH_MAX_BYTES;
+}
+
+/* Return MAP_DUMP_BATCH_FALLBACK to continue with individual lookups. */
+static int dump_map_batch(int fd, void *key, void *value,
+ struct bpf_map_info *info, struct btf *btf,
+ json_writer_t *wtr, unsigned int *num_elems)
+{
+ /* HASH batch cursors are u32 bucket indices, independent of key size. */
+ __u32 batch = 0, next_batch = 0;
+ __u32 capacity, max_capacity, count, i;
+ void *keys = NULL, *values = NULL, *buf;
+ bool first = true, can_restart = true;
+ int err;
+
+ max_capacity = MAP_DUMP_BATCH_MAX_BYTES / (info->key_size + info->value_size);
+ max_capacity = min(max_capacity, info->max_entries);
+ capacity = min(max_capacity, MAP_DUMP_BATCH_SIZE);
+resize:
+ buf = realloc(keys, (size_t)capacity * info->key_size);
+ if (!buf) {
+ err = ENOMEM;
+ goto error;
+ }
+ keys = buf;
+ buf = realloc(values, (size_t)capacity * info->value_size);
+ if (!buf) {
+ err = ENOMEM;
+ goto error;
+ }
+ values = buf;
+
+ while (true) {
+ count = capacity;
+ err = bpf_map_lookup_batch(fd, first ? NULL : &batch,
+ &next_batch, keys, values, &count, NULL);
+ err = err ? errno : 0;
+ /*
+ * Older kernels reject the command before updating count. Do not
+ * inspect the buffers on these errors, or restart after progress.
+ */
+ if (can_restart && (err == EINVAL || err == EOPNOTSUPP ||
+ err == 524 /* ENOTSUPP */)) {
+ err = MAP_DUMP_BATCH_FALLBACK;
+ goto out;
+ }
+ can_restart = false;
+ if (err == ENOSPC) {
+ if (capacity == max_capacity) {
+ err = MAP_DUMP_BATCH_FALLBACK;
+ goto out;
+ }
+ capacity += min(capacity, max_capacity - capacity);
+ /* Preserve the input cursor: the oversized bucket was not read. */
+ goto resize;
+ }
+ /* In particular, EFAULT can leave count and the buffers invalid. */
+ if (err && err != ENOENT)
+ goto error;
+ for (i = 0; i < count; i++) {
+ /*
+ * Keep the alignment provided by individual lookups, including
+ * for BTF types whose map key/value size is not aligned.
+ */
+ memcpy(key, keys + (size_t)i * info->key_size, info->key_size);
+ memcpy(value, values + (size_t)i * info->value_size, info->value_size);
+ print_map_elem(key, value, info, btf, wtr, NULL, 0);
+ (*num_elems)++;
+ }
+ if (err == ENOENT) {
+ err = 0;
+ goto out;
+ }
+ first = false;
+ batch = next_batch;
+ }
+error:
+ fprintf(stderr, "Error: can't lookup map batch: %s\n", bpf_strerror(err));
+ err = -1;
+out:
+ free(keys);
+ free(values);
+ return err;
+}
+
static int maps_have_btf(int *fds, int nb_fds)
{
struct bpf_map_info info = {};
@@ -895,6 +993,14 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr,
p_info("Warning: cannot read values from %s map with value_size != 8",
map_type_str);
}
+ if (map_dump_can_batch(info)) {
+ err = dump_map_batch(fd, key, value, info, btf, wtr, &num_elems);
+ if (err != MAP_DUMP_BATCH_FALLBACK)
+ goto end_dump;
+ /* Continue after the last emitted key rather than restarting. */
+ if (num_elems)
+ prev_key = key;
+ }
while (true) {
err = bpf_map_get_next_key(fd, prev_key, key);
if (err) {
@@ -911,6 +1017,7 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr,
prev_key = key;
}
+end_dump:
if (wtr) {
jsonw_end_array(wtr); /* elements */
if (show_header)
--
2.56.0
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH bpf-next v7 2/2] selftests/bpf: Check bpftool batch map dump contents
2026-11-03 19:01 [PATCH bpf-next v7 0/2] bpftool: Batch hash map dumps with bounded buffers Tianyi Chen
2026-11-03 19:01 ` [PATCH bpf-next v7 1/2] bpftool: Use batch lookups for hash map dumps Tianyi Chen
@ 2026-11-03 19:01 ` Tianyi Chen
1 sibling, 0 replies; 4+ messages in thread
From: Tianyi Chen @ 2026-11-03 19:01 UTC (permalink / raw)
To: bpf; +Cc: qmo, andrii, eddyz87, ihor.solodrai, linux-kselftest, daniel, ast,
hi
Exercise hash map dumps around the initial batch size and across
multiple batches, including empty and single-entry maps. Compare each
complete unordered key/value set with the input data in plain, JSON
and pretty JSON output.
Cover one-byte keys, three-byte values and BTF-formatted maps to catch
cursor sizing, buffer alignment and formatting regressions.
Also cover a sparse hash map whose maximum capacity exceeds the batch
buffer budget.
Acked-by: Quentin Monnet <qmo@kernel.org>
Assisted-by: LLM
Signed-off-by: Tianyi Chen <hi@tychen.cc>
---
.../bpf/prog_tests/bpftool_map_batch.c | 191 ++++++++++++++++++
1 file changed, 191 insertions(+)
create mode 100644 tools/testing/selftests/bpf/prog_tests/bpftool_map_batch.c
diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_map_batch.c b/tools/testing/selftests/bpf/prog_tests/bpftool_map_batch.c
new file mode 100644
index 0000000000000..5387de2ba79fd
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/bpftool_map_batch.c
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+#include <bpftool_helpers.h>
+#include <bpf/btf.h>
+#include <ctype.h>
+
+#define MAX_ENTRIES 1025
+#define RECORD_SIZE 256
+#define OUTPUT_SIZE (MAX_ENTRIES * RECORD_SIZE + 1024)
+
+struct dump_case {
+ const char *name;
+ unsigned int count;
+ unsigned int key_size;
+ unsigned int value_size;
+ bool btf;
+ unsigned int max_entries;
+};
+
+static void hex_bytes(char *out, const void *data, unsigned int size, bool json)
+{
+ const unsigned char *bytes = data;
+ unsigned int i;
+
+ if (json)
+ *out++ = '[';
+ for (i = 0; i < size; i++) {
+ if (json && i)
+ *out++ = ',';
+ out += sprintf(out, json ? "\"0x%02x\"" : "%02x", bytes[i]);
+ }
+ if (json)
+ *out++ = ']';
+ *out = '\0';
+}
+
+static void expected_record(char *record, const struct dump_case *test,
+ unsigned int index, bool json)
+{
+ __u32 key = index, value = index * 37 + 11;
+ unsigned char short_key = index;
+ char key_hex[64], value_hex[64], formatted[96];
+
+ hex_bytes(key_hex, test->key_size == 1 ? (void *)&short_key : &key,
+ test->key_size, json);
+ hex_bytes(value_hex, &value, test->value_size, json);
+ snprintf(formatted, sizeof(formatted), "{\"key\":%u,\"value\":%u}",
+ key, value);
+ if (json && test->btf)
+ snprintf(record, RECORD_SIZE,
+ "{\"key\":%s,\"value\":%s,\"formatted\":%s}",
+ key_hex, value_hex, formatted);
+ else if (json)
+ snprintf(record, RECORD_SIZE, "{\"key\":%s,\"value\":%s}",
+ key_hex, value_hex);
+ else if (test->btf)
+ snprintf(record, RECORD_SIZE, "%s", formatted);
+ else
+ snprintf(record, RECORD_SIZE, "key:%svalue:%s", key_hex, value_hex);
+}
+
+static void check_dump(const struct dump_case *test, __u32 id, bool json, bool pretty)
+{
+ bool array = json || test->btf;
+ char command[MAX_BPFTOOL_CMD_LEN], expected[RECORD_SIZE], footer[64];
+ bool seen[MAX_ENTRIES] = {};
+ char *output, *src, *dst, *cursor;
+ unsigned int i, n;
+ int err;
+
+ output = calloc(1, OUTPUT_SIZE);
+ if (!ASSERT_OK_PTR(output, "alloc_output"))
+ return;
+ snprintf(command, sizeof(command), "%smap dump id %u",
+ pretty ? "-p " : json ? "-j " : "", id);
+ err = get_bpftool_command_output(command, output, OUTPUT_SIZE);
+ if (!ASSERT_OK(err, "map_dump"))
+ goto out;
+ /*
+ * Ignore presentation whitespace, but compare complete records and all
+ * punctuation. Expected contents come only from the input data, never
+ * from another map walk or bpftool invocation.
+ */
+ for (src = output, dst = output; *src; src++)
+ if (!isspace((unsigned char)*src))
+ *dst++ = *src;
+ *dst = '\0';
+ cursor = output;
+ if (array) {
+ if (!ASSERT_EQ(*cursor, '[', "array_start"))
+ goto out;
+ cursor++;
+ }
+ for (n = 0; n < test->count; n++) {
+ if (array && n) {
+ if (!ASSERT_EQ(*cursor, ',', "record_separator"))
+ goto out;
+ cursor++;
+ }
+ for (i = 0; i < test->count; i++) {
+ if (seen[i])
+ continue;
+ expected_record(expected, test, i, json);
+ if (!strncmp(cursor, expected, strlen(expected)))
+ break;
+ }
+ if (!ASSERT_LT(i, test->count, "unique_expected_record"))
+ goto out;
+ seen[i] = true;
+ cursor += strlen(expected);
+ }
+ if (array) {
+ ASSERT_STREQ(cursor, "]", "array_end_and_count");
+ } else {
+ snprintf(footer, sizeof(footer), "Found%uelement%s", test->count,
+ test->count == 1 ? "" : "s");
+ ASSERT_STREQ(cursor, footer, "plain_count");
+ }
+out:
+ free(output);
+}
+
+static void run_dump_case(const struct dump_case *test)
+{
+ LIBBPF_OPTS(bpf_map_create_opts, opts);
+ struct bpf_map_info info = {};
+ __u32 info_len = sizeof(info);
+ struct btf *btf = NULL;
+ unsigned int i;
+ int fd = -1;
+
+ if (test->btf) {
+ btf = btf__new_empty();
+ if (!ASSERT_OK_PTR(btf, "btf_new"))
+ return;
+ if (!ASSERT_EQ(btf__add_int(btf, "unsigned int", 4, 0), 1,
+ "btf_int") ||
+ !ASSERT_OK(btf__load_into_kernel(btf), "btf_load"))
+ goto out;
+ opts.btf_fd = btf__fd(btf);
+ opts.btf_key_type_id = 1;
+ opts.btf_value_type_id = 1;
+ }
+ if (test->max_entries)
+ opts.map_flags = BPF_F_NO_PREALLOC;
+ fd = bpf_map_create(BPF_MAP_TYPE_HASH, "dump_batch", test->key_size,
+ test->value_size, test->max_entries ?: (test->count ?: 1), &opts);
+ if (!ASSERT_OK_FD(fd, "map_create"))
+ goto out;
+ for (i = 0; i < test->count; i++) {
+ __u32 key = i, value = i * 37 + 11;
+ unsigned char short_key = i;
+ void *key_ptr = test->key_size == 1 ? (void *)&short_key : &key;
+
+ if (!ASSERT_OK(bpf_map_update_elem(fd, key_ptr, &value, BPF_ANY),
+ "map_update"))
+ goto out;
+ }
+ if (!ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &info_len), "map_info"))
+ goto out;
+ check_dump(test, info.id, false, false);
+ check_dump(test, info.id, true, false);
+ check_dump(test, info.id, true, true);
+out:
+ if (fd >= 0)
+ close(fd);
+ btf__free(btf);
+}
+
+void test_bpftool_map_batch(void)
+{
+ static const struct dump_case cases[] = {
+ { "empty", 0, 4, 4 },
+ { "single", 1, 4, 4 },
+ { "below_batch", 255, 4, 4 },
+ { "exact_batch", 256, 4, 4 },
+ { "above_batch", 257, 4, 4 },
+ { "multiple_batches", 1025, 4, 4 },
+ { "large_sparse", 1025, 4, 4, false, 1U << 20 },
+ { "one_byte_key", 256, 1, 4 },
+ { "odd_value_size", 257, 4, 3 },
+ { "btf_empty", 0, 4, 4, true },
+ { "btf_single", 1, 4, 4, true },
+ { "btf_multiple_batches", 1025, 4, 4, true },
+ };
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(cases); i++)
+ if (test__start_subtest(cases[i].name))
+ run_dump_case(&cases[i]);
+}
--
2.56.0
^ permalink raw reply related [flat|nested] 4+ messages in thread