Re: [PATCH 5/5] RFC: Initial iwtrace utility

[James Prestwood <prestwoj@gmail.com>]

Hi Denis,

On 12/26/23 10:09 PM, Denis Kenzior wrote:
> eBPF based tracing, similar to iwmon.  This is meant as an alternative
> to the nlmon based netlink tracing and can be used on kernels where
> nlmon might not be available.
All looks good to me. This seems like a nice utility if the support is 
more widespread than nlmon.
> ---
>   Makefile.am           |  46 ++++++---
>   configure.ac          |  11 +++
>   monitor/iwtrace.bpf.c | 194 +++++++++++++++++++++++++++++++++++++
>   monitor/iwtrace.c     | 218 ++++++++++++++++++++++++++++++++++++++++++
>   monitor/iwtrace.h     |  12 +++
>   5 files changed, 466 insertions(+), 15 deletions(-)
>   create mode 100644 monitor/iwtrace.bpf.c
>   create mode 100644 monitor/iwtrace.c
>   create mode 100644 monitor/iwtrace.h
>
> diff --git a/Makefile.am b/Makefile.am
> index 5ed6ab37164b..601db513d1e3 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -327,23 +327,39 @@ endif
>   endif
>   
>   if MONITOR
> +MONITOR_SOURCES = linux/nl80211.h \
> +				monitor/nlmon.h monitor/nlmon.c \
> +				monitor/pcap.h monitor/pcap.c \
> +				monitor/display.h monitor/display.c \
> +				src/ie.h src/ie.c \
> +				src/wscutil.h src/wscutil.c \
> +				src/mpdu.h src/mpdu.c \
> +				src/util.h src/util.c \
> +				src/crypto.h src/crypto.c \
> +				src/watchlist.h src/watchlist.c \
> +				src/eapolutil.h src/eapolutil.c \
> +				src/nl80211cmd.h src/nl80211cmd.c \
> +				src/p2putil.c src/p2putil.h \
> +				src/anqputil.h src/anqputil.c \
> +				src/band.h src/band.c
>   bin_PROGRAMS += monitor/iwmon
>   
> -monitor_iwmon_SOURCES = monitor/main.c linux/nl80211.h \
> -					monitor/nlmon.h monitor/nlmon.c \
> -					monitor/pcap.h monitor/pcap.c \
> -					monitor/display.h monitor/display.c \
> -					src/ie.h src/ie.c \
> -					src/wscutil.h src/wscutil.c \
> -					src/mpdu.h src/mpdu.c \
> -					src/util.h src/util.c \
> -					src/crypto.h src/crypto.c \
> -					src/watchlist.h src/watchlist.c \
> -					src/eapolutil.h src/eapolutil.c \
> -					src/nl80211cmd.h src/nl80211cmd.c \
> -					src/p2putil.c src/p2putil.h \
> -					src/anqputil.h src/anqputil.c \
> -					src/band.h src/band.c
> +if BPF
> +bin_PROGRAMS += monitor/iwtrace
> +
> +monitor/iwtrace.bpf.o: monitor/iwtrace.bpf.c
> +	clang -g -O2 -target bpf -c monitor/iwtrace.bpf.c -o monitor/iwtrace.tmp.bpf.o
> +	bpftool gen object monitor/iwtrace.bpf.o monitor/iwtrace.tmp.bpf.o
> +
> +monitor/iwtrace.skel.h: monitor/iwtrace.bpf.o
> +	bpftool gen skeleton monitor/iwtrace.bpf.o > monitor/iwtrace.skel.h
> +
> +monitor_iwtrace_SOURCES = monitor/iwtrace.c monitor/iwtrace.h \
> +				monitor/iwtrace.skel.h $(MONITOR_SOURCES)
> +monitor_iwtrace_LDADD = $(ell_ldadd) $(LIBBPF_LIBS)
> +endif
> +
> +monitor_iwmon_SOURCES = monitor/main.c $(MONITOR_SOURCES)
>   monitor_iwmon_LDADD = $(ell_ldadd)
>   
>   if MANUAL_PAGES
> diff --git a/configure.ac b/configure.ac
> index d84e035c2f5f..bb7bfedcf80f 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -200,6 +200,17 @@ AC_ARG_ENABLE([monitor], AS_HELP_STRING([--disable-monitor],
>   					[enable_monitor=${enableval}])
>   AM_CONDITIONAL(MONITOR, test "${enable_monitor}" != "no")
>   
> +AC_ARG_ENABLE([bpf], AS_HELP_STRING([--disable-bpf],
> +				[don't enable bpf based tracing]),
> +					[enable_bpf=${enableval}])
> +if (test "${enable_bpf}" != "no"); then
> +	PKG_CHECK_MODULES(LIBBPF, libbpf, dummy=yes,
> +			AC_MSG_ERROR(libbpf is required))
> +	AC_SUBST(LIBBPF_CFLAGS)
> +	AC_SUBST(LIBBPF_LIBS)
> +fi
> +AM_CONDITIONAL(BPF, test "${enable_bpf}" != "no")
> +
>   AC_ARG_ENABLE([dbus-policy], AS_HELP_STRING([--disable-dbus-policy],
>   				[don't install D-Bus system policy files]),
>   					[enable_dbus_policy=${enableval}])
> diff --git a/monitor/iwtrace.bpf.c b/monitor/iwtrace.bpf.c
> new file mode 100644
> index 000000000000..251b54fc2a41
> --- /dev/null
> +++ b/monitor/iwtrace.bpf.c
> @@ -0,0 +1,194 @@
> +/*
> + * Embedded Linux library
> + * Copyright (C) 2023  Cruise, LLC
> + *
> + * SPDX-License-Identifier: GPL-2.0-only
> + */
> +
> +#include <stdint.h>
> +#include <string.h>
> +#include <linux/types.h>
> +#include <linux/bpf.h>
> +#include <linux/netlink.h>
> +#include <bpf/bpf_helpers.h>
> +#include <bpf/bpf_tracing.h>
> +#include <bpf/bpf_core_read.h>
> +#include "iwtrace.h"
> +
> +char LICENSE[] SEC("license") = "GPL";
> +
> +struct sock;
> +struct netlink_ext_ack;
> +
> +struct {
> +	__uint(type, BPF_MAP_TYPE_RINGBUF);
> +	__uint(max_entries, 256 * 1024);
> +} rb SEC(".maps");
> +
> +struct sock {
> +	uint16_t sk_protocol;
> +};
> +
> +struct sk_buff {
> +	unsigned int len, data_len;
> +	unsigned char *data;
> +	struct net_device *dev;
> +	union {
> +		struct sock		*sk;
> +		int			ip_defrag_offset;
> +	};
> +};
> +
> +struct capture_256 {
> +	struct metadata meta;
> +	uint8_t packet[256 - sizeof(struct metadata)];
> +} __attribute__ ((packed));
> +
> +struct capture_1k {
> +	struct metadata meta;
> +	uint8_t packet[1024 - sizeof(struct metadata)];
> +} __attribute__ ((packed));
> +
> +struct capture_4k {
> +	struct metadata meta;
> +	uint8_t packet[4096 - sizeof(struct metadata)];
> +} __attribute__ ((packed));
> +
> +struct capture_8k {
> +	struct metadata meta;
> +	uint8_t packet[8192 - sizeof(struct metadata)];
> +} __attribute__ ((packed));
> +
> +struct capture_16k {
> +	struct metadata meta;
> +	uint8_t packet[16384];
> +} __attribute__ ((packed));
> +
> +static void metadata_fill(struct metadata *meta, const struct sk_buff *skb)
> +{
> +	struct sock *sk = __builtin_preserve_access_index(skb->sk);
> +
> +	meta->timestamp = bpf_ktime_get_boot_ns();
> +	meta->len = __builtin_preserve_access_index(skb->len);
> +	meta->protocol = __builtin_preserve_access_index(sk->sk_protocol);
> +}
> +
> +static int capture_common(const struct sk_buff *skb)
> +{
> +	uint16_t len = __builtin_preserve_access_index(skb->len);
> +	const void *data = __builtin_preserve_access_index(skb->data);
> +
> +	/*
> +	 * bpf_ringbuf_reserve is currently limited to a known constant
> +	 * value, and cannot handle values that are not constant (even if
> +	 * bounded).  bpf_ringbuf_output might be suitable, but no metadata
> +	 * could be prepended if that is used.  Another alternative is to use
> +	 * a perf buffer, but it is per-CPU and might result in packets being
> +	 * processed out of order.  We trick the validator by using several
> +	 * well known structure sizes (256/1k/4k/8k/16k) in order to save on
> +	 * memory space, but the resultant program is larger than it would be
> +	 * if dynamic sizing was supported.
> +	 */
> +	if (len <= 256 - sizeof(struct metadata)) {
> +		struct capture_256 *c256 = bpf_ringbuf_reserve(&rb,
> +					sizeof(struct capture_256), 0);
> +
> +		if (!c256)
> +			return 0;
> +
> +		metadata_fill(&c256->meta, skb);
> +
> +		if (bpf_probe_read_kernel(c256->packet, len, data) < 0)
> +			bpf_ringbuf_discard(c256, BPF_RB_NO_WAKEUP);
> +		else
> +			bpf_ringbuf_submit(c256, 0);
> +
> +		return 0;
> +	}
> +
> +	if (len <= sizeof(struct capture_1k) - sizeof(struct metadata)) {
> +		struct capture_1k *c1k = bpf_ringbuf_reserve(&rb,
> +						sizeof(struct capture_1k), 0);
> +
> +		if (!c1k)
> +			return 0;
> +
> +		metadata_fill(&c1k->meta, skb);
> +
> +		if (bpf_probe_read_kernel(c1k->packet, len, data) < 0)
> +			bpf_ringbuf_discard(c1k, BPF_RB_NO_WAKEUP);
> +		else
> +			bpf_ringbuf_submit(c1k, 0);
> +
> +		return 0;
> +	}
> +
> +	if (len <= sizeof(struct capture_4k) - sizeof(struct metadata)) {
> +		struct capture_4k *c4k = bpf_ringbuf_reserve(&rb,
> +						sizeof(struct capture_4k), 0);
> +
> +		if (!c4k)
> +			return 0;
> +
> +		metadata_fill(&c4k->meta, skb);
> +
> +		if (bpf_probe_read_kernel(c4k->packet, len, data) < 0)
> +			bpf_ringbuf_discard(c4k, BPF_RB_NO_WAKEUP);
> +		else
> +			bpf_ringbuf_submit(c4k, 0);
> +
> +		return 0;
> +	}
> +
> +	if (len <= sizeof(struct capture_8k) - sizeof(struct metadata)) {
> +		struct capture_8k *c8k = bpf_ringbuf_reserve(&rb,
> +						sizeof(struct capture_8k), 0);
> +
> +		if (!c8k)
> +			return 0;
> +
> +		metadata_fill(&c8k->meta, skb);
> +
> +		if (bpf_probe_read_kernel(c8k->packet, len, data) < 0)
> +			bpf_ringbuf_discard(c8k, BPF_RB_NO_WAKEUP);
> +		else
> +			bpf_ringbuf_submit(c8k, 0);
> +
> +		return 0;
> +	}
> +
> +	/* 16384 is the largest packet size for genl currently */
> +	if (len <= 16384) {
> +		struct capture_16k *c16k = bpf_ringbuf_reserve(&rb,
> +						sizeof(struct capture_16k), 0);
> +
> +		if (!c16k)
> +			return 0;
> +
> +		metadata_fill(&c16k->meta, skb);
> +
> +		if (bpf_probe_read_kernel(c16k->packet, len, data) < 0)
> +			bpf_ringbuf_discard(c16k, BPF_RB_NO_WAKEUP);
> +		else
> +			bpf_ringbuf_submit(c16k, 0);
> +
> +		return 0;
> +	}
> +
> +	return 0;
> +}
> +
> +SEC("fentry/__netlink_sendskb")
> +int BPF_PROG(trace___netlink_sendskb, struct sock *sk, struct sk_buff *skb)
> +{
> +	return capture_common(skb);
> +}
> +
> +SEC("fentry/netlink_rcv_skb")
> +int BPF_PROG(trace_netlink_rcv_skb, struct sk_buff *skb,
> +					int (*cb)(struct sk_buff *,
> +                                                   struct nlmsghdr *,
> +                                                   struct netlink_ext_ack *))
> +{
> +	return capture_common(skb);
> +}
> diff --git a/monitor/iwtrace.c b/monitor/iwtrace.c
> new file mode 100644
> index 000000000000..ceff71753463
> --- /dev/null
> +++ b/monitor/iwtrace.c
> @@ -0,0 +1,218 @@
> +/*
> + * Copyright (C) 2023  Cruise, LLC
> + *
> + * SPDX-License-Identifier: LGPL-2.1-or-later
> + */
> +
> +#ifdef HAVE_CONFIG_H
> +#include <config.h>
> +#endif
> +
> +#define _GNU_SOURCE
> +#include <argp.h>
> +#include <signal.h>
> +#include <stdio.h>
> +#include <time.h>
> +#include <inttypes.h>
> +#include <arpa/inet.h>
> +#include <sys/resource.h>
> +#include <bpf/libbpf.h>
> +#include <linux/rtnetlink.h>
> +#include "iwtrace.h"
> +#include "iwtrace.skel.h"
> +#include "monitor/nlmon.h"
> +#include "monitor/pcap.h"
> +#include "monitor/display.h"
> +
> +#include <ell/ell.h>
> +
> +static struct iwtrace_bpf *skel;
> +static struct ring_buffer *rb;
> +static struct l_io *io;
> +static struct l_genl *genl;
> +static struct nlmon_config config;
> +static const char *writer_path;
> +static struct nlmon *nlmon;
> +
> +#ifndef ARPHRD_NETLINK
> +#define ARPHRD_NETLINK	824
> +#endif
> +
> +static int libbpf_print_fn(enum libbpf_print_level level,
> +				const char *format, va_list args)
> +{
> +	return vfprintf(stderr, format, args);
> +}
> +
> +static int handle_packet(void *ctx, void *data, size_t size)
> +{
> +	struct metadata *meta = data;
> +	struct timeval tv;
> +
> +	data += sizeof(struct metadata);
> +
> +	tv.tv_sec = meta->timestamp / L_NSEC_PER_SEC;
> +	tv.tv_usec = (meta->timestamp % L_NSEC_PER_SEC) / L_NSEC_PER_USEC;
> +
> +	switch (meta->protocol) {
> +	case NETLINK_ROUTE:
> +		nlmon_print_rtnl(nlmon, &tv, data, meta->len);
> +		break;
> +	case NETLINK_GENERIC:
> +		nlmon_print_genl(nlmon, &tv, data, meta->len);
> +		break;
> +	}
> +
> +	return 0;
> +}
> +
> +static bool ringbuf_receive(struct l_io *io, void *user_data)
> +{
> +	ring_buffer__poll(rb, 0);
> +
> +	return true;
> +}
> +
> +static void nl80211_appeared(const struct l_genl_family_info *info,
> +					void *user_data)
> +{
> +	int err;
> +
> +	err = iwtrace_bpf__attach(skel);
> +	if (err) {
> +		fprintf(stderr, "Unable to attach eBPF program\n");
> +		goto failed;
> +	}
> +
> +	rb = ring_buffer__new(bpf_map__fd(skel->maps.rb),
> +					handle_packet, NULL, NULL);
> +	if (!rb) {
> +		fprintf(stderr, "Failed to create ringbuffer\n");
> +		goto failed;
> +	}
> +
> +	nlmon = nlmon_open(l_genl_family_info_get_id(info),
> +						writer_path, &config);
> +	if (!nlmon)
> +		goto failed;
> +
> +	io = l_io_new(bpf_map__fd(skel->maps.rb));
> +	l_io_set_close_on_destroy(io, false);
> +	l_io_set_read_handler(io, ringbuf_receive, NULL, NULL);
> +
> +	return;
> +failed:
> +	l_main_quit();
> +}
> +
> +static void signal_handler(uint32_t signo, void *user_data)
> +{
> +	switch (signo) {
> +	case SIGINT:
> +	case SIGTERM:
> +		l_main_quit();
> +		break;
> +	}
> +}
> +
> +static void usage(void)
> +{
> +	printf("iwtrace - Wireless monitor using eBPF\n"
> +		"Usage:\n");
> +	printf("\tiwtrace [options]\n");
> +	printf("Options:\n"
> +		"\t-w, --write <file>     Write netlink PCAP trace file\n"
> +		"\t-n, --nortnl           Don't show RTNL output\n"
> +		"\t-y, --nowiphy          Don't show 'New Wiphy' output\n"
> +		"\t-s, --noscan           Don't show scan result output\n"
> +		"\t-e, --noies            Don't show IEs except SSID\n"
> +		"\t-h, --help             Show help options\n");
> +}
> +
> +static const struct option main_options[] = {
> +	{ "write",     required_argument, NULL, 'w' },
> +	{ "nortnl",    no_argument,       NULL, 'n' },
> +	{ "nowiphy",   no_argument,       NULL, 'y' },
> +	{ "noscan",    no_argument,       NULL, 's' },
> +	{ "noies",     no_argument,       NULL, 'e' },
> +	{ "version",   no_argument,       NULL, 'v' },
> +	{ "help",      no_argument,       NULL, 'h' },
> +	{ }
> +};
> +
> +int main(int argc, char *argv[])
> +{
> +	int exit_status = EXIT_FAILURE;
> +
> +	for (;;) {
> +		int opt;
> +
> +		opt = getopt_long(argc, argv, "w:nvhyse",
> +						main_options, NULL);
> +		if (opt < 0)
> +			break;
> +
> +		switch (opt) {
> +		case 'w':
> +			writer_path = optarg;
> +			break;
> +		case 'n':
> +			config.nortnl = true;
> +			break;
> +		case 'y':
> +			config.nowiphy = true;
> +			break;
> +		case 's':
> +			config.noscan = true;
> +			break;
> +		case 'e':
> +			config.noies = true;
> +			break;
> +		case 'v':
> +			printf("%s\n", VERSION);
> +			return EXIT_SUCCESS;
> +		case 'h':
> +			usage();
> +			return EXIT_SUCCESS;
> +		default:
> +			return EXIT_FAILURE;
> +		}
> +	}
> +
> +	if (argc - optind > 0) {
> +		fprintf(stderr, "Invalid command line parameters\n");
> +		return EXIT_FAILURE;
> +	}
> +
> +	libbpf_set_print(libbpf_print_fn);
> +
> +	skel = iwtrace_bpf__open_and_load();
> +	if (!skel)
> +		return EXIT_FAILURE;
> +
> +	fprintf(stdout, "Wireless monitor (eBPF) ver %s\n", VERSION);
> +
> +	if (!l_main_init())
> +		goto init_failed;
> +
> +	genl = l_genl_new();
> +	if (!genl) {
> +		fprintf(stderr, "Failed to open generic netlink socket\n");
> +		goto genl_failed;
> +	}
> +
> +	l_genl_request_family(genl, "nl80211", nl80211_appeared, NULL, NULL);
> +	exit_status = l_main_run_with_signal(signal_handler, NULL);
> +
> +	l_genl_unref(genl);
> +	nlmon_close(nlmon);
> +	l_io_destroy(io);
> +	ring_buffer__free(rb);
> +
> +genl_failed:
> +	l_main_exit();
> +init_failed:
> +	iwtrace_bpf__destroy(skel);
> +
> +	return exit_status;
> +}
> diff --git a/monitor/iwtrace.h b/monitor/iwtrace.h
> new file mode 100644
> index 000000000000..0df95f2ea989
> --- /dev/null
> +++ b/monitor/iwtrace.h
> @@ -0,0 +1,12 @@
> +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
> +/* Copyright (c) 2020 Facebook */
> +#ifndef __BOOTSTRAP_H
> +#define __BOOTSTRAP_H
> +
> +struct metadata {
> +	uint64_t timestamp;
> +	uint16_t len;
> +	uint16_t protocol;
> +} __attribute__ ((packed));
> +
> +#endif /* __BOOTSTRAP_H */