Re: [PATCH v3] libbpf: kprobe.multi: Filter with available_filter_functions

[Jiri Olsa <olsajiri@gmail.com>]

On Wed, May 24, 2023 at 04:41:54PM +0800, Jackie Liu wrote:
> From: Jackie Liu <liuyun01@kylinos.cn>
> 
> When using regular expression matching with "kprobe multi", it scans all
> the functions under "/proc/kallsyms" that can be matched. However, not all
> of them can be traced by kprobe.multi. If any one of the functions fails
> to be traced, it will result in the failure of all functions. The best
> approach is to filter out the functions that cannot be traced to ensure
> proper tracking of the functions.
> 
> Use available_filter_functions check first, if failed, fallback to
> kallsyms.
> 
> Here is the test eBPF program [1].
> [1] https://github.com/JackieLiu1/ketones/commit/a9e76d1ba57390e533b8b3eadde97f7a4535e867
> 
> Signed-off-by: Jackie Liu <liuyun01@kylinos.cn>
> ---
>  v1: 0.27s user 5.09s system 99% cpu 5.392 total
>  v2: 0.37s user 1.54s system 98% cpu 1.947 total
>  v3: 0.10s user 0.98s system 97% cpu 1.107 total
> 
>  I saw that reading available_filter_functions takes 0.98s and kallsyms only
>  takes 0.12s.  There is a big difference in performance between them.
> 
>  tools/lib/bpf/libbpf.c          | 80 ++++++++++++++++++++++++++++++---
>  tools/lib/bpf/libbpf_internal.h |  4 +-
>  2 files changed, 77 insertions(+), 7 deletions(-)
> 
> diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
> index ad1ec893b41b..f3e3c92bdf8a 100644
> --- a/tools/lib/bpf/libbpf.c
> +++ b/tools/lib/bpf/libbpf.c
> @@ -10417,13 +10417,14 @@ static bool glob_match(const char *str, const char *pat)
>  struct kprobe_multi_resolve {
>  	const char *pattern;
>  	unsigned long *addrs;
> +	const char **syms;
>  	size_t cap;
>  	size_t cnt;
>  };
>  
>  static int
> -resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
> -			const char *sym_name, void *ctx)
> +kallsyms_resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
> +				 const char *sym_name, void *ctx)
>  {
>  	struct kprobe_multi_resolve *res = ctx;
>  	int err;
> @@ -10440,6 +10441,69 @@ resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
>  	return 0;
>  }
>  
> +static int resolve_kprobe_multi_cb(const char *sym_name, void *ctx)
> +{
> +	struct kprobe_multi_resolve *res = ctx;
> +	int err;
> +
> +	if (!glob_match(sym_name, res->pattern))
> +		return 0;
> +
> +	err = libbpf_ensure_mem((void **) &res->syms, &res->cap, sizeof(const char *),
> +				res->cnt + 1);
> +	if (err)
> +		return err;
> +
> +	res->syms[res->cnt++] = strdup(sym_name);

we need to check the strdup return value

> +	return 0;
> +}
> +
> +int libbpf_available_filter_functions_parse(available_filter_functions_cb_t cb,
> +					    void *ctx)

might be too long, maybe we could just use 'ftrace' instead, so we'd have:

   libbpf_ftrace_parse
   libbpf_kallsyms_parse

that might be too terse, but AFAIK we don't parse any other ftrace file

> +{
> +	char sym_name[256];
> +	FILE *f;
> +	int ret, err = 0;
> +
> +	f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
> +	if (!f) {
> +		pr_warn("failed to open available_filter_functions, fallback to /proc/kallsyms.\n");
> +		goto fallback;
> +	}
> +
> +	while (true) {
> +		ret = fscanf(f, "%s%*[^\n]\n", sym_name);
> +		if (ret == EOF && feof(f))
> +			break;
> +		if (ret != 1) {
> +			pr_warn("failed to read available_filter_functions entry: %d\n",
> +				ret);
> +			break;
> +		}
> +
> +		err = cb(sym_name, ctx);
> +		if (err)
> +			break;
> +	}
> +
> +	fclose(f);
> +	return err;
> +
> +fallback:
> +	return libbpf_kallsyms_parse(kallsyms_resolve_kprobe_multi_cb, ctx);

let's do the kallsyms fallback in bpf_program__attach_kprobe_multi_opts,
something like:

	err = libbpf_available_filter_functions_parse(resolve_kprobe_multi_cb, &res);
	if (err)
		err = libbpf_kallsyms_parse(kallsyms_resolve_kprobe_multi_cb, &res);
	if (err)
		goto error;


> +}
> +
> +static void kprobe_multi_resolve_resource_free(struct kprobe_multi_resolve *res)

s/kprobe_multi_resolve_resource_free/kprobe_multi_resolve_free/

> +{
> +	if (res->syms) {
> +		while (res->cnt)
> +			free((char *)res->syms[--res->cnt]);
> +		free(res->syms);
> +	} else {
> +		free(res->addrs);
> +	}
> +}
> +
>  struct bpf_link *
>  bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
>  				      const char *pattern,
> @@ -10476,14 +10540,18 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
>  		return libbpf_err_ptr(-EINVAL);
>  
>  	if (pattern) {
> -		err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
> +		err = libbpf_available_filter_functions_parse(resolve_kprobe_multi_cb,
> +							      &res);
>  		if (err)
>  			goto error;
>  		if (!res.cnt) {
>  			err = -ENOENT;
>  			goto error;
>  		}
> -		addrs = res.addrs;
> +		if (res.syms)
> +			syms = res.syms;
> +		else
> +			addrs = res.addrs;

this could be just:

		syms = res.syms;
		addrs = res.addrs;

once (cnt > 0) then we have either syms or addrs defined
and the other is NULL

>  		cnt = res.cnt;
>  	}
>  
> @@ -10511,12 +10579,12 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
>  		goto error;
>  	}
>  	link->fd = link_fd;
> -	free(res.addrs);
> +	kprobe_multi_resolve_resource_free(&res);
>  	return link;
>  
>  error:
>  	free(link);
> -	free(res.addrs);
> +	kprobe_multi_resolve_resource_free(&res);
>  	return libbpf_err_ptr(err);
>  }
>  
> diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
> index e4d05662a96c..fdf6b464481f 100644
> --- a/tools/lib/bpf/libbpf_internal.h
> +++ b/tools/lib/bpf/libbpf_internal.h
> @@ -481,8 +481,10 @@ __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
>  
>  typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,
>  			     const char *sym_name, void *ctx);
> -
>  int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *arg);
> +typedef int (*available_filter_functions_cb_t)(const char *sym_name, void *ctx);
> +int libbpf_available_filter_functions_parse(available_filter_functions_cb_t cb,
> +					    void *arg);

let's kee it static, we don't have any other callers

thanks,
jirka