* [PATCH 0/2] bpf: Add kretprobe fallback for kprobe multi link @ 2025-12-18 13:06 liujing40 2025-12-18 13:06 ` [PATCH 1/2] bpf: Prepare for kprobe multi link fallback patch liujing40 2025-12-18 13:06 ` [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link liujing40 0 siblings, 2 replies; 10+ messages in thread From: liujing40 @ 2025-12-18 13:06 UTC (permalink / raw) To: ast, daniel, andrii, mhiramat, martin.lau, eddyz87, song, yonghong.song, john.fastabend, kpsingh, sdf, haoluo, jolsa Cc: bpf, linux-kernel, linux-trace-kernel, liujing40 This patch provides a fallback implementation of kprobe multi link using the traditional kretprobe API When fprobe is not available. This ensures compatibility with older kernels or platforms where fprobe support is not compiled in. Uses kretprobe's entry_handler and handler callbacks to simulate fprobe's entry/exit functionality.The API remains identical to fprobe-based implementation, allowing userspace tools to work transparently with either backend. Cookie support, both entry and return probes, and session handling are fully supported. Jing Liu (2): bpf: Prepare for kprobe multi link fallback patch bpf: Implement kretprobe fallback for kprobe multi link kernel/trace/bpf_trace.c | 603 ++++++++++++++++++++++++++++----------- 1 file changed, 443 insertions(+), 160 deletions(-) -- 2.25.1 ^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 1/2] bpf: Prepare for kprobe multi link fallback patch 2025-12-18 13:06 [PATCH 0/2] bpf: Add kretprobe fallback for kprobe multi link liujing40 @ 2025-12-18 13:06 ` liujing40 2025-12-18 13:06 ` [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link liujing40 1 sibling, 0 replies; 10+ messages in thread From: liujing40 @ 2025-12-18 13:06 UTC (permalink / raw) To: ast, daniel, andrii, mhiramat, martin.lau, eddyz87, song, yonghong.song, john.fastabend, kpsingh, sdf, haoluo, jolsa Cc: bpf, linux-kernel, linux-trace-kernel, liujing40 This moves some functions and structs around to make the following patch easier to read. Signed-off-by: Jing Liu <liujing40@xiaomi.com> --- kernel/trace/bpf_trace.c | 304 +++++++++++++++++++-------------------- 1 file changed, 152 insertions(+), 152 deletions(-) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index fe28d86f7c35..1fd07c10378f 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2291,67 +2291,6 @@ struct bpf_kprobe_multi_run_ctx { unsigned long entry_ip; }; -struct user_syms { - const char **syms; - char *buf; -}; - -#ifndef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS -static DEFINE_PER_CPU(struct pt_regs, bpf_kprobe_multi_pt_regs); -#define bpf_kprobe_multi_pt_regs_ptr() this_cpu_ptr(&bpf_kprobe_multi_pt_regs) -#else -#define bpf_kprobe_multi_pt_regs_ptr() (NULL) -#endif - -static unsigned long ftrace_get_entry_ip(unsigned long fentry_ip) -{ - unsigned long ip = ftrace_get_symaddr(fentry_ip); - - return ip ? : fentry_ip; -} - -static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt) -{ - unsigned long __user usymbol; - const char **syms = NULL; - char *buf = NULL, *p; - int err = -ENOMEM; - unsigned int i; - - syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); - if (!syms) - goto error; - - buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); - if (!buf) - goto error; - - for (p = buf, i = 0; i < cnt; i++) { - if (__get_user(usymbol, usyms + i)) { - err = -EFAULT; - goto error; - } - err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN); - if (err == KSYM_NAME_LEN) - err = -E2BIG; - if (err < 0) - goto error; - syms[i] = p; - p += err + 1; - } - - us->syms = syms; - us->buf = buf; - return 0; - -error: - if (err) { - kvfree(syms); - kvfree(buf); - } - return err; -} - static void kprobe_multi_put_modules(struct module **mods, u32 cnt) { u32 i; @@ -2360,12 +2299,6 @@ static void kprobe_multi_put_modules(struct module **mods, u32 cnt) module_put(mods[i]); } -static void free_user_syms(struct user_syms *us) -{ - kvfree(us->syms); - kvfree(us->buf); -} - static void bpf_kprobe_multi_link_release(struct bpf_link *link) { struct bpf_kprobe_multi_link *kmulti_link; @@ -2469,6 +2402,152 @@ static const struct bpf_link_ops bpf_kprobe_multi_link_lops = { #endif }; +static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx) +{ + struct bpf_kprobe_multi_run_ctx *run_ctx; + + run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, + session_ctx.run_ctx); + return run_ctx->entry_ip; +} + +struct modules_array { + struct module **mods; + int mods_cnt; + int mods_cap; +}; + +static int add_module(struct modules_array *arr, struct module *mod) +{ + struct module **mods; + + if (arr->mods_cnt == arr->mods_cap) { + arr->mods_cap = max(16, arr->mods_cap * 3 / 2); + mods = krealloc_array(arr->mods, arr->mods_cap, sizeof(*mods), GFP_KERNEL); + if (!mods) + return -ENOMEM; + arr->mods = mods; + } + + arr->mods[arr->mods_cnt] = mod; + arr->mods_cnt++; + return 0; +} + +static bool has_module(struct modules_array *arr, struct module *mod) +{ + int i; + + for (i = arr->mods_cnt - 1; i >= 0; i--) { + if (arr->mods[i] == mod) + return true; + } + return false; +} + +static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt) +{ + struct modules_array arr = {}; + u32 i, err = 0; + + for (i = 0; i < addrs_cnt; i++) { + bool skip_add = false; + struct module *mod; + + scoped_guard(rcu) { + mod = __module_address(addrs[i]); + /* Either no module or it's already stored */ + if (!mod || has_module(&arr, mod)) { + skip_add = true; + break; /* scoped_guard */ + } + if (!try_module_get(mod)) + err = -EINVAL; + } + if (skip_add) + continue; + if (err) + break; + err = add_module(&arr, mod); + if (err) { + module_put(mod); + break; + } + } + + /* We return either err < 0 in case of error, ... */ + if (err) { + kprobe_multi_put_modules(arr.mods, arr.mods_cnt); + kfree(arr.mods); + return err; + } + + /* or number of modules found if everything is ok. */ + *mods = arr.mods; + return arr.mods_cnt; +} + +struct user_syms { + const char **syms; + char *buf; +}; + +#ifndef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS +static DEFINE_PER_CPU(struct pt_regs, bpf_kprobe_multi_pt_regs); +#define bpf_kprobe_multi_pt_regs_ptr() this_cpu_ptr(&bpf_kprobe_multi_pt_regs) +#else +#define bpf_kprobe_multi_pt_regs_ptr() (NULL) +#endif + +static unsigned long ftrace_get_entry_ip(unsigned long fentry_ip) +{ + unsigned long ip = ftrace_get_symaddr(fentry_ip); + + return ip ? : fentry_ip; +} + +static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt) +{ + unsigned long __user usymbol; + const char **syms = NULL; + char *buf = NULL, *p; + int err = -ENOMEM; + unsigned int i; + + syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); + if (!syms) + goto error; + + buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); + if (!buf) + goto error; + + for (p = buf, i = 0; i < cnt; i++) { + if (__get_user(usymbol, usyms + i)) { + err = -EFAULT; + goto error; + } + err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN); + if (err == KSYM_NAME_LEN) + err = -E2BIG; + if (err < 0) + goto error; + syms[i] = p; + p += err + 1; + } + + us->syms = syms; + us->buf = buf; + return 0; + +error: + if (err) { + kvfree(syms); + kvfree(buf); + } + return err; +} + static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv) { const struct bpf_kprobe_multi_link *link = priv; @@ -2520,15 +2599,6 @@ static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx) return *cookie; } -static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx) -{ - struct bpf_kprobe_multi_run_ctx *run_ctx; - - run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, - session_ctx.run_ctx); - return run_ctx->entry_ip; -} - static __always_inline int kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, unsigned long entry_ip, struct ftrace_regs *fregs, @@ -2597,6 +2667,12 @@ kprobe_multi_link_exit_handler(struct fprobe *fp, unsigned long fentry_ip, fregs, true, data); } +static void free_user_syms(struct user_syms *us) +{ + kvfree(us->syms); + kvfree(us->buf); +} + static int symbols_cmp_r(const void *a, const void *b, const void *priv) { const char **str_a = (const char **) a; @@ -2627,82 +2703,6 @@ static void symbols_swap_r(void *a, void *b, int size, const void *priv) } } -struct modules_array { - struct module **mods; - int mods_cnt; - int mods_cap; -}; - -static int add_module(struct modules_array *arr, struct module *mod) -{ - struct module **mods; - - if (arr->mods_cnt == arr->mods_cap) { - arr->mods_cap = max(16, arr->mods_cap * 3 / 2); - mods = krealloc_array(arr->mods, arr->mods_cap, sizeof(*mods), GFP_KERNEL); - if (!mods) - return -ENOMEM; - arr->mods = mods; - } - - arr->mods[arr->mods_cnt] = mod; - arr->mods_cnt++; - return 0; -} - -static bool has_module(struct modules_array *arr, struct module *mod) -{ - int i; - - for (i = arr->mods_cnt - 1; i >= 0; i--) { - if (arr->mods[i] == mod) - return true; - } - return false; -} - -static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt) -{ - struct modules_array arr = {}; - u32 i, err = 0; - - for (i = 0; i < addrs_cnt; i++) { - bool skip_add = false; - struct module *mod; - - scoped_guard(rcu) { - mod = __module_address(addrs[i]); - /* Either no module or it's already stored */ - if (!mod || has_module(&arr, mod)) { - skip_add = true; - break; /* scoped_guard */ - } - if (!try_module_get(mod)) - err = -EINVAL; - } - if (skip_add) - continue; - if (err) - break; - err = add_module(&arr, mod); - if (err) { - module_put(mod); - break; - } - } - - /* We return either err < 0 in case of error, ... */ - if (err) { - kprobe_multi_put_modules(arr.mods, arr.mods_cnt); - kfree(arr.mods); - return err; - } - - /* or number of modules found if everything is ok. */ - *mods = arr.mods; - return arr.mods_cnt; -} - static int addrs_check_error_injection_list(unsigned long *addrs, u32 cnt) { u32 i; -- 2.25.1 ^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link 2025-12-18 13:06 [PATCH 0/2] bpf: Add kretprobe fallback for kprobe multi link liujing40 2025-12-18 13:06 ` [PATCH 1/2] bpf: Prepare for kprobe multi link fallback patch liujing40 @ 2025-12-18 13:06 ` liujing40 2025-12-18 13:33 ` bot+bpf-ci ` (2 more replies) 1 sibling, 3 replies; 10+ messages in thread From: liujing40 @ 2025-12-18 13:06 UTC (permalink / raw) To: ast, daniel, andrii, mhiramat, martin.lau, eddyz87, song, yonghong.song, john.fastabend, kpsingh, sdf, haoluo, jolsa Cc: bpf, linux-kernel, linux-trace-kernel, liujing40 When fprobe is not available, provide a fallback implementation of kprobe_multi using the traditional kretprobe API. Uses kretprobe's entry_handler and handler callbacks to simulate fprobe's entry/exit functionality. Signed-off-by: Jing Liu <liujing40@xiaomi.com> --- kernel/trace/bpf_trace.c | 307 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 295 insertions(+), 12 deletions(-) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 1fd07c10378f..426a1c627508 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2274,12 +2274,44 @@ struct bpf_session_run_ctx { void *data; }; -#ifdef CONFIG_FPROBE +#if defined(CONFIG_FPROBE) || defined(CONFIG_KRETPROBES) +#ifndef CONFIG_FPROBE +struct bpf_kprobe { + struct bpf_kprobe_multi_link *link; + u64 cookie; + struct kretprobe rp; +}; + +static void bpf_kprobe_unregister(struct bpf_kprobe *kps, u32 cnt) +{ + for (int i = 0; i < cnt; i++) + unregister_kretprobe(&kps[i].rp); +} + +static int bpf_kprobe_register(struct bpf_kprobe *kps, u32 cnt) +{ + int ret = 0, i; + + for (i = 0; i < cnt; i++) { + ret = register_kretprobe(&kps[i].rp); + if (ret < 0) { + bpf_kprobe_unregister(kps, i); + break; + } + } + return ret; +} +#endif + struct bpf_kprobe_multi_link { struct bpf_link link; +#ifdef CONFIG_FPROBE struct fprobe fp; unsigned long *addrs; u64 *cookies; +#else + struct bpf_kprobe *kprobes; +#endif u32 cnt; u32 mods_cnt; struct module **mods; @@ -2287,7 +2319,11 @@ struct bpf_kprobe_multi_link { struct bpf_kprobe_multi_run_ctx { struct bpf_session_run_ctx session_ctx; +#ifdef CONFIG_FPROBE struct bpf_kprobe_multi_link *link; +#else + struct bpf_kprobe *kprobe; +#endif unsigned long entry_ip; }; @@ -2304,7 +2340,11 @@ static void bpf_kprobe_multi_link_release(struct bpf_link *link) struct bpf_kprobe_multi_link *kmulti_link; kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); +#ifdef CONFIG_FPROBE unregister_fprobe(&kmulti_link->fp); +#else + bpf_kprobe_unregister(kmulti_link->kprobes, kmulti_link->cnt); +#endif kprobe_multi_put_modules(kmulti_link->mods, kmulti_link->mods_cnt); } @@ -2313,8 +2353,12 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link) struct bpf_kprobe_multi_link *kmulti_link; kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); +#ifdef CONFIG_FPROBE kvfree(kmulti_link->addrs); kvfree(kmulti_link->cookies); +#else + kvfree(kmulti_link->kprobes); +#endif kfree(kmulti_link->mods); kfree(kmulti_link); } @@ -2326,6 +2370,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs); struct bpf_kprobe_multi_link *kmulti_link; u32 ucount = info->kprobe_multi.count; + bool kallsyms_show = kallsyms_show_value(current_cred()); int err = 0, i; if (!uaddrs ^ !ucount) @@ -2336,7 +2381,12 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); info->kprobe_multi.count = kmulti_link->cnt; info->kprobe_multi.flags = kmulti_link->link.flags; +#ifdef CONFIG_FPROBE info->kprobe_multi.missed = kmulti_link->fp.nmissed; +#else + for (i = 0; i < kmulti_link->cnt; i++) + info->kprobe_multi.missed += kmulti_link->kprobes[i].rp.nmissed; +#endif if (!uaddrs) return 0; @@ -2345,6 +2395,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, else ucount = kmulti_link->cnt; +#ifdef CONFIG_FPROBE if (ucookies) { if (kmulti_link->cookies) { if (copy_to_user(ucookies, kmulti_link->cookies, ucount * sizeof(u64))) @@ -2357,7 +2408,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, } } - if (kallsyms_show_value(current_cred())) { + if (kallsyms_show) { if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64))) return -EFAULT; } else { @@ -2366,6 +2417,16 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, return -EFAULT; } } +#else + for (i = 0; i < ucount; i++) { + if (ucookies && put_user(kmulti_link->kprobes[i].cookie, ucookies + i)) + return -EFAULT; + + if (put_user(kallsyms_show ? (uintptr_t)kmulti_link->kprobes[i].rp.kp.addr : 0, + uaddrs + i)) + return -EFAULT; + } +#endif return err; } @@ -2374,21 +2435,32 @@ static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) { struct bpf_kprobe_multi_link *kmulti_link; + unsigned long kprobe_multi_missed = 0; kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); +#ifdef CONFIG_FPROBE + kprobe_multi_missed = kmulti_link->fp.nmissed; +#else + for (int i = 0; i < kmulti_link->cnt; i++) + kprobe_multi_missed += kmulti_link->kprobes[i].rp.nmissed; +#endif seq_printf(seq, "kprobe_cnt:\t%u\n" "missed:\t%lu\n", kmulti_link->cnt, - kmulti_link->fp.nmissed); + kprobe_multi_missed); seq_printf(seq, "%s\t %s\n", "cookie", "func"); for (int i = 0; i < kmulti_link->cnt; i++) { - seq_printf(seq, - "%llu\t %pS\n", - kmulti_link->cookies[i], - (void *)kmulti_link->addrs[i]); +#ifdef CONFIG_FPROBE + u64 cookie = kmulti_link->cookies[i]; + void *addr = (void *)kmulti_link->addrs[i]; +#else + u64 cookie = kmulti_link->kprobes[i].cookie; + void *addr = (void *)kmulti_link->kprobes[i].rp.kp.addr; +#endif + seq_printf(seq, "%llu\t %pS\n", cookie, addr); } } #endif @@ -2445,17 +2517,22 @@ static bool has_module(struct modules_array *arr, struct module *mod) return false; } -static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt) +static int get_modules_for_addrs(struct bpf_kprobe_multi_link *link) { struct modules_array arr = {}; u32 i, err = 0; - for (i = 0; i < addrs_cnt; i++) { + for (i = 0; i < link->cnt; i++) { bool skip_add = false; struct module *mod; +#ifdef CONFIG_FPROBE + unsigned long addr = link->addrs[i]; +#else + unsigned long addr = (unsigned long)link->kprobes[i].rp.kp.addr; +#endif scoped_guard(rcu) { - mod = __module_address(addrs[i]); + mod = __module_address(addr); /* Either no module or it's already stored */ if (!mod || has_module(&arr, mod)) { skip_add = true; @@ -2483,10 +2560,11 @@ static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u3 } /* or number of modules found if everything is ok. */ - *mods = arr.mods; + link->mods = arr.mods; return arr.mods_cnt; } +#ifdef CONFIG_FPROBE struct user_syms { const char **syms; char *buf; @@ -2843,7 +2921,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr link); } - err = get_modules_for_addrs(&link->mods, addrs, cnt); + err = get_modules_for_addrs(link); if (err < 0) { bpf_link_cleanup(&link_primer); return err; @@ -2866,6 +2944,211 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr return err; } #else /* !CONFIG_FPROBE */ +static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx) +{ + struct bpf_kprobe_multi_run_ctx *run_ctx; + + run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, + session_ctx.run_ctx); + return run_ctx->kprobe->cookie; +} + +static __always_inline int +kprobe_multi_link_prog_run(struct bpf_kprobe *kprobe, unsigned long entry_ip, + struct pt_regs *regs, bool is_return, void *data) +{ + struct bpf_kprobe_multi_link *link = kprobe->link; + struct bpf_kprobe_multi_run_ctx run_ctx = { + .session_ctx = { + .is_return = is_return, + .data = data, + }, + .kprobe = kprobe, + .entry_ip = entry_ip, + }; + struct bpf_run_ctx *old_run_ctx; + int err; + + cant_sleep(); + + if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { + bpf_prog_inc_misses_counter(link->link.prog); + err = 1; + goto out; + } + + rcu_read_lock(); + migrate_disable(); + old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); + err = bpf_prog_run(link->link.prog, regs); + bpf_reset_run_ctx(old_run_ctx); + migrate_enable(); + rcu_read_unlock(); + + out: + __this_cpu_dec(bpf_prog_active); + return err; +} + +static int +kprobe_multi_link_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + struct kretprobe *rp = get_kretprobe(ri); + struct bpf_kprobe *kprobe; + int err; + + if (unlikely(!rp)) + return 1; + + kprobe = container_of(rp, struct bpf_kprobe, rp); + err = kprobe_multi_link_prog_run(kprobe, get_entry_ip((uintptr_t)rp->kp.addr), + regs, false, ri->data); + return is_kprobe_session(kprobe->link->link.prog) ? err : 0; +} + +static int +kprobe_multi_link_exit_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + struct kretprobe *rp = get_kretprobe(ri); + struct bpf_kprobe *kprobe; + + if (unlikely(!rp)) + return 0; + + kprobe = container_of(rp, struct bpf_kprobe, rp); + kprobe_multi_link_prog_run(kprobe, get_entry_ip((uintptr_t)rp->kp.addr), + regs, true, ri->data); + return 0; +} + +int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) +{ + struct bpf_kprobe_multi_link *link = NULL; + struct bpf_link_primer link_primer; + struct bpf_kprobe *kprobes = NULL; + u32 flags, cnt; + u64 __user *ucookies; + unsigned long __user *uaddrs; + unsigned long __user *usyms; + int err, i; + + /* no support for 32bit archs yet */ + if (sizeof(u64) != sizeof(void *)) + return -EOPNOTSUPP; + + if (attr->link_create.flags) + return -EINVAL; + + if (!is_kprobe_multi(prog)) + return -EINVAL; + + /* Writing to context is not allowed for kprobes. */ + if (prog->aux->kprobe_write_ctx) + return -EINVAL; + + flags = attr->link_create.kprobe_multi.flags; + if (flags & ~BPF_F_KPROBE_MULTI_RETURN) + return -EINVAL; + + uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs); + usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms); + if (!!uaddrs == !!usyms) + return -EINVAL; + + cnt = attr->link_create.kprobe_multi.cnt; + if (!cnt) + return -EINVAL; + if (cnt > MAX_KPROBE_MULTI_CNT) + return -E2BIG; + + ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies); + kprobes = kvcalloc(cnt, sizeof(*kprobes), GFP_KERNEL); + link = kzalloc(sizeof(*link), GFP_KERNEL); + if (!link || !kprobes) { + err = -ENOMEM; + goto error; + } + + for (i = 0; i < cnt; i++) { + unsigned long addr; + + if (uaddrs) { + if (__get_user(addr, uaddrs + i)) { + err = -EFAULT; + goto error; + } + } else { + unsigned long __user usymbol; + char buf[KSYM_NAME_LEN]; + + if (__get_user(usymbol, usyms + i)) { + err = -EFAULT; + goto error; + } + err = strncpy_from_user(buf, (const char __user *) usymbol, KSYM_NAME_LEN); + if (err == KSYM_NAME_LEN) + err = -E2BIG; + if (err < 0) + goto error; + + addr = kallsyms_lookup_name(buf); + if (!addr) + goto error; + } + if (prog->kprobe_override && !within_error_injection_list(addr)) { + err = -EINVAL; + goto error; + } + if (ucookies && __get_user(kprobes[i].cookie, ucookies + i)) { + err = -EFAULT; + goto error; + } + + kprobes[i].link = link; + kprobes[i].rp.kp.addr = (kprobe_opcode_t *)addr; + + if (!(flags & BPF_F_KPROBE_MULTI_RETURN)) + kprobes[i].rp.entry_handler = kprobe_multi_link_handler; + if ((flags & BPF_F_KPROBE_MULTI_RETURN) || is_kprobe_session(prog)) + kprobes[i].rp.handler = kprobe_multi_link_exit_handler; + if (is_kprobe_session(prog)) + kprobes[i].rp.data_size = sizeof(u64); + } + + bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI, + &bpf_kprobe_multi_link_lops, prog, attr->link_create.attach_type); + + err = bpf_link_prime(&link->link, &link_primer); + if (err) + goto error; + + link->kprobes = kprobes; + link->cnt = cnt; + link->link.flags = flags; + + err = get_modules_for_addrs(link); + if (err < 0) { + bpf_link_cleanup(&link_primer); + return err; + } + link->mods_cnt = err; + + err = bpf_kprobe_register(kprobes, cnt); + if (err) { + kprobe_multi_put_modules(link->mods, link->mods_cnt); + bpf_link_cleanup(&link_primer); + return err; + } + + return bpf_link_settle(&link_primer); + +error: + kvfree(kprobes); + kfree(link); + return err; +} +#endif +#else /* !CONFIG_FPROBE && !CONFIG_KRETPROBES*/ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { return -EOPNOTSUPP; -- 2.25.1 ^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link 2025-12-18 13:06 ` [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link liujing40 @ 2025-12-18 13:33 ` bot+bpf-ci 2025-12-18 17:53 ` Alexei Starovoitov 2025-12-19 1:57 ` Menglong Dong 2 siblings, 0 replies; 10+ messages in thread From: bot+bpf-ci @ 2025-12-18 13:33 UTC (permalink / raw) To: liujing.root, ast, daniel, andrii, mhiramat, martin.lau, eddyz87, song, yonghong.song, john.fastabend, kpsingh, sdf, haoluo, jolsa Cc: bpf, linux-kernel, linux-trace-kernel, liujing40, ast, andrii, daniel, martin.lau, eddyz87, yonghong.song, clm, ihor.solodrai [-- Attachment #1: Type: text/plain, Size: 2128 bytes --] > diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c > index 1fd07c10378f..426a1c627508 100644 > --- a/kernel/trace/bpf_trace.c > +++ b/kernel/trace/bpf_trace.c [ ... ] > +int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) > +{ > + struct bpf_kprobe_multi_link *link = NULL; > + struct bpf_link_primer link_primer; > + struct bpf_kprobe *kprobes = NULL; > + u32 flags, cnt; > + u64 __user *ucookies; > + unsigned long __user *uaddrs; > + unsigned long __user *usyms; > + int err, i; [ ... ] > + for (i = 0; i < cnt; i++) { > + unsigned long addr; > + > + if (uaddrs) { > + if (__get_user(addr, uaddrs + i)) { > + err = -EFAULT; > + goto error; > + } > + } else { > + unsigned long __user usymbol; > + char buf[KSYM_NAME_LEN]; > + > + if (__get_user(usymbol, usyms + i)) { > + err = -EFAULT; > + goto error; > + } > + err = strncpy_from_user(buf, (const char __user *) usymbol, KSYM_NAME_LEN); > + if (err == KSYM_NAME_LEN) > + err = -E2BIG; > + if (err < 0) > + goto error; > + > + addr = kallsyms_lookup_name(buf); > + if (!addr) > + goto error; When kallsyms_lookup_name() returns 0 (symbol not found), is err being set to an error code before jumping to the error label? It looks like err will still hold the positive return value from strncpy_from_user() (the string length), which would be returned to the caller as a success indication. The error label at the end of the function does: error: kvfree(kprobes); kfree(link); return err; This could cause the syscall to return a positive value to userspace instead of an error, and also leak a prog reference since link_create() only calls bpf_prog_put() when ret < 0. > + } > + if (prog->kprobe_override && !within_error_injection_list(addr)) { > + err = -EINVAL; > + goto error; > + } [ ... ] --- AI reviewed your patch. Please fix the bug or email reply why it's not a bug. See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md CI run summary: https://github.com/kernel-patches/bpf/actions/runs/20338242683 ^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link 2025-12-18 13:06 ` [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link liujing40 2025-12-18 13:33 ` bot+bpf-ci @ 2025-12-18 17:53 ` Alexei Starovoitov 2025-12-18 21:09 ` Steven Rostedt 2025-12-19 1:57 ` Menglong Dong 2 siblings, 1 reply; 10+ messages in thread From: Alexei Starovoitov @ 2025-12-18 17:53 UTC (permalink / raw) To: liujing40 Cc: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, Masami Hiramatsu, Martin KaFai Lau, Eduard, Song Liu, Yonghong Song, John Fastabend, KP Singh, Stanislav Fomichev, Hao Luo, Jiri Olsa, bpf, LKML, linux-trace-kernel, liujing40 On Thu, Dec 18, 2025 at 5:07 AM liujing40 <liujing.root@gmail.com> wrote: > > When fprobe is not available, provide a fallback implementation of > kprobe_multi using the traditional kretprobe API. > > Uses kretprobe's entry_handler and handler callbacks to simulate fprobe's > entry/exit functionality. > > Signed-off-by: Jing Liu <liujing40@xiaomi.com> > --- > kernel/trace/bpf_trace.c | 307 +++++++++++++++++++++++++++++++++++++-- > 1 file changed, 295 insertions(+), 12 deletions(-) > > diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c > index 1fd07c10378f..426a1c627508 100644 > --- a/kernel/trace/bpf_trace.c > +++ b/kernel/trace/bpf_trace.c > @@ -2274,12 +2274,44 @@ struct bpf_session_run_ctx { > void *data; > }; > > -#ifdef CONFIG_FPROBE > +#if defined(CONFIG_FPROBE) || defined(CONFIG_KRETPROBES) > +#ifndef CONFIG_FPROBE > +struct bpf_kprobe { > + struct bpf_kprobe_multi_link *link; > + u64 cookie; > + struct kretprobe rp; > +}; > + > +static void bpf_kprobe_unregister(struct bpf_kprobe *kps, u32 cnt) > +{ > + for (int i = 0; i < cnt; i++) > + unregister_kretprobe(&kps[i].rp); > +} Nack. This is not a good idea. unregister_kretprobe() calls synchronize_rcu(). So the above loop will cause soft lockups for sure. pw-bot: cr ^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link 2025-12-18 17:53 ` Alexei Starovoitov @ 2025-12-18 21:09 ` Steven Rostedt 2025-12-22 8:00 ` liujing40 0 siblings, 1 reply; 10+ messages in thread From: Steven Rostedt @ 2025-12-18 21:09 UTC (permalink / raw) To: Alexei Starovoitov Cc: liujing40, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, Masami Hiramatsu, Martin KaFai Lau, Eduard, Song Liu, Yonghong Song, John Fastabend, KP Singh, Stanislav Fomichev, Hao Luo, Jiri Olsa, bpf, LKML, linux-trace-kernel, liujing40 On Thu, 18 Dec 2025 09:53:16 -0800 Alexei Starovoitov <alexei.starovoitov@gmail.com> wrote: > > +static void bpf_kprobe_unregister(struct bpf_kprobe *kps, u32 cnt) > > +{ > > + for (int i = 0; i < cnt; i++) > > + unregister_kretprobe(&kps[i].rp); > > +} > > Nack. > This is not a good idea. > unregister_kretprobe() calls synchronize_rcu(). > So the above loop will cause soft lockups for sure. Looks like it could be replaced with: unregister_kretprobes(kps, cnt); Which unregisters an array of kreptrobes and does a single synchronize_rcu(). -- Steve ^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link 2025-12-18 21:09 ` Steven Rostedt @ 2025-12-22 8:00 ` liujing40 0 siblings, 0 replies; 10+ messages in thread From: liujing40 @ 2025-12-22 8:00 UTC (permalink / raw) To: rostedt Cc: alexei.starovoitov, andrii, ast, bpf, daniel, eddyz87, haoluo, john.fastabend, jolsa, kpsingh, linux-kernel, linux-trace-kernel, liujing.root, liujing40, martin.lau, mhiramat, sdf, song, yonghong.song On Thu, 18 Dec 2025 16:09:25 -0500 Steven Rostedt <rostedt@goodmis.org> wrote: > On Thu, 18 Dec 2025 09:53:16 -0800 > Alexei Starovoitov <alexei.starovoitov@gmail.com> wrote: > > > > +static void bpf_kprobe_unregister(struct bpf_kprobe *kps, u32 cnt) > > > +{ > > > + for (int i = 0; i < cnt; i++) > > > + unregister_kretprobe(&kps[i].rp); > > > +} > > > > Nack. > > This is not a good idea. > > unregister_kretprobe() calls synchronize_rcu(). > > So the above loop will cause soft lockups for sure. > > Looks like it could be replaced with: > > unregister_kretprobes(kps, cnt); > > Which unregisters an array of kreptrobes and does a single > synchronize_rcu(). > > -- Steve Thanks for the suggestion. I will refactor the loop to use unregister_kretprobes() for a single RCU sync. ^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link 2025-12-18 13:06 ` [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link liujing40 2025-12-18 13:33 ` bot+bpf-ci 2025-12-18 17:53 ` Alexei Starovoitov @ 2025-12-19 1:57 ` Menglong Dong 2025-12-22 8:02 ` liujing40 2 siblings, 1 reply; 10+ messages in thread From: Menglong Dong @ 2025-12-19 1:57 UTC (permalink / raw) To: ast, daniel, andrii, mhiramat, martin.lau, eddyz87, song, yonghong.song, john.fastabend, kpsingh, sdf, haoluo, jolsa, liujing40 Cc: bpf, linux-kernel, linux-trace-kernel, liujing40 On 2025/12/18 21:06 liujing40 <liujing.root@gmail.com> write: > When fprobe is not available, provide a fallback implementation of > kprobe_multi using the traditional kretprobe API. > > Uses kretprobe's entry_handler and handler callbacks to simulate fprobe's > entry/exit functionality. > > Signed-off-by: Jing Liu <liujing40@xiaomi.com> > --- > kernel/trace/bpf_trace.c | 307 +++++++++++++++++++++++++++++++++++++-- > 1 file changed, 295 insertions(+), 12 deletions(-) > > diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c > index 1fd07c10378f..426a1c627508 100644 > --- a/kernel/trace/bpf_trace.c > +++ b/kernel/trace/bpf_trace.c > @@ -2274,12 +2274,44 @@ struct bpf_session_run_ctx { > void *data; > }; > > -#ifdef CONFIG_FPROBE > +#if defined(CONFIG_FPROBE) || defined(CONFIG_KRETPROBES) > +#ifndef CONFIG_FPROBE > +struct bpf_kprobe { > + struct bpf_kprobe_multi_link *link; > + u64 cookie; > + struct kretprobe rp; > +}; > + > +static void bpf_kprobe_unregister(struct bpf_kprobe *kps, u32 cnt) > +{ > + for (int i = 0; i < cnt; i++) > + unregister_kretprobe(&kps[i].rp); > +} > + > +static int bpf_kprobe_register(struct bpf_kprobe *kps, u32 cnt) > +{ > + int ret = 0, i; > + > + for (i = 0; i < cnt; i++) { > + ret = register_kretprobe(&kps[i].rp); > + if (ret < 0) { > + bpf_kprobe_unregister(kps, i); > + break; > + } > + } > + return ret; > +} Hi, Jing. I don't see the point of the fallback logic. If we want to use the kprobe-multi, we enable CONFIG_FPROBE. Is there any reason that we can't enable CONFIG_FPROBE? As you said, for a "old kernel", I think we don't introduce new feature for an old kernel. Besides, did you measure the performance of attaching bench? You will register a kretprobe for each of the target. AFAIK, the kprobe will use ftrace for optimization if we hook the entry of the target function. So I suspect it will be quite slow here. Thanks! Menglong Dong > +#endif > + > struct bpf_kprobe_multi_link { > struct bpf_link link; > +#ifdef CONFIG_FPROBE > struct fprobe fp; > unsigned long *addrs; > u64 *cookies; > +#else > + struct bpf_kprobe *kprobes; > +#endif > u32 cnt; > u32 mods_cnt; > struct module **mods; > @@ -2287,7 +2319,11 @@ struct bpf_kprobe_multi_link { > > struct bpf_kprobe_multi_run_ctx { > struct bpf_session_run_ctx session_ctx; > +#ifdef CONFIG_FPROBE > struct bpf_kprobe_multi_link *link; > +#else > + struct bpf_kprobe *kprobe; > +#endif > unsigned long entry_ip; > }; > > @@ -2304,7 +2340,11 @@ static void bpf_kprobe_multi_link_release(struct bpf_link *link) > struct bpf_kprobe_multi_link *kmulti_link; > > kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); > +#ifdef CONFIG_FPROBE > unregister_fprobe(&kmulti_link->fp); > +#else > + bpf_kprobe_unregister(kmulti_link->kprobes, kmulti_link->cnt); > +#endif > kprobe_multi_put_modules(kmulti_link->mods, kmulti_link->mods_cnt); > } > > @@ -2313,8 +2353,12 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link) > struct bpf_kprobe_multi_link *kmulti_link; > > kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); > +#ifdef CONFIG_FPROBE > kvfree(kmulti_link->addrs); > kvfree(kmulti_link->cookies); > +#else > + kvfree(kmulti_link->kprobes); > +#endif > kfree(kmulti_link->mods); > kfree(kmulti_link); > } > @@ -2326,6 +2370,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, > u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs); > struct bpf_kprobe_multi_link *kmulti_link; > u32 ucount = info->kprobe_multi.count; > + bool kallsyms_show = kallsyms_show_value(current_cred()); > int err = 0, i; > > if (!uaddrs ^ !ucount) > @@ -2336,7 +2381,12 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, > kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); > info->kprobe_multi.count = kmulti_link->cnt; > info->kprobe_multi.flags = kmulti_link->link.flags; > +#ifdef CONFIG_FPROBE > info->kprobe_multi.missed = kmulti_link->fp.nmissed; > +#else > + for (i = 0; i < kmulti_link->cnt; i++) > + info->kprobe_multi.missed += kmulti_link->kprobes[i].rp.nmissed; > +#endif > > if (!uaddrs) > return 0; > @@ -2345,6 +2395,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, > else > ucount = kmulti_link->cnt; > > +#ifdef CONFIG_FPROBE > if (ucookies) { > if (kmulti_link->cookies) { > if (copy_to_user(ucookies, kmulti_link->cookies, ucount * sizeof(u64))) > @@ -2357,7 +2408,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, > } > } > > - if (kallsyms_show_value(current_cred())) { > + if (kallsyms_show) { > if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64))) > return -EFAULT; > } else { > @@ -2366,6 +2417,16 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, > return -EFAULT; > } > } > +#else > + for (i = 0; i < ucount; i++) { > + if (ucookies && put_user(kmulti_link->kprobes[i].cookie, ucookies + i)) > + return -EFAULT; > + > + if (put_user(kallsyms_show ? (uintptr_t)kmulti_link->kprobes[i].rp.kp.addr : 0, > + uaddrs + i)) > + return -EFAULT; > + } > +#endif > return err; > } > > @@ -2374,21 +2435,32 @@ static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link, > struct seq_file *seq) > { > struct bpf_kprobe_multi_link *kmulti_link; > + unsigned long kprobe_multi_missed = 0; > > kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); > +#ifdef CONFIG_FPROBE > + kprobe_multi_missed = kmulti_link->fp.nmissed; > +#else > + for (int i = 0; i < kmulti_link->cnt; i++) > + kprobe_multi_missed += kmulti_link->kprobes[i].rp.nmissed; > +#endif > > seq_printf(seq, > "kprobe_cnt:\t%u\n" > "missed:\t%lu\n", > kmulti_link->cnt, > - kmulti_link->fp.nmissed); > + kprobe_multi_missed); > > seq_printf(seq, "%s\t %s\n", "cookie", "func"); > for (int i = 0; i < kmulti_link->cnt; i++) { > - seq_printf(seq, > - "%llu\t %pS\n", > - kmulti_link->cookies[i], > - (void *)kmulti_link->addrs[i]); > +#ifdef CONFIG_FPROBE > + u64 cookie = kmulti_link->cookies[i]; > + void *addr = (void *)kmulti_link->addrs[i]; > +#else > + u64 cookie = kmulti_link->kprobes[i].cookie; > + void *addr = (void *)kmulti_link->kprobes[i].rp.kp.addr; > +#endif > + seq_printf(seq, "%llu\t %pS\n", cookie, addr); > } > } > #endif > @@ -2445,17 +2517,22 @@ static bool has_module(struct modules_array *arr, struct module *mod) > return false; > } > > -static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt) > +static int get_modules_for_addrs(struct bpf_kprobe_multi_link *link) > { > struct modules_array arr = {}; > u32 i, err = 0; > > - for (i = 0; i < addrs_cnt; i++) { > + for (i = 0; i < link->cnt; i++) { > bool skip_add = false; > struct module *mod; > +#ifdef CONFIG_FPROBE > + unsigned long addr = link->addrs[i]; > +#else > + unsigned long addr = (unsigned long)link->kprobes[i].rp.kp.addr; > +#endif > > scoped_guard(rcu) { > - mod = __module_address(addrs[i]); > + mod = __module_address(addr); > /* Either no module or it's already stored */ > if (!mod || has_module(&arr, mod)) { > skip_add = true; > @@ -2483,10 +2560,11 @@ static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u3 > } > > /* or number of modules found if everything is ok. */ > - *mods = arr.mods; > + link->mods = arr.mods; > return arr.mods_cnt; > } > > +#ifdef CONFIG_FPROBE > struct user_syms { > const char **syms; > char *buf; > @@ -2843,7 +2921,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr > link); > } > > - err = get_modules_for_addrs(&link->mods, addrs, cnt); > + err = get_modules_for_addrs(link); > if (err < 0) { > bpf_link_cleanup(&link_primer); > return err; > @@ -2866,6 +2944,211 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr > return err; > } > #else /* !CONFIG_FPROBE */ > +static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx) > +{ > + struct bpf_kprobe_multi_run_ctx *run_ctx; > + > + run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, > + session_ctx.run_ctx); > + return run_ctx->kprobe->cookie; > +} > + > +static __always_inline int > +kprobe_multi_link_prog_run(struct bpf_kprobe *kprobe, unsigned long entry_ip, > + struct pt_regs *regs, bool is_return, void *data) > +{ > + struct bpf_kprobe_multi_link *link = kprobe->link; > + struct bpf_kprobe_multi_run_ctx run_ctx = { > + .session_ctx = { > + .is_return = is_return, > + .data = data, > + }, > + .kprobe = kprobe, > + .entry_ip = entry_ip, > + }; > + struct bpf_run_ctx *old_run_ctx; > + int err; > + > + cant_sleep(); > + > + if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { > + bpf_prog_inc_misses_counter(link->link.prog); > + err = 1; > + goto out; > + } > + > + rcu_read_lock(); > + migrate_disable(); > + old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); > + err = bpf_prog_run(link->link.prog, regs); > + bpf_reset_run_ctx(old_run_ctx); > + migrate_enable(); > + rcu_read_unlock(); > + > + out: > + __this_cpu_dec(bpf_prog_active); > + return err; > +} > + > +static int > +kprobe_multi_link_handler(struct kretprobe_instance *ri, struct pt_regs *regs) > +{ > + struct kretprobe *rp = get_kretprobe(ri); > + struct bpf_kprobe *kprobe; > + int err; > + > + if (unlikely(!rp)) > + return 1; > + > + kprobe = container_of(rp, struct bpf_kprobe, rp); > + err = kprobe_multi_link_prog_run(kprobe, get_entry_ip((uintptr_t)rp->kp.addr), > + regs, false, ri->data); > + return is_kprobe_session(kprobe->link->link.prog) ? err : 0; > +} > + > +static int > +kprobe_multi_link_exit_handler(struct kretprobe_instance *ri, struct pt_regs *regs) > +{ > + struct kretprobe *rp = get_kretprobe(ri); > + struct bpf_kprobe *kprobe; > + > + if (unlikely(!rp)) > + return 0; > + > + kprobe = container_of(rp, struct bpf_kprobe, rp); > + kprobe_multi_link_prog_run(kprobe, get_entry_ip((uintptr_t)rp->kp.addr), > + regs, true, ri->data); > + return 0; > +} > + > +int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) > +{ > + struct bpf_kprobe_multi_link *link = NULL; > + struct bpf_link_primer link_primer; > + struct bpf_kprobe *kprobes = NULL; > + u32 flags, cnt; > + u64 __user *ucookies; > + unsigned long __user *uaddrs; > + unsigned long __user *usyms; > + int err, i; > + > + /* no support for 32bit archs yet */ > + if (sizeof(u64) != sizeof(void *)) > + return -EOPNOTSUPP; > + > + if (attr->link_create.flags) > + return -EINVAL; > + > + if (!is_kprobe_multi(prog)) > + return -EINVAL; > + > + /* Writing to context is not allowed for kprobes. */ > + if (prog->aux->kprobe_write_ctx) > + return -EINVAL; > + > + flags = attr->link_create.kprobe_multi.flags; > + if (flags & ~BPF_F_KPROBE_MULTI_RETURN) > + return -EINVAL; > + > + uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs); > + usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms); > + if (!!uaddrs == !!usyms) > + return -EINVAL; > + > + cnt = attr->link_create.kprobe_multi.cnt; > + if (!cnt) > + return -EINVAL; > + if (cnt > MAX_KPROBE_MULTI_CNT) > + return -E2BIG; > + > + ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies); > + kprobes = kvcalloc(cnt, sizeof(*kprobes), GFP_KERNEL); > + link = kzalloc(sizeof(*link), GFP_KERNEL); > + if (!link || !kprobes) { > + err = -ENOMEM; > + goto error; > + } > + > + for (i = 0; i < cnt; i++) { > + unsigned long addr; > + > + if (uaddrs) { > + if (__get_user(addr, uaddrs + i)) { > + err = -EFAULT; > + goto error; > + } > + } else { > + unsigned long __user usymbol; > + char buf[KSYM_NAME_LEN]; > + > + if (__get_user(usymbol, usyms + i)) { > + err = -EFAULT; > + goto error; > + } > + err = strncpy_from_user(buf, (const char __user *) usymbol, KSYM_NAME_LEN); > + if (err == KSYM_NAME_LEN) > + err = -E2BIG; > + if (err < 0) > + goto error; > + > + addr = kallsyms_lookup_name(buf); > + if (!addr) > + goto error; > + } > + if (prog->kprobe_override && !within_error_injection_list(addr)) { > + err = -EINVAL; > + goto error; > + } > + if (ucookies && __get_user(kprobes[i].cookie, ucookies + i)) { > + err = -EFAULT; > + goto error; > + } > + > + kprobes[i].link = link; > + kprobes[i].rp.kp.addr = (kprobe_opcode_t *)addr; > + > + if (!(flags & BPF_F_KPROBE_MULTI_RETURN)) > + kprobes[i].rp.entry_handler = kprobe_multi_link_handler; > + if ((flags & BPF_F_KPROBE_MULTI_RETURN) || is_kprobe_session(prog)) > + kprobes[i].rp.handler = kprobe_multi_link_exit_handler; > + if (is_kprobe_session(prog)) > + kprobes[i].rp.data_size = sizeof(u64); > + } > + > + bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI, > + &bpf_kprobe_multi_link_lops, prog, attr->link_create.attach_type); > + > + err = bpf_link_prime(&link->link, &link_primer); > + if (err) > + goto error; > + > + link->kprobes = kprobes; > + link->cnt = cnt; > + link->link.flags = flags; > + > + err = get_modules_for_addrs(link); > + if (err < 0) { > + bpf_link_cleanup(&link_primer); > + return err; > + } > + link->mods_cnt = err; > + > + err = bpf_kprobe_register(kprobes, cnt); > + if (err) { > + kprobe_multi_put_modules(link->mods, link->mods_cnt); > + bpf_link_cleanup(&link_primer); > + return err; > + } > + > + return bpf_link_settle(&link_primer); > + > +error: > + kvfree(kprobes); > + kfree(link); > + return err; > +} > +#endif > +#else /* !CONFIG_FPROBE && !CONFIG_KRETPROBES*/ > int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) > { > return -EOPNOTSUPP; > -- > 2.25.1 > > > ^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link 2025-12-19 1:57 ` Menglong Dong @ 2025-12-22 8:02 ` liujing40 2025-12-22 17:15 ` Steven Rostedt 0 siblings, 1 reply; 10+ messages in thread From: liujing40 @ 2025-12-22 8:02 UTC (permalink / raw) To: menglong.dong Cc: andrii, ast, bpf, daniel, eddyz87, haoluo, john.fastabend, jolsa, kpsingh, linux-kernel, linux-trace-kernel, liujing.root, liujing40, martin.lau, mhiramat, sdf, song, yonghong.song On Fri, 19 Dec 2025 09:57:37 +0800 Menglong Dong <menglong.dong@linux.dev> wrote: > On 2025/12/18 21:06 liujing40 <liujing.root@gmail.com> write: > > When fprobe is not available, provide a fallback implementation of > > kprobe_multi using the traditional kretprobe API. > > > > Uses kretprobe's entry_handler and handler callbacks to simulate fprobe's > > entry/exit functionality. > > > > Signed-off-by: Jing Liu <liujing40@xiaomi.com> > > --- > > kernel/trace/bpf_trace.c | 307 +++++++++++++++++++++++++++++++++++++-- > > 1 file changed, 295 insertions(+), 12 deletions(-) > > > > diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c > > index 1fd07c10378f..426a1c627508 100644 > > --- a/kernel/trace/bpf_trace.c > > +++ b/kernel/trace/bpf_trace.c > > @@ -2274,12 +2274,44 @@ struct bpf_session_run_ctx { > > void *data; > > }; > > > > -#ifdef CONFIG_FPROBE > > +#if defined(CONFIG_FPROBE) || defined(CONFIG_KRETPROBES) > > +#ifndef CONFIG_FPROBE > > +struct bpf_kprobe { > > + struct bpf_kprobe_multi_link *link; > > + u64 cookie; > > + struct kretprobe rp; > > +}; > > + > > +static void bpf_kprobe_unregister(struct bpf_kprobe *kps, u32 cnt) > > +{ > > + for (int i = 0; i < cnt; i++) > > + unregister_kretprobe(&kps[i].rp); > > +} > > + > > +static int bpf_kprobe_register(struct bpf_kprobe *kps, u32 cnt) > > +{ > > + int ret = 0, i; > > + > > + for (i = 0; i < cnt; i++) { > > + ret = register_kretprobe(&kps[i].rp); > > + if (ret < 0) { > > + bpf_kprobe_unregister(kps, i); > > + break; > > + } > > + } > > + return ret; > > +} > > Hi, Jing. I don't see the point of the fallback logic. If we want to > use the kprobe-multi, we enable CONFIG_FPROBE. Is there any reason > that we can't enable CONFIG_FPROBE? As you said, for a "old kernel", > I think we don't introduce new feature for an old kernel. > > Besides, did you measure the performance of attaching bench? > You will register a kretprobe for each of the target. AFAIK, the > kprobe will use ftrace for optimization if we hook the entry of the > target function. So I suspect it will be quite slow here. > > Thanks! > Menglong Dong The Dynamic ftrace feature is not enabled in Android for security reasons, forcing us to fall back on kretprobe. https://source.android.com/docs/core/tests/debug/ftrace#dftrace I will provide the benchmark test results as soon as possible. ^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link 2025-12-22 8:02 ` liujing40 @ 2025-12-22 17:15 ` Steven Rostedt 0 siblings, 0 replies; 10+ messages in thread From: Steven Rostedt @ 2025-12-22 17:15 UTC (permalink / raw) To: liujing40 Cc: menglong.dong, andrii, ast, bpf, daniel, eddyz87, haoluo, john.fastabend, jolsa, kpsingh, linux-kernel, linux-trace-kernel, liujing40, martin.lau, mhiramat, sdf, song, yonghong.song On Mon, 22 Dec 2025 16:02:53 +0800 liujing40 <liujing.root@gmail.com> wrote: > The Dynamic ftrace feature is not enabled in Android for security reasons, > forcing us to fall back on kretprobe. Really? I would say kretprobe is a much bigger security risk than ftrace. Ftrace only attaches to a set of defined functions and anything that is enabled is displayed in /sys/kernel/tracing/enabled_functions (for security reasons!) Whereas kretprobe can attach to anything, and call anything. Not to mention, there's no way to know if a kretprobe is there or not. So rootkits that would use this can most definitely go under the wire, whereas they can't with ftrace. So if they disable ftrace for security reasons, they most definitely should be disabling kprobes! -- Steve > https://source.android.com/docs/core/tests/debug/ftrace#dftrace > > I will provide the benchmark test results as soon as possible. ^ permalink raw reply [flat|nested] 10+ messages in thread
end of thread, other threads:[~2025-12-22 17:14 UTC | newest] Thread overview: 10+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2025-12-18 13:06 [PATCH 0/2] bpf: Add kretprobe fallback for kprobe multi link liujing40 2025-12-18 13:06 ` [PATCH 1/2] bpf: Prepare for kprobe multi link fallback patch liujing40 2025-12-18 13:06 ` [PATCH 2/2] bpf: Implement kretprobe fallback for kprobe multi link liujing40 2025-12-18 13:33 ` bot+bpf-ci 2025-12-18 17:53 ` Alexei Starovoitov 2025-12-18 21:09 ` Steven Rostedt 2025-12-22 8:00 ` liujing40 2025-12-19 1:57 ` Menglong Dong 2025-12-22 8:02 ` liujing40 2025-12-22 17:15 ` Steven Rostedt
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).