pahole and gcc-14 issues

pahole and gcc-14 issues

[Alexei Starovoitov]

Hi All,

Looks like pahole fails to deduplicate BTF when kernel and
kernel module are built with gcc-14.
I see this issue with various kernel .config-s on bpf and
bpf-next trees.
I tried pahole 1.28 and the latest master. Same issues.

BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
When built with gcc-13 it has 454 types.
So something is confusing dedup logic.
Would be great if dedup experts can take a look,
since this dedup issue is breaking a lot of selftests/bpf.

Also vmlinux.h generated out of the kernel compiled with gcc-13
and out of the kernel compiled with gcc-14 shows these differences:

--- vmlinux13.h    2025-04-24 21:33:50.556884372 -0700
+++ vmlinux14.h    2025-04-24 21:39:10.310488992 -0700
@@ -148815,7 +148815,6 @@
 extern int hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum
hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
 extern void hid_bpf_release_context(struct hid_bpf_ctx *ctx) __weak __ksym;
 extern int hid_bpf_try_input_report(struct hid_bpf_ctx *ctx, enum
hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
-extern bool scx_bpf_consume(u64 dsq_id) __weak __ksym;
 extern int scx_bpf_cpu_node(s32 cpu) __weak __ksym;
 extern struct rq *scx_bpf_cpu_rq(s32 cpu) __weak __ksym;
 extern u32 scx_bpf_cpuperf_cap(s32 cpu) __weak __ksym;
@@ -148825,12 +148824,8 @@
 extern void scx_bpf_destroy_dsq(u64 dsq_id) __weak __ksym;
 extern void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64
slice, u64 enq_flags) __weak __ksym;
 extern void scx_bpf_dispatch_cancel(void) __weak __ksym;
-extern bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq
*it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
__ksym;
-extern void scx_bpf_dispatch_from_dsq_set_slice(struct
bpf_iter_scx_dsq *it__iter, u64 slice) __weak __ksym;
 extern void scx_bpf_dispatch_from_dsq_set_vtime(struct
bpf_iter_scx_dsq *it__iter, u64 vtime) __weak __ksym;
 extern u32 scx_bpf_dispatch_nr_slots(void) __weak __ksym;
-extern void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
-extern bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq
*it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
__ksym;
 extern void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64
slice, u64 enq_flags) __weak __ksym;
 extern void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64
dsq_id, u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
 extern bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak __ksym;

gcc-14's kernel is clearly wrong.
These 5 kfuncs still exist in the kernel.
I manually checked there is no if __GNUC__ > 13 in the code.
Also:
nm bld/vmlinux|grep -w scx_bpf_consume
ffffffff8159d4b0 T scx_bpf_consume
ffffffff8120ea81 t scx_bpf_consume.cold

I suspect the second issue is not related to the dedup problem.
All 5 missing kfuncs have ".cold" optimized bodies.
But ".cold" maybe a red herring, since
nm bld/vmlinux|grep -w scx_bpf_dispatch
ffffffff8159d020 T scx_bpf_dispatch
ffffffff8120ea0f t scx_bpf_dispatch.cold
but this kfunc is present in vmlinux14.h

If it makes a difference I have these configs:
# CONFIG_DEBUG_INFO_DWARF4 is not set
# CONFIG_DEBUG_INFO_DWARF5 is not set
# CONFIG_DEBUG_INFO_REDUCED is not set
CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
# CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
# CONFIG_DEBUG_INFO_SPLIT is not set
CONFIG_DEBUG_INFO_BTF=y
CONFIG_PAHOLE_HAS_SPLIT_BTF=y
CONFIG_DEBUG_INFO_BTF_MODULES=y

Re: pahole and gcc-14 issues

[Alan Maguire]

On 25/04/2025 15:50, Alexei Starovoitov wrote:
> Hi All,
> 
> Looks like pahole fails to deduplicate BTF when kernel and
> kernel module are built with gcc-14.
> I see this issue with various kernel .config-s on bpf and
> bpf-next trees.
> I tried pahole 1.28 and the latest master. Same issues.
> 
> BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
> When built with gcc-13 it has 454 types.
> So something is confusing dedup logic.
> Would be great if dedup experts can take a look,
> since this dedup issue is breaking a lot of selftests/bpf.
> 
> Also vmlinux.h generated out of the kernel compiled with gcc-13
> and out of the kernel compiled with gcc-14 shows these differences:
> 
> --- vmlinux13.h    2025-04-24 21:33:50.556884372 -0700
> +++ vmlinux14.h    2025-04-24 21:39:10.310488992 -0700
> @@ -148815,7 +148815,6 @@
>  extern int hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum
> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
>  extern void hid_bpf_release_context(struct hid_bpf_ctx *ctx) __weak __ksym;
>  extern int hid_bpf_try_input_report(struct hid_bpf_ctx *ctx, enum
> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
> -extern bool scx_bpf_consume(u64 dsq_id) __weak __ksym;
>  extern int scx_bpf_cpu_node(s32 cpu) __weak __ksym;
>  extern struct rq *scx_bpf_cpu_rq(s32 cpu) __weak __ksym;
>  extern u32 scx_bpf_cpuperf_cap(s32 cpu) __weak __ksym;
> @@ -148825,12 +148824,8 @@
>  extern void scx_bpf_destroy_dsq(u64 dsq_id) __weak __ksym;
>  extern void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64
> slice, u64 enq_flags) __weak __ksym;
>  extern void scx_bpf_dispatch_cancel(void) __weak __ksym;
> -extern bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq
> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
> __ksym;
> -extern void scx_bpf_dispatch_from_dsq_set_slice(struct
> bpf_iter_scx_dsq *it__iter, u64 slice) __weak __ksym;
>  extern void scx_bpf_dispatch_from_dsq_set_vtime(struct
> bpf_iter_scx_dsq *it__iter, u64 vtime) __weak __ksym;
>  extern u32 scx_bpf_dispatch_nr_slots(void) __weak __ksym;
> -extern void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
> u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
> -extern bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq
> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
> __ksym;
>  extern void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64
> slice, u64 enq_flags) __weak __ksym;
>  extern void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64
> dsq_id, u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
>  extern bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
> struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak __ksym;
> 
> gcc-14's kernel is clearly wrong.
> These 5 kfuncs still exist in the kernel.
> I manually checked there is no if __GNUC__ > 13 in the code.
> Also:
> nm bld/vmlinux|grep -w scx_bpf_consume
> ffffffff8159d4b0 T scx_bpf_consume
> ffffffff8120ea81 t scx_bpf_consume.cold
> 
> I suspect the second issue is not related to the dedup problem.
> All 5 missing kfuncs have ".cold" optimized bodies.
> But ".cold" maybe a red herring, since
> nm bld/vmlinux|grep -w scx_bpf_dispatch
> ffffffff8159d020 T scx_bpf_dispatch
> ffffffff8120ea0f t scx_bpf_dispatch.cold
> but this kfunc is present in vmlinux14.h
> 
> If it makes a difference I have these configs:
> # CONFIG_DEBUG_INFO_DWARF4 is not set
> # CONFIG_DEBUG_INFO_DWARF5 is not set
> # CONFIG_DEBUG_INFO_REDUCED is not set
> CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
> # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
> # CONFIG_DEBUG_INFO_SPLIT is not set
> CONFIG_DEBUG_INFO_BTF=y
> CONFIG_PAHOLE_HAS_SPLIT_BTF=y
> CONFIG_DEBUG_INFO_BTF_MODULES=y

thanks for the report! I've just reproduced this now with gcc 14; my
initial theory was it might be DWARF5-related, but dedup issues occur
for modules with CONFIG_DEBUG_INFO_DWARF4=y also. I'm seeing task_struct
duplicates in module BTF among other things, so I will try and dig
further and report back when I find something. Like you I suspect the
issues with missing kfuncs are different; may be an issue with our logic
handling inconsistent functions getting confused by the .cold
components. But right now understanding dedup issues is the top priority.

Alan

Re: pahole and gcc-14 issues

[Andrii Nakryiko]

On Fri, Apr 25, 2025 at 10:50 AM Alan Maguire <alan.maguire@oracle.com> wrote:
>
> On 25/04/2025 15:50, Alexei Starovoitov wrote:
> > Hi All,
> >
> > Looks like pahole fails to deduplicate BTF when kernel and
> > kernel module are built with gcc-14.
> > I see this issue with various kernel .config-s on bpf and
> > bpf-next trees.
> > I tried pahole 1.28 and the latest master. Same issues.
> >
> > BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
> > When built with gcc-13 it has 454 types.
> > So something is confusing dedup logic.
> > Would be great if dedup experts can take a look,
> > since this dedup issue is breaking a lot of selftests/bpf.
> >
> > Also vmlinux.h generated out of the kernel compiled with gcc-13
> > and out of the kernel compiled with gcc-14 shows these differences:
> >
> > --- vmlinux13.h    2025-04-24 21:33:50.556884372 -0700
> > +++ vmlinux14.h    2025-04-24 21:39:10.310488992 -0700
> > @@ -148815,7 +148815,6 @@
> >  extern int hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum
> > hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
> >  extern void hid_bpf_release_context(struct hid_bpf_ctx *ctx) __weak __ksym;
> >  extern int hid_bpf_try_input_report(struct hid_bpf_ctx *ctx, enum
> > hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
> > -extern bool scx_bpf_consume(u64 dsq_id) __weak __ksym;
> >  extern int scx_bpf_cpu_node(s32 cpu) __weak __ksym;
> >  extern struct rq *scx_bpf_cpu_rq(s32 cpu) __weak __ksym;
> >  extern u32 scx_bpf_cpuperf_cap(s32 cpu) __weak __ksym;
> > @@ -148825,12 +148824,8 @@
> >  extern void scx_bpf_destroy_dsq(u64 dsq_id) __weak __ksym;
> >  extern void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64
> > slice, u64 enq_flags) __weak __ksym;
> >  extern void scx_bpf_dispatch_cancel(void) __weak __ksym;
> > -extern bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq
> > *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
> > __ksym;
> > -extern void scx_bpf_dispatch_from_dsq_set_slice(struct
> > bpf_iter_scx_dsq *it__iter, u64 slice) __weak __ksym;
> >  extern void scx_bpf_dispatch_from_dsq_set_vtime(struct
> > bpf_iter_scx_dsq *it__iter, u64 vtime) __weak __ksym;
> >  extern u32 scx_bpf_dispatch_nr_slots(void) __weak __ksym;
> > -extern void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
> > u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
> > -extern bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq
> > *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
> > __ksym;
> >  extern void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64
> > slice, u64 enq_flags) __weak __ksym;
> >  extern void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64
> > dsq_id, u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
> >  extern bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
> > struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak __ksym;
> >
> > gcc-14's kernel is clearly wrong.
> > These 5 kfuncs still exist in the kernel.
> > I manually checked there is no if __GNUC__ > 13 in the code.
> > Also:
> > nm bld/vmlinux|grep -w scx_bpf_consume
> > ffffffff8159d4b0 T scx_bpf_consume
> > ffffffff8120ea81 t scx_bpf_consume.cold
> >
> > I suspect the second issue is not related to the dedup problem.
> > All 5 missing kfuncs have ".cold" optimized bodies.
> > But ".cold" maybe a red herring, since
> > nm bld/vmlinux|grep -w scx_bpf_dispatch
> > ffffffff8159d020 T scx_bpf_dispatch
> > ffffffff8120ea0f t scx_bpf_dispatch.cold
> > but this kfunc is present in vmlinux14.h
> >
> > If it makes a difference I have these configs:
> > # CONFIG_DEBUG_INFO_DWARF4 is not set
> > # CONFIG_DEBUG_INFO_DWARF5 is not set
> > # CONFIG_DEBUG_INFO_REDUCED is not set
> > CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
> > # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
> > # CONFIG_DEBUG_INFO_SPLIT is not set
> > CONFIG_DEBUG_INFO_BTF=y
> > CONFIG_PAHOLE_HAS_SPLIT_BTF=y
> > CONFIG_DEBUG_INFO_BTF_MODULES=y
>
> thanks for the report! I've just reproduced this now with gcc 14; my
> initial theory was it might be DWARF5-related, but dedup issues occur
> for modules with CONFIG_DEBUG_INFO_DWARF4=y also. I'm seeing task_struct
> duplicates in module BTF among other things, so I will try and dig
> further and report back when I find something. Like you I suspect the

This is a bizarre case. I have a custom small tool that recursively
traverses two parallel subgraphs of BTF types and prints anything that
differs between them ([0]). (I had to disable distilled BTF to make
use of this, the issue is present both with distilled BTF and
without).

I see that struct sock both in vmlinux and bpf_testmod.ko are
*IDENTICAL*. There is no difference I could detect. So very weird. I'm
thinking of bisecting, as this didn't happen before with exactly the
same compiler and pahole, so this must be a kernel-side change.

  [0] https://github.com/anakryiko/libbpf-bootstrap/tree/btfdiff-hack

> issues with missing kfuncs are different; may be an issue with our logic
> handling inconsistent functions getting confused by the .cold
> components. But right now understanding dedup issues is the top priority.
>
> Alan

Re: pahole and gcc-14 issues

[Alan Maguire]

On 25/04/2025 18:58, Andrii Nakryiko wrote:
> On Fri, Apr 25, 2025 at 10:50 AM Alan Maguire <alan.maguire@oracle.com> wrote:
>>
>> On 25/04/2025 15:50, Alexei Starovoitov wrote:
>>> Hi All,
>>>
>>> Looks like pahole fails to deduplicate BTF when kernel and
>>> kernel module are built with gcc-14.
>>> I see this issue with various kernel .config-s on bpf and
>>> bpf-next trees.
>>> I tried pahole 1.28 and the latest master. Same issues.
>>>
>>> BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
>>> When built with gcc-13 it has 454 types.
>>> So something is confusing dedup logic.
>>> Would be great if dedup experts can take a look,
>>> since this dedup issue is breaking a lot of selftests/bpf.
>>>
>>> Also vmlinux.h generated out of the kernel compiled with gcc-13
>>> and out of the kernel compiled with gcc-14 shows these differences:
>>>
>>> --- vmlinux13.h    2025-04-24 21:33:50.556884372 -0700
>>> +++ vmlinux14.h    2025-04-24 21:39:10.310488992 -0700
>>> @@ -148815,7 +148815,6 @@
>>>  extern int hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum
>>> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
>>>  extern void hid_bpf_release_context(struct hid_bpf_ctx *ctx) __weak __ksym;
>>>  extern int hid_bpf_try_input_report(struct hid_bpf_ctx *ctx, enum
>>> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
>>> -extern bool scx_bpf_consume(u64 dsq_id) __weak __ksym;
>>>  extern int scx_bpf_cpu_node(s32 cpu) __weak __ksym;
>>>  extern struct rq *scx_bpf_cpu_rq(s32 cpu) __weak __ksym;
>>>  extern u32 scx_bpf_cpuperf_cap(s32 cpu) __weak __ksym;
>>> @@ -148825,12 +148824,8 @@
>>>  extern void scx_bpf_destroy_dsq(u64 dsq_id) __weak __ksym;
>>>  extern void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64
>>> slice, u64 enq_flags) __weak __ksym;
>>>  extern void scx_bpf_dispatch_cancel(void) __weak __ksym;
>>> -extern bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq
>>> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
>>> __ksym;
>>> -extern void scx_bpf_dispatch_from_dsq_set_slice(struct
>>> bpf_iter_scx_dsq *it__iter, u64 slice) __weak __ksym;
>>>  extern void scx_bpf_dispatch_from_dsq_set_vtime(struct
>>> bpf_iter_scx_dsq *it__iter, u64 vtime) __weak __ksym;
>>>  extern u32 scx_bpf_dispatch_nr_slots(void) __weak __ksym;
>>> -extern void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
>>> u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
>>> -extern bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq
>>> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
>>> __ksym;
>>>  extern void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64
>>> slice, u64 enq_flags) __weak __ksym;
>>>  extern void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64
>>> dsq_id, u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
>>>  extern bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
>>> struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak __ksym;
>>>
>>> gcc-14's kernel is clearly wrong.
>>> These 5 kfuncs still exist in the kernel.
>>> I manually checked there is no if __GNUC__ > 13 in the code.
>>> Also:
>>> nm bld/vmlinux|grep -w scx_bpf_consume
>>> ffffffff8159d4b0 T scx_bpf_consume
>>> ffffffff8120ea81 t scx_bpf_consume.cold
>>>
>>> I suspect the second issue is not related to the dedup problem.
>>> All 5 missing kfuncs have ".cold" optimized bodies.
>>> But ".cold" maybe a red herring, since
>>> nm bld/vmlinux|grep -w scx_bpf_dispatch
>>> ffffffff8159d020 T scx_bpf_dispatch
>>> ffffffff8120ea0f t scx_bpf_dispatch.cold
>>> but this kfunc is present in vmlinux14.h
>>>
>>> If it makes a difference I have these configs:
>>> # CONFIG_DEBUG_INFO_DWARF4 is not set
>>> # CONFIG_DEBUG_INFO_DWARF5 is not set
>>> # CONFIG_DEBUG_INFO_REDUCED is not set
>>> CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
>>> # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
>>> # CONFIG_DEBUG_INFO_SPLIT is not set
>>> CONFIG_DEBUG_INFO_BTF=y
>>> CONFIG_PAHOLE_HAS_SPLIT_BTF=y
>>> CONFIG_DEBUG_INFO_BTF_MODULES=y
>>
>> thanks for the report! I've just reproduced this now with gcc 14; my
>> initial theory was it might be DWARF5-related, but dedup issues occur
>> for modules with CONFIG_DEBUG_INFO_DWARF4=y also. I'm seeing task_struct
>> duplicates in module BTF among other things, so I will try and dig
>> further and report back when I find something. Like you I suspect the
> 
> This is a bizarre case. I have a custom small tool that recursively
> traverses two parallel subgraphs of BTF types and prints anything that
> differs between them ([0]). (I had to disable distilled BTF to make
> use of this, the issue is present both with distilled BTF and
> without).
> 
> I see that struct sock both in vmlinux and bpf_testmod.ko are
> *IDENTICAL*. There is no difference I could detect. So very weird. I'm
> thinking of bisecting, as this didn't happen before with exactly the
> same compiler and pahole, so this must be a kernel-side change.
> 
>   [0] https://github.com/anakryiko/libbpf-bootstrap/tree/btfdiff-hack
> 

thanks for the pointer to this! My initial suspicion was that we had
some sort of dups of slightly-differently-defined primitive types that
bubbled up through multiple structs in the module case since the level
of duplication is so high; a colleague ran across something like this
recently and indeed if I dump vmlinux BTF in C format I see:

typedef unsigned char u8___2;

...along with the original u8 definition:

typedef unsigned char __u8;
typedef __u8 u8;

However on checking I didn't find any references to the "wrong" u8, so I
don't think it is the cause (the definition comes from
crypto/jitterentropy.c so as a .c redefinition it's less likely to cause
chaos across multiple CUs).

Perhaps we should be thinking of cases where "#ifdef MODULE" leads to
different structure content, maybe something changed that results in
that leaking into core kernel structures like task_struct. Haven't had
any luck finding a common culprit across duplicated structures yet..

Alan

Re: pahole and gcc-14 issues

[Andrii Nakryiko]

On Fri, Apr 25, 2025 at 1:36 PM Alan Maguire <alan.maguire@oracle.com> wrote:
>
> On 25/04/2025 18:58, Andrii Nakryiko wrote:
> > On Fri, Apr 25, 2025 at 10:50 AM Alan Maguire <alan.maguire@oracle.com> wrote:
> >>
> >> On 25/04/2025 15:50, Alexei Starovoitov wrote:
> >>> Hi All,
> >>>
> >>> Looks like pahole fails to deduplicate BTF when kernel and
> >>> kernel module are built with gcc-14.
> >>> I see this issue with various kernel .config-s on bpf and
> >>> bpf-next trees.
> >>> I tried pahole 1.28 and the latest master. Same issues.
> >>>
> >>> BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
> >>> When built with gcc-13 it has 454 types.
> >>> So something is confusing dedup logic.
> >>> Would be great if dedup experts can take a look,
> >>> since this dedup issue is breaking a lot of selftests/bpf.
> >>>
> >>> Also vmlinux.h generated out of the kernel compiled with gcc-13
> >>> and out of the kernel compiled with gcc-14 shows these differences:
> >>>
> >>> --- vmlinux13.h    2025-04-24 21:33:50.556884372 -0700
> >>> +++ vmlinux14.h    2025-04-24 21:39:10.310488992 -0700
> >>> @@ -148815,7 +148815,6 @@
> >>>  extern int hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum
> >>> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
> >>>  extern void hid_bpf_release_context(struct hid_bpf_ctx *ctx) __weak __ksym;
> >>>  extern int hid_bpf_try_input_report(struct hid_bpf_ctx *ctx, enum
> >>> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
> >>> -extern bool scx_bpf_consume(u64 dsq_id) __weak __ksym;
> >>>  extern int scx_bpf_cpu_node(s32 cpu) __weak __ksym;
> >>>  extern struct rq *scx_bpf_cpu_rq(s32 cpu) __weak __ksym;
> >>>  extern u32 scx_bpf_cpuperf_cap(s32 cpu) __weak __ksym;
> >>> @@ -148825,12 +148824,8 @@
> >>>  extern void scx_bpf_destroy_dsq(u64 dsq_id) __weak __ksym;
> >>>  extern void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64
> >>> slice, u64 enq_flags) __weak __ksym;
> >>>  extern void scx_bpf_dispatch_cancel(void) __weak __ksym;
> >>> -extern bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq
> >>> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
> >>> __ksym;
> >>> -extern void scx_bpf_dispatch_from_dsq_set_slice(struct
> >>> bpf_iter_scx_dsq *it__iter, u64 slice) __weak __ksym;
> >>>  extern void scx_bpf_dispatch_from_dsq_set_vtime(struct
> >>> bpf_iter_scx_dsq *it__iter, u64 vtime) __weak __ksym;
> >>>  extern u32 scx_bpf_dispatch_nr_slots(void) __weak __ksym;
> >>> -extern void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
> >>> u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
> >>> -extern bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq
> >>> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
> >>> __ksym;
> >>>  extern void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64
> >>> slice, u64 enq_flags) __weak __ksym;
> >>>  extern void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64
> >>> dsq_id, u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
> >>>  extern bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
> >>> struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak __ksym;
> >>>
> >>> gcc-14's kernel is clearly wrong.
> >>> These 5 kfuncs still exist in the kernel.
> >>> I manually checked there is no if __GNUC__ > 13 in the code.
> >>> Also:
> >>> nm bld/vmlinux|grep -w scx_bpf_consume
> >>> ffffffff8159d4b0 T scx_bpf_consume
> >>> ffffffff8120ea81 t scx_bpf_consume.cold
> >>>
> >>> I suspect the second issue is not related to the dedup problem.
> >>> All 5 missing kfuncs have ".cold" optimized bodies.
> >>> But ".cold" maybe a red herring, since
> >>> nm bld/vmlinux|grep -w scx_bpf_dispatch
> >>> ffffffff8159d020 T scx_bpf_dispatch
> >>> ffffffff8120ea0f t scx_bpf_dispatch.cold
> >>> but this kfunc is present in vmlinux14.h
> >>>
> >>> If it makes a difference I have these configs:
> >>> # CONFIG_DEBUG_INFO_DWARF4 is not set
> >>> # CONFIG_DEBUG_INFO_DWARF5 is not set
> >>> # CONFIG_DEBUG_INFO_REDUCED is not set
> >>> CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
> >>> # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
> >>> # CONFIG_DEBUG_INFO_SPLIT is not set
> >>> CONFIG_DEBUG_INFO_BTF=y
> >>> CONFIG_PAHOLE_HAS_SPLIT_BTF=y
> >>> CONFIG_DEBUG_INFO_BTF_MODULES=y
> >>
> >> thanks for the report! I've just reproduced this now with gcc 14; my
> >> initial theory was it might be DWARF5-related, but dedup issues occur
> >> for modules with CONFIG_DEBUG_INFO_DWARF4=y also. I'm seeing task_struct
> >> duplicates in module BTF among other things, so I will try and dig
> >> further and report back when I find something. Like you I suspect the
> >
> > This is a bizarre case. I have a custom small tool that recursively
> > traverses two parallel subgraphs of BTF types and prints anything that
> > differs between them ([0]). (I had to disable distilled BTF to make
> > use of this, the issue is present both with distilled BTF and
> > without).
> >
> > I see that struct sock both in vmlinux and bpf_testmod.ko are
> > *IDENTICAL*. There is no difference I could detect. So very weird. I'm
> > thinking of bisecting, as this didn't happen before with exactly the
> > same compiler and pahole, so this must be a kernel-side change.
> >
> >   [0] https://github.com/anakryiko/libbpf-bootstrap/tree/btfdiff-hack
> >
>
> thanks for the pointer to this! My initial suspicion was that we had
> some sort of dups of slightly-differently-defined primitive types that
> bubbled up through multiple structs in the module case since the level
> of duplication is so high; a colleague ran across something like this
> recently and indeed if I dump vmlinux BTF in C format I see:
>
> typedef unsigned char u8___2;
>
> ...along with the original u8 definition:
>
> typedef unsigned char __u8;
> typedef __u8 u8;

Are you sure you are not dumping distilled BTF?

>
> However on checking I didn't find any references to the "wrong" u8, so I
> don't think it is the cause (the definition comes from
> crypto/jitterentropy.c so as a .c redefinition it's less likely to cause
> chaos across multiple CUs).
>
> Perhaps we should be thinking of cases where "#ifdef MODULE" leads to
> different structure content, maybe something changed that results in
> that leaking into core kernel structures like task_struct. Haven't had
> any luck finding a common culprit across duplicated structures yet..
>
> Alan

Re: pahole and gcc-14 issues

[Alan Maguire]

On 25/04/2025 21:41, Andrii Nakryiko wrote:
> On Fri, Apr 25, 2025 at 1:36 PM Alan Maguire <alan.maguire@oracle.com> wrote:
>>
>> On 25/04/2025 18:58, Andrii Nakryiko wrote:
>>> On Fri, Apr 25, 2025 at 10:50 AM Alan Maguire <alan.maguire@oracle.com> wrote:
>>>>
>>>> On 25/04/2025 15:50, Alexei Starovoitov wrote:
>>>>> Hi All,
>>>>>
>>>>> Looks like pahole fails to deduplicate BTF when kernel and
>>>>> kernel module are built with gcc-14.
>>>>> I see this issue with various kernel .config-s on bpf and
>>>>> bpf-next trees.
>>>>> I tried pahole 1.28 and the latest master. Same issues.
>>>>>
>>>>> BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
>>>>> When built with gcc-13 it has 454 types.
>>>>> So something is confusing dedup logic.
>>>>> Would be great if dedup experts can take a look,
>>>>> since this dedup issue is breaking a lot of selftests/bpf.
>>>>>
>>>>> Also vmlinux.h generated out of the kernel compiled with gcc-13
>>>>> and out of the kernel compiled with gcc-14 shows these differences:
>>>>>
>>>>> --- vmlinux13.h    2025-04-24 21:33:50.556884372 -0700
>>>>> +++ vmlinux14.h    2025-04-24 21:39:10.310488992 -0700
>>>>> @@ -148815,7 +148815,6 @@
>>>>>  extern int hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum
>>>>> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
>>>>>  extern void hid_bpf_release_context(struct hid_bpf_ctx *ctx) __weak __ksym;
>>>>>  extern int hid_bpf_try_input_report(struct hid_bpf_ctx *ctx, enum
>>>>> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
>>>>> -extern bool scx_bpf_consume(u64 dsq_id) __weak __ksym;
>>>>>  extern int scx_bpf_cpu_node(s32 cpu) __weak __ksym;
>>>>>  extern struct rq *scx_bpf_cpu_rq(s32 cpu) __weak __ksym;
>>>>>  extern u32 scx_bpf_cpuperf_cap(s32 cpu) __weak __ksym;
>>>>> @@ -148825,12 +148824,8 @@
>>>>>  extern void scx_bpf_destroy_dsq(u64 dsq_id) __weak __ksym;
>>>>>  extern void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64
>>>>> slice, u64 enq_flags) __weak __ksym;
>>>>>  extern void scx_bpf_dispatch_cancel(void) __weak __ksym;
>>>>> -extern bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq
>>>>> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
>>>>> __ksym;
>>>>> -extern void scx_bpf_dispatch_from_dsq_set_slice(struct
>>>>> bpf_iter_scx_dsq *it__iter, u64 slice) __weak __ksym;
>>>>>  extern void scx_bpf_dispatch_from_dsq_set_vtime(struct
>>>>> bpf_iter_scx_dsq *it__iter, u64 vtime) __weak __ksym;
>>>>>  extern u32 scx_bpf_dispatch_nr_slots(void) __weak __ksym;
>>>>> -extern void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
>>>>> u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
>>>>> -extern bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq
>>>>> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
>>>>> __ksym;
>>>>>  extern void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64
>>>>> slice, u64 enq_flags) __weak __ksym;
>>>>>  extern void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64
>>>>> dsq_id, u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
>>>>>  extern bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
>>>>> struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak __ksym;
>>>>>
>>>>> gcc-14's kernel is clearly wrong.
>>>>> These 5 kfuncs still exist in the kernel.
>>>>> I manually checked there is no if __GNUC__ > 13 in the code.
>>>>> Also:
>>>>> nm bld/vmlinux|grep -w scx_bpf_consume
>>>>> ffffffff8159d4b0 T scx_bpf_consume
>>>>> ffffffff8120ea81 t scx_bpf_consume.cold
>>>>>
>>>>> I suspect the second issue is not related to the dedup problem.
>>>>> All 5 missing kfuncs have ".cold" optimized bodies.
>>>>> But ".cold" maybe a red herring, since
>>>>> nm bld/vmlinux|grep -w scx_bpf_dispatch
>>>>> ffffffff8159d020 T scx_bpf_dispatch
>>>>> ffffffff8120ea0f t scx_bpf_dispatch.cold
>>>>> but this kfunc is present in vmlinux14.h
>>>>>
>>>>> If it makes a difference I have these configs:
>>>>> # CONFIG_DEBUG_INFO_DWARF4 is not set
>>>>> # CONFIG_DEBUG_INFO_DWARF5 is not set
>>>>> # CONFIG_DEBUG_INFO_REDUCED is not set
>>>>> CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
>>>>> # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
>>>>> # CONFIG_DEBUG_INFO_SPLIT is not set
>>>>> CONFIG_DEBUG_INFO_BTF=y
>>>>> CONFIG_PAHOLE_HAS_SPLIT_BTF=y
>>>>> CONFIG_DEBUG_INFO_BTF_MODULES=y
>>>>
>>>> thanks for the report! I've just reproduced this now with gcc 14; my
>>>> initial theory was it might be DWARF5-related, but dedup issues occur
>>>> for modules with CONFIG_DEBUG_INFO_DWARF4=y also. I'm seeing task_struct
>>>> duplicates in module BTF among other things, so I will try and dig
>>>> further and report back when I find something. Like you I suspect the
>>>
>>> This is a bizarre case. I have a custom small tool that recursively
>>> traverses two parallel subgraphs of BTF types and prints anything that
>>> differs between them ([0]). (I had to disable distilled BTF to make
>>> use of this, the issue is present both with distilled BTF and
>>> without).
>>>
>>> I see that struct sock both in vmlinux and bpf_testmod.ko are
>>> *IDENTICAL*. There is no difference I could detect. So very weird. I'm
>>> thinking of bisecting, as this didn't happen before with exactly the
>>> same compiler and pahole, so this must be a kernel-side change.
>>>
>>>   [0] https://github.com/anakryiko/libbpf-bootstrap/tree/btfdiff-hack
>>>
>>
>> thanks for the pointer to this! My initial suspicion was that we had
>> some sort of dups of slightly-differently-defined primitive types that
>> bubbled up through multiple structs in the module case since the level
>> of duplication is so high; a colleague ran across something like this
>> recently and indeed if I dump vmlinux BTF in C format I see:
>>
>> typedef unsigned char u8___2;
>>
>> ...along with the original u8 definition:
>>
>> typedef unsigned char __u8;
>> typedef __u8 u8;
> 
> Are you sure you are not dumping distilled BTF?
>

nope, that's in vmlinux BTF, originating from crypto/jitterentropy.c

> This is the commit introducing a regression:
>
> eb0ece16027f ("Merge tag 'mm-stable-2025-03-30-16-52' of
> git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm")
>
> Yes, it's a "merge commit", but there is a lot of code introduced in
> it. Among it:
>
> + /*
> +  * Use __typeof_unqual__() when available.
> +  *
> +  * XXX: Remove test for __CHECKER__ once
> +  * sparse learns about __typeof_unqual__().
> +  */
> + #if CC_HAS_TYPEOF_UNQUAL && !defined(__CHECKER__)
> + # define USE_TYPEOF_UNQUAL 1
> + #endif
> +
> + /*
> +  * Define TYPEOF_UNQUAL() to use __typeof_unqual__() as typeof
> +  * operator when available, to return an unqualified type of the exp.
> +  */
> + #if defined(USE_TYPEOF_UNQUAL)
> + # define TYPEOF_UNQUAL(exp) __typeof_unqual__(exp)
> + #else
> + # define TYPEOF_UNQUAL(exp) __typeof__(exp)
> + #endif
> +
>
>
> And that's exactly what causes this divergence. Commenting out that
> USE_TYPEOF_UNQUAL #define fixes issues.
>
> As to why that causes a problem. I suspect __typeof_unqual__() changes
> how GCC generates DWARF information within any given compilation unit
> (CU). Libbpf's BTF dedup relies on a property that compiler won't have
> duplicate definitions of exactly the same type (i.e., DWARF itself
> can't have two `struct blah` definitions), without which it's not
> possible to deduplicate entire clusters of self-referencing BTF types.
> It seems like typeof_unqual breaks this somehow.
>
> We need to compare DWARF with and without TYPEOF_UNQUAL and see what
> the differences are and how we can prevent or accommodate them.

Great find! As you suspect the handling in BTF btf_dedup_is_equiv()
covers two cases handling multiple instances of objects in DWARF;
duplicate arrays and duplicate structs. In this case however we are for
some reason winding up with multiple copies of structures containing
duplicate pointers in DWARF which have different type ids, which however
both point at the same target type. Adding the following to BTF dedup
accordingly solves the cascade of dedup issues for me:

diff --git a/src/btf.c b/src/btf.c
index eea99c7..2155dd9 100644
--- a/src/btf.c
+++ b/src/btf.c
@@ -4379,6 +4379,18 @@ static bool btf_dedup_identical_structs(struct
btf_dedup *d, __u32 id1, __u32 id
        return true;
 }

+static bool btf_dedup_identical_ptrs(struct btf_dedup *d, __u32 id1,
__u32 id2)
+{
+       struct btf_type *t1, *t2;
+
+       t1 = btf_type_by_id(d->btf, id1);
+       t2 = btf_type_by_id(d->btf, id2);
+
+       if (btf_kind(t1) != BTF_KIND_PTR || btf_kind(t1) != btf_kind(t2))
+               return false;
+       return t1->type == t2->type;
+}
+
 /*
  * Check equivalence of BTF type graph formed by candidate struct/union
(we'll
  * call it "candidate graph" in this description for brevity) to a type
graph
@@ -4511,6 +4523,8 @@ static int btf_dedup_is_equiv(struct btf_dedup *d,
__u32 cand_id,
                 */
                if (btf_dedup_identical_structs(d, hypot_type_id, cand_id))
                        return 1;
+               if (btf_dedup_identical_ptrs(d, hypot_type_id, cand_id))
+                       return 1;
                return 0;
        }


Now why this new behaviour results from the inclusion of typeof_unqual()
is something I can't explain yet. Requires more digging to understand
exactly what's going on..

Alan

Re: pahole and gcc-14 issues

[Alan Maguire]

On 26/04/2025 18:28, Alan Maguire wrote:
> On 25/04/2025 21:41, Andrii Nakryiko wrote:
>> On Fri, Apr 25, 2025 at 1:36 PM Alan Maguire <alan.maguire@oracle.com> wrote:
>>>
>>> On 25/04/2025 18:58, Andrii Nakryiko wrote:
>>>> On Fri, Apr 25, 2025 at 10:50 AM Alan Maguire <alan.maguire@oracle.com> wrote:
>>>>>
>>>>> On 25/04/2025 15:50, Alexei Starovoitov wrote:
>>>>>> Hi All,
>>>>>>
>>>>>> Looks like pahole fails to deduplicate BTF when kernel and
>>>>>> kernel module are built with gcc-14.
>>>>>> I see this issue with various kernel .config-s on bpf and
>>>>>> bpf-next trees.
>>>>>> I tried pahole 1.28 and the latest master. Same issues.
>>>>>>
>>>>>> BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
>>>>>> When built with gcc-13 it has 454 types.
>>>>>> So something is confusing dedup logic.
>>>>>> Would be great if dedup experts can take a look,
>>>>>> since this dedup issue is breaking a lot of selftests/bpf.
>>>>>>
>>>>>> Also vmlinux.h generated out of the kernel compiled with gcc-13
>>>>>> and out of the kernel compiled with gcc-14 shows these differences:
>>>>>>
>>>>>> --- vmlinux13.h    2025-04-24 21:33:50.556884372 -0700
>>>>>> +++ vmlinux14.h    2025-04-24 21:39:10.310488992 -0700
>>>>>> @@ -148815,7 +148815,6 @@
>>>>>>  extern int hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum
>>>>>> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
>>>>>>  extern void hid_bpf_release_context(struct hid_bpf_ctx *ctx) __weak __ksym;
>>>>>>  extern int hid_bpf_try_input_report(struct hid_bpf_ctx *ctx, enum
>>>>>> hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
>>>>>> -extern bool scx_bpf_consume(u64 dsq_id) __weak __ksym;
>>>>>>  extern int scx_bpf_cpu_node(s32 cpu) __weak __ksym;
>>>>>>  extern struct rq *scx_bpf_cpu_rq(s32 cpu) __weak __ksym;
>>>>>>  extern u32 scx_bpf_cpuperf_cap(s32 cpu) __weak __ksym;
>>>>>> @@ -148825,12 +148824,8 @@
>>>>>>  extern void scx_bpf_destroy_dsq(u64 dsq_id) __weak __ksym;
>>>>>>  extern void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64
>>>>>> slice, u64 enq_flags) __weak __ksym;
>>>>>>  extern void scx_bpf_dispatch_cancel(void) __weak __ksym;
>>>>>> -extern bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq
>>>>>> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
>>>>>> __ksym;
>>>>>> -extern void scx_bpf_dispatch_from_dsq_set_slice(struct
>>>>>> bpf_iter_scx_dsq *it__iter, u64 slice) __weak __ksym;
>>>>>>  extern void scx_bpf_dispatch_from_dsq_set_vtime(struct
>>>>>> bpf_iter_scx_dsq *it__iter, u64 vtime) __weak __ksym;
>>>>>>  extern u32 scx_bpf_dispatch_nr_slots(void) __weak __ksym;
>>>>>> -extern void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
>>>>>> u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
>>>>>> -extern bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq
>>>>>> *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
>>>>>> __ksym;
>>>>>>  extern void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64
>>>>>> slice, u64 enq_flags) __weak __ksym;
>>>>>>  extern void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64
>>>>>> dsq_id, u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
>>>>>>  extern bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
>>>>>> struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak __ksym;
>>>>>>
>>>>>> gcc-14's kernel is clearly wrong.
>>>>>> These 5 kfuncs still exist in the kernel.
>>>>>> I manually checked there is no if __GNUC__ > 13 in the code.
>>>>>> Also:
>>>>>> nm bld/vmlinux|grep -w scx_bpf_consume
>>>>>> ffffffff8159d4b0 T scx_bpf_consume
>>>>>> ffffffff8120ea81 t scx_bpf_consume.cold
>>>>>>
>>>>>> I suspect the second issue is not related to the dedup problem.
>>>>>> All 5 missing kfuncs have ".cold" optimized bodies.
>>>>>> But ".cold" maybe a red herring, since
>>>>>> nm bld/vmlinux|grep -w scx_bpf_dispatch
>>>>>> ffffffff8159d020 T scx_bpf_dispatch
>>>>>> ffffffff8120ea0f t scx_bpf_dispatch.cold
>>>>>> but this kfunc is present in vmlinux14.h
>>>>>>
>>>>>> If it makes a difference I have these configs:
>>>>>> # CONFIG_DEBUG_INFO_DWARF4 is not set
>>>>>> # CONFIG_DEBUG_INFO_DWARF5 is not set
>>>>>> # CONFIG_DEBUG_INFO_REDUCED is not set
>>>>>> CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
>>>>>> # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
>>>>>> # CONFIG_DEBUG_INFO_SPLIT is not set
>>>>>> CONFIG_DEBUG_INFO_BTF=y
>>>>>> CONFIG_PAHOLE_HAS_SPLIT_BTF=y
>>>>>> CONFIG_DEBUG_INFO_BTF_MODULES=y
>>>>>
>>>>> thanks for the report! I've just reproduced this now with gcc 14; my
>>>>> initial theory was it might be DWARF5-related, but dedup issues occur
>>>>> for modules with CONFIG_DEBUG_INFO_DWARF4=y also. I'm seeing task_struct
>>>>> duplicates in module BTF among other things, so I will try and dig
>>>>> further and report back when I find something. Like you I suspect the
>>>>
>>>> This is a bizarre case. I have a custom small tool that recursively
>>>> traverses two parallel subgraphs of BTF types and prints anything that
>>>> differs between them ([0]). (I had to disable distilled BTF to make
>>>> use of this, the issue is present both with distilled BTF and
>>>> without).
>>>>
>>>> I see that struct sock both in vmlinux and bpf_testmod.ko are
>>>> *IDENTICAL*. There is no difference I could detect. So very weird. I'm
>>>> thinking of bisecting, as this didn't happen before with exactly the
>>>> same compiler and pahole, so this must be a kernel-side change.
>>>>
>>>>   [0] https://github.com/anakryiko/libbpf-bootstrap/tree/btfdiff-hack
>>>>
>>>
>>> thanks for the pointer to this! My initial suspicion was that we had
>>> some sort of dups of slightly-differently-defined primitive types that
>>> bubbled up through multiple structs in the module case since the level
>>> of duplication is so high; a colleague ran across something like this
>>> recently and indeed if I dump vmlinux BTF in C format I see:
>>>
>>> typedef unsigned char u8___2;
>>>
>>> ...along with the original u8 definition:
>>>
>>> typedef unsigned char __u8;
>>> typedef __u8 u8;
>>
>> Are you sure you are not dumping distilled BTF?
>>
> 
> nope, that's in vmlinux BTF, originating from crypto/jitterentropy.c
> 
>> This is the commit introducing a regression:
>>
>> eb0ece16027f ("Merge tag 'mm-stable-2025-03-30-16-52' of
>> git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm")
>>
>> Yes, it's a "merge commit", but there is a lot of code introduced in
>> it. Among it:
>>
>> + /*
>> +  * Use __typeof_unqual__() when available.
>> +  *
>> +  * XXX: Remove test for __CHECKER__ once
>> +  * sparse learns about __typeof_unqual__().
>> +  */
>> + #if CC_HAS_TYPEOF_UNQUAL && !defined(__CHECKER__)
>> + # define USE_TYPEOF_UNQUAL 1
>> + #endif
>> +
>> + /*
>> +  * Define TYPEOF_UNQUAL() to use __typeof_unqual__() as typeof
>> +  * operator when available, to return an unqualified type of the exp.
>> +  */
>> + #if defined(USE_TYPEOF_UNQUAL)
>> + # define TYPEOF_UNQUAL(exp) __typeof_unqual__(exp)
>> + #else
>> + # define TYPEOF_UNQUAL(exp) __typeof__(exp)
>> + #endif
>> +
>>
>>
>> And that's exactly what causes this divergence. Commenting out that
>> USE_TYPEOF_UNQUAL #define fixes issues.
>>
>> As to why that causes a problem. I suspect __typeof_unqual__() changes
>> how GCC generates DWARF information within any given compilation unit
>> (CU). Libbpf's BTF dedup relies on a property that compiler won't have
>> duplicate definitions of exactly the same type (i.e., DWARF itself
>> can't have two `struct blah` definitions), without which it's not
>> possible to deduplicate entire clusters of self-referencing BTF types.
>> It seems like typeof_unqual breaks this somehow.
>>
>> We need to compare DWARF with and without TYPEOF_UNQUAL and see what
>> the differences are and how we can prevent or accommodate them.
> 
> Great find! As you suspect the handling in BTF btf_dedup_is_equiv()
> covers two cases handling multiple instances of objects in DWARF;
> duplicate arrays and duplicate structs. In this case however we are for
> some reason winding up with multiple copies of structures containing
> duplicate pointers in DWARF which have different type ids, which however
> both point at the same target type. Adding the following to BTF dedup
> accordingly solves the cascade of dedup issues for me:
> 
> diff --git a/src/btf.c b/src/btf.c
> index eea99c7..2155dd9 100644
> --- a/src/btf.c
> +++ b/src/btf.c
> @@ -4379,6 +4379,18 @@ static bool btf_dedup_identical_structs(struct
> btf_dedup *d, __u32 id1, __u32 id
>         return true;
>  }
> 
> +static bool btf_dedup_identical_ptrs(struct btf_dedup *d, __u32 id1,
> __u32 id2)
> +{
> +       struct btf_type *t1, *t2;
> +
> +       t1 = btf_type_by_id(d->btf, id1);
> +       t2 = btf_type_by_id(d->btf, id2);
> +
> +       if (btf_kind(t1) != BTF_KIND_PTR || btf_kind(t1) != btf_kind(t2))
> +               return false;
> +       return t1->type == t2->type;
> +}
> +
>  /*
>   * Check equivalence of BTF type graph formed by candidate struct/union
> (we'll
>   * call it "candidate graph" in this description for brevity) to a type
> graph
> @@ -4511,6 +4523,8 @@ static int btf_dedup_is_equiv(struct btf_dedup *d,
> __u32 cand_id,
>                  */
>                 if (btf_dedup_identical_structs(d, hypot_type_id, cand_id))
>                         return 1;
> +               if (btf_dedup_identical_ptrs(d, hypot_type_id, cand_id))
> +                       return 1;
>                 return 0;
>         }
> 
> 
> Now why this new behaviour results from the inclusion of typeof_unqual()
> is something I can't explain yet. Requires more digging to understand
> exactly what's going on..
>

The connection here seems to be around __percpu - annotations; so for
example bpf_prog has the "int __percpu *active" field, and with the
typeof_unqual() change, duplicate pointer declarations in BTF lead to
the dedup issues. As far as DWARF representations go, we can compare
good vs bad below.

In the good case (no dedup issues) the bpf_prog active member looks like
this:

 <2><155b7b>: Abbrev Number: 4 (DW_TAG_member)
    <155b7c>   DW_AT_name        : (indirect string, offset: 0xeaa2c):
active
    <155b80>   DW_AT_decl_file   : 137
    <155b81>   DW_AT_decl_line   : 1654
    <155b83>   DW_AT_decl_column : 17
    <155b84>   DW_AT_type        : <0x14decc>
    <155b88>   DW_AT_data_member_location: 40

...i.e. it is a pointer:

 <1><14decc>: Abbrev Number: 3 (DW_TAG_pointer_type)
    <14decd>   DW_AT_byte_size   : 8
    <14dece>   DW_AT_type        : <0x147ddd>

...which points at an int:

 <1><147ddd>: Abbrev Number: 216 (DW_TAG_base_type)
    <147ddf>   DW_AT_byte_size   : 4
    <147de0>   DW_AT_encoding    : 5    (signed)
    <147de1>   DW_AT_name        : int

In the bad case, the bpf_prog active member:

<2><3d594>: Abbrev Number: 4 (DW_TAG_member)
    <3d595>   DW_AT_name        : (indirect string, offset: 0x3b976): active
    <3d599>   DW_AT_decl_file   : 125
    <3d59a>   DW_AT_decl_line   : 1654
    <3d59c>   DW_AT_decl_column : 17
    <3d59d>   DW_AT_type        : <0x4bd05>

...is a pointer:


 <1><4bd05>: Abbrev Number: 58 (DW_TAG_pointer_type)
    <4bd06>   DW_AT_byte_size   : 8
    <4bd07>   DW_AT_address_class: 2
    <4bd08>   DW_AT_type        : <0x301cd>

...which points at an int

 <1><301cd>: Abbrev Number: 214 (DW_TAG_base_type)
    <301cf>   DW_AT_byte_size   : 4
    <301d0>   DW_AT_encoding    : 5     (signed)
    <301d1>   DW_AT_name        : int
    <301d5>   DW_AT_name        : int

...but note the the DW_AT_address_class attribute in the latter case and
the two DW_AT_name values. We don't use that address attribute in pahole
as far as I can see, but it might be enough to cause problems.

Alan

Re: pahole and gcc-14 issues

[Arnaldo Carvalho de Melo]

On Mon, Apr 28, 2025 at 04:21:14PM +0100, Alan Maguire wrote:
> In the bad case, the bpf_prog active member:
 
> <2><3d594>: Abbrev Number: 4 (DW_TAG_member)
>     <3d595>   DW_AT_name        : (indirect string, offset: 0x3b976): active
>     <3d599>   DW_AT_decl_file   : 125
>     <3d59a>   DW_AT_decl_line   : 1654
>     <3d59c>   DW_AT_decl_column : 17
>     <3d59d>   DW_AT_type        : <0x4bd05>
 
> ...is a pointer:
> 
>  <1><4bd05>: Abbrev Number: 58 (DW_TAG_pointer_type)
>     <4bd06>   DW_AT_byte_size   : 8
>     <4bd07>   DW_AT_address_class: 2
>     <4bd08>   DW_AT_type        : <0x301cd>
 
> ...which points at an int
 
>  <1><301cd>: Abbrev Number: 214 (DW_TAG_base_type)
>     <301cf>   DW_AT_byte_size   : 4
>     <301d0>   DW_AT_encoding    : 5     (signed)
>     <301d1>   DW_AT_name        : int
>     <301d5>   DW_AT_name        : int
 
> ...but note the the DW_AT_address_class attribute in the latter case and
> the two DW_AT_name values. We don't use that address attribute in pahole
> as far as I can see, but it might be enough to cause problems.

Looks like broken DWARF, there should be just one DW_AT_name per
DW_TAG_base_type, what is the language for the CU where the bad cases
appear? Is some sort of LTO being used?

- Arnaldo

Re: pahole and gcc-14 issues

[Arnaldo Carvalho de Melo]

On Mon, Apr 28, 2025 at 04:36:07PM -0300, Arnaldo Carvalho de Melo wrote:
> On Mon, Apr 28, 2025 at 04:21:14PM +0100, Alan Maguire wrote:
> > In the bad case, the bpf_prog active member:
>  
> > <2><3d594>: Abbrev Number: 4 (DW_TAG_member)
> >     <3d595>   DW_AT_name        : (indirect string, offset: 0x3b976): active
> >     <3d599>   DW_AT_decl_file   : 125
> >     <3d59a>   DW_AT_decl_line   : 1654
> >     <3d59c>   DW_AT_decl_column : 17
> >     <3d59d>   DW_AT_type        : <0x4bd05>
>  
> > ...is a pointer:
> > 
> >  <1><4bd05>: Abbrev Number: 58 (DW_TAG_pointer_type)
> >     <4bd06>   DW_AT_byte_size   : 8
> >     <4bd07>   DW_AT_address_class: 2
> >     <4bd08>   DW_AT_type        : <0x301cd>
>  
> > ...which points at an int
>  
> >  <1><301cd>: Abbrev Number: 214 (DW_TAG_base_type)
> >     <301cf>   DW_AT_byte_size   : 4
> >     <301d0>   DW_AT_encoding    : 5     (signed)
> >     <301d1>   DW_AT_name        : int
> >     <301d5>   DW_AT_name        : int
>  
> > ...but note the the DW_AT_address_class attribute in the latter case and
> > the two DW_AT_name values. We don't use that address attribute in pahole
> > as far as I can see, but it might be enough to cause problems.

I just looked at a vmlinux built with gcc 15 and we have those:

 <1><7812a39>: Abbrev Number: 7 (DW_TAG_structure_type)
    <7812a3a>   DW_AT_name        : (indirect string, offset: 0xc0e4): alloc_tag_counters
    <7812a3e>   DW_AT_byte_size   : 16
    <7812a3f>   DW_AT_decl_file   : 76
    <7812a40>   DW_AT_decl_line   : 18
    <7812a41>   DW_AT_decl_column : 8
    <7812a42>   DW_AT_sibling     : <0x7812a61>
 <2><7812a46>: Abbrev Number: 1 (DW_TAG_member)
    <7812a47>   DW_AT_name        : (indirect string, offset: 0x3c288f): bytes
    <7812a4b>   DW_AT_decl_file   : 76
    <7812a4c>   DW_AT_decl_line   : 19
    <7812a4d>   DW_AT_decl_column : 6
    <7812a4e>   DW_AT_type        : <0x780f217>
    <7812a52>   DW_AT_data_member_location: 0
 <2><7812a53>: Abbrev Number: 1 (DW_TAG_member)
    <7812a54>   DW_AT_name        : (indirect string, offset: 0x255770): calls
    <7812a58>   DW_AT_decl_file   : 76
    <7812a59>   DW_AT_decl_line   : 20
    <7812a5a>   DW_AT_decl_column : 6
    <7812a5b>   DW_AT_type        : <0x780f217>
    <7812a5f>   DW_AT_data_member_location: 8

 <1><7812a88>: Abbrev Number: 80 (DW_TAG_pointer_type)
    <7812a89>   DW_AT_byte_size   : 8
    <7812a89>   DW_AT_address_class: 2
    <7812a89>   DW_AT_type        : <0x7812a39>

 <1><7812a61>: Abbrev Number: 25 (DW_TAG_structure_type)
    <7812a62>   DW_AT_name        : (indirect string, offset: 0x585bd1): alloc_tag
    <7812a66>   DW_AT_byte_size   : 40
    <7812a67>   DW_AT_alignment   : 8
    <7812a68>   DW_AT_decl_file   : 76
    <7812a69>   DW_AT_decl_line   : 28
    <7812a6a>   DW_AT_decl_column : 8
    <7812a6a>   DW_AT_sibling     : <0x7812a88>
 <2><7812a6e>: Abbrev Number: 54 (DW_TAG_member)
    <7812a6f>   DW_AT_name        : ct
    <7812a72>   DW_AT_decl_file   : 76
    <7812a73>   DW_AT_decl_line   : 29
    <7812a74>   DW_AT_decl_column : 19
    <7812a75>   DW_AT_type        : <0x78129ea>
    <7812a79>   DW_AT_alignment   : 8
    <7812a79>   DW_AT_data_member_location: 0
 <2><7812a7a>: Abbrev Number: 1 (DW_TAG_member)
    <7812a7b>   DW_AT_name        : (indirect string, offset: 0x565566): counters
    <7812a7f>   DW_AT_decl_file   : 76
    <7812a80>   DW_AT_decl_line   : 30
    <7812a81>   DW_AT_decl_column : 38
    <7812a82>   DW_AT_type        : <0x7812a88>
    <7812a86>   DW_AT_data_member_location: 32

struct alloc_tag {
        struct codetag                  ct;
        struct alloc_tag_counters __percpu      *counters;
} __aligned(8);

Its the __percpu, something else to catch in the DWARF loader and then
to use when pretty printing.

⬢ [acme@toolbx linux]$ pahole -F dwarf -C alloc_tag ../build/v6.15.0-rc2+/vmlinux
struct alloc_tag {
	struct codetag             ct __attribute__((__aligned__(8))); /*     0    32 */
	struct alloc_tag_counters * counters;            /*    32     8 */

	/* size: 40, cachelines: 1, members: 2 */
	/* forced alignments: 1 */
	/* last cacheline: 40 bytes */
} __attribute__((__aligned__(8)));
⬢ [acme@toolbx linux]$

- Arnaldo
 
> Looks like broken DWARF, there should be just one DW_AT_name per
> DW_TAG_base_type, what is the language for the CU where the bad cases
> appear? Is some sort of LTO being used?

Re: pahole and gcc-14 issues

[Alexei Starovoitov]

On Mon, Apr 28, 2025 at 8:21 AM Alan Maguire <alan.maguire@oracle.com> wrote:
>
>  <1><4bd05>: Abbrev Number: 58 (DW_TAG_pointer_type)
>     <4bd06>   DW_AT_byte_size   : 8
>     <4bd07>   DW_AT_address_class: 2
>     <4bd08>   DW_AT_type        : <0x301cd>
>
> ...which points at an int
>
>  <1><301cd>: Abbrev Number: 214 (DW_TAG_base_type)
>     <301cf>   DW_AT_byte_size   : 4
>     <301d0>   DW_AT_encoding    : 5     (signed)
>     <301d1>   DW_AT_name        : int
>     <301d5>   DW_AT_name        : int
>
> ...but note the the DW_AT_address_class attribute in the latter case and
> the two DW_AT_name values. We don't use that address attribute in pahole
> as far as I can see, but it might be enough to cause problems.

DW_AT_address_class is there because it's an actual address space
qualifier in C. The dwarf is correct, but I thought pahole
will ignore it while converting to BTF, so it shouldn't matter
from dedup pov.

And since dedup is working for vmlinux BTF, I doubt there are CUs
where the same type is represented with different dwarf id-s.
Otherwise dedup wouldn't have worked for vmlinux.

DW_AT_name is concerning. Sounds like it's a gcc bug, but it
shouldn't be causing dedup issues for modules.

So what is the workaround?

We need to find it asap. Since at present we cannot build
kernels with gcc-14, since modules won't dedup BTF.
Hence a bunch of selftests/bpf are failing.
We want to upgrade BPF CI to gcc-14 to catch nginx-like issues,
but we cannot until this pahole/dedup issue is resolved.

Re: pahole and gcc-14 issues

[Andrii Nakryiko]

On Mon, Apr 28, 2025 at 3:12 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Mon, Apr 28, 2025 at 8:21 AM Alan Maguire <alan.maguire@oracle.com> wrote:
> >
> >  <1><4bd05>: Abbrev Number: 58 (DW_TAG_pointer_type)
> >     <4bd06>   DW_AT_byte_size   : 8
> >     <4bd07>   DW_AT_address_class: 2
> >     <4bd08>   DW_AT_type        : <0x301cd>
> >
> > ...which points at an int
> >
> >  <1><301cd>: Abbrev Number: 214 (DW_TAG_base_type)
> >     <301cf>   DW_AT_byte_size   : 4
> >     <301d0>   DW_AT_encoding    : 5     (signed)
> >     <301d1>   DW_AT_name        : int
> >     <301d5>   DW_AT_name        : int
> >
> > ...but note the the DW_AT_address_class attribute in the latter case and
> > the two DW_AT_name values. We don't use that address attribute in pahole
> > as far as I can see, but it might be enough to cause problems.
>
> DW_AT_address_class is there because it's an actual address space
> qualifier in C. The dwarf is correct, but I thought pahole
> will ignore it while converting to BTF, so it shouldn't matter
> from dedup pov.
>
> And since dedup is working for vmlinux BTF, I doubt there are CUs
> where the same type is represented with different dwarf id-s.
> Otherwise dedup wouldn't have worked for vmlinux.
>
> DW_AT_name is concerning. Sounds like it's a gcc bug, but it
> shouldn't be causing dedup issues for modules.
>
> So what is the workaround?

I'm thinking of generalizing Alan's proposed fix so that all our
existing special equality cases (arrays, identical structs, and now
pointers to identical types) are handled a bit more generically. I'll
try to get a patch out later tonight.

>
> We need to find it asap. Since at present we cannot build
> kernels with gcc-14, since modules won't dedup BTF.
> Hence a bunch of selftests/bpf are failing.
> We want to upgrade BPF CI to gcc-14 to catch nginx-like issues,
> but we cannot until this pahole/dedup issue is resolved.

Re: pahole and gcc-14 issues

[Andrii Nakryiko]

On Mon, Apr 28, 2025 at 5:33 PM Andrii Nakryiko
<andrii.nakryiko@gmail.com> wrote:
>
> On Mon, Apr 28, 2025 at 3:12 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
> >
> > On Mon, Apr 28, 2025 at 8:21 AM Alan Maguire <alan.maguire@oracle.com> wrote:
> > >
> > >  <1><4bd05>: Abbrev Number: 58 (DW_TAG_pointer_type)
> > >     <4bd06>   DW_AT_byte_size   : 8
> > >     <4bd07>   DW_AT_address_class: 2
> > >     <4bd08>   DW_AT_type        : <0x301cd>
> > >
> > > ...which points at an int
> > >
> > >  <1><301cd>: Abbrev Number: 214 (DW_TAG_base_type)
> > >     <301cf>   DW_AT_byte_size   : 4
> > >     <301d0>   DW_AT_encoding    : 5     (signed)
> > >     <301d1>   DW_AT_name        : int
> > >     <301d5>   DW_AT_name        : int
> > >
> > > ...but note the the DW_AT_address_class attribute in the latter case and
> > > the two DW_AT_name values. We don't use that address attribute in pahole
> > > as far as I can see, but it might be enough to cause problems.
> >
> > DW_AT_address_class is there because it's an actual address space
> > qualifier in C. The dwarf is correct, but I thought pahole
> > will ignore it while converting to BTF, so it shouldn't matter
> > from dedup pov.
> >
> > And since dedup is working for vmlinux BTF, I doubt there are CUs
> > where the same type is represented with different dwarf id-s.
> > Otherwise dedup wouldn't have worked for vmlinux.
> >
> > DW_AT_name is concerning. Sounds like it's a gcc bug, but it
> > shouldn't be causing dedup issues for modules.
> >
> > So what is the workaround?
>
> I'm thinking of generalizing Alan's proposed fix so that all our
> existing special equality cases (arrays, identical structs, and now
> pointers to identical types) are handled a bit more generically. I'll
> try to get a patch out later tonight.

So I ran out of time, but I'm thinking something like below. It
results in identical bpf_testmod.ko compared to Alan's proposed fix,
so perhaps we should just go with the simpler approach. But this one
should stand the test of time a bit better.

In any case, I'd like to be able to handle not just PTR -> TYPE chain,
but also some PTR -> MODIFIER* -> TYPE chains at the very least.
Because any const in the chain will throw off Alan's heuristic.

I'll try to benchmark and polish the patch tomorrow and post it for
proper review.


diff --git a/src/btf.c b/src/btf.c
index e9673c0ecbe7..e4a3e3183742 100644
--- a/src/btf.c
+++ b/src/btf.c
@@ -4310,6 +4310,8 @@ static bool btf_dedup_identical_arrays(struct
btf_dedup *d, __u32 id1, __u32 id2
  return btf_equal_array(t1, t2);
 }

+static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1,
__u32 id2);
+
 /* Check if given two types are identical STRUCT/UNION definitions */
 static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32
id1, __u32 id2)
 {
@@ -4329,14 +4331,93 @@ static bool btf_dedup_identical_structs(struct
btf_dedup *d, __u32 id1, __u32 id
  m1 = btf_members(t1);
  m2 = btf_members(t2);
  for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
- if (m1->type != m2->type &&
-     !btf_dedup_identical_arrays(d, m1->type, m2->type) &&
-     !btf_dedup_identical_structs(d, m1->type, m2->type))
+ if (m1->type != m2->type && !btf_dedup_identical_types(d, m1->type, m2->type))
+ return false;
+ }
+ return true;
+}
+
+static bool btf_dedup_identical_fnprotos(struct btf_dedup *d, __u32
id1, __u32 id2)
+{
+ const struct btf_param *p1, *p2;
+ struct btf_type *t1, *t2;
+ int n, i;
+
+ t1 = btf_type_by_id(d->btf, id1);
+ t2 = btf_type_by_id(d->btf, id2);
+
+ if (!btf_is_func_proto(t1) || !btf_is_func_proto(t2))
+ return false;
+
+ if (!btf_compat_fnproto(t1, t2))
+ return false;
+
+ if (!btf_dedup_identical_types(d, t1->type, t2->type))
+ return false;
+
+ p1 = btf_params(t1);
+ p2 = btf_params(t2);
+ for (i = 0, n = btf_vlen(t1); i < n; i++, p1++, p2++) {
+ if (p1->type != p2->type && !btf_dedup_identical_types(d, p1->type, p2->type))
  return false;
  }
  return true;
 }

+static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1,
__u32 id2)
+{
+ int max_depth = 32;
+
+ while (max_depth-- > 0) {
+ struct btf_type *t1, *t2;
+ int k1, k2;
+
+ t1 = btf_type_by_id(d->btf, id1);
+ t2 = btf_type_by_id(d->btf, id2);
+
+ k1 = btf_kind(t1);
+ k2 = btf_kind(t2);
+ if (k1 != k2)
+ return false;
+
+ switch (k1) {
+ case BTF_KIND_UNKN: /* VOID */
+ return true;
+ case BTF_KIND_INT:
+ return btf_equal_int_tag(t1, t2);
+ case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
+ return btf_compat_enum(t1, t2);
+ case BTF_KIND_FWD:
+ case BTF_KIND_FLOAT:
+ return btf_equal_common(t1, t2);
+ case BTF_KIND_CONST:
+ case BTF_KIND_VOLATILE:
+ case BTF_KIND_RESTRICT:
+ case BTF_KIND_PTR:
+ case BTF_KIND_TYPEDEF:
+ case BTF_KIND_FUNC:
+ case BTF_KIND_TYPE_TAG:
+ if (t1->info != t2->info)
+ return 0;
+ id1 = t1->type;
+ id2 = t2->type;
+ continue;
+ case BTF_KIND_ARRAY:
+ return btf_equal_array(t1, t2);
+ case BTF_KIND_STRUCT:
+ case BTF_KIND_UNION:
+ return btf_dedup_identical_structs(d, id1, id2);
+ case BTF_KIND_FUNC_PROTO:
+ return btf_dedup_identical_fnprotos(d, id1, id2);
+ default:
+ return false;
+ }
+ }
+ return false;
+}
+
+
 /*
  * Check equivalence of BTF type graph formed by candidate struct/union (we'll
  * call it "candidate graph" in this description for brevity) to a type graph
@@ -4458,8 +4539,6 @@ static int btf_dedup_is_equiv(struct btf_dedup
*d, __u32 cand_id,
  * types within a single CU. So work around that by explicitly
  * allowing identical array types here.
  */
- if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id))
- return 1;
  /* It turns out that similar situation can happen with
  * struct/union sometimes, sigh... Handle the case where
  * structs/unions are exactly the same, down to the referenced
@@ -4467,7 +4546,7 @@ static int btf_dedup_is_equiv(struct btf_dedup
*d, __u32 cand_id,
  * types are different, but equivalent) is *way more*
  * complicated and requires a many-to-many equivalence mapping.
  */
- if (btf_dedup_identical_structs(d, hypot_type_id, cand_id))
+ if (btf_dedup_identical_types(d, hypot_type_id, cand_id))
  return 1;
  return 0;
  }

>
> >
> > We need to find it asap. Since at present we cannot build
> > kernels with gcc-14, since modules won't dedup BTF.
> > Hence a bunch of selftests/bpf are failing.
> > We want to upgrade BPF CI to gcc-14 to catch nginx-like issues,
> > but we cannot until this pahole/dedup issue is resolved.

Re: pahole and gcc-14 issues

[Andrii Nakryiko]

On Mon, Apr 28, 2025 at 11:59 PM Andrii Nakryiko
<andrii.nakryiko@gmail.com> wrote:
>
> On Mon, Apr 28, 2025 at 5:33 PM Andrii Nakryiko
> <andrii.nakryiko@gmail.com> wrote:
> >
> > On Mon, Apr 28, 2025 at 3:12 PM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> > >
> > > On Mon, Apr 28, 2025 at 8:21 AM Alan Maguire <alan.maguire@oracle.com> wrote:
> > > >
> > > >  <1><4bd05>: Abbrev Number: 58 (DW_TAG_pointer_type)
> > > >     <4bd06>   DW_AT_byte_size   : 8
> > > >     <4bd07>   DW_AT_address_class: 2
> > > >     <4bd08>   DW_AT_type        : <0x301cd>
> > > >
> > > > ...which points at an int
> > > >
> > > >  <1><301cd>: Abbrev Number: 214 (DW_TAG_base_type)
> > > >     <301cf>   DW_AT_byte_size   : 4
> > > >     <301d0>   DW_AT_encoding    : 5     (signed)
> > > >     <301d1>   DW_AT_name        : int
> > > >     <301d5>   DW_AT_name        : int
> > > >
> > > > ...but note the the DW_AT_address_class attribute in the latter case and
> > > > the two DW_AT_name values. We don't use that address attribute in pahole
> > > > as far as I can see, but it might be enough to cause problems.
> > >
> > > DW_AT_address_class is there because it's an actual address space
> > > qualifier in C. The dwarf is correct, but I thought pahole
> > > will ignore it while converting to BTF, so it shouldn't matter
> > > from dedup pov.
> > >
> > > And since dedup is working for vmlinux BTF, I doubt there are CUs
> > > where the same type is represented with different dwarf id-s.
> > > Otherwise dedup wouldn't have worked for vmlinux.
> > >
> > > DW_AT_name is concerning. Sounds like it's a gcc bug, but it
> > > shouldn't be causing dedup issues for modules.
> > >
> > > So what is the workaround?
> >
> > I'm thinking of generalizing Alan's proposed fix so that all our
> > existing special equality cases (arrays, identical structs, and now
> > pointers to identical types) are handled a bit more generically. I'll
> > try to get a patch out later tonight.
>
> So I ran out of time, but I'm thinking something like below. It
> results in identical bpf_testmod.ko compared to Alan's proposed fix,
> so perhaps we should just go with the simpler approach. But this one
> should stand the test of time a bit better.
>
> In any case, I'd like to be able to handle not just PTR -> TYPE chain,
> but also some PTR -> MODIFIER* -> TYPE chains at the very least.
> Because any const in the chain will throw off Alan's heuristic.
>

Ok, so sleeping on this a bit more, I'm hesitant to do this more
generic approach, as now we'll be running a risk of potentially
looping indefinitely (the max_depth check I added doesn't completely
prevent this).

So, Alan, do you mind sending your proposed patch for formal review
and BPF CI testing? Just please use btf_is_ptr() check instead of
explicit kind equality, thanks!

> I'll try to benchmark and polish the patch tomorrow and post it for
> proper review.
>
>
> diff --git a/src/btf.c b/src/btf.c
> index e9673c0ecbe7..e4a3e3183742 100644
> --- a/src/btf.c
> +++ b/src/btf.c
> @@ -4310,6 +4310,8 @@ static bool btf_dedup_identical_arrays(struct
> btf_dedup *d, __u32 id1, __u32 id2
>   return btf_equal_array(t1, t2);
>  }
>
> +static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1,
> __u32 id2);
> +
>  /* Check if given two types are identical STRUCT/UNION definitions */
>  static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32
> id1, __u32 id2)
>  {
> @@ -4329,14 +4331,93 @@ static bool btf_dedup_identical_structs(struct
> btf_dedup *d, __u32 id1, __u32 id
>   m1 = btf_members(t1);
>   m2 = btf_members(t2);
>   for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
> - if (m1->type != m2->type &&
> -     !btf_dedup_identical_arrays(d, m1->type, m2->type) &&
> -     !btf_dedup_identical_structs(d, m1->type, m2->type))
> + if (m1->type != m2->type && !btf_dedup_identical_types(d, m1->type, m2->type))
> + return false;
> + }
> + return true;
> +}
> +
> +static bool btf_dedup_identical_fnprotos(struct btf_dedup *d, __u32
> id1, __u32 id2)
> +{
> + const struct btf_param *p1, *p2;
> + struct btf_type *t1, *t2;
> + int n, i;
> +
> + t1 = btf_type_by_id(d->btf, id1);
> + t2 = btf_type_by_id(d->btf, id2);
> +
> + if (!btf_is_func_proto(t1) || !btf_is_func_proto(t2))
> + return false;
> +
> + if (!btf_compat_fnproto(t1, t2))
> + return false;
> +
> + if (!btf_dedup_identical_types(d, t1->type, t2->type))
> + return false;
> +
> + p1 = btf_params(t1);
> + p2 = btf_params(t2);
> + for (i = 0, n = btf_vlen(t1); i < n; i++, p1++, p2++) {
> + if (p1->type != p2->type && !btf_dedup_identical_types(d, p1->type, p2->type))
>   return false;
>   }
>   return true;
>  }
>
> +static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1,
> __u32 id2)
> +{
> + int max_depth = 32;
> +
> + while (max_depth-- > 0) {
> + struct btf_type *t1, *t2;
> + int k1, k2;
> +
> + t1 = btf_type_by_id(d->btf, id1);
> + t2 = btf_type_by_id(d->btf, id2);
> +
> + k1 = btf_kind(t1);
> + k2 = btf_kind(t2);
> + if (k1 != k2)
> + return false;
> +
> + switch (k1) {
> + case BTF_KIND_UNKN: /* VOID */
> + return true;
> + case BTF_KIND_INT:
> + return btf_equal_int_tag(t1, t2);
> + case BTF_KIND_ENUM:
> + case BTF_KIND_ENUM64:
> + return btf_compat_enum(t1, t2);
> + case BTF_KIND_FWD:
> + case BTF_KIND_FLOAT:
> + return btf_equal_common(t1, t2);
> + case BTF_KIND_CONST:
> + case BTF_KIND_VOLATILE:
> + case BTF_KIND_RESTRICT:
> + case BTF_KIND_PTR:
> + case BTF_KIND_TYPEDEF:
> + case BTF_KIND_FUNC:
> + case BTF_KIND_TYPE_TAG:
> + if (t1->info != t2->info)
> + return 0;
> + id1 = t1->type;
> + id2 = t2->type;
> + continue;
> + case BTF_KIND_ARRAY:
> + return btf_equal_array(t1, t2);
> + case BTF_KIND_STRUCT:
> + case BTF_KIND_UNION:
> + return btf_dedup_identical_structs(d, id1, id2);
> + case BTF_KIND_FUNC_PROTO:
> + return btf_dedup_identical_fnprotos(d, id1, id2);
> + default:
> + return false;
> + }
> + }
> + return false;
> +}
> +
> +
>  /*
>   * Check equivalence of BTF type graph formed by candidate struct/union (we'll
>   * call it "candidate graph" in this description for brevity) to a type graph
> @@ -4458,8 +4539,6 @@ static int btf_dedup_is_equiv(struct btf_dedup
> *d, __u32 cand_id,
>   * types within a single CU. So work around that by explicitly
>   * allowing identical array types here.
>   */
> - if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id))
> - return 1;
>   /* It turns out that similar situation can happen with
>   * struct/union sometimes, sigh... Handle the case where
>   * structs/unions are exactly the same, down to the referenced
> @@ -4467,7 +4546,7 @@ static int btf_dedup_is_equiv(struct btf_dedup
> *d, __u32 cand_id,
>   * types are different, but equivalent) is *way more*
>   * complicated and requires a many-to-many equivalence mapping.
>   */
> - if (btf_dedup_identical_structs(d, hypot_type_id, cand_id))
> + if (btf_dedup_identical_types(d, hypot_type_id, cand_id))
>   return 1;
>   return 0;
>   }
>
> >
> > >
> > > We need to find it asap. Since at present we cannot build
> > > kernels with gcc-14, since modules won't dedup BTF.
> > > Hence a bunch of selftests/bpf are failing.
> > > We want to upgrade BPF CI to gcc-14 to catch nginx-like issues,
> > > but we cannot until this pahole/dedup issue is resolved.

Re: pahole and gcc-14 issues

[Alan Maguire]

On 29/04/2025 16:37, Andrii Nakryiko wrote:
> On Mon, Apr 28, 2025 at 11:59 PM Andrii Nakryiko
> <andrii.nakryiko@gmail.com> wrote:
>>
>> On Mon, Apr 28, 2025 at 5:33 PM Andrii Nakryiko
>> <andrii.nakryiko@gmail.com> wrote:
>>>
>>> On Mon, Apr 28, 2025 at 3:12 PM Alexei Starovoitov
>>> <alexei.starovoitov@gmail.com> wrote:
>>>>
>>>> On Mon, Apr 28, 2025 at 8:21 AM Alan Maguire <alan.maguire@oracle.com> wrote:
>>>>>
>>>>>  <1><4bd05>: Abbrev Number: 58 (DW_TAG_pointer_type)
>>>>>     <4bd06>   DW_AT_byte_size   : 8
>>>>>     <4bd07>   DW_AT_address_class: 2
>>>>>     <4bd08>   DW_AT_type        : <0x301cd>
>>>>>
>>>>> ...which points at an int
>>>>>
>>>>>  <1><301cd>: Abbrev Number: 214 (DW_TAG_base_type)
>>>>>     <301cf>   DW_AT_byte_size   : 4
>>>>>     <301d0>   DW_AT_encoding    : 5     (signed)
>>>>>     <301d1>   DW_AT_name        : int
>>>>>     <301d5>   DW_AT_name        : int
>>>>>
>>>>> ...but note the the DW_AT_address_class attribute in the latter case and
>>>>> the two DW_AT_name values. We don't use that address attribute in pahole
>>>>> as far as I can see, but it might be enough to cause problems.
>>>>
>>>> DW_AT_address_class is there because it's an actual address space
>>>> qualifier in C. The dwarf is correct, but I thought pahole
>>>> will ignore it while converting to BTF, so it shouldn't matter
>>>> from dedup pov.
>>>>
>>>> And since dedup is working for vmlinux BTF, I doubt there are CUs
>>>> where the same type is represented with different dwarf id-s.
>>>> Otherwise dedup wouldn't have worked for vmlinux.
>>>>
>>>> DW_AT_name is concerning. Sounds like it's a gcc bug, but it
>>>> shouldn't be causing dedup issues for modules.
>>>>
>>>> So what is the workaround?
>>>
>>> I'm thinking of generalizing Alan's proposed fix so that all our
>>> existing special equality cases (arrays, identical structs, and now
>>> pointers to identical types) are handled a bit more generically. I'll
>>> try to get a patch out later tonight.
>>
>> So I ran out of time, but I'm thinking something like below. It
>> results in identical bpf_testmod.ko compared to Alan's proposed fix,
>> so perhaps we should just go with the simpler approach. But this one
>> should stand the test of time a bit better.
>>
>> In any case, I'd like to be able to handle not just PTR -> TYPE chain,
>> but also some PTR -> MODIFIER* -> TYPE chains at the very least.
>> Because any const in the chain will throw off Alan's heuristic.
>>
> 
> Ok, so sleeping on this a bit more, I'm hesitant to do this more
> generic approach, as now we'll be running a risk of potentially
> looping indefinitely (the max_depth check I added doesn't completely
> prevent this).
> 
> So, Alan, do you mind sending your proposed patch for formal review
> and BPF CI testing? Just please use btf_is_ptr() check instead of
> explicit kind equality, thanks!
>

Sure, will do! Was about to reply that I'd tested your patch and all
worked well. I'm a bit worried we may end up playing whack-a-mole with
issues in this area so I do like the idea of a more generic approach,
but that doesn't have to happen immediately.

Note that the missing kfunc issue that Alexei also reported appears to
be a separate problem, and I'm seeing that despite having dedup-related
fixes. My suspicion that the associated .cold functions were triggering
our inconsistent function detection in pahole seems more likely given
that the following change:

diff --git a/btf_encoder.c b/btf_encoder.c
index 0bc2334..a72c9c3 100644
--- a/btf_encoder.c
+++ b/btf_encoder.c
@@ -1456,7 +1456,7 @@ static void elf_functions__collect_function(struct
elf_functions *functions, GEl
                return;

        name = elf_sym__name(sym, functions->symtab);
-       if (!name)
+       if (!name || strstr(name, ".cold"))
                return;

        func = &functions->entries[functions->cnt];


...appears to be enough to make those kfuncs reappear in BTF.

I think ideally we should have a better means of mapping from ELF
function -> DWARF rather than purely via name prefix. That part requires
a bit more thought, hopefuly we can get that fixed ASAP too.

Alan


>> I'll try to benchmark and polish the patch tomorrow and post it for
>> proper review.
>>
>>
>> diff --git a/src/btf.c b/src/btf.c
>> index e9673c0ecbe7..e4a3e3183742 100644
>> --- a/src/btf.c
>> +++ b/src/btf.c
>> @@ -4310,6 +4310,8 @@ static bool btf_dedup_identical_arrays(struct
>> btf_dedup *d, __u32 id1, __u32 id2
>>   return btf_equal_array(t1, t2);
>>  }
>>
>> +static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1,
>> __u32 id2);
>> +
>>  /* Check if given two types are identical STRUCT/UNION definitions */
>>  static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32
>> id1, __u32 id2)
>>  {
>> @@ -4329,14 +4331,93 @@ static bool btf_dedup_identical_structs(struct
>> btf_dedup *d, __u32 id1, __u32 id
>>   m1 = btf_members(t1);
>>   m2 = btf_members(t2);
>>   for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
>> - if (m1->type != m2->type &&
>> -     !btf_dedup_identical_arrays(d, m1->type, m2->type) &&
>> -     !btf_dedup_identical_structs(d, m1->type, m2->type))
>> + if (m1->type != m2->type && !btf_dedup_identical_types(d, m1->type, m2->type))
>> + return false;
>> + }
>> + return true;
>> +}
>> +
>> +static bool btf_dedup_identical_fnprotos(struct btf_dedup *d, __u32
>> id1, __u32 id2)
>> +{
>> + const struct btf_param *p1, *p2;
>> + struct btf_type *t1, *t2;
>> + int n, i;
>> +
>> + t1 = btf_type_by_id(d->btf, id1);
>> + t2 = btf_type_by_id(d->btf, id2);
>> +
>> + if (!btf_is_func_proto(t1) || !btf_is_func_proto(t2))
>> + return false;
>> +
>> + if (!btf_compat_fnproto(t1, t2))
>> + return false;
>> +
>> + if (!btf_dedup_identical_types(d, t1->type, t2->type))
>> + return false;
>> +
>> + p1 = btf_params(t1);
>> + p2 = btf_params(t2);
>> + for (i = 0, n = btf_vlen(t1); i < n; i++, p1++, p2++) {
>> + if (p1->type != p2->type && !btf_dedup_identical_types(d, p1->type, p2->type))
>>   return false;
>>   }
>>   return true;
>>  }
>>
>> +static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1,
>> __u32 id2)
>> +{
>> + int max_depth = 32;
>> +
>> + while (max_depth-- > 0) {
>> + struct btf_type *t1, *t2;
>> + int k1, k2;
>> +
>> + t1 = btf_type_by_id(d->btf, id1);
>> + t2 = btf_type_by_id(d->btf, id2);
>> +
>> + k1 = btf_kind(t1);
>> + k2 = btf_kind(t2);
>> + if (k1 != k2)
>> + return false;
>> +
>> + switch (k1) {
>> + case BTF_KIND_UNKN: /* VOID */
>> + return true;
>> + case BTF_KIND_INT:
>> + return btf_equal_int_tag(t1, t2);
>> + case BTF_KIND_ENUM:
>> + case BTF_KIND_ENUM64:
>> + return btf_compat_enum(t1, t2);
>> + case BTF_KIND_FWD:
>> + case BTF_KIND_FLOAT:
>> + return btf_equal_common(t1, t2);
>> + case BTF_KIND_CONST:
>> + case BTF_KIND_VOLATILE:
>> + case BTF_KIND_RESTRICT:
>> + case BTF_KIND_PTR:
>> + case BTF_KIND_TYPEDEF:
>> + case BTF_KIND_FUNC:
>> + case BTF_KIND_TYPE_TAG:
>> + if (t1->info != t2->info)
>> + return 0;
>> + id1 = t1->type;
>> + id2 = t2->type;
>> + continue;
>> + case BTF_KIND_ARRAY:
>> + return btf_equal_array(t1, t2);
>> + case BTF_KIND_STRUCT:
>> + case BTF_KIND_UNION:
>> + return btf_dedup_identical_structs(d, id1, id2);
>> + case BTF_KIND_FUNC_PROTO:
>> + return btf_dedup_identical_fnprotos(d, id1, id2);
>> + default:
>> + return false;
>> + }
>> + }
>> + return false;
>> +}
>> +
>> +
>>  /*
>>   * Check equivalence of BTF type graph formed by candidate struct/union (we'll
>>   * call it "candidate graph" in this description for brevity) to a type graph
>> @@ -4458,8 +4539,6 @@ static int btf_dedup_is_equiv(struct btf_dedup
>> *d, __u32 cand_id,
>>   * types within a single CU. So work around that by explicitly
>>   * allowing identical array types here.
>>   */
>> - if (btf_dedup_identical_arrays(d, hypot_type_id, cand_id))
>> - return 1;
>>   /* It turns out that similar situation can happen with
>>   * struct/union sometimes, sigh... Handle the case where
>>   * structs/unions are exactly the same, down to the referenced
>> @@ -4467,7 +4546,7 @@ static int btf_dedup_is_equiv(struct btf_dedup
>> *d, __u32 cand_id,
>>   * types are different, but equivalent) is *way more*
>>   * complicated and requires a many-to-many equivalence mapping.
>>   */
>> - if (btf_dedup_identical_structs(d, hypot_type_id, cand_id))
>> + if (btf_dedup_identical_types(d, hypot_type_id, cand_id))
>>   return 1;
>>   return 0;
>>   }
>>
>>>
>>>>
>>>> We need to find it asap. Since at present we cannot build
>>>> kernels with gcc-14, since modules won't dedup BTF.
>>>> Hence a bunch of selftests/bpf are failing.
>>>> We want to upgrade BPF CI to gcc-14 to catch nginx-like issues,
>>>> but we cannot until this pahole/dedup issue is resolved.
> 

Re: pahole and gcc-14 issues

[Eduard Zingerman]

Andrii Nakryiko <andrii.nakryiko@gmail.com> writes:

[...]

> Ok, so sleeping on this a bit more, I'm hesitant to do this more
> generic approach, as now we'll be running a risk of potentially
> looping indefinitely (the max_depth check I added doesn't completely
> prevent this).

Is that different from what happens already?

  /* Check if given two types are identical STRUCT/UNION definitions */
  static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2)
  {
          [...]
          for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
                  if (...
                      !btf_dedup_identical_structs(d, m1->type, m2->type)) // <-- recursion,
                          return false;                                    //     afaiu it is unbounded.
          }
          return true;
  }

E.g. adding an array to mark visited types and stop descend should stop
inifinite recurusion. I agree with Alan, the patch you shared in the
evening looks interesting. (But it was butchered by gmail,
fixing patches by hand is a bit annoyiong, maybe just attach such files?).

[...]

Re: pahole and gcc-14 issues

[Andrii Nakryiko]

On Tue, Apr 29, 2025 at 12:30 PM Eduard Zingerman <eddyz87@gmail.com> wrote:
>
> Andrii Nakryiko <andrii.nakryiko@gmail.com> writes:
>
> [...]
>
> > Ok, so sleeping on this a bit more, I'm hesitant to do this more
> > generic approach, as now we'll be running a risk of potentially
> > looping indefinitely (the max_depth check I added doesn't completely
> > prevent this).
>
> Is that different from what happens already?

yes, it's different, we don't follow pointers. that identical_structs
recurse only for embedded/nested structs, not structs-by-pointer.
idential_arrays also doesn't follow any references

so it could lead to loop only if BTF is broken and we have one struct
embedding another, while that other embeds its parent, which isn't
legal in BTF and C (so whatever)

>
>   /* Check if given two types are identical STRUCT/UNION definitions */
>   static bool btf_dedup_identical_structs(struct btf_dedup *d, __u32 id1, __u32 id2)
>   {
>           [...]
>           for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
>                   if (...
>                       !btf_dedup_identical_structs(d, m1->type, m2->type)) // <-- recursion,
>                           return false;                                    //     afaiu it is unbounded.
>           }
>           return true;
>   }
>
> E.g. adding an array to mark visited types and stop descend should stop
> inifinite recurusion. I agree with Alan, the patch you shared in the
> evening looks interesting. (But it was butchered by gmail,
> fixing patches by hand is a bit annoyiong, maybe just attach such files?).

yeah, I know that gmail butchers diffs, I just wanted to give a rough
idea, didn't expect anyone wanting to apply it (plus I did it on top
of Github-based libbpf inside pahole's libbpf submodule)

But here you go if you want to play with it ([0]).

And yes, "visited" marks are the solution, but I was thinking that if
we implement a pre-processing deduplication step as we discussed
offline, we won't need to do any of this, so didn't want to pursue
this further. But we can talk about this, of course. So far this
generality doesn't buy us anything, I got byte-for-byte identical
bpf_testmod.ko with Alan's and my changes all the same.

  [0] https://gist.github.com/anakryiko/fd1c84dcad91141d27d8bd33453521d1

>
> [...]

Re: pahole and gcc-14 issues

[Eduard Zingerman]

Andrii Nakryiko <andrii.nakryiko@gmail.com> writes:

[...]

>> Is that different from what happens already?
>
> yes, it's different, we don't follow pointers. that identical_structs
> recurse only for embedded/nested structs, not structs-by-pointer.
> idential_arrays also doesn't follow any references
>
> so it could lead to loop only if BTF is broken and we have one struct
> embedding another, while that other embeds its parent, which isn't
> legal in BTF and C (so whatever)

Missed that, thank you for explaining.

[...]

> And yes, "visited" marks are the solution, but I was thinking that if
> we implement a pre-processing deduplication step as we discussed
> offline, we won't need to do any of this, so didn't want to pursue
> this further.

From logical pov the pre-processing step would do the same thing, right?
Follow pointer/array/qualifier chains and merge structs that can be
trivially merged.

[...]

>   [0] https://gist.github.com/anakryiko/fd1c84dcad91141d27d8bd33453521d1

Thanks, I actually cleaned up and applied what you posted in email :)

Re: pahole and gcc-14 issues

[Alexei Starovoitov]

On Tue, Apr 29, 2025 at 12:50 PM Andrii Nakryiko
<andrii.nakryiko@gmail.com> wrote:
>
>
> But here you go if you want to play with it ([0]).
>
> And yes, "visited" marks are the solution, but I was thinking that if
> we implement a pre-processing deduplication step as we discussed
> offline, we won't need to do any of this, so didn't want to pursue
> this further. But we can talk about this, of course. So far this
> generality doesn't buy us anything, I got byte-for-byte identical
> bpf_testmod.ko with Alan's and my changes all the same.
>
>   [0] https://gist.github.com/anakryiko/fd1c84dcad91141d27d8bd33453521d1

I like it. I think it's worth following up with that.
Why do you think max_depth is not enough?
Because of
btf_dedup_identical_types ->
  btf_dedup_identical_structs ->
    btf_dedup_identical_types
?
Then pass &max_depth from btf_dedup_is_equiv() ?

The visited mark seems overkill.

Re: pahole and gcc-14 issues

[Andrii Nakryiko]

On Tue, Apr 29, 2025 at 1:56 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Tue, Apr 29, 2025 at 12:50 PM Andrii Nakryiko
> <andrii.nakryiko@gmail.com> wrote:
> >
> >
> > But here you go if you want to play with it ([0]).
> >
> > And yes, "visited" marks are the solution, but I was thinking that if
> > we implement a pre-processing deduplication step as we discussed
> > offline, we won't need to do any of this, so didn't want to pursue
> > this further. But we can talk about this, of course. So far this
> > generality doesn't buy us anything, I got byte-for-byte identical
> > bpf_testmod.ko with Alan's and my changes all the same.
> >
> >   [0] https://gist.github.com/anakryiko/fd1c84dcad91141d27d8bd33453521d1
>
> I like it. I think it's worth following up with that.
> Why do you think max_depth is not enough?
> Because of
> btf_dedup_identical_types ->
>   btf_dedup_identical_structs ->
>     btf_dedup_identical_types
> ?
> Then pass &max_depth from btf_dedup_is_equiv() ?
>

yep, that was another option I considered, just have a "global max depth".

OK, I'll update my patch and do a bit more benchmarking. I want to see
if this doesn't add much to the overall dedup time. Will post
something later (probably tomorrow)

> The visited mark seems overkill.

Re: pahole and gcc-14 issues

[Andrii Nakryiko]

On Fri, Apr 25, 2025 at 10:58 AM Andrii Nakryiko
<andrii.nakryiko@gmail.com> wrote:
>
> On Fri, Apr 25, 2025 at 10:50 AM Alan Maguire <alan.maguire@oracle.com> wrote:
> >
> > On 25/04/2025 15:50, Alexei Starovoitov wrote:
> > > Hi All,
> > >
> > > Looks like pahole fails to deduplicate BTF when kernel and
> > > kernel module are built with gcc-14.
> > > I see this issue with various kernel .config-s on bpf and
> > > bpf-next trees.
> > > I tried pahole 1.28 and the latest master. Same issues.
> > >
> > > BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
> > > When built with gcc-13 it has 454 types.
> > > So something is confusing dedup logic.
> > > Would be great if dedup experts can take a look,
> > > since this dedup issue is breaking a lot of selftests/bpf.
> > >
> > > Also vmlinux.h generated out of the kernel compiled with gcc-13
> > > and out of the kernel compiled with gcc-14 shows these differences:
> > >
> > > --- vmlinux13.h    2025-04-24 21:33:50.556884372 -0700
> > > +++ vmlinux14.h    2025-04-24 21:39:10.310488992 -0700
> > > @@ -148815,7 +148815,6 @@
> > >  extern int hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum
> > > hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
> > >  extern void hid_bpf_release_context(struct hid_bpf_ctx *ctx) __weak __ksym;
> > >  extern int hid_bpf_try_input_report(struct hid_bpf_ctx *ctx, enum
> > > hid_report_type type, u8 *buf, const size_t buf__sz) __weak __ksym;
> > > -extern bool scx_bpf_consume(u64 dsq_id) __weak __ksym;
> > >  extern int scx_bpf_cpu_node(s32 cpu) __weak __ksym;
> > >  extern struct rq *scx_bpf_cpu_rq(s32 cpu) __weak __ksym;
> > >  extern u32 scx_bpf_cpuperf_cap(s32 cpu) __weak __ksym;
> > > @@ -148825,12 +148824,8 @@
> > >  extern void scx_bpf_destroy_dsq(u64 dsq_id) __weak __ksym;
> > >  extern void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64
> > > slice, u64 enq_flags) __weak __ksym;
> > >  extern void scx_bpf_dispatch_cancel(void) __weak __ksym;
> > > -extern bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq
> > > *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
> > > __ksym;
> > > -extern void scx_bpf_dispatch_from_dsq_set_slice(struct
> > > bpf_iter_scx_dsq *it__iter, u64 slice) __weak __ksym;
> > >  extern void scx_bpf_dispatch_from_dsq_set_vtime(struct
> > > bpf_iter_scx_dsq *it__iter, u64 vtime) __weak __ksym;
> > >  extern u32 scx_bpf_dispatch_nr_slots(void) __weak __ksym;
> > > -extern void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
> > > u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
> > > -extern bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq
> > > *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak
> > > __ksym;
> > >  extern void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64
> > > slice, u64 enq_flags) __weak __ksym;
> > >  extern void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64
> > > dsq_id, u64 slice, u64 vtime, u64 enq_flags) __weak __ksym;
> > >  extern bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
> > > struct task_struct *p, u64 dsq_id, u64 enq_flags) __weak __ksym;
> > >
> > > gcc-14's kernel is clearly wrong.
> > > These 5 kfuncs still exist in the kernel.
> > > I manually checked there is no if __GNUC__ > 13 in the code.
> > > Also:
> > > nm bld/vmlinux|grep -w scx_bpf_consume
> > > ffffffff8159d4b0 T scx_bpf_consume
> > > ffffffff8120ea81 t scx_bpf_consume.cold
> > >
> > > I suspect the second issue is not related to the dedup problem.
> > > All 5 missing kfuncs have ".cold" optimized bodies.
> > > But ".cold" maybe a red herring, since
> > > nm bld/vmlinux|grep -w scx_bpf_dispatch
> > > ffffffff8159d020 T scx_bpf_dispatch
> > > ffffffff8120ea0f t scx_bpf_dispatch.cold
> > > but this kfunc is present in vmlinux14.h
> > >
> > > If it makes a difference I have these configs:
> > > # CONFIG_DEBUG_INFO_DWARF4 is not set
> > > # CONFIG_DEBUG_INFO_DWARF5 is not set
> > > # CONFIG_DEBUG_INFO_REDUCED is not set
> > > CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
> > > # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
> > > # CONFIG_DEBUG_INFO_SPLIT is not set
> > > CONFIG_DEBUG_INFO_BTF=y
> > > CONFIG_PAHOLE_HAS_SPLIT_BTF=y
> > > CONFIG_DEBUG_INFO_BTF_MODULES=y
> >
> > thanks for the report! I've just reproduced this now with gcc 14; my
> > initial theory was it might be DWARF5-related, but dedup issues occur
> > for modules with CONFIG_DEBUG_INFO_DWARF4=y also. I'm seeing task_struct
> > duplicates in module BTF among other things, so I will try and dig
> > further and report back when I find something. Like you I suspect the
>
> This is a bizarre case. I have a custom small tool that recursively
> traverses two parallel subgraphs of BTF types and prints anything that
> differs between them ([0]). (I had to disable distilled BTF to make
> use of this, the issue is present both with distilled BTF and
> without).
>
> I see that struct sock both in vmlinux and bpf_testmod.ko are
> *IDENTICAL*. There is no difference I could detect. So very weird. I'm
> thinking of bisecting, as this didn't happen before with exactly the
> same compiler and pahole, so this must be a kernel-side change.

Ok, so while you guys had fun chasing distilled BTF quirks... ;)

This is the commit introducing a regression:

eb0ece16027f ("Merge tag 'mm-stable-2025-03-30-16-52' of
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm")

Yes, it's a "merge commit", but there is a lot of code introduced in
it. Among it:

+ /*
+  * Use __typeof_unqual__() when available.
+  *
+  * XXX: Remove test for __CHECKER__ once
+  * sparse learns about __typeof_unqual__().
+  */
+ #if CC_HAS_TYPEOF_UNQUAL && !defined(__CHECKER__)
+ # define USE_TYPEOF_UNQUAL 1
+ #endif
+
+ /*
+  * Define TYPEOF_UNQUAL() to use __typeof_unqual__() as typeof
+  * operator when available, to return an unqualified type of the exp.
+  */
+ #if defined(USE_TYPEOF_UNQUAL)
+ # define TYPEOF_UNQUAL(exp) __typeof_unqual__(exp)
+ #else
+ # define TYPEOF_UNQUAL(exp) __typeof__(exp)
+ #endif
+


And that's exactly what causes this divergence. Commenting out that
USE_TYPEOF_UNQUAL #define fixes issues.

As to why that causes a problem. I suspect __typeof_unqual__() changes
how GCC generates DWARF information within any given compilation unit
(CU). Libbpf's BTF dedup relies on a property that compiler won't have
duplicate definitions of exactly the same type (i.e., DWARF itself
can't have two `struct blah` definitions), without which it's not
possible to deduplicate entire clusters of self-referencing BTF types.
It seems like typeof_unqual breaks this somehow.

We need to compare DWARF with and without TYPEOF_UNQUAL and see what
the differences are and how we can prevent or accommodate them.

>
>   [0] https://github.com/anakryiko/libbpf-bootstrap/tree/btfdiff-hack
>
> > issues with missing kfuncs are different; may be an issue with our logic
> > handling inconsistent functions getting confused by the .cold
> > components. But right now understanding dedup issues is the top priority.
> >
> > Alan

Re: pahole and gcc-14 issues

[Eduard Zingerman]

Alexei Starovoitov <alexei.starovoitov@gmail.com> writes:

> Hi All,
>
> Looks like pahole fails to deduplicate BTF when kernel and
> kernel module are built with gcc-14.
> I see this issue with various kernel .config-s on bpf and
> bpf-next trees.
> I tried pahole 1.28 and the latest master. Same issues.
>
> BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
> When built with gcc-13 it has 454 types.
> So something is confusing dedup logic.
> Would be great if dedup experts can take a look,
> since this dedup issue is breaking a lot of selftests/bpf.

It does not look like the problem is with dedup.
Quick glance at structure definitions does not show any duplications,
just much more structs compared to clang:

  $ bpftool btf dump file objs-gcc/bpf_testmod.ko format raw | grep STRUCT | wc -l
  351
  $ bpftool btf dump file objs-clang/bpf_testmod.ko format raw | grep STRUCT | wc -l
  38

Comparing raw C dumps for btf_testmod.ko it looks like much more
structures are injected to distilled base, e.g. the following type is
present in gcc generated .ko and is absent in clang generated:

  struct taskstats {
          long: 64;
          long: 64;
          long: 64;
          long: 64;
          ... lots of long: 64; ...
  }

  $ bpftool btf dump file objs-gcc/bpf_testmod.ko format c \
    | awk '/^struct .* \{/ {s=1;c=0;o=0} \
      {l=match($0, "long: 64;")} \
      s && l {c++} \
      s && !l {o++} \
      s {print $0} \
      s && /^\}/ { if (o==2 && c) print "distilled?\n"}' \
      | grep 'distilled\?' | wc -l
  408
  $ bpftool btf dump file objs-clang/bpf_testmod.ko format c | awk ... | wc -l
  33

Re: pahole and gcc-14 issues

[Eduard Zingerman]

Eduard Zingerman <eddyz87@gmail.com> writes:

> Alexei Starovoitov <alexei.starovoitov@gmail.com> writes:
>
>> Hi All,
>>
>> Looks like pahole fails to deduplicate BTF when kernel and
>> kernel module are built with gcc-14.
>> I see this issue with various kernel .config-s on bpf and
>> bpf-next trees.
>> I tried pahole 1.28 and the latest master. Same issues.
>>
>> BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
>> When built with gcc-13 it has 454 types.
>> So something is confusing dedup logic.
>> Would be great if dedup experts can take a look,
>> since this dedup issue is breaking a lot of selftests/bpf.
>
> It does not look like the problem is with dedup.
> Quick glance at structure definitions does not show any duplications,
> just much more structs compared to clang:

Or maybe it is.
For example, task_struct is added to .ko BTF generated by gcc, but not
clang. This can only happen if dedup fails to merge structures in base
and module btf, right?

Here is an interesting observation:

$ bpftool btf dump file ~/tmp/objs-gcc/bpf_testmod.ko format c | awk '/struct task_struct \{/ {s=1} s {print $0} /^\}/ {s=0}' > ~/tmp/task_struct.ko.c

$ bpftool btf dump file ~/tmp/objs-gcc/vmlinux format c | awk '/struct task_struct \{/ {s=1} s {print $0} /^\}/ {s=0}' > ~/tmp/task_struct.vmlinux.c

$ diff -pruN ~/tmp/task_struct.ko.c ~/tmp/task_struct.vmlinux.c
--- /home/ezingerman/tmp/task_struct.ko.c       2025-04-25 12:37:48.312480603 -0700
+++ /home/ezingerman/tmp/task_struct.vmlinux.c  2025-04-25 12:38:03.096644654 -0700
@@ -18,7 +18,6 @@ struct task_struct {
        int static_prio;
        int normal_prio;
        unsigned int rt_priority;
-       long: 0;
        struct sched_entity se;
        struct sched_rt_entity rt;
        struct sched_dl_entity dl;
@@ -46,7 +45,6 @@ struct task_struct {
        short unsigned int migration_flags;
        int rcu_read_lock_nesting;
        union rcu_special rcu_read_unlock_special;
-       long: 0;
        struct list_head rcu_node_entry;
        struct rcu_node *rcu_blocked_node;
        long unsigned int rcu_tasks_nvcsw;
@@ -55,16 +53,13 @@ struct task_struct {
        int rcu_tasks_idle_cpu;
        struct list_head rcu_tasks_holdout_list;
        int rcu_tasks_exit_cpu;
-       long: 0;
        struct list_head rcu_tasks_exit_list;
        int trc_reader_nesting;
        int trc_ipi_to_cpu;
        union rcu_special trc_reader_special;
-       long: 0;
        struct list_head trc_holdout_list;
        struct list_head trc_blkd_node;
        int trc_blkd_cpu;
-       long: 0;
        struct sched_info sched_info;
        struct list_head tasks;
        struct plist_node pushable_tasks;
@@ -166,7 +161,6 @@ struct task_struct {
        struct mutex_waiter *blocked_on;
        struct mutex *blocker_mutex;
        int non_block_count;
-       long: 0;
        struct irqtrace_events irqtrace;
        unsigned int hardirq_threaded;
        u64 hardirq_chain_key;
@@ -277,7 +271,6 @@ struct task_struct {
        __u64 __mce_reserved: 62;
        struct callback_head mce_kill_me;
        int mce_count;
-       long: 0;
        struct llist_head kretprobe_instances;
        struct llist_head rethooks;
        struct callback_head l1d_flush_kill;

Re: pahole and gcc-14 issues

[Alexei Starovoitov]

On Fri, Apr 25, 2025 at 12:43 PM Eduard Zingerman <eddyz87@gmail.com> wrote:
>
> Eduard Zingerman <eddyz87@gmail.com> writes:
>
> > Alexei Starovoitov <alexei.starovoitov@gmail.com> writes:
> >
> >> Hi All,
> >>
> >> Looks like pahole fails to deduplicate BTF when kernel and
> >> kernel module are built with gcc-14.
> >> I see this issue with various kernel .config-s on bpf and
> >> bpf-next trees.
> >> I tried pahole 1.28 and the latest master. Same issues.
> >>
> >> BTF in bpf_testmod.ko built with gcc-14 has 2849 types.
> >> When built with gcc-13 it has 454 types.
> >> So something is confusing dedup logic.
> >> Would be great if dedup experts can take a look,
> >> since this dedup issue is breaking a lot of selftests/bpf.
> >
> > It does not look like the problem is with dedup.
> > Quick glance at structure definitions does not show any duplications,
> > just much more structs compared to clang:
>
> Or maybe it is.
> For example, task_struct is added to .ko BTF generated by gcc, but not
> clang. This can only happen if dedup fails to merge structures in base
> and module btf, right?
>
> Here is an interesting observation:
>
> $ bpftool btf dump file ~/tmp/objs-gcc/bpf_testmod.ko format c | awk '/struct task_struct \{/ {s=1} s {print $0} /^\}/ {s=0}' > ~/tmp/task_struct.ko.c
>
> $ bpftool btf dump file ~/tmp/objs-gcc/vmlinux format c | awk '/struct task_struct \{/ {s=1} s {print $0} /^\}/ {s=0}' > ~/tmp/task_struct.vmlinux.c
>
> $ diff -pruN ~/tmp/task_struct.ko.c ~/tmp/task_struct.vmlinux.c
> --- /home/ezingerman/tmp/task_struct.ko.c       2025-04-25 12:37:48.312480603 -0700
> +++ /home/ezingerman/tmp/task_struct.vmlinux.c  2025-04-25 12:38:03.096644654 -0700
> @@ -18,7 +18,6 @@ struct task_struct {
>         int static_prio;
>         int normal_prio;
>         unsigned int rt_priority;
> -       long: 0;
>         struct sched_entity se;

I reproed this issue with default .ko build that includes:
--btf_features=distilled_base

Once I disabled it and did
bpftool btf dump file ./bpf_testmod.ko --base-btf .../vmlinux format c
the task_struct from vmlinux.h and from testmod.h became exactly the same.
So it sounds like the 3rd issue :)
bpftool dump of distilled btf needs work.

Re: pahole and gcc-14 issues

[Eduard Zingerman]

Alexei Starovoitov <alexei.starovoitov@gmail.com> writes:

[...]

>> $ diff -pruN ~/tmp/task_struct.ko.c ~/tmp/task_struct.vmlinux.c
>> --- /home/ezingerman/tmp/task_struct.ko.c       2025-04-25 12:37:48.312480603 -0700
>> +++ /home/ezingerman/tmp/task_struct.vmlinux.c  2025-04-25 12:38:03.096644654 -0700
>> @@ -18,7 +18,6 @@ struct task_struct {
>>         int static_prio;
>>         int normal_prio;
>>         unsigned int rt_priority;
>> -       long: 0;
>>         struct sched_entity se;
>
> I reproed this issue with default .ko build that includes:
> --btf_features=distilled_base
>
> Once I disabled it and did
> bpftool btf dump file ./bpf_testmod.ko --base-btf .../vmlinux format c
> the task_struct from vmlinux.h and from testmod.h became exactly the same.
> So it sounds like the 3rd issue :)
> bpftool dump of distilled btf needs work.

Mystery upon mystery.
Here is a continuation of the last one.
This is raw BTF for .ko:

  [509] STRUCT 'task_struct' size=10496 vlen=268
          ...
          'rt_priority' type_id=3 bits_offset=960
          ...

And this is raw BTF for vmlinux:

  [1] INT 'long unsigned int' size=8 bits_offset=0 nr_bits=64 encoding=(none)
  [2] CONST '(anon)' type_id=1
  [3] VOLATILE '(anon)' type_id=2
  ...
  [9] INT 'unsigned int' size=4 bits_offset=0 nr_bits=32 encoding=(none)
  ...
  [111] STRUCT 'task_struct' size=10496 vlen=268
          ...
          'rt_priority' type_id=9 bits_offset=960
          ...

Note type conflict 'volatile const long' vs 'unsigned int'.
Either something is very broken or I completely messed up the build.

Re: pahole and gcc-14 issues

[Alexei Starovoitov]

On Fri, Apr 25, 2025 at 1:25 PM Eduard Zingerman <eddyz87@gmail.com> wrote:
>
> Alexei Starovoitov <alexei.starovoitov@gmail.com> writes:
>
> [...]
>
> >> $ diff -pruN ~/tmp/task_struct.ko.c ~/tmp/task_struct.vmlinux.c
> >> --- /home/ezingerman/tmp/task_struct.ko.c       2025-04-25 12:37:48.312480603 -0700
> >> +++ /home/ezingerman/tmp/task_struct.vmlinux.c  2025-04-25 12:38:03.096644654 -0700
> >> @@ -18,7 +18,6 @@ struct task_struct {
> >>         int static_prio;
> >>         int normal_prio;
> >>         unsigned int rt_priority;
> >> -       long: 0;
> >>         struct sched_entity se;
> >
> > I reproed this issue with default .ko build that includes:
> > --btf_features=distilled_base
> >
> > Once I disabled it and did
> > bpftool btf dump file ./bpf_testmod.ko --base-btf .../vmlinux format c
> > the task_struct from vmlinux.h and from testmod.h became exactly the same.
> > So it sounds like the 3rd issue :)
> > bpftool dump of distilled btf needs work.
>
> Mystery upon mystery.
> Here is a continuation of the last one.
> This is raw BTF for .ko:
>
>   [509] STRUCT 'task_struct' size=10496 vlen=268
>           ...
>           'rt_priority' type_id=3 bits_offset=960
>           ...
>
> And this is raw BTF for vmlinux:
>
>   [1] INT 'long unsigned int' size=8 bits_offset=0 nr_bits=64 encoding=(none)
>   [2] CONST '(anon)' type_id=1
>   [3] VOLATILE '(anon)' type_id=2
>   ...
>   [9] INT 'unsigned int' size=4 bits_offset=0 nr_bits=32 encoding=(none)
>   ...
>   [111] STRUCT 'task_struct' size=10496 vlen=268
>           ...
>           'rt_priority' type_id=9 bits_offset=960
>           ...
>
> Note type conflict 'volatile const long' vs 'unsigned int'.
> Either something is very broken or I completely messed up the build.

You didn't turn off distilled_base :)

Re: pahole and gcc-14 issues

[Eduard Zingerman]

Eduard Zingerman <eddyz87@gmail.com> writes:

[...]

>> Once I disabled it and did
>> bpftool btf dump file ./bpf_testmod.ko --base-btf .../vmlinux format c
>> the task_struct from vmlinux.h and from testmod.h became exactly the same.
>> So it sounds like the 3rd issue :)
>> bpftool dump of distilled btf needs work.
>
> Mystery upon mystery.
> Here is a continuation of the last one.
> This is raw BTF for .ko:
>
>   [509] STRUCT 'task_struct' size=10496 vlen=268
>           ...
>           'rt_priority' type_id=3 bits_offset=960
>           ...
>

Nah, this refers to id in the distilled base.
Sorry for the noise.

[...]