[PATCH dwarves 0/3] pahole: support nonstandard btf_features

[PATCH dwarves 0/3] pahole: support nonstandard btf_features

[Alan Maguire]

This small series allows the user to specify --btf_features=all along
with non-standard features such as --btf_features=reproducible_build .
Features are documented as standard - so participating in "all" -
or non-standard - such as "reproducible_build".

Tests are updated to use --btf_features=all,reproducible_build.

Series is applicable on next/ branch.

Signed-off-by: Alan Maguire <alan.maguire@oracle.com>

Alan Maguire (3):
  pahole: allow --btf_features to not participate in "all"
  pahole: add reproducible_build to --btf_features
  tests/reproducible_build: use --btf_features=all,reproducible_build

 man-pages/pahole.1          | 10 +++++++++-
 pahole.c                    | 37 ++++++++++++++++++++++++-------------
 tests/reproducible_build.sh |  2 +-
 3 files changed, 34 insertions(+), 15 deletions(-)

-- 
2.39.3

[PATCH dwarves 1/3] pahole: allow --btf_features to not participate in "all"

[Alan Maguire]

Specifying --btf_features=all enables all supported BTF features.
However there are some features that are non-standard, so we should
support a way to use them in --btf_features but not participate in
the set of features enabled by "--btf_features=all".  As part of this,
also support all used in a list of --btf_features, i.e.

--btf_features=all,nonstandard_feature

Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
---
 man-pages/pahole.1 |  2 +-
 pahole.c           | 36 +++++++++++++++++++++++-------------
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/man-pages/pahole.1 b/man-pages/pahole.1
index 2be165d..2c08e97 100644
--- a/man-pages/pahole.1
+++ b/man-pages/pahole.1
@@ -290,7 +290,7 @@ Allow using all the BTF features supported by pahole.
 
 .TP
 .B \-\-btf_features=FEATURE_LIST
-Encode BTF using the specified feature list, or specify 'all' for all features supported.  This option can be used as an alternative to unsing multiple BTF-related options. Supported features are
+Encode BTF using the specified feature list, or specify 'all' for all standard features supported.  This option can be used as an alternative to unsing multiple BTF-related options. Supported standard features are
 
 .nf
 	encode_force       Ignore invalid symbols when encoding BTF; for example
diff --git a/pahole.c b/pahole.c
index 77772bb..890ef81 100644
--- a/pahole.c
+++ b/pahole.c
@@ -1266,23 +1266,26 @@ ARGP_PROGRAM_VERSION_HOOK_DEF = dwarves_print_version;
  * BTF encoding apply; we encode type/decl tags, do not encode
  * floats, etc.  This ensures backwards compatibility.
  */
-#define BTF_FEATURE(name, alias, default_value)			\
-	{ #name, #alias, &conf_load.alias, default_value }
+#define BTF_FEATURE(name, alias, default_value, enable_for_all)		\
+	{ #name, #alias, &conf_load.alias, default_value, enable_for_all }
 
 struct btf_feature {
 	const char      *name;
 	const char      *option_alias;
 	bool		*conf_value;
 	bool		default_value;
+	bool		enable_for_all;	/* some nonstandard features may not
+					 * be enabled for --btf_features=all
+					 */
 } btf_features[] = {
-	BTF_FEATURE(encode_force, btf_encode_force, false),
-	BTF_FEATURE(var, skip_encoding_btf_vars, true),
-	BTF_FEATURE(float, btf_gen_floats, false),
-	BTF_FEATURE(decl_tag, skip_encoding_btf_decl_tag, true),
-	BTF_FEATURE(type_tag, skip_encoding_btf_type_tag, true),
-	BTF_FEATURE(enum64, skip_encoding_btf_enum64, true),
-	BTF_FEATURE(optimized_func, btf_gen_optimized, false),
-	BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false),
+	BTF_FEATURE(encode_force, btf_encode_force, false, true),
+	BTF_FEATURE(var, skip_encoding_btf_vars, true, true),
+	BTF_FEATURE(float, btf_gen_floats, false, true),
+	BTF_FEATURE(decl_tag, skip_encoding_btf_decl_tag, true, true),
+	BTF_FEATURE(type_tag, skip_encoding_btf_type_tag, true, true),
+	BTF_FEATURE(enum64, skip_encoding_btf_enum64, true, true),
+	BTF_FEATURE(optimized_func, btf_gen_optimized, false, true),
+	BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false, true),
 };
 
 #define BTF_MAX_FEATURE_STR	1024
@@ -1350,8 +1353,10 @@ static void parse_btf_features(const char *features, bool strict)
 	if (strcmp(features, "all") == 0) {
 		int i;
 
-		for (i = 0; i < ARRAY_SIZE(btf_features); i++)
-			enable_btf_feature(&btf_features[i]);
+		for (i = 0; i < ARRAY_SIZE(btf_features); i++) {
+			if (btf_features[i].enable_for_all)
+				enable_btf_feature(&btf_features[i]);
+		}
 		return;
 	}
 
@@ -1361,7 +1366,12 @@ static void parse_btf_features(const char *features, bool strict)
 		struct btf_feature *feature = find_btf_feature(feature_name);
 
 		if (!feature) {
-			if (strict) {
+			/* --btf_features=all,nonstandard_feature should be
+			 * allowed.
+			 */
+			if (strcmp(feature_name, "all") == 0) {
+				parse_btf_features(feature_name, strict);
+			} else if (strict) {
 				fprintf(stderr, "Feature '%s' in '%s' is not supported.  Supported BTF features are:\n",
 					feature_name, features);
 				show_supported_btf_features(stderr);
-- 
2.39.3

Re: [PATCH dwarves 1/3] pahole: allow --btf_features to not participate in "all"

[Andrii Nakryiko]

On Tue, Apr 16, 2024 at 7:38 AM Alan Maguire <alan.maguire@oracle.com> wrote:
>
> Specifying --btf_features=all enables all supported BTF features.
> However there are some features that are non-standard, so we should
> support a way to use them in --btf_features but not participate in
> the set of features enabled by "--btf_features=all".  As part of this,
> also support all used in a list of --btf_features, i.e.
>
> --btf_features=all,nonstandard_feature
>
> Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
> ---
>  man-pages/pahole.1 |  2 +-
>  pahole.c           | 36 +++++++++++++++++++++++-------------
>  2 files changed, 24 insertions(+), 14 deletions(-)
>
> diff --git a/man-pages/pahole.1 b/man-pages/pahole.1
> index 2be165d..2c08e97 100644
> --- a/man-pages/pahole.1
> +++ b/man-pages/pahole.1
> @@ -290,7 +290,7 @@ Allow using all the BTF features supported by pahole.
>
>  .TP
>  .B \-\-btf_features=FEATURE_LIST
> -Encode BTF using the specified feature list, or specify 'all' for all features supported.  This option can be used as an alternative to unsing multiple BTF-related options. Supported features are
> +Encode BTF using the specified feature list, or specify 'all' for all standard features supported.  This option can be used as an alternative to unsing multiple BTF-related options. Supported standard features are
>
>  .nf
>         encode_force       Ignore invalid symbols when encoding BTF; for example
> diff --git a/pahole.c b/pahole.c
> index 77772bb..890ef81 100644
> --- a/pahole.c
> +++ b/pahole.c
> @@ -1266,23 +1266,26 @@ ARGP_PROGRAM_VERSION_HOOK_DEF = dwarves_print_version;
>   * BTF encoding apply; we encode type/decl tags, do not encode
>   * floats, etc.  This ensures backwards compatibility.
>   */
> -#define BTF_FEATURE(name, alias, default_value)                        \
> -       { #name, #alias, &conf_load.alias, default_value }
> +#define BTF_FEATURE(name, alias, default_value, enable_for_all)                \
> +       { #name, #alias, &conf_load.alias, default_value, enable_for_all }
>
>  struct btf_feature {
>         const char      *name;
>         const char      *option_alias;
>         bool            *conf_value;
>         bool            default_value;
> +       bool            enable_for_all; /* some nonstandard features may not
> +                                        * be enabled for --btf_features=all
> +                                        */
>  } btf_features[] = {
> -       BTF_FEATURE(encode_force, btf_encode_force, false),
> -       BTF_FEATURE(var, skip_encoding_btf_vars, true),
> -       BTF_FEATURE(float, btf_gen_floats, false),
> -       BTF_FEATURE(decl_tag, skip_encoding_btf_decl_tag, true),
> -       BTF_FEATURE(type_tag, skip_encoding_btf_type_tag, true),
> -       BTF_FEATURE(enum64, skip_encoding_btf_enum64, true),
> -       BTF_FEATURE(optimized_func, btf_gen_optimized, false),
> -       BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false),
> +       BTF_FEATURE(encode_force, btf_encode_force, false, true),
> +       BTF_FEATURE(var, skip_encoding_btf_vars, true, true),
> +       BTF_FEATURE(float, btf_gen_floats, false, true),
> +       BTF_FEATURE(decl_tag, skip_encoding_btf_decl_tag, true, true),
> +       BTF_FEATURE(type_tag, skip_encoding_btf_type_tag, true, true),
> +       BTF_FEATURE(enum64, skip_encoding_btf_enum64, true, true),
> +       BTF_FEATURE(optimized_func, btf_gen_optimized, false, true),
> +       BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false, true),
>  };

maybe keep those special features in a separate array instead?

it is kind of weird to see --btf_features=all,and_then_some, maybe it
makes sense to have --btf_extras or something for those?

>
>  #define BTF_MAX_FEATURE_STR    1024
> @@ -1350,8 +1353,10 @@ static void parse_btf_features(const char *features, bool strict)
>         if (strcmp(features, "all") == 0) {
>                 int i;
>
> -               for (i = 0; i < ARRAY_SIZE(btf_features); i++)
> -                       enable_btf_feature(&btf_features[i]);
> +               for (i = 0; i < ARRAY_SIZE(btf_features); i++) {
> +                       if (btf_features[i].enable_for_all)
> +                               enable_btf_feature(&btf_features[i]);
> +               }
>                 return;
>         }
>
> @@ -1361,7 +1366,12 @@ static void parse_btf_features(const char *features, bool strict)
>                 struct btf_feature *feature = find_btf_feature(feature_name);
>
>                 if (!feature) {
> -                       if (strict) {
> +                       /* --btf_features=all,nonstandard_feature should be
> +                        * allowed.
> +                        */
> +                       if (strcmp(feature_name, "all") == 0) {
> +                               parse_btf_features(feature_name, strict);
> +                       } else if (strict) {
>                                 fprintf(stderr, "Feature '%s' in '%s' is not supported.  Supported BTF features are:\n",
>                                         feature_name, features);
>                                 show_supported_btf_features(stderr);
> --
> 2.39.3
>
>

Re: [PATCH dwarves 1/3] pahole: allow --btf_features to not participate in "all"

[Alan Maguire]

On 19/04/2024 01:06, Andrii Nakryiko wrote:
> On Tue, Apr 16, 2024 at 7:38 AM Alan Maguire <alan.maguire@oracle.com> wrote:
>>
>> Specifying --btf_features=all enables all supported BTF features.
>> However there are some features that are non-standard, so we should
>> support a way to use them in --btf_features but not participate in
>> the set of features enabled by "--btf_features=all".  As part of this,
>> also support all used in a list of --btf_features, i.e.
>>
>> --btf_features=all,nonstandard_feature
>>
>> Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
>> ---
>>  man-pages/pahole.1 |  2 +-
>>  pahole.c           | 36 +++++++++++++++++++++++-------------
>>  2 files changed, 24 insertions(+), 14 deletions(-)
>>
>> diff --git a/man-pages/pahole.1 b/man-pages/pahole.1
>> index 2be165d..2c08e97 100644
>> --- a/man-pages/pahole.1
>> +++ b/man-pages/pahole.1
>> @@ -290,7 +290,7 @@ Allow using all the BTF features supported by pahole.
>>
>>  .TP
>>  .B \-\-btf_features=FEATURE_LIST
>> -Encode BTF using the specified feature list, or specify 'all' for all features supported.  This option can be used as an alternative to unsing multiple BTF-related options. Supported features are
>> +Encode BTF using the specified feature list, or specify 'all' for all standard features supported.  This option can be used as an alternative to unsing multiple BTF-related options. Supported standard features are
>>
>>  .nf
>>         encode_force       Ignore invalid symbols when encoding BTF; for example
>> diff --git a/pahole.c b/pahole.c
>> index 77772bb..890ef81 100644
>> --- a/pahole.c
>> +++ b/pahole.c
>> @@ -1266,23 +1266,26 @@ ARGP_PROGRAM_VERSION_HOOK_DEF = dwarves_print_version;
>>   * BTF encoding apply; we encode type/decl tags, do not encode
>>   * floats, etc.  This ensures backwards compatibility.
>>   */
>> -#define BTF_FEATURE(name, alias, default_value)                        \
>> -       { #name, #alias, &conf_load.alias, default_value }
>> +#define BTF_FEATURE(name, alias, default_value, enable_for_all)                \
>> +       { #name, #alias, &conf_load.alias, default_value, enable_for_all }
>>
>>  struct btf_feature {
>>         const char      *name;
>>         const char      *option_alias;
>>         bool            *conf_value;
>>         bool            default_value;
>> +       bool            enable_for_all; /* some nonstandard features may not
>> +                                        * be enabled for --btf_features=all
>> +                                        */
>>  } btf_features[] = {
>> -       BTF_FEATURE(encode_force, btf_encode_force, false),
>> -       BTF_FEATURE(var, skip_encoding_btf_vars, true),
>> -       BTF_FEATURE(float, btf_gen_floats, false),
>> -       BTF_FEATURE(decl_tag, skip_encoding_btf_decl_tag, true),
>> -       BTF_FEATURE(type_tag, skip_encoding_btf_type_tag, true),
>> -       BTF_FEATURE(enum64, skip_encoding_btf_enum64, true),
>> -       BTF_FEATURE(optimized_func, btf_gen_optimized, false),
>> -       BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false),
>> +       BTF_FEATURE(encode_force, btf_encode_force, false, true),
>> +       BTF_FEATURE(var, skip_encoding_btf_vars, true, true),
>> +       BTF_FEATURE(float, btf_gen_floats, false, true),
>> +       BTF_FEATURE(decl_tag, skip_encoding_btf_decl_tag, true, true),
>> +       BTF_FEATURE(type_tag, skip_encoding_btf_type_tag, true, true),
>> +       BTF_FEATURE(enum64, skip_encoding_btf_enum64, true, true),
>> +       BTF_FEATURE(optimized_func, btf_gen_optimized, false, true),
>> +       BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false, true),
>>  };
> 
> maybe keep those special features in a separate array instead?
> 

yeah, maybe that or have BTF_STANDARD_FEATURE() and BTF_EXTRA_FEATURE()
macros to clarify the difference.

> it is kind of weird to see --btf_features=all,and_then_some, maybe it
> makes sense to have --btf_extras or something for those?
>

yeah, the "all" is a problem. Arnaldo and I talked a bit about using
"default" or "standard" instead of "all"; the problem with default is
that there's already a notion of default BTF values - these are the ones
we get without specifying any --btf_features or other BTF-related flags.

Having an "extras" feature option (drawing from a separate option array)
as you suggest might be a cleaner way to make this distinction. What do
others think?

>>
>>  #define BTF_MAX_FEATURE_STR    1024
>> @@ -1350,8 +1353,10 @@ static void parse_btf_features(const char *features, bool strict)
>>         if (strcmp(features, "all") == 0) {
>>                 int i;
>>
>> -               for (i = 0; i < ARRAY_SIZE(btf_features); i++)
>> -                       enable_btf_feature(&btf_features[i]);
>> +               for (i = 0; i < ARRAY_SIZE(btf_features); i++) {
>> +                       if (btf_features[i].enable_for_all)
>> +                               enable_btf_feature(&btf_features[i]);
>> +               }
>>                 return;
>>         }
>>
>> @@ -1361,7 +1366,12 @@ static void parse_btf_features(const char *features, bool strict)
>>                 struct btf_feature *feature = find_btf_feature(feature_name);
>>
>>                 if (!feature) {
>> -                       if (strict) {
>> +                       /* --btf_features=all,nonstandard_feature should be
>> +                        * allowed.
>> +                        */
>> +                       if (strcmp(feature_name, "all") == 0) {
>> +                               parse_btf_features(feature_name, strict);
>> +                       } else if (strict) {
>>                                 fprintf(stderr, "Feature '%s' in '%s' is not supported.  Supported BTF features are:\n",
>>                                         feature_name, features);
>>                                 show_supported_btf_features(stderr);
>> --
>> 2.39.3
>>
>>

[PATCH dwarves 2/3] pahole: add reproducible_build to --btf_features

[Alan Maguire]

...as a non-standard feature, so it will not be enabled for
"--btf_features=all"

Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
---
 man-pages/pahole.1 | 8 ++++++++
 pahole.c           | 1 +
 2 files changed, 9 insertions(+)

diff --git a/man-pages/pahole.1 b/man-pages/pahole.1
index 2c08e97..64de343 100644
--- a/man-pages/pahole.1
+++ b/man-pages/pahole.1
@@ -310,6 +310,14 @@ Encode BTF using the specified feature list, or specify 'all' for all standard f
 	                   in different CUs.
 .fi
 
+Supported non-standard features (not enabled for 'all')
+
+.nf
+	reproducible_build Ensure generated BTF is consistent every time;
+	                   without this parallel BTF encoding can result in
+	                   inconsistent BTF ids.
+.fi
+
 So for example, specifying \-\-btf_encode=var,enum64 will result in a BTF encoding that (as well as encoding basic BTF information) will contain variables and enum64 values.
 
 .TP
diff --git a/pahole.c b/pahole.c
index 890ef81..38cc636 100644
--- a/pahole.c
+++ b/pahole.c
@@ -1286,6 +1286,7 @@ struct btf_feature {
 	BTF_FEATURE(enum64, skip_encoding_btf_enum64, true, true),
 	BTF_FEATURE(optimized_func, btf_gen_optimized, false, true),
 	BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false, true),
+	BTF_FEATURE(reproducible_build, reproducible_build, false, false),
 };
 
 #define BTF_MAX_FEATURE_STR	1024
-- 
2.39.3

Re: [PATCH dwarves 2/3] pahole: add reproducible_build to --btf_features

[Arnaldo Carvalho de Melo]

On Tue, Apr 16, 2024 at 03:37:17PM +0100, Alan Maguire wrote:
> ...as a non-standard feature, so it will not be enabled for
> "--btf_features=all"

How did you test this?

⬢[acme@toolbox pahole]$ pahole --btf_features_strict=bgasd
Feature 'bgasd' in 'bgasd' is not supported.  Supported BTF features are:
encode_force,var,float,decl_tag,type_tag,enum64,optimized_func,consistent_func,reproducible_build
⬢[acme@toolbox pahole]$ pahole -j --btf_features=all,reproducible_build --btf_encode_detached=vmlinux.btf.parallel.reproducible_build-via-btf_features vmlinux
⬢[acme@toolbox pahole]$ bpftool btf dump file vmlinux.btf.parallel.reproducible_build-via-btf_features > output.vmlinux.btf.parallel.reproducible_build-via-btf_features
⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.serial output.vmlinux.btf.parallel.reproducible
⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.parallel.reproducible_build output.vmlinux.btf.parallel.reproducible_build-via-btf_features | head
--- output.vmlinux.btf.parallel.reproducible_build	2024-04-16 12:20:28.513462223 -0300
+++ output.vmlinux.btf.parallel.reproducible_build-via-btf_features	2024-04-16 12:23:37.792962930 -0300
@@ -265,7 +265,7 @@
 	'target' type_id=33 bits_offset=32
 	'key' type_id=43 bits_offset=64
 [164] PTR '(anon)' type_id=163
-[165] PTR '(anon)' type_id=35751
+[165] PTR '(anon)' type_id=14983
 [166] STRUCT 'static_key' size=16 vlen=2
 	'enabled' type_id=88 bits_offset=0
⬢[acme@toolbox pahole]$

I'm double checking things now...

- Arnaldo
 
> Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
> ---
>  man-pages/pahole.1 | 8 ++++++++
>  pahole.c           | 1 +
>  2 files changed, 9 insertions(+)
> 
> diff --git a/man-pages/pahole.1 b/man-pages/pahole.1
> index 2c08e97..64de343 100644
> --- a/man-pages/pahole.1
> +++ b/man-pages/pahole.1
> @@ -310,6 +310,14 @@ Encode BTF using the specified feature list, or specify 'all' for all standard f
>  	                   in different CUs.
>  .fi
>  
> +Supported non-standard features (not enabled for 'all')
> +
> +.nf
> +	reproducible_build Ensure generated BTF is consistent every time;
> +	                   without this parallel BTF encoding can result in
> +	                   inconsistent BTF ids.
> +.fi
> +
>  So for example, specifying \-\-btf_encode=var,enum64 will result in a BTF encoding that (as well as encoding basic BTF information) will contain variables and enum64 values.
>  
>  .TP
> diff --git a/pahole.c b/pahole.c
> index 890ef81..38cc636 100644
> --- a/pahole.c
> +++ b/pahole.c
> @@ -1286,6 +1286,7 @@ struct btf_feature {
>  	BTF_FEATURE(enum64, skip_encoding_btf_enum64, true, true),
>  	BTF_FEATURE(optimized_func, btf_gen_optimized, false, true),
>  	BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false, true),
> +	BTF_FEATURE(reproducible_build, reproducible_build, false, false),
>  };
>  
>  #define BTF_MAX_FEATURE_STR	1024

Re: [PATCH dwarves 2/3] pahole: add reproducible_build to --btf_features

[Alan Maguire]

On 16/04/2024 16:24, Arnaldo Carvalho de Melo wrote:
> On Tue, Apr 16, 2024 at 03:37:17PM +0100, Alan Maguire wrote:
>> ...as a non-standard feature, so it will not be enabled for
>> "--btf_features=all"
> 
> How did you test this?
> 
> ⬢[acme@toolbox pahole]$ pahole --btf_features_strict=bgasd
> Feature 'bgasd' in 'bgasd' is not supported.  Supported BTF features are:
> encode_force,var,float,decl_tag,type_tag,enum64,optimized_func,consistent_func,reproducible_build
> ⬢[acme@toolbox pahole]$ pahole -j --btf_features=all,reproducible_build --btf_encode_detached=vmlinux.btf.parallel.reproducible_build-via-btf_features vmlinux
> ⬢[acme@toolbox pahole]$ bpftool btf dump file vmlinux.btf.parallel.reproducible_build-via-btf_features > output.vmlinux.btf.parallel.reproducible_build-via-btf_features
> ⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.serial output.vmlinux.btf.parallel.reproducible
> ⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.parallel.reproducible_build output.vmlinux.btf.parallel.reproducible_build-via-btf_features | head
> --- output.vmlinux.btf.parallel.reproducible_build	2024-04-16 12:20:28.513462223 -0300
> +++ output.vmlinux.btf.parallel.reproducible_build-via-btf_features	2024-04-16 12:23:37.792962930 -0300
> @@ -265,7 +265,7 @@
>  	'target' type_id=33 bits_offset=32
>  	'key' type_id=43 bits_offset=64
>  [164] PTR '(anon)' type_id=163
> -[165] PTR '(anon)' type_id=35751
> +[165] PTR '(anon)' type_id=14983
>  [166] STRUCT 'static_key' size=16 vlen=2
>  	'enabled' type_id=88 bits_offset=0
> ⬢[acme@toolbox pahole]$
> 
> I'm double checking things now...
>

The test worked for me on x86_64/aarch64. Did you test with patch 3
applied? Because the test in its original state prior to patch 3 sets
--reproducible_build before setting --btf_features=all, you won't get a
reproducible build since the command line is saying "enable reproducible
builds but also enable standard features only"; the second action undoes
the first. switching to using --btf_features=all,reproducible_build
fixes things for me.
> - Arnaldo
>  
>> Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
>> ---
>>  man-pages/pahole.1 | 8 ++++++++
>>  pahole.c           | 1 +
>>  2 files changed, 9 insertions(+)
>>
>> diff --git a/man-pages/pahole.1 b/man-pages/pahole.1
>> index 2c08e97..64de343 100644
>> --- a/man-pages/pahole.1
>> +++ b/man-pages/pahole.1
>> @@ -310,6 +310,14 @@ Encode BTF using the specified feature list, or specify 'all' for all standard f
>>  	                   in different CUs.
>>  .fi
>>  
>> +Supported non-standard features (not enabled for 'all')
>> +
>> +.nf
>> +	reproducible_build Ensure generated BTF is consistent every time;
>> +	                   without this parallel BTF encoding can result in
>> +	                   inconsistent BTF ids.
>> +.fi
>> +
>>  So for example, specifying \-\-btf_encode=var,enum64 will result in a BTF encoding that (as well as encoding basic BTF information) will contain variables and enum64 values.
>>  
>>  .TP
>> diff --git a/pahole.c b/pahole.c
>> index 890ef81..38cc636 100644
>> --- a/pahole.c
>> +++ b/pahole.c
>> @@ -1286,6 +1286,7 @@ struct btf_feature {
>>  	BTF_FEATURE(enum64, skip_encoding_btf_enum64, true, true),
>>  	BTF_FEATURE(optimized_func, btf_gen_optimized, false, true),
>>  	BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false, true),
>> +	BTF_FEATURE(reproducible_build, reproducible_build, false, false),
>>  };
>>  
>>  #define BTF_MAX_FEATURE_STR	1024
> 
> 
> 

Re: [PATCH dwarves 2/3] pahole: add reproducible_build to --btf_features

[Arnaldo Carvalho de Melo]

On Tue, Apr 16, 2024 at 04:41:20PM +0100, Alan Maguire wrote:
> On 16/04/2024 16:24, Arnaldo Carvalho de Melo wrote:
> > On Tue, Apr 16, 2024 at 03:37:17PM +0100, Alan Maguire wrote:
> >> ...as a non-standard feature, so it will not be enabled for
> >> "--btf_features=all"
> > 
> > How did you test this?
> > 
> > ⬢[acme@toolbox pahole]$ pahole --btf_features_strict=bgasd
> > Feature 'bgasd' in 'bgasd' is not supported.  Supported BTF features are:
> > encode_force,var,float,decl_tag,type_tag,enum64,optimized_func,consistent_func,reproducible_build
> > ⬢[acme@toolbox pahole]$ pahole -j --btf_features=all,reproducible_build --btf_encode_detached=vmlinux.btf.parallel.reproducible_build-via-btf_features vmlinux
> > ⬢[acme@toolbox pahole]$ bpftool btf dump file vmlinux.btf.parallel.reproducible_build-via-btf_features > output.vmlinux.btf.parallel.reproducible_build-via-btf_features
> > ⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.serial output.vmlinux.btf.parallel.reproducible
> > ⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.parallel.reproducible_build output.vmlinux.btf.parallel.reproducible_build-via-btf_features | head
> > --- output.vmlinux.btf.parallel.reproducible_build	2024-04-16 12:20:28.513462223 -0300
> > +++ output.vmlinux.btf.parallel.reproducible_build-via-btf_features	2024-04-16 12:23:37.792962930 -0300
> > @@ -265,7 +265,7 @@
> >  	'target' type_id=33 bits_offset=32
> >  	'key' type_id=43 bits_offset=64
> >  [164] PTR '(anon)' type_id=163
> > -[165] PTR '(anon)' type_id=35751
> > +[165] PTR '(anon)' type_id=14983
> >  [166] STRUCT 'static_key' size=16 vlen=2
> >  	'enabled' type_id=88 bits_offset=0
> > ⬢[acme@toolbox pahole]$
> > 
> > I'm double checking things now...
 
> The test worked for me on x86_64/aarch64. Did you test with patch 3
> applied? Because the test in its original state prior to patch 3 sets

With all patches applied:

⬢[acme@toolbox pahole]$ git log --oneline -5
ecd3b0852ab1f1ff (HEAD -> master) tests/reproducible_build: use --btf_features=all,reproducible_build
f9c8f5856b2aafea pahole: Add reproducible_build to --btf_features
0412978e8e6f8f76 pahole: Allow --btf_features to not participate in "all"
a9738ddc828d5ea0 tests/reproducible_build: Use --btf_features=all when encoding
d7edf9ae0388fb97 tests/reproducible_build: Validate the vmlinux file
⬢[acme@toolbox pahole]$ 

I made some mistake:

⬢[acme@toolbox pahole]$ pahole --btf_encode_detached=vmlinux.btf.serial --btf_features=all vmlinux
⬢[acme@toolbox pahole]$ pahole --btf_encode_detached=vmlinux.btf.parallel --btf_features=all -j vmlinux
⬢[acme@toolbox pahole]$ pahole --btf_encode_detached=vmlinux.btf.parallel--reproducible_build --btf_features=all --reproducible_build -j vmlinux
⬢[acme@toolbox pahole]$ pahole --btf_encode_detached=vmlinux.btf.parallel--btf_features=all,reproducible_build --btf_features=all,reproducible_build -j vmlinux
⬢[acme@toolbox pahole]$ bpftool btf dump file vmlinux.btf.serial > output.vmlinux.btf.serial
⬢[acme@toolbox pahole]$ bpftool btf dump file vmlinux.btf.parallel > output.vmlinux.btf.parallel
⬢[acme@toolbox pahole]$ bpftool btf dump file vmlinux.btf.parallel--reproducible_build > output.vmlinux.btf.parallel--reproducible_build
⬢[acme@toolbox pahole]$ bpftool btf dump file vmlinux.btf.parallel--btf_features=all,reproducible_build > output.vmlinux.btf.parallel--btf_features=all,reproducible_build
⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.serial output.vmlinux.btf.parallel | wc -l
629165
⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.serial output.vmlinux.btf.parallel--reproducible_build | wc -l
0
⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.serial output.vmlinux.btf.parallel--btf_features=all,reproducible_build | wc -l
0
⬢[acme@toolbox pahole]$ diff -u output.vmlinux.btf.parallel--reproducible_build output.vmlinux.btf.parallel--btf_features=all,reproducible_build | wc -l
0
⬢[acme@toolbox pahole]$

And of course:

⬢[acme@toolbox pahole]$ time tests/reproducible_build.sh  vmlinux
Parallel reproducible DWARF Loading/Serial BTF encoding: Ok

real	1m25.241s
user	3m10.144s
sys	0m48.104s
⬢[acme@toolbox pahole]$

> --reproducible_build before setting --btf_features=all, you won't get a
> reproducible build since the command line is saying "enable reproducible
> builds but also enable standard features only"; the second action undoes

Makes sense.

> the first. switching to using --btf_features=all,reproducible_build
> fixes things for me.

Ok, all tested now, will push to the 'next' branch, waiting for Eduard's
review.

- Arnaldo

> > - Arnaldo
> >  
> >> Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
> >> ---
> >>  man-pages/pahole.1 | 8 ++++++++
> >>  pahole.c           | 1 +
> >>  2 files changed, 9 insertions(+)
> >>
> >> diff --git a/man-pages/pahole.1 b/man-pages/pahole.1
> >> index 2c08e97..64de343 100644
> >> --- a/man-pages/pahole.1
> >> +++ b/man-pages/pahole.1
> >> @@ -310,6 +310,14 @@ Encode BTF using the specified feature list, or specify 'all' for all standard f
> >>  	                   in different CUs.
> >>  .fi
> >>  
> >> +Supported non-standard features (not enabled for 'all')
> >> +
> >> +.nf
> >> +	reproducible_build Ensure generated BTF is consistent every time;
> >> +	                   without this parallel BTF encoding can result in
> >> +	                   inconsistent BTF ids.
> >> +.fi
> >> +
> >>  So for example, specifying \-\-btf_encode=var,enum64 will result in a BTF encoding that (as well as encoding basic BTF information) will contain variables and enum64 values.
> >>  
> >>  .TP
> >> diff --git a/pahole.c b/pahole.c
> >> index 890ef81..38cc636 100644
> >> --- a/pahole.c
> >> +++ b/pahole.c
> >> @@ -1286,6 +1286,7 @@ struct btf_feature {
> >>  	BTF_FEATURE(enum64, skip_encoding_btf_enum64, true, true),
> >>  	BTF_FEATURE(optimized_func, btf_gen_optimized, false, true),
> >>  	BTF_FEATURE(consistent_func, skip_encoding_btf_inconsistent_proto, false, true),
> >> +	BTF_FEATURE(reproducible_build, reproducible_build, false, false),
> >>  };
> >>  
> >>  #define BTF_MAX_FEATURE_STR	1024
> > 
> > 
> > 

[PATCH dwarves 3/3] tests/reproducible_build: use --btf_features=all,reproducible_build

[Alan Maguire]

...as this will test enabling all standard features plus a non-standard
one.

Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
---
 tests/reproducible_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/reproducible_build.sh b/tests/reproducible_build.sh
index 8cc36fe..e2f8360 100755
--- a/tests/reproducible_build.sh
+++ b/tests/reproducible_build.sh
@@ -29,7 +29,7 @@ nr_proc=$(getconf _NPROCESSORS_ONLN)
 
 for threads in $(seq $nr_proc) ; do
 	test -n "$VERBOSE" && echo $threads threads encoding
-	pahole -j$threads --reproducible_build --btf_features=all --btf_encode_detached=$outdir/vmlinux.btf.parallel.reproducible $vmlinux &
+	pahole -j$threads --btf_features=all,reproducible_build --btf_encode_detached=$outdir/vmlinux.btf.parallel.reproducible $vmlinux &
 	pahole=$!
 	# HACK: Wait a bit for pahole to start its threads
 	sleep 0.3s
-- 
2.39.3