[PATCH 01/11] meta/lib/oe/reproducible.py: gitsm:// works just as fine as git:// for timestamps

[PATCH 01/11] meta/lib/oe/reproducible.py: gitsm:// works just as fine as git:// for timestamps

[Alexander Kanavin]

This in particular addresses vulkan-samples reproducibility which made me scratch my
head for a while.

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 meta/lib/oe/reproducible.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oe/reproducible.py b/meta/lib/oe/reproducible.py
index 421bb12f54..0fb02ccdb0 100644
--- a/meta/lib/oe/reproducible.py
+++ b/meta/lib/oe/reproducible.py
@@ -47,7 +47,7 @@ def find_git_folder(d, sourcedir):
     return None
 
 def get_source_date_epoch_from_git(d, sourcedir):
-    if not "git://" in d.getVar('SRC_URI'):
+    if not "git://" in d.getVar('SRC_URI') and not "gitsm://" in d.getVar('SRC_URI'):
         return None
 
     gitpath = find_git_folder(d, sourcedir)
-- 
2.29.2

[PATCH 02/11] kea: fix reproducibility

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 ...er_unittest_support.cc-do-not-write-.patch | 27 +++++++++++++++++
 meta/recipes-connectivity/kea/kea_1.7.10.bb   | 29 +++++++++++--------
 2 files changed, 44 insertions(+), 12 deletions(-)
 create mode 100644 meta/recipes-connectivity/kea/files/0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch

diff --git a/meta/recipes-connectivity/kea/files/0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch b/meta/recipes-connectivity/kea/files/0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch
new file mode 100644
index 0000000000..226bc5b311
--- /dev/null
+++ b/meta/recipes-connectivity/kea/files/0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch
@@ -0,0 +1,27 @@
+From 9985a03f13da4d7bb0a433f7305d2ffae3d82a27 Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Tue, 10 Nov 2020 15:57:03 +0000
+Subject: [PATCH] src/lib/log/logger_unittest_support.cc: do not write build
+ path into binary
+
+This breaks reproducibility and is needed only in unit testing.
+
+Upstream-Status: Inappropriate [oe-core specific]
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ src/lib/log/logger_unittest_support.cc | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/lib/log/logger_unittest_support.cc b/src/lib/log/logger_unittest_support.cc
+index 58dbef8..9a2929c 100644
+--- a/src/lib/log/logger_unittest_support.cc
++++ b/src/lib/log/logger_unittest_support.cc
+@@ -84,7 +84,7 @@ void initLogger(isc::log::Severity severity, int dbglevel) {
+     const char* localfile = getenv("KEA_LOGGER_LOCALMSG");
+ 
+     // Set a directory for creating lockfiles when running tests
+-    setenv("KEA_LOCKFILE_DIR", TOP_BUILDDIR, 0);
++    //setenv("KEA_LOCKFILE_DIR", TOP_BUILDDIR, 0);
+ 
+     // Initialize logging
+     initLogger(root, isc::log::DEBUG, isc::log::MAX_DEBUG_LEVEL, localfile);
diff --git a/meta/recipes-connectivity/kea/kea_1.7.10.bb b/meta/recipes-connectivity/kea/kea_1.7.10.bb
index 1d011ace78..c9a11908e5 100644
--- a/meta/recipes-connectivity/kea/kea_1.7.10.bb
+++ b/meta/recipes-connectivity/kea/kea_1.7.10.bb
@@ -7,18 +7,18 @@ LIC_FILES_CHKSUM = "file://COPYING;md5=68d95543d2096459290a4e6b9ceccffa"
 
 DEPENDS = "boost log4cplus openssl"
 
-SRC_URI = "\
-    http://ftp.isc.org/isc/kea/${PV}/${BP}.tar.gz \
-    file://0001-keactrl.in-create-var-lib-kea-and-var-run-kea-folder.patch \
-    file://kea-dhcp4.service \
-    file://kea-dhcp6.service \
-    file://kea-dhcp-ddns.service \
-    file://kea-dhcp4-server \
-    file://kea-dhcp6-server \
-    file://kea-dhcp-ddns-server \
-    file://fix-multilib-conflict.patch \
-    file://fix_pid_keactrl.patch \
-"
+SRC_URI = "http://ftp.isc.org/isc/kea/${PV}/${BP}.tar.gz \
+           file://0001-keactrl.in-create-var-lib-kea-and-var-run-kea-folder.patch \
+           file://kea-dhcp4.service \
+           file://kea-dhcp6.service \
+           file://kea-dhcp-ddns.service \
+           file://kea-dhcp4-server \
+           file://kea-dhcp6-server \
+           file://kea-dhcp-ddns-server \
+           file://fix-multilib-conflict.patch \
+           file://fix_pid_keactrl.patch \
+           file://0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch \
+           "
 SRC_URI[sha256sum] = "4e121f0e58b175a827581c69cb1d60778647049fa47f142940dddc9ce58f3c82"
 
 inherit autotools systemd update-rc.d upstream-version-is-even
@@ -50,6 +50,11 @@ do_configure_prepend() {
     sed -i "s:@abs_top_srcdir@:@abs_top_srcdir_placeholder@:g" ${S}/src/bin/admin/kea-admin.in
 }
 
+# patch out build host paths for reproducibility
+do_compile_prepend_class-target() {
+        sed -i -e "s,${WORKDIR},,g" ${B}/config.report
+}
+
 do_install_append() {
     install -d ${D}${sysconfdir}/init.d
     install -d ${D}${systemd_system_unitdir}
-- 
2.29.2

Re: [OE-core] [PATCH 02/11] kea: fix reproducibility

[Khem Raj]

On Thu, Dec 3, 2020 at 5:37 AM Alexander Kanavin <alex.kanavin@gmail.com> wrote:
>
> Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
> ---
>  ...er_unittest_support.cc-do-not-write-.patch | 27 +++++++++++++++++
>  meta/recipes-connectivity/kea/kea_1.7.10.bb   | 29 +++++++++++--------
>  2 files changed, 44 insertions(+), 12 deletions(-)
>  create mode 100644 meta/recipes-connectivity/kea/files/0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch
>
> diff --git a/meta/recipes-connectivity/kea/files/0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch b/meta/recipes-connectivity/kea/files/0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch
> new file mode 100644
> index 0000000000..226bc5b311
> --- /dev/null
> +++ b/meta/recipes-connectivity/kea/files/0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch
> @@ -0,0 +1,27 @@
> +From 9985a03f13da4d7bb0a433f7305d2ffae3d82a27 Mon Sep 17 00:00:00 2001
> +From: Alexander Kanavin <alex.kanavin@gmail.com>
> +Date: Tue, 10 Nov 2020 15:57:03 +0000
> +Subject: [PATCH] src/lib/log/logger_unittest_support.cc: do not write build
> + path into binary
> +
> +This breaks reproducibility and is needed only in unit testing.
> +
> +Upstream-Status: Inappropriate [oe-core specific]
> +Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
> +---
> + src/lib/log/logger_unittest_support.cc | 2 +-
> + 1 file changed, 1 insertion(+), 1 deletion(-)
> +
> +diff --git a/src/lib/log/logger_unittest_support.cc b/src/lib/log/logger_unittest_support.cc
> +index 58dbef8..9a2929c 100644
> +--- a/src/lib/log/logger_unittest_support.cc
> ++++ b/src/lib/log/logger_unittest_support.cc
> +@@ -84,7 +84,7 @@ void initLogger(isc::log::Severity severity, int dbglevel) {
> +     const char* localfile = getenv("KEA_LOGGER_LOCALMSG");
> +
> +     // Set a directory for creating lockfiles when running tests
> +-    setenv("KEA_LOCKFILE_DIR", TOP_BUILDDIR, 0);
> ++    //setenv("KEA_LOCKFILE_DIR", TOP_BUILDDIR, 0);
> +

does this test then work or do we also need to ensure that
KEA_LOCKFILE_DIR is set to somewhere in /run dir
via Compiler flags or some such

> +     // Initialize logging
> +     initLogger(root, isc::log::DEBUG, isc::log::MAX_DEBUG_LEVEL, localfile);
> diff --git a/meta/recipes-connectivity/kea/kea_1.7.10.bb b/meta/recipes-connectivity/kea/kea_1.7.10.bb
> index 1d011ace78..c9a11908e5 100644
> --- a/meta/recipes-connectivity/kea/kea_1.7.10.bb
> +++ b/meta/recipes-connectivity/kea/kea_1.7.10.bb
> @@ -7,18 +7,18 @@ LIC_FILES_CHKSUM = "file://COPYING;md5=68d95543d2096459290a4e6b9ceccffa"
>
>  DEPENDS = "boost log4cplus openssl"
>
> -SRC_URI = "\
> -    http://ftp.isc.org/isc/kea/${PV}/${BP}.tar.gz \
> -    file://0001-keactrl.in-create-var-lib-kea-and-var-run-kea-folder.patch \
> -    file://kea-dhcp4.service \
> -    file://kea-dhcp6.service \
> -    file://kea-dhcp-ddns.service \
> -    file://kea-dhcp4-server \
> -    file://kea-dhcp6-server \
> -    file://kea-dhcp-ddns-server \
> -    file://fix-multilib-conflict.patch \
> -    file://fix_pid_keactrl.patch \
> -"
> +SRC_URI = "http://ftp.isc.org/isc/kea/${PV}/${BP}.tar.gz \
> +           file://0001-keactrl.in-create-var-lib-kea-and-var-run-kea-folder.patch \
> +           file://kea-dhcp4.service \
> +           file://kea-dhcp6.service \
> +           file://kea-dhcp-ddns.service \
> +           file://kea-dhcp4-server \
> +           file://kea-dhcp6-server \
> +           file://kea-dhcp-ddns-server \
> +           file://fix-multilib-conflict.patch \
> +           file://fix_pid_keactrl.patch \
> +           file://0001-src-lib-log-logger_unittest_support.cc-do-not-write-.patch \
> +           "
>  SRC_URI[sha256sum] = "4e121f0e58b175a827581c69cb1d60778647049fa47f142940dddc9ce58f3c82"
>
>  inherit autotools systemd update-rc.d upstream-version-is-even
> @@ -50,6 +50,11 @@ do_configure_prepend() {
>      sed -i "s:@abs_top_srcdir@:@abs_top_srcdir_placeholder@:g" ${S}/src/bin/admin/kea-admin.in
>  }
>
> +# patch out build host paths for reproducibility
> +do_compile_prepend_class-target() {
> +        sed -i -e "s,${WORKDIR},,g" ${B}/config.report
> +}
> +
>  do_install_append() {
>      install -d ${D}${sysconfdir}/init.d
>      install -d ${D}${systemd_system_unitdir}
> --
> 2.29.2
>
>
> 
>

Re: [OE-core] [PATCH 02/11] kea: fix reproducibility

[Alexander Kanavin]

[-- Attachment #1: Type: text/plain, Size: 440 bytes --]

On Thu, 3 Dec 2020 at 20:07, Khem Raj <raj.khem@gmail.com> wrote:

>
> does this test then work or do we also need to ensure that
> KEA_LOCKFILE_DIR is set to somewhere in /run dir
> via Compiler flags or some such
>

As we are not executing the test, it does not matter, and it would not have
worked on target anyway as it hardcodes the build path. If someone wants to
work on the tests, they can look into it properly.

Alex

[-- Attachment #2: Type: text/html, Size: 745 bytes --]

Re: [OE-core] [PATCH 02/11] kea: fix reproducibility

[Khem Raj]

On Thu, Dec 3, 2020 at 11:12 AM Alexander Kanavin
<alex.kanavin@gmail.com> wrote:
>
> On Thu, 3 Dec 2020 at 20:07, Khem Raj <raj.khem@gmail.com> wrote:
>>
>>
>> does this test then work or do we also need to ensure that
>> KEA_LOCKFILE_DIR is set to somewhere in /run dir
>> via Compiler flags or some such
>
>
> As we are not executing the test, it does not matter, and it would not have worked on target anyway as it hardcodes the build path. If someone wants to work on the tests, they can look into it properly.
>

but say if you were. Would it make the problem more obscure than what
it is now. So while here, perhaps defining KEA_LOCKFILE_DIR to point
to /run would be a prudent thing to do if it's already not done.


> Alex

Re: [OE-core] [PATCH 02/11] kea: fix reproducibility

[Alexander Kanavin]

[-- Attachment #1: Type: text/plain, Size: 467 bytes --]

On Thu, 3 Dec 2020 at 20:19, Khem Raj <raj.khem@gmail.com> wrote:

>
> but say if you were. Would it make the problem more obscure than what
> it is now. So while here, perhaps defining KEA_LOCKFILE_DIR to point
> to /run would be a prudent thing to do if it's already not done.
>

Apologies, but no. I am not defining something that I have no way to test
the correctness of, and I am not going to bring up the kea tests on target
to achieve that.

Alex

[-- Attachment #2: Type: text/html, Size: 780 bytes --]

Re: [OE-core] [PATCH 02/11] kea: fix reproducibility

[Khem Raj]

On Thu, Dec 3, 2020 at 11:22 AM Alexander Kanavin
<alex.kanavin@gmail.com> wrote:
>
> On Thu, 3 Dec 2020 at 20:19, Khem Raj <raj.khem@gmail.com> wrote:
>>
>>
>> but say if you were. Would it make the problem more obscure than what
>> it is now. So while here, perhaps defining KEA_LOCKFILE_DIR to point
>> to /run would be a prudent thing to do if it's already not done.
>
>
> Apologies, but no. I am not defining something that I have no way to test the correctness of, and I am not going to bring up the kea tests on target to achieve that.
>

Looking at code it seems defaults are /var/run/kea so perhaps we are
fine by not setting it explicitly here,  I know systemd wants to move
away from /var/run but that's perhaps an independent issue

> Alex

[PATCH 03/11] llvm: fix reproducibility

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 meta/recipes-devtools/llvm/llvm_git.bb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/meta/recipes-devtools/llvm/llvm_git.bb b/meta/recipes-devtools/llvm/llvm_git.bb
index 4c2d490315..b146d0e6e3 100644
--- a/meta/recipes-devtools/llvm/llvm_git.bb
+++ b/meta/recipes-devtools/llvm/llvm_git.bb
@@ -99,6 +99,11 @@ do_configure_prepend() {
 	sed -ri "s#lib/${LLVM_DIR}#${baselib}/${LLVM_DIR}#g" ${S}/tools/llvm-config/llvm-config.cpp
 }
 
+# patch out build host paths for reproducibility
+do_compile_prepend_class-target() {
+        sed -i -e "s,${WORKDIR},,g" ${B}/tools/llvm-config/BuildVariables.inc
+}
+
 do_compile() {
 	ninja -v ${PARALLEL_MAKE}
 }
-- 
2.29.2

[PATCH 04/11] ruby: fix reproducibility

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 ...e.in-do-not-write-host-cross-cc-item.patch | 32 +++++++++++++++++++
 meta/recipes-devtools/ruby/ruby_2.7.2.bb      |  1 +
 2 files changed, 33 insertions(+)
 create mode 100644 meta/recipes-devtools/ruby/ruby/0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch

diff --git a/meta/recipes-devtools/ruby/ruby/0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch b/meta/recipes-devtools/ruby/ruby/0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch
new file mode 100644
index 0000000000..826daf2cda
--- /dev/null
+++ b/meta/recipes-devtools/ruby/ruby/0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch
@@ -0,0 +1,32 @@
+From 2368d07660a93a2c41d63f3ab6054ca4daeef820 Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Tue, 17 Nov 2020 18:31:40 +0000
+Subject: [PATCH] template/Makefile.in: do not write host cross-cc items into
+ target config
+
+This helps reproducibility.
+
+Upstream-Status: Inapproppriate [oe-core specific]
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ template/Makefile.in | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/template/Makefile.in b/template/Makefile.in
+index 10dc826..940ee07 100644
+--- a/template/Makefile.in
++++ b/template/Makefile.in
+@@ -657,11 +657,11 @@ mjit_config.h:
+ 	echo '#endif'; \
+ 	quote MJIT_MIN_HEADER_NAME "$(MJIT_MIN_HEADER_NAME)"; \
+ 	sep=,; \
+-	quote "MJIT_CC_COMMON  " $(MJIT_CC); \
++	quote "MJIT_CC_COMMON  " ; \
+ 	quote "MJIT_CFLAGS      MJIT_ARCHFLAG" $(MJIT_CFLAGS); \
+ 	quote "MJIT_OPTFLAGS   " $(MJIT_OPTFLAGS); \
+ 	quote "MJIT_DEBUGFLAGS " $(MJIT_DEBUGFLAGS); \
+-	quote "MJIT_LDSHARED   " $(MJIT_LDSHARED); \
++	quote "MJIT_LDSHARED   " ; \
+ 	quote "MJIT_DLDFLAGS    MJIT_ARCHFLAG" $(MJIT_DLDFLAGS); \
+ 	quote "MJIT_LIBS       " $(LIBRUBYARG_SHARED); \
+ 	quote 'PRELOADENV       "@PRELOADENV@"'; \
diff --git a/meta/recipes-devtools/ruby/ruby_2.7.2.bb b/meta/recipes-devtools/ruby/ruby_2.7.2.bb
index 055ea9343f..db6d672985 100644
--- a/meta/recipes-devtools/ruby/ruby_2.7.2.bb
+++ b/meta/recipes-devtools/ruby/ruby_2.7.2.bb
@@ -6,6 +6,7 @@ SRC_URI += " \
            file://remove_has_include_macros.patch \
            file://run-ptest \
            file://0001-Modify-shebang-of-libexec-y2racc-and-libexec-racc2y.patch \
+           file://0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch \
            "
 
 SRC_URI[md5sum] = "2d4a28dcfa38352a627a597f6057c465"
-- 
2.29.2

Re: [OE-core] [PATCH 04/11] ruby: fix reproducibility

[Khem Raj]

On Thu, Dec 3, 2020 at 5:37 AM Alexander Kanavin <alex.kanavin@gmail.com> wrote:
>
> Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
> ---
>  ...e.in-do-not-write-host-cross-cc-item.patch | 32 +++++++++++++++++++
>  meta/recipes-devtools/ruby/ruby_2.7.2.bb      |  1 +
>  2 files changed, 33 insertions(+)
>  create mode 100644 meta/recipes-devtools/ruby/ruby/0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch
>
> diff --git a/meta/recipes-devtools/ruby/ruby/0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch b/meta/recipes-devtools/ruby/ruby/0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch
> new file mode 100644
> index 0000000000..826daf2cda
> --- /dev/null
> +++ b/meta/recipes-devtools/ruby/ruby/0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch
> @@ -0,0 +1,32 @@
> +From 2368d07660a93a2c41d63f3ab6054ca4daeef820 Mon Sep 17 00:00:00 2001
> +From: Alexander Kanavin <alex.kanavin@gmail.com>
> +Date: Tue, 17 Nov 2020 18:31:40 +0000
> +Subject: [PATCH] template/Makefile.in: do not write host cross-cc items into
> + target config
> +
> +This helps reproducibility.
> +
> +Upstream-Status: Inapproppriate [oe-core specific]
> +Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
> +---
> + template/Makefile.in | 4 ++--
> + 1 file changed, 2 insertions(+), 2 deletions(-)
> +
> +diff --git a/template/Makefile.in b/template/Makefile.in
> +index 10dc826..940ee07 100644
> +--- a/template/Makefile.in
> ++++ b/template/Makefile.in
> +@@ -657,11 +657,11 @@ mjit_config.h:
> +       echo '#endif'; \
> +       quote MJIT_MIN_HEADER_NAME "$(MJIT_MIN_HEADER_NAME)"; \
> +       sep=,; \
> +-      quote "MJIT_CC_COMMON  " $(MJIT_CC); \
> ++      quote "MJIT_CC_COMMON  " ; \

where is this template used ? is it used on target ? is it possible to
just all bare compiler name then since it will still work on target
although not as important as if it was needed during cross builds.

> +       quote "MJIT_CFLAGS      MJIT_ARCHFLAG" $(MJIT_CFLAGS); \
> +       quote "MJIT_OPTFLAGS   " $(MJIT_OPTFLAGS); \
> +       quote "MJIT_DEBUGFLAGS " $(MJIT_DEBUGFLAGS); \
> +-      quote "MJIT_LDSHARED   " $(MJIT_LDSHARED); \
> ++      quote "MJIT_LDSHARED   " ; \
> +       quote "MJIT_DLDFLAGS    MJIT_ARCHFLAG" $(MJIT_DLDFLAGS); \
> +       quote "MJIT_LIBS       " $(LIBRUBYARG_SHARED); \
> +       quote 'PRELOADENV       "@PRELOADENV@"'; \
> diff --git a/meta/recipes-devtools/ruby/ruby_2.7.2.bb b/meta/recipes-devtools/ruby/ruby_2.7.2.bb
> index 055ea9343f..db6d672985 100644
> --- a/meta/recipes-devtools/ruby/ruby_2.7.2.bb
> +++ b/meta/recipes-devtools/ruby/ruby_2.7.2.bb
> @@ -6,6 +6,7 @@ SRC_URI += " \
>             file://remove_has_include_macros.patch \
>             file://run-ptest \
>             file://0001-Modify-shebang-of-libexec-y2racc-and-libexec-racc2y.patch \
> +           file://0001-template-Makefile.in-do-not-write-host-cross-cc-item.patch \
>             "
>
>  SRC_URI[md5sum] = "2d4a28dcfa38352a627a597f6057c465"
> --
> 2.29.2
>
>
> 
>

Re: [OE-core] [PATCH 04/11] ruby: fix reproducibility

[Alexander Kanavin]

[-- Attachment #1: Type: text/plain, Size: 579 bytes --]

On Thu, 3 Dec 2020 at 20:16, Khem Raj <raj.khem@gmail.com> wrote:

>
> where is this template used ? is it used on target ? is it possible to
> just all bare compiler name then since it will still work on target
> although not as important as if it was needed during cross builds.
>

I don't think it is used on target, as harcoded host items would surely not
have worked. Again, the goal here is not to ensure that target toolchain
and target tests work (a large and not necessarily useful task), but merely
to avoid host paths leaking into target packages.

Alex

[-- Attachment #2: Type: text/html, Size: 889 bytes --]

Re: [OE-core] [PATCH 04/11] ruby: fix reproducibility

[Khem Raj]

On Thu, Dec 3, 2020 at 11:20 AM Alexander Kanavin
<alex.kanavin@gmail.com> wrote:
>
> On Thu, 3 Dec 2020 at 20:16, Khem Raj <raj.khem@gmail.com> wrote:
>>
>>
>> where is this template used ? is it used on target ? is it possible to
>> just all bare compiler name then since it will still work on target
>> although not as important as if it was needed during cross builds.
>
>
> I don't think it is used on target, as harcoded host items would surely not have worked. Again, the goal here is not to ensure that target toolchain and target tests work (a large and not necessarily useful task), but merely to avoid host paths leaking into target packages.
>

I see that, I just want to ensure that deferring the problems does not
make them harder to detect in the future at the same time. It's easy
to fix them in the context than to revisit later.
> Alex

[PATCH 05/11] webkitgtk: fix reproducibility

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 meta/recipes-sato/webkit/webkitgtk_2.30.2.bb | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/meta/recipes-sato/webkit/webkitgtk_2.30.2.bb b/meta/recipes-sato/webkit/webkitgtk_2.30.2.bb
index 58b66c0f66..31370f3239 100644
--- a/meta/recipes-sato/webkit/webkitgtk_2.30.2.bb
+++ b/meta/recipes-sato/webkit/webkitgtk_2.30.2.bb
@@ -134,3 +134,15 @@ GI_DATA_ENABLED_libc-musl_armv7ve = "False"
 
 # Can't be built with ccache
 CCACHE_DISABLE = "1"
+
+PACKAGE_PREPROCESS_FUNCS += "src_package_preprocess"
+src_package_preprocess () {
+        # Trim build paths from comments in generated sources to ensure reproducibility
+        sed -i -e "s,${WORKDIR},,g" \
+            ${B}/DerivedSources/webkit2gtk/webkit2/*.cpp \
+            ${B}/DerivedSources/ForwardingHeaders/JavaScriptCore/*.h \
+            ${B}/DerivedSources/JavaScriptCore/*.h \
+            ${B}/DerivedSources/JavaScriptCore/yarr/*.h \
+            ${B}/DerivedSources/MiniBrowser/*.c
+}
+
-- 
2.29.2

[PATCH 06/11] ffmpeg: fix reproducibility

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 ...e-assembly-with-full-path-from-sourc.patch | 97 +++++++++++++++++++
 .../recipes-multimedia/ffmpeg/ffmpeg_4.3.1.bb |  6 ++
 2 files changed, 103 insertions(+)
 create mode 100644 meta/recipes-multimedia/ffmpeg/ffmpeg/0001-libavutil-include-assembly-with-full-path-from-sourc.patch

diff --git a/meta/recipes-multimedia/ffmpeg/ffmpeg/0001-libavutil-include-assembly-with-full-path-from-sourc.patch b/meta/recipes-multimedia/ffmpeg/ffmpeg/0001-libavutil-include-assembly-with-full-path-from-sourc.patch
new file mode 100644
index 0000000000..3b503c49c9
--- /dev/null
+++ b/meta/recipes-multimedia/ffmpeg/ffmpeg/0001-libavutil-include-assembly-with-full-path-from-sourc.patch
@@ -0,0 +1,97 @@
+From 24a58d70cbb3997e471366bd5afe54be9007bfb1 Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Tue, 10 Nov 2020 15:32:14 +0000
+Subject: [PATCH] libavutil: include assembly with full path from source root
+
+Otherwise nasm writes the full host-specific paths into .o
+output, which breaks binary reproducibility.
+
+Upstream-Status: Pending
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ libavutil/x86/cpuid.asm      | 2 +-
+ libavutil/x86/emms.asm       | 2 +-
+ libavutil/x86/fixed_dsp.asm  | 2 +-
+ libavutil/x86/float_dsp.asm  | 2 +-
+ libavutil/x86/lls.asm        | 2 +-
+ libavutil/x86/pixelutils.asm | 2 +-
+ 6 files changed, 6 insertions(+), 6 deletions(-)
+
+diff --git a/libavutil/x86/cpuid.asm b/libavutil/x86/cpuid.asm
+index c3f7866..766f77f 100644
+--- a/libavutil/x86/cpuid.asm
++++ b/libavutil/x86/cpuid.asm
+@@ -21,7 +21,7 @@
+ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ ;******************************************************************************
+ 
+-%include "x86util.asm"
++%include "libavutil/x86/x86util.asm"
+ 
+ SECTION .text
+ 
+diff --git a/libavutil/x86/emms.asm b/libavutil/x86/emms.asm
+index 8611762..df84f22 100644
+--- a/libavutil/x86/emms.asm
++++ b/libavutil/x86/emms.asm
+@@ -18,7 +18,7 @@
+ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ ;******************************************************************************
+ 
+-%include "x86util.asm"
++%include "libavutil/x86/x86util.asm"
+ 
+ SECTION .text
+ 
+diff --git a/libavutil/x86/fixed_dsp.asm b/libavutil/x86/fixed_dsp.asm
+index 979dd5c..2f41185 100644
+--- a/libavutil/x86/fixed_dsp.asm
++++ b/libavutil/x86/fixed_dsp.asm
+@@ -20,7 +20,7 @@
+ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ ;******************************************************************************
+ 
+-%include "x86util.asm"
++%include "libavutil/x86/x86util.asm"
+ 
+ SECTION .text
+ 
+diff --git a/libavutil/x86/float_dsp.asm b/libavutil/x86/float_dsp.asm
+index 517fd63..b773e61 100644
+--- a/libavutil/x86/float_dsp.asm
++++ b/libavutil/x86/float_dsp.asm
+@@ -20,7 +20,7 @@
+ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ ;******************************************************************************
+ 
+-%include "x86util.asm"
++%include "libavutil/x86/x86util.asm"
+ 
+ SECTION_RODATA 32
+ pd_reverse: dd 7, 6, 5, 4, 3, 2, 1, 0
+diff --git a/libavutil/x86/lls.asm b/libavutil/x86/lls.asm
+index 317fba6..d2526d1 100644
+--- a/libavutil/x86/lls.asm
++++ b/libavutil/x86/lls.asm
+@@ -20,7 +20,7 @@
+ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ ;******************************************************************************
+ 
+-%include "x86util.asm"
++%include "libavutil/x86/x86util.asm"
+ 
+ SECTION .text
+ 
+diff --git a/libavutil/x86/pixelutils.asm b/libavutil/x86/pixelutils.asm
+index 36c57c5..8b45ead 100644
+--- a/libavutil/x86/pixelutils.asm
++++ b/libavutil/x86/pixelutils.asm
+@@ -21,7 +21,7 @@
+ ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ ;******************************************************************************
+ 
+-%include "x86util.asm"
++%include "libavutil/x86/x86util.asm"
+ 
+ SECTION .text
+ 
diff --git a/meta/recipes-multimedia/ffmpeg/ffmpeg_4.3.1.bb b/meta/recipes-multimedia/ffmpeg/ffmpeg_4.3.1.bb
index 72c2fe16ec..97b2d21d31 100644
--- a/meta/recipes-multimedia/ffmpeg/ffmpeg_4.3.1.bb
+++ b/meta/recipes-multimedia/ffmpeg/ffmpeg_4.3.1.bb
@@ -26,6 +26,7 @@ LIC_FILES_CHKSUM = "file://COPYING.GPLv2;md5=b234ee4d69f5fce4486a80fdaf4a4263 \
 SRC_URI = "https://www.ffmpeg.org/releases/${BP}.tar.xz \
            file://mips64_cpu_detection.patch \
            file://0001-lavf-srt-fix-build-fail-when-used-the-libsrt-1.4.1.patch \
+           file://0001-libavutil-include-assembly-with-full-path-from-sourc.patch \
            "
 SRC_URI[sha256sum] = "ad009240d46e307b4e03a213a0f49c11b650e445b1f8be0dda2a9212b34d2ffb"
 
@@ -130,6 +131,11 @@ do_configure() {
     ${S}/configure ${EXTRA_OECONF}
 }
 
+# patch out build host paths for reproducibility
+do_compile_prepend_class-target() {
+        sed -i -e "s,${WORKDIR},,g" ${B}/config.h
+}
+
 PACKAGES =+ "libavcodec \
              libavdevice \
              libavfilter \
-- 
2.29.2

[PATCH 07/11] piglit: fix reproducibility

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 ...file.py-make-test-lists-reproducible.patch | 31 +++++++++++++
 ...gen_tcs-tes_input_tests.py-do-not-ha.patch | 44 +++++++++++++++++++
 ...lizer.py-make-.gz-files-reproducible.patch | 30 +++++++++++++
 ...sort-the-file-list-before-working-on.patch | 28 ++++++++++++
 ...t-shader.c-do-not-hardcode-build-pat.patch | 30 +++++++++++++
 meta/recipes-graphics/piglit/piglit_git.bb    |  6 +++
 6 files changed, 169 insertions(+)
 create mode 100644 meta/recipes-graphics/piglit/piglit/0001-framework-profile.py-make-test-lists-reproducible.patch
 create mode 100644 meta/recipes-graphics/piglit/piglit/0001-generated_tests-gen_tcs-tes_input_tests.py-do-not-ha.patch
 create mode 100644 meta/recipes-graphics/piglit/piglit/0001-serializer.py-make-.gz-files-reproducible.patch
 create mode 100644 meta/recipes-graphics/piglit/piglit/0001-tests-shader.py-sort-the-file-list-before-working-on.patch
 create mode 100644 meta/recipes-graphics/piglit/piglit/0002-tests-util-piglit-shader.c-do-not-hardcode-build-pat.patch

diff --git a/meta/recipes-graphics/piglit/piglit/0001-framework-profile.py-make-test-lists-reproducible.patch b/meta/recipes-graphics/piglit/piglit/0001-framework-profile.py-make-test-lists-reproducible.patch
new file mode 100644
index 0000000000..cc9482c047
--- /dev/null
+++ b/meta/recipes-graphics/piglit/piglit/0001-framework-profile.py-make-test-lists-reproducible.patch
@@ -0,0 +1,31 @@
+From 9086d42df1f3134bafcfe33ff16db7bbb9d9a0fd Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Mon, 30 Nov 2020 23:08:22 +0000
+Subject: [PATCH] framework/profile.py: make test lists reproducible
+
+These are created with os.walk, which yields different
+order depending on where it's run.
+
+Upstream-Status: Pending
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ framework/profile.py | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/framework/profile.py b/framework/profile.py
+index c210e535e..9b5d51d68 100644
+--- a/framework/profile.py
++++ b/framework/profile.py
+@@ -528,7 +528,11 @@ class TestProfile(object):
+                 else:
+                     opts[n] = self.test_list[n]
+         else:
+-            opts = self.test_list  # pylint: disable=redefined-variable-type
++            opts = collections.OrderedDict()
++            test_keys = list(self.test_list.keys())
++            test_keys.sort()
++            for k in test_keys:
++                opts[k] = self.test_list[k]
+ 
+         for k, v in self.filters.run(opts.items()):
+             yield k, v
diff --git a/meta/recipes-graphics/piglit/piglit/0001-generated_tests-gen_tcs-tes_input_tests.py-do-not-ha.patch b/meta/recipes-graphics/piglit/piglit/0001-generated_tests-gen_tcs-tes_input_tests.py-do-not-ha.patch
new file mode 100644
index 0000000000..8704f98500
--- /dev/null
+++ b/meta/recipes-graphics/piglit/piglit/0001-generated_tests-gen_tcs-tes_input_tests.py-do-not-ha.patch
@@ -0,0 +1,44 @@
+From 1b23539aece156f6fe0789cb988f22e5915228f6 Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Tue, 10 Nov 2020 17:12:32 +0000
+Subject: [PATCH 1/2] generated_tests/gen_tcs/tes_input_tests.py: do not
+ hardcode the full binary path
+
+This helps reproducibility.
+
+Upstream-Status: Pending
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ generated_tests/gen_tcs_input_tests.py | 2 +-
+ generated_tests/gen_tes_input_tests.py | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/generated_tests/gen_tcs_input_tests.py b/generated_tests/gen_tcs_input_tests.py
+index face4f19a..e36671af4 100644
+--- a/generated_tests/gen_tcs_input_tests.py
++++ b/generated_tests/gen_tcs_input_tests.py
+@@ -272,7 +272,7 @@ class Test(object):
+             relative probe rgb (0.75, 0.75) (0.0, 1.0, 0.0)
+             """)
+ 
+-        test = test.format(self=self, generator_command=" ".join(sys.argv))
++        test = test.format(self=self, generator_command="generated_tests/gen_tcs_input_tests.py")
+ 
+         filename = self.filename()
+         dirname = os.path.dirname(filename)
+diff --git a/generated_tests/gen_tes_input_tests.py b/generated_tests/gen_tes_input_tests.py
+index 3d847b5cc..954840b20 100644
+--- a/generated_tests/gen_tes_input_tests.py
++++ b/generated_tests/gen_tes_input_tests.py
+@@ -301,7 +301,7 @@ class Test(object):
+             relative probe rgb (0.75, 0.75) (0.0, 1.0, 0.0)
+             """)
+ 
+-        test = test.format(self=self, generator_command=" ".join(sys.argv))
++        test = test.format(self=self, generator_command="generated_tests/gen_tes_input_tests.py")
+ 
+         filename = self.filename()
+         dirname = os.path.dirname(filename)
+-- 
+2.17.1
+
diff --git a/meta/recipes-graphics/piglit/piglit/0001-serializer.py-make-.gz-files-reproducible.patch b/meta/recipes-graphics/piglit/piglit/0001-serializer.py-make-.gz-files-reproducible.patch
new file mode 100644
index 0000000000..2efba6f866
--- /dev/null
+++ b/meta/recipes-graphics/piglit/piglit/0001-serializer.py-make-.gz-files-reproducible.patch
@@ -0,0 +1,30 @@
+From 1919bb7f4072d73dcbb64d0e06eff5b04529c3db Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Mon, 16 Nov 2020 18:01:02 +0000
+Subject: [PATCH] serializer.py: make .gz files reproducible
+
+.gz format contains mtime of the compressed data, and
+SOURCE_DATE_EPOCH is the standard way to make it reproducuble.
+
+Upstream-Status: Pending
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ tests/serializer.py | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/tests/serializer.py b/tests/serializer.py
+index bd14bc3db..bc5b45d7f 100644
+--- a/tests/serializer.py
++++ b/tests/serializer.py
+@@ -138,7 +138,10 @@ def serializer(name, profile, outfile):
+                 et.SubElement(env, 'env', name=k, value=v)
+ 
+     tree = et.ElementTree(root)
+-    with gzip.open(outfile, 'wb') as f:
++    reproducible_mtime = None
++    if 'SOURCE_DATE_EPOCH' in os.environ:
++        reproducible_mtime=os.environ['SOURCE_DATE_EPOCH']
++    with gzip.GzipFile(outfile, 'wb', mtime=reproducible_mtime) as f:
+         tree.write(f, encoding='utf-8', xml_declaration=True)
+ 
+ 
diff --git a/meta/recipes-graphics/piglit/piglit/0001-tests-shader.py-sort-the-file-list-before-working-on.patch b/meta/recipes-graphics/piglit/piglit/0001-tests-shader.py-sort-the-file-list-before-working-on.patch
new file mode 100644
index 0000000000..8321be8490
--- /dev/null
+++ b/meta/recipes-graphics/piglit/piglit/0001-tests-shader.py-sort-the-file-list-before-working-on.patch
@@ -0,0 +1,28 @@
+From 5bf89c6a314952313b2b762fff0d5501fe57ac53 Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Wed, 2 Dec 2020 21:21:52 +0000
+Subject: [PATCH] tests/shader.py: sort the file list before working on it
+
+This allows later xml output to be reproducible.
+
+Upstream-Status: Pending
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ tests/shader.py | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/tests/shader.py b/tests/shader.py
+index 849273660..e6e65d1ba 100644
+--- a/tests/shader.py
++++ b/tests/shader.py
+@@ -52,7 +52,9 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
+ for group, files in shader_tests.items():
+     assert group not in profile.test_list, 'duplicate group: {}'.format(group)
+ 
+-    # We'll end up with a list of tuples, split that into two lists
++    # This makes the xml output reproducible, as os.walk() order is random
++    files.sort()
++    # We'll end up with a list of tuples, split that into two list
+     files, installedfiles = list(zip(*files))
+     files = list(files)
+     installedfiles = list(installedfiles)
diff --git a/meta/recipes-graphics/piglit/piglit/0002-tests-util-piglit-shader.c-do-not-hardcode-build-pat.patch b/meta/recipes-graphics/piglit/piglit/0002-tests-util-piglit-shader.c-do-not-hardcode-build-pat.patch
new file mode 100644
index 0000000000..16c7c5c803
--- /dev/null
+++ b/meta/recipes-graphics/piglit/piglit/0002-tests-util-piglit-shader.c-do-not-hardcode-build-pat.patch
@@ -0,0 +1,30 @@
+From 1c67250308a92d4991ed05d9d240090ab84accae Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Tue, 10 Nov 2020 17:13:50 +0000
+Subject: [PATCH 2/2] tests/util/piglit-shader.c: do not hardcode build path
+ into target binary
+
+This helps reproducibilty.
+
+Upstream-Status: Inappropriate [oe-core specific]
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ tests/util/piglit-shader.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/tests/util/piglit-shader.c b/tests/util/piglit-shader.c
+index 4fd68d21e..c9ea8295e 100644
+--- a/tests/util/piglit-shader.c
++++ b/tests/util/piglit-shader.c
+@@ -73,7 +73,7 @@ piglit_compile_shader(GLenum target, const char *filename)
+ 
+ 	source_dir = getenv("PIGLIT_SOURCE_DIR");
+ 	if (source_dir == NULL) {
+-		source_dir = SOURCE_DIR;
++		source_dir = ".";
+ 	}
+ 
+ 	snprintf(filename_with_path, FILENAME_MAX - 1,
+-- 
+2.17.1
+
diff --git a/meta/recipes-graphics/piglit/piglit_git.bb b/meta/recipes-graphics/piglit/piglit_git.bb
index a9d1d39dfe..a441699800 100644
--- a/meta/recipes-graphics/piglit/piglit_git.bb
+++ b/meta/recipes-graphics/piglit/piglit_git.bb
@@ -8,6 +8,11 @@ SRC_URI = "git://gitlab.freedesktop.org/mesa/piglit.git;protocol=https \
            file://0001-cmake-install-bash-completions-in-the-right-place.patch \
            file://0001-cmake-use-proper-WAYLAND_INCLUDE_DIRS-variable.patch \
            file://0001-Add-a-missing-include-for-htobe32-definition.patch \
+           file://0001-generated_tests-gen_tcs-tes_input_tests.py-do-not-ha.patch \
+           file://0002-tests-util-piglit-shader.c-do-not-hardcode-build-pat.patch \
+           file://0001-serializer.py-make-.gz-files-reproducible.patch \
+           file://0001-framework-profile.py-make-test-lists-reproducible.patch \
+           file://0001-tests-shader.py-sort-the-file-list-before-working-on.patch \
            "
 UPSTREAM_CHECK_COMMITS = "1"
 
@@ -37,6 +42,7 @@ PACKAGECONFIG ??= "${@bb.utils.filter('DISTRO_FEATURES', 'x11', d)}"
 PACKAGECONFIG[freeglut] = "-DPIGLIT_USE_GLUT=1,-DPIGLIT_USE_GLUT=0,freeglut,"
 PACKAGECONFIG[x11] = "-DPIGLIT_BUILD_GL_TESTS=ON,-DPIGLIT_BUILD_GL_TESTS=OFF,${X11_DEPS}, ${X11_RDEPS}"
 
+export PIGLIT_BUILD_DIR = "../../../../git"
 
 do_configure_prepend() {
    if [ "${@bb.utils.contains('PACKAGECONFIG', 'freeglut', 'yes', 'no', d)}" = "no" ]; then
-- 
2.29.2

[PATCH 08/11] serf: do not install the static library

[Alexander Kanavin]

scons is using host ar to create it, which may or may not
be built with reproducible option by default. Rather than
patch scons for the benefit of a single recipe, let's
just not install the .a, which is unused anyway.

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 meta/recipes-support/serf/serf_1.3.9.bb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/meta/recipes-support/serf/serf_1.3.9.bb b/meta/recipes-support/serf/serf_1.3.9.bb
index 6a27f12102..2fbf96f997 100644
--- a/meta/recipes-support/serf/serf_1.3.9.bb
+++ b/meta/recipes-support/serf/serf_1.3.9.bb
@@ -30,4 +30,9 @@ EXTRA_OESCONS = " \
                   OPENSSL="${STAGING_EXECPREFIXDIR}" \
                   "
 
+# scons creates non-reproducible archives
+do_install_append() {
+	rm ${D}/${libdir}/*.a
+}
+
 BBCLASSEXTEND = "native nativesdk"
-- 
2.29.2

Re: [OE-core] [PATCH 08/11] serf: do not install the static library

[Khem Raj]

On Thu, Dec 3, 2020 at 5:37 AM Alexander Kanavin <alex.kanavin@gmail.com> wrote:
>
> scons is using host ar to create it, which may or may not
> be built with reproducible option by default. Rather than
> patch scons for the benefit of a single recipe, let's
> just not install the .a, which is unused anyway.
>

honestly, scons should be fixed here. Since it will be used in more
than one places
using host ar is never a good option for cross builds.
if you dare reluctant to fix scons properly that's fine but at least
create a tracker bug for it.

> Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
> ---
>  meta/recipes-support/serf/serf_1.3.9.bb | 5 +++++
>  1 file changed, 5 insertions(+)
>
> diff --git a/meta/recipes-support/serf/serf_1.3.9.bb b/meta/recipes-support/serf/serf_1.3.9.bb
> index 6a27f12102..2fbf96f997 100644
> --- a/meta/recipes-support/serf/serf_1.3.9.bb
> +++ b/meta/recipes-support/serf/serf_1.3.9.bb
> @@ -30,4 +30,9 @@ EXTRA_OESCONS = " \
>                    OPENSSL="${STAGING_EXECPREFIXDIR}" \
>                    "
>
> +# scons creates non-reproducible archives
> +do_install_append() {
> +       rm ${D}/${libdir}/*.a
> +}
> +
>  BBCLASSEXTEND = "native nativesdk"
> --
> 2.29.2
>
>
> 
>

Re: [OE-core] [PATCH 08/11] serf: do not install the static library

[Alexander Kanavin]

[-- Attachment #1: Type: text/plain, Size: 396 bytes --]

On Thu, 3 Dec 2020 at 20:30, Khem Raj <raj.khem@gmail.com> wrote:

>
> honestly, scons should be fixed here. Since it will be used in more
> than one places
> using host ar is never a good option for cross builds.
> if you dare reluctant to fix scons properly that's fine but at least
> create a tracker bug for it.
>

There you go:
https://github.com/SCons/scons/issues/3838

Alex

[-- Attachment #2: Type: text/html, Size: 777 bytes --]

[PATCH 09/11] llvm: sort the lists in generated source reproducibibly

[Alexander Kanavin]

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 ...r-sort-ClassInfo-lists-by-name-as-we.patch | 31 +++++++++++++++++++
 meta/recipes-devtools/llvm/llvm_git.bb        |  3 +-
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 meta/recipes-devtools/llvm/llvm/0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch

diff --git a/meta/recipes-devtools/llvm/llvm/0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch b/meta/recipes-devtools/llvm/llvm/0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch
new file mode 100644
index 0000000000..20eea060b1
--- /dev/null
+++ b/meta/recipes-devtools/llvm/llvm/0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch
@@ -0,0 +1,31 @@
+From 86940d87026432683fb6741cd8a34d3b9b18e40d Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Fri, 27 Nov 2020 10:11:08 +0000
+Subject: [PATCH] AsmMatcherEmitter: sort ClassInfo lists by name as well
+
+Otherwise, there are instances which are identical in
+every other field and therefore sort non-reproducibly
+(which breaks binary and source reproducibiliy).
+
+Upstream-Status: Pending
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ llvm/utils/TableGen/AsmMatcherEmitter.cpp | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+index ccf0959389b..1f801e83b7d 100644
+--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
++++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+@@ -359,7 +359,10 @@ public:
+     // name of a class shouldn't be significant. However, some of the backends
+     // accidentally rely on this behaviour, so it will have to stay like this
+     // until they are fixed.
+-    return ValueName < RHS.ValueName;
++    if (ValueName != RHS.ValueName)
++        return ValueName < RHS.ValueName;
++    // All else being equal, we should sort by name, for source and binary reproducibility
++    return Name < RHS.Name;
+   }
+ };
+ 
diff --git a/meta/recipes-devtools/llvm/llvm_git.bb b/meta/recipes-devtools/llvm/llvm_git.bb
index b146d0e6e3..43395f8cfc 100644
--- a/meta/recipes-devtools/llvm/llvm_git.bb
+++ b/meta/recipes-devtools/llvm/llvm_git.bb
@@ -33,7 +33,8 @@ SRCREV = "ef32c611aa214dea855364efd7ba451ec5ec3f74"
 SRC_URI = "git://github.com/llvm/llvm-project.git;branch=${BRANCH} \
            file://0006-llvm-TargetLibraryInfo-Undefine-libc-functions-if-th.patch;striplevel=2 \
            file://0007-llvm-allow-env-override-of-exe-path.patch;striplevel=2 \
-          "
+           file://0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch;striplevel=2 \
+           "
 
 UPSTREAM_CHECK_GITTAGREGEX = "llvmorg-(?P<pver>\d+(\.\d+)+)"
 
-- 
2.29.2

Re: [OE-core] [PATCH 09/11] llvm: sort the lists in generated source reproducibibly

[Khem Raj]

On Thu, Dec 3, 2020 at 5:37 AM Alexander Kanavin <alex.kanavin@gmail.com> wrote:
>
> Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
> ---
>  ...r-sort-ClassInfo-lists-by-name-as-we.patch | 31 +++++++++++++++++++
>  meta/recipes-devtools/llvm/llvm_git.bb        |  3 +-
>  2 files changed, 33 insertions(+), 1 deletion(-)
>  create mode 100644 meta/recipes-devtools/llvm/llvm/0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch
>
> diff --git a/meta/recipes-devtools/llvm/llvm/0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch b/meta/recipes-devtools/llvm/llvm/0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch
> new file mode 100644
> index 0000000000..20eea060b1
> --- /dev/null
> +++ b/meta/recipes-devtools/llvm/llvm/0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch
> @@ -0,0 +1,31 @@
> +From 86940d87026432683fb6741cd8a34d3b9b18e40d Mon Sep 17 00:00:00 2001
> +From: Alexander Kanavin <alex.kanavin@gmail.com>
> +Date: Fri, 27 Nov 2020 10:11:08 +0000
> +Subject: [PATCH] AsmMatcherEmitter: sort ClassInfo lists by name as well
> +
> +Otherwise, there are instances which are identical in
> +every other field and therefore sort non-reproducibly
> +(which breaks binary and source reproducibiliy).
> +
> +Upstream-Status: Pending
> +Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
> +---
> + llvm/utils/TableGen/AsmMatcherEmitter.cpp | 5 ++++-
> + 1 file changed, 4 insertions(+), 1 deletion(-)
> +
> +diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
> +index ccf0959389b..1f801e83b7d 100644
> +--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
> ++++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
> +@@ -359,7 +359,10 @@ public:
> +     // name of a class shouldn't be significant. However, some of the backends
> +     // accidentally rely on this behaviour, so it will have to stay like this
> +     // until they are fixed.
> +-    return ValueName < RHS.ValueName;
> ++    if (ValueName != RHS.ValueName)
> ++        return ValueName < RHS.ValueName;
> ++    // All else being equal, we should sort by name, for source and binary reproducibility
> ++    return Name < RHS.Name;

this change looks good to me. it would be good to post it upstream as
well via phabricator to get additional reviews from LLVM community as
well.

> +   }
> + };
> +
> diff --git a/meta/recipes-devtools/llvm/llvm_git.bb b/meta/recipes-devtools/llvm/llvm_git.bb
> index b146d0e6e3..43395f8cfc 100644
> --- a/meta/recipes-devtools/llvm/llvm_git.bb
> +++ b/meta/recipes-devtools/llvm/llvm_git.bb
> @@ -33,7 +33,8 @@ SRCREV = "ef32c611aa214dea855364efd7ba451ec5ec3f74"
>  SRC_URI = "git://github.com/llvm/llvm-project.git;branch=${BRANCH} \
>             file://0006-llvm-TargetLibraryInfo-Undefine-libc-functions-if-th.patch;striplevel=2 \
>             file://0007-llvm-allow-env-override-of-exe-path.patch;striplevel=2 \
> -          "
> +           file://0001-AsmMatcherEmitter-sort-ClassInfo-lists-by-name-as-we.patch;striplevel=2 \
> +           "
>
>  UPSTREAM_CHECK_GITTAGREGEX = "llvmorg-(?P<pver>\d+(\.\d+)+)"
>
> --
> 2.29.2
>
>
> 
>

[PATCH 10/11] selftest/reproducible: enable world reproducibility test

[Alexander Kanavin]

Add systemd and pam distro features, and commercial license flag
to include more recipes into the world set.

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 meta/lib/oeqa/selftest/cases/reproducible.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
index a7ef336143..6faeedb544 100644
--- a/meta/lib/oeqa/selftest/cases/reproducible.py
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -105,7 +105,7 @@ class DiffoscopeTests(OESelftestTestCase):
 
 class ReproducibleTests(OESelftestTestCase):
     package_classes = ['deb', 'ipk']
-    images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline']
+    images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'world']
     save_results = False
     if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ:
         save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT']
@@ -176,6 +176,8 @@ class ReproducibleTests(OESelftestTestCase):
             PACKAGE_CLASSES = "{package_classes}"
             INHIBIT_PACKAGE_STRIP = "1"
             TMPDIR = "{tmpdir}"
+            LICENSE_FLAGS_WHITELIST = "commercial"
+            DISTRO_FEATURES_append = ' systemd pam'
             ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes),
                         tmpdir=tmpdir)
 
-- 
2.29.2

[PATCH 11/11] selftest/reproducible: add an exclusion list for items that are not yet reproducible

[Alexander Kanavin]

Hopefully over time this list will be reduced to an empty one.

Non-reproducible excluded packages are not given to diffoscope and do not cause a
failure, but still saved side-by-side with non-reproducible failing ones to make
investigation easier.

Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
---
 meta/lib/oeqa/selftest/cases/reproducible.py | 82 +++++++++++++++++++-
 1 file changed, 79 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
index 6faeedb544..e3597d5081 100644
--- a/meta/lib/oeqa/selftest/cases/reproducible.py
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -17,6 +17,72 @@ import stat
 import os
 import datetime
 
+# For sample packages, see:
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/
+# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/
+# (both packages/ and packages-excluded/)
+exclude_packages = [
+	'acpica-src',
+	'babeltrace2-ptest',
+	'bootchart2-doc',
+	'cups',
+	'cwautomacros',
+	'dtc',
+	'efivar',
+	'epiphany',
+	'gcr',
+	'git',
+	'glide',
+	'go-dep',
+	'go-helloworld',
+	'go-runtime',
+	'go_',
+	'groff',
+	'gst-devtools',
+	'gstreamer1.0-python',
+	'gtk-doc',
+	'igt-gpu-tools',
+        'kernel-devsrc',
+	'libaprutil',
+	'libcap-ng',
+	'libhandy-1-src',
+	'libid3tag',
+	'libproxy',
+	'libsecret-dev',
+	'libsecret-src',
+	'lttng-tools-dbg',
+	'lttng-tools-ptest',
+	'ltp',
+	'meson',
+	'ovmf-shell-efi',
+	'parted-ptest',
+	'perf',
+	'python3-cython',
+	'qemu',
+	'quilt-ptest',
+	'rsync',
+	'ruby',
+	'spirv-tools-dev',
+	'swig',
+	'syslinux-misc',
+	'systemd-bootchart',
+	'valgrind-ptest',
+	'vim',
+	'watchdog',
+	'xmlto',
+	'xorg-minimal-fonts'
+	]
+
+def is_excluded(package):
+    package_name = os.path.basename(package)
+    for i in exclude_packages:
+        if package_name.startswith(i):
+            return True
+    return False
+
 MISSING = 'MISSING'
 DIFFERENT = 'DIFFERENT'
 SAME = 'SAME'
@@ -39,6 +105,7 @@ class PackageCompareResults(object):
         self.total = []
         self.missing = []
         self.different = []
+        self.different_excluded = []
         self.same = []
 
     def add_result(self, r):
@@ -46,7 +113,10 @@ class PackageCompareResults(object):
         if r.status == MISSING:
             self.missing.append(r)
         elif r.status == DIFFERENT:
-            self.different.append(r)
+            if is_excluded(r.reference):
+                self.different_excluded.append(r)
+            else:
+                self.different.append(r)
         else:
             self.same.append(r)
 
@@ -54,10 +124,11 @@ class PackageCompareResults(object):
         self.total.sort()
         self.missing.sort()
         self.different.sort()
+        self.different_excluded.sort()
         self.same.sort()
 
     def __str__(self):
-        return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total))
+        return 'same=%i different=%i different_excluded=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total))
 
 def compare_file(reference, test, diffutils_sysroot):
     result = CompareResult()
@@ -237,6 +308,7 @@ class ReproducibleTests(OESelftestTestCase):
 
                 self.write_package_list(package_class, 'missing', result.missing)
                 self.write_package_list(package_class, 'different', result.different)
+                self.write_package_list(package_class, 'different_excluded', result.different_excluded)
                 self.write_package_list(package_class, 'same', result.same)
 
                 if self.save_results:
@@ -244,8 +316,12 @@ class ReproducibleTests(OESelftestTestCase):
                         self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)]))
                         self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)]))
 
+                    for d in result.different_excluded:
+                        self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)]))
+                        self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)]))
+
                 if result.missing or result.different:
-                    fails.append("The following %s packages are missing or different: %s" %
+                    fails.append("The following %s packages are missing or different and not in exclusion list: %s" %
                             (c, '\n'.join(r.test for r in (result.missing + result.different))))
 
         # Clean up empty directories
-- 
2.29.2