* [Buildroot] Build reproducibility @ 2013-08-30 8:31 Jérôme Pouiller 2013-08-30 8:31 ` [Buildroot] [PATCH] Fix build reproducibility in Make 3.82 Jérôme Pouiller 2013-08-30 11:59 ` [Buildroot] Build reproducibility Thomas De Schampheleire 0 siblings, 2 replies; 22+ messages in thread From: Jérôme Pouiller @ 2013-08-30 8:31 UTC (permalink / raw) To: buildroot Hello, I noticed a problem with reproducibility of build. If I list targets (using make show-targets) on Fedora and on Ubuntu, targets are on sorted on same orders. Indeed, result of $(wildcard *.mk) are not the same on these systems. It seems Make does not sort anymore result of wildcard since 3.82: http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260. Following patch should fix problem. ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] [PATCH] Fix build reproducibility in Make 3.82 2013-08-30 8:31 [Buildroot] Build reproducibility Jérôme Pouiller @ 2013-08-30 8:31 ` Jérôme Pouiller 2013-09-03 6:13 ` Arnout Vandecappelle 2013-08-30 11:59 ` [Buildroot] Build reproducibility Thomas De Schampheleire 1 sibling, 1 reply; 22+ messages in thread From: Jérôme Pouiller @ 2013-08-30 8:31 UTC (permalink / raw) To: buildroot Make 3.82 do not sort anymore result of wildcards (see http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break build reproducibility. This patch sort results of wildcards to ensure reproducibility. Signed-off-by: J?r?me Pouiller <jezz@sysmic.org> --- Makefile | 2 +- boot/common.mk | 2 +- docs/manual/manual.mk | 2 +- fs/common.mk | 2 +- linux/linux.mk | 2 +- package/efl/efl.mk | 2 +- package/freescale-imx/freescale-imx.mk | 2 +- package/gcc/gcc.mk | 2 +- package/gtk2-themes/gtk2-themes.mk | 2 +- package/matchbox/matchbox.mk | 2 +- package/opengl/opengl.mk | 2 +- package/qt5/qt5.mk | 2 +- package/x11r7/x11r7.mk | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 84d160a..5d6315a 100644 --- a/Makefile +++ b/Makefile @@ -337,7 +337,7 @@ ifneq ($(PACKAGE_OVERRIDE_FILE),) -include $(PACKAGE_OVERRIDE_FILE) endif -include package/*/*.mk +include $(sort $(wildcard package/*/*.mk)) include boot/common.mk include linux/linux.mk diff --git a/boot/common.mk b/boot/common.mk index b315fe8..3021e51 100644 --- a/boot/common.mk +++ b/boot/common.mk @@ -1 +1 @@ -include boot/*/*.mk +include $(sort $(wildcard boot/*/*.mk)) diff --git a/docs/manual/manual.mk b/docs/manual/manual.mk index 4906bc8..bcbedad 100644 --- a/docs/manual/manual.mk +++ b/docs/manual/manual.mk @@ -53,5 +53,5 @@ $(1)-clean: .PHONY: $(1) $(1)-clean manual-update-lists endef -MANUAL_SOURCES = $(wildcard docs/manual/*.txt) $(wildcard docs/images/*) +MANUAL_SOURCES = $(sort $(wildcard docs/manual/*.txt) $(wildcard docs/images/*)) $(eval $(call GENDOC,manual)) diff --git a/fs/common.mk b/fs/common.mk index 36da2dd..4dab7ea 100644 --- a/fs/common.mk +++ b/fs/common.mk @@ -94,4 +94,4 @@ define ROOTFS_TARGET $(call ROOTFS_TARGET_INTERNAL,$(1),$(call UPPERCASE,$(1))) endef -include fs/*/*.mk +include $(sort $(wildcard fs/*/*.mk)) diff --git a/linux/linux.mk b/linux/linux.mk index 578d535..b9d7028 100644 --- a/linux/linux.mk +++ b/linux/linux.mk @@ -281,7 +281,7 @@ define LINUX_INSTALL_TARGET_CMDS $(LINUX_INSTALL_HOST_TOOLS) endef -include linux/linux-ext-*.mk +include $(sort $(wildcard linux/linux-ext-*.mk)) $(eval $(generic-package)) diff --git a/package/efl/efl.mk b/package/efl/efl.mk index f284c93..1de68cf 100644 --- a/package/efl/efl.mk +++ b/package/efl/efl.mk @@ -1,3 +1,3 @@ EFL_VERSION = 1.7.7 -include package/efl/*/*.mk +include $(sort $(wildcard package/efl/*/*.mk)) diff --git a/package/freescale-imx/freescale-imx.mk b/package/freescale-imx/freescale-imx.mk index 9f63ce3..c8f99b6 100644 --- a/package/freescale-imx/freescale-imx.mk +++ b/package/freescale-imx/freescale-imx.mk @@ -9,5 +9,5 @@ FREESCALE_IMX_VERSION = 1.1.0 # No official download site from freescale, just this mirror FREESCALE_IMX_MIRROR_SITE = http://download.ossystems.com.br/bsp/freescale/source -include package/freescale-imx/*/*.mk +include $(sort $(wildcard package/freescale-imx/*/*.mk)) diff --git a/package/gcc/gcc.mk b/package/gcc/gcc.mk index cd6efed..66af425 100644 --- a/package/gcc/gcc.mk +++ b/package/gcc/gcc.mk @@ -215,4 +215,4 @@ HOST_GCC_COMMON_CONF_OPT += \ --with-long-double-128 endif -include package/gcc/*/*.mk +include $(sort $(wildcard package/gcc/*/*.mk)) diff --git a/package/gtk2-themes/gtk2-themes.mk b/package/gtk2-themes/gtk2-themes.mk index 6f31c6d..fc66698 100644 --- a/package/gtk2-themes/gtk2-themes.mk +++ b/package/gtk2-themes/gtk2-themes.mk @@ -1 +1 @@ -include package/gtk2-themes/*/*.mk +include $(sort $(wildcard package/gtk2-themes/*/*.mk)) diff --git a/package/matchbox/matchbox.mk b/package/matchbox/matchbox.mk index abda915..fff4add 100644 --- a/package/matchbox/matchbox.mk +++ b/package/matchbox/matchbox.mk @@ -1,4 +1,4 @@ ifeq ($(BR2_PACKAGE_MATCHBOX),y) -include package/matchbox/*/*.mk +include $(sort $(wildcard package/matchbox/*/*.mk)) TARGETS+=matchbox-lib matchbox-wm endif diff --git a/package/opengl/opengl.mk b/package/opengl/opengl.mk index 68df3b1..abf96d5 100644 --- a/package/opengl/opengl.mk +++ b/package/opengl/opengl.mk @@ -1 +1 @@ -include package/opengl/*/*.mk +include $(sort $(wildcard package/opengl/*/*.mk)) diff --git a/package/qt5/qt5.mk b/package/qt5/qt5.mk index 9e71c83..fce0ca3 100644 --- a/package/qt5/qt5.mk +++ b/package/qt5/qt5.mk @@ -1,6 +1,6 @@ QT5_VERSION = 5.0.2 QT5_SITE = http://download.qt-project.org/archive/qt/5.0/$(QT5_VERSION)/submodules/ -include package/qt5/*/*.mk +include $(sort $(wildcard package/qt5/*/*.mk)) define QT5_LA_PRL_FILES_FIXUP for i in $$(find $(STAGING_DIR)/usr/lib* -name "libQt5*.la"); do \ diff --git a/package/x11r7/x11r7.mk b/package/x11r7/x11r7.mk index ea0363f..85ecbc5 100644 --- a/package/x11r7/x11r7.mk +++ b/package/x11r7/x11r7.mk @@ -1 +1 @@ -include package/x11r7/*/*.mk +include $(sort $(wildcard package/x11r7/*/*.mk)) -- 1.7.9.5 ^ permalink raw reply related [flat|nested] 22+ messages in thread
* [Buildroot] [PATCH] Fix build reproducibility in Make 3.82 2013-08-30 8:31 ` [Buildroot] [PATCH] Fix build reproducibility in Make 3.82 Jérôme Pouiller @ 2013-09-03 6:13 ` Arnout Vandecappelle 2013-09-03 8:45 ` [Buildroot] [PATCH v2] " Jérôme Pouiller 0 siblings, 1 reply; 22+ messages in thread From: Arnout Vandecappelle @ 2013-09-03 6:13 UTC (permalink / raw) To: buildroot On 08/30/13 10:31, J?r?me Pouiller wrote: > Make 3.82 do not sort anymore result of wildcards (see > http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break > build reproducibility. > > This patch sort results of wildcards to ensure reproducibility. > > Signed-off-by: J?r?me Pouiller <jezz@sysmic.org> Acked-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be> You missed one in support/dependencies/dependencies.mk, but that one doesn't make a difference for the build result. Also the one of the manual isn't really necessary; it is only used to check for rebuilding the manual and doesn't affect the way the manual is built. But both of these are not really important differences, hence my ack. Regards, Arnout > --- > Makefile | 2 +- > boot/common.mk | 2 +- > docs/manual/manual.mk | 2 +- > fs/common.mk | 2 +- > linux/linux.mk | 2 +- > package/efl/efl.mk | 2 +- > package/freescale-imx/freescale-imx.mk | 2 +- > package/gcc/gcc.mk | 2 +- > package/gtk2-themes/gtk2-themes.mk | 2 +- > package/matchbox/matchbox.mk | 2 +- > package/opengl/opengl.mk | 2 +- > package/qt5/qt5.mk | 2 +- > package/x11r7/x11r7.mk | 2 +- > 13 files changed, 13 insertions(+), 13 deletions(-) > > diff --git a/Makefile b/Makefile > index 84d160a..5d6315a 100644 > --- a/Makefile > +++ b/Makefile > @@ -337,7 +337,7 @@ ifneq ($(PACKAGE_OVERRIDE_FILE),) > -include $(PACKAGE_OVERRIDE_FILE) > endif > > -include package/*/*.mk > +include $(sort $(wildcard package/*/*.mk)) > > include boot/common.mk > include linux/linux.mk > diff --git a/boot/common.mk b/boot/common.mk > index b315fe8..3021e51 100644 > --- a/boot/common.mk > +++ b/boot/common.mk > @@ -1 +1 @@ > -include boot/*/*.mk > +include $(sort $(wildcard boot/*/*.mk)) > diff --git a/docs/manual/manual.mk b/docs/manual/manual.mk > index 4906bc8..bcbedad 100644 > --- a/docs/manual/manual.mk > +++ b/docs/manual/manual.mk > @@ -53,5 +53,5 @@ $(1)-clean: > .PHONY: $(1) $(1)-clean manual-update-lists > endef > > -MANUAL_SOURCES = $(wildcard docs/manual/*.txt) $(wildcard docs/images/*) > +MANUAL_SOURCES = $(sort $(wildcard docs/manual/*.txt) $(wildcard docs/images/*)) > $(eval $(call GENDOC,manual)) > diff --git a/fs/common.mk b/fs/common.mk > index 36da2dd..4dab7ea 100644 > --- a/fs/common.mk > +++ b/fs/common.mk > @@ -94,4 +94,4 @@ define ROOTFS_TARGET > $(call ROOTFS_TARGET_INTERNAL,$(1),$(call UPPERCASE,$(1))) > endef > > -include fs/*/*.mk > +include $(sort $(wildcard fs/*/*.mk)) > diff --git a/linux/linux.mk b/linux/linux.mk > index 578d535..b9d7028 100644 > --- a/linux/linux.mk > +++ b/linux/linux.mk > @@ -281,7 +281,7 @@ define LINUX_INSTALL_TARGET_CMDS > $(LINUX_INSTALL_HOST_TOOLS) > endef > > -include linux/linux-ext-*.mk > +include $(sort $(wildcard linux/linux-ext-*.mk)) > > $(eval $(generic-package)) > > diff --git a/package/efl/efl.mk b/package/efl/efl.mk > index f284c93..1de68cf 100644 > --- a/package/efl/efl.mk > +++ b/package/efl/efl.mk > @@ -1,3 +1,3 @@ > EFL_VERSION = 1.7.7 > > -include package/efl/*/*.mk > +include $(sort $(wildcard package/efl/*/*.mk)) > diff --git a/package/freescale-imx/freescale-imx.mk b/package/freescale-imx/freescale-imx.mk > index 9f63ce3..c8f99b6 100644 > --- a/package/freescale-imx/freescale-imx.mk > +++ b/package/freescale-imx/freescale-imx.mk > @@ -9,5 +9,5 @@ FREESCALE_IMX_VERSION = 1.1.0 > # No official download site from freescale, just this mirror > FREESCALE_IMX_MIRROR_SITE = http://download.ossystems.com.br/bsp/freescale/source > > -include package/freescale-imx/*/*.mk > +include $(sort $(wildcard package/freescale-imx/*/*.mk)) > > diff --git a/package/gcc/gcc.mk b/package/gcc/gcc.mk > index cd6efed..66af425 100644 > --- a/package/gcc/gcc.mk > +++ b/package/gcc/gcc.mk > @@ -215,4 +215,4 @@ HOST_GCC_COMMON_CONF_OPT += \ > --with-long-double-128 > endif > > -include package/gcc/*/*.mk > +include $(sort $(wildcard package/gcc/*/*.mk)) > diff --git a/package/gtk2-themes/gtk2-themes.mk b/package/gtk2-themes/gtk2-themes.mk > index 6f31c6d..fc66698 100644 > --- a/package/gtk2-themes/gtk2-themes.mk > +++ b/package/gtk2-themes/gtk2-themes.mk > @@ -1 +1 @@ > -include package/gtk2-themes/*/*.mk > +include $(sort $(wildcard package/gtk2-themes/*/*.mk)) > diff --git a/package/matchbox/matchbox.mk b/package/matchbox/matchbox.mk > index abda915..fff4add 100644 > --- a/package/matchbox/matchbox.mk > +++ b/package/matchbox/matchbox.mk > @@ -1,4 +1,4 @@ > ifeq ($(BR2_PACKAGE_MATCHBOX),y) > -include package/matchbox/*/*.mk > +include $(sort $(wildcard package/matchbox/*/*.mk)) > TARGETS+=matchbox-lib matchbox-wm > endif > diff --git a/package/opengl/opengl.mk b/package/opengl/opengl.mk > index 68df3b1..abf96d5 100644 > --- a/package/opengl/opengl.mk > +++ b/package/opengl/opengl.mk > @@ -1 +1 @@ > -include package/opengl/*/*.mk > +include $(sort $(wildcard package/opengl/*/*.mk)) > diff --git a/package/qt5/qt5.mk b/package/qt5/qt5.mk > index 9e71c83..fce0ca3 100644 > --- a/package/qt5/qt5.mk > +++ b/package/qt5/qt5.mk > @@ -1,6 +1,6 @@ > QT5_VERSION = 5.0.2 > QT5_SITE = http://download.qt-project.org/archive/qt/5.0/$(QT5_VERSION)/submodules/ > -include package/qt5/*/*.mk > +include $(sort $(wildcard package/qt5/*/*.mk)) > > define QT5_LA_PRL_FILES_FIXUP > for i in $$(find $(STAGING_DIR)/usr/lib* -name "libQt5*.la"); do \ > diff --git a/package/x11r7/x11r7.mk b/package/x11r7/x11r7.mk > index ea0363f..85ecbc5 100644 > --- a/package/x11r7/x11r7.mk > +++ b/package/x11r7/x11r7.mk > @@ -1 +1 @@ > -include package/x11r7/*/*.mk > +include $(sort $(wildcard package/x11r7/*/*.mk)) > -- Arnout Vandecappelle arnout at mind be Senior Embedded Software Architect +32-16-286500 Essensium/Mind http://www.mind.be G.Geenslaan 9, 3001 Leuven, Belgium BE 872 984 063 RPR Leuven LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle GPG fingerprint: 7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] [PATCH v2] Fix build reproducibility in Make 3.82 2013-09-03 6:13 ` Arnout Vandecappelle @ 2013-09-03 8:45 ` Jérôme Pouiller 2013-09-03 9:31 ` Arnout Vandecappelle 2013-09-07 6:06 ` Peter Korsgaard 0 siblings, 2 replies; 22+ messages in thread From: Jérôme Pouiller @ 2013-09-03 8:45 UTC (permalink / raw) To: buildroot Make 3.82 do not sort anymore result of wildcards (see http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break build reproducibility. This patch sort results of wildcards to ensure reproducibility. Signed-off-by: J?r?me Pouiller <jezz@sysmic.org> --- V2: Add forgotten file support/dependencies/dependencies.mk. Even if it does not make difference for the build result, I prefer to be consistent. Makefile | 2 +- boot/common.mk | 2 +- docs/manual/manual.mk | 2 +- fs/common.mk | 2 +- linux/linux.mk | 2 +- package/efl/efl.mk | 2 +- package/freescale-imx/freescale-imx.mk | 2 +- package/gcc/gcc.mk | 2 +- package/gtk2-themes/gtk2-themes.mk | 2 +- package/matchbox/matchbox.mk | 2 +- package/opengl/opengl.mk | 2 +- package/qt5/qt5.mk | 2 +- package/x11r7/x11r7.mk | 2 +- support/dependencies/dependencies.mk | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 09faeba..f547af5 100644 --- a/Makefile +++ b/Makefile @@ -333,7 +333,7 @@ ifneq ($(PACKAGE_OVERRIDE_FILE),) -include $(PACKAGE_OVERRIDE_FILE) endif -include package/*/*.mk +include $(sort $(wildcard package/*/*.mk)) include boot/common.mk include linux/linux.mk diff --git a/boot/common.mk b/boot/common.mk index b315fe8..3021e51 100644 --- a/boot/common.mk +++ b/boot/common.mk @@ -1 +1 @@ -include boot/*/*.mk +include $(sort $(wildcard boot/*/*.mk)) diff --git a/docs/manual/manual.mk b/docs/manual/manual.mk index 4906bc8..bcbedad 100644 --- a/docs/manual/manual.mk +++ b/docs/manual/manual.mk @@ -53,5 +53,5 @@ $(1)-clean: .PHONY: $(1) $(1)-clean manual-update-lists endef -MANUAL_SOURCES = $(wildcard docs/manual/*.txt) $(wildcard docs/images/*) +MANUAL_SOURCES = $(sort $(wildcard docs/manual/*.txt) $(wildcard docs/images/*)) $(eval $(call GENDOC,manual)) diff --git a/fs/common.mk b/fs/common.mk index 36da2dd..4dab7ea 100644 --- a/fs/common.mk +++ b/fs/common.mk @@ -94,4 +94,4 @@ define ROOTFS_TARGET $(call ROOTFS_TARGET_INTERNAL,$(1),$(call UPPERCASE,$(1))) endef -include fs/*/*.mk +include $(sort $(wildcard fs/*/*.mk)) diff --git a/linux/linux.mk b/linux/linux.mk index 025fd51..8508a54 100644 --- a/linux/linux.mk +++ b/linux/linux.mk @@ -281,7 +281,7 @@ define LINUX_INSTALL_TARGET_CMDS $(LINUX_INSTALL_HOST_TOOLS) endef -include linux/linux-ext-*.mk +include $(sort $(wildcard linux/linux-ext-*.mk)) $(eval $(generic-package)) diff --git a/package/efl/efl.mk b/package/efl/efl.mk index f284c93..1de68cf 100644 --- a/package/efl/efl.mk +++ b/package/efl/efl.mk @@ -1,3 +1,3 @@ EFL_VERSION = 1.7.7 -include package/efl/*/*.mk +include $(sort $(wildcard package/efl/*/*.mk)) diff --git a/package/freescale-imx/freescale-imx.mk b/package/freescale-imx/freescale-imx.mk index 9f63ce3..c8f99b6 100644 --- a/package/freescale-imx/freescale-imx.mk +++ b/package/freescale-imx/freescale-imx.mk @@ -9,5 +9,5 @@ FREESCALE_IMX_VERSION = 1.1.0 # No official download site from freescale, just this mirror FREESCALE_IMX_MIRROR_SITE = http://download.ossystems.com.br/bsp/freescale/source -include package/freescale-imx/*/*.mk +include $(sort $(wildcard package/freescale-imx/*/*.mk)) diff --git a/package/gcc/gcc.mk b/package/gcc/gcc.mk index cd6efed..66af425 100644 --- a/package/gcc/gcc.mk +++ b/package/gcc/gcc.mk @@ -215,4 +215,4 @@ HOST_GCC_COMMON_CONF_OPT += \ --with-long-double-128 endif -include package/gcc/*/*.mk +include $(sort $(wildcard package/gcc/*/*.mk)) diff --git a/package/gtk2-themes/gtk2-themes.mk b/package/gtk2-themes/gtk2-themes.mk index 6f31c6d..fc66698 100644 --- a/package/gtk2-themes/gtk2-themes.mk +++ b/package/gtk2-themes/gtk2-themes.mk @@ -1 +1 @@ -include package/gtk2-themes/*/*.mk +include $(sort $(wildcard package/gtk2-themes/*/*.mk)) diff --git a/package/matchbox/matchbox.mk b/package/matchbox/matchbox.mk index abda915..fff4add 100644 --- a/package/matchbox/matchbox.mk +++ b/package/matchbox/matchbox.mk @@ -1,4 +1,4 @@ ifeq ($(BR2_PACKAGE_MATCHBOX),y) -include package/matchbox/*/*.mk +include $(sort $(wildcard package/matchbox/*/*.mk)) TARGETS+=matchbox-lib matchbox-wm endif diff --git a/package/opengl/opengl.mk b/package/opengl/opengl.mk index 68df3b1..abf96d5 100644 --- a/package/opengl/opengl.mk +++ b/package/opengl/opengl.mk @@ -1 +1 @@ -include package/opengl/*/*.mk +include $(sort $(wildcard package/opengl/*/*.mk)) diff --git a/package/qt5/qt5.mk b/package/qt5/qt5.mk index 9e71c83..fce0ca3 100644 --- a/package/qt5/qt5.mk +++ b/package/qt5/qt5.mk @@ -1,6 +1,6 @@ QT5_VERSION = 5.0.2 QT5_SITE = http://download.qt-project.org/archive/qt/5.0/$(QT5_VERSION)/submodules/ -include package/qt5/*/*.mk +include $(sort $(wildcard package/qt5/*/*.mk)) define QT5_LA_PRL_FILES_FIXUP for i in $$(find $(STAGING_DIR)/usr/lib* -name "libQt5*.la"); do \ diff --git a/package/x11r7/x11r7.mk b/package/x11r7/x11r7.mk index ea0363f..85ecbc5 100644 --- a/package/x11r7/x11r7.mk +++ b/package/x11r7/x11r7.mk @@ -1 +1 @@ -include package/x11r7/*/*.mk +include $(sort $(wildcard package/x11r7/*/*.mk)) diff --git a/support/dependencies/dependencies.mk b/support/dependencies/dependencies.mk index 4a220e0..149b8e5 100644 --- a/support/dependencies/dependencies.mk +++ b/support/dependencies/dependencies.mk @@ -14,7 +14,7 @@ DEPENDENCIES_HOST_PREREQ := define suitable-host-package $(shell support/dependencies/check-host-$(1).sh $(2)) endef --include support/dependencies/check-host-*.mk +-include $(sort $(wildcard support/dependencies/check-host-*.mk)) ifeq ($(BR2_STRIP_sstrip),y) DEPENDENCIES_HOST_PREREQ+=host-sstrip -- 1.7.9.5 ^ permalink raw reply related [flat|nested] 22+ messages in thread
* [Buildroot] [PATCH v2] Fix build reproducibility in Make 3.82 2013-09-03 8:45 ` [Buildroot] [PATCH v2] " Jérôme Pouiller @ 2013-09-03 9:31 ` Arnout Vandecappelle 2013-09-07 6:06 ` Peter Korsgaard 1 sibling, 0 replies; 22+ messages in thread From: Arnout Vandecappelle @ 2013-09-03 9:31 UTC (permalink / raw) To: buildroot On 09/03/13 10:45, J?r?me Pouiller wrote: > Make 3.82 do not sort anymore result of wildcards (see > http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break > build reproducibility. > > This patch sort results of wildcards to ensure reproducibility. > > Signed-off-by: J?r?me Pouiller<jezz@sysmic.org> You forgot Acked-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be> :-) > --- > V2: > Add forgotten file support/dependencies/dependencies.mk. Even if it > does not make difference for the build result, I prefer to be > consistent. I agree! Regards, Arnout -- Arnout Vandecappelle arnout at mind be Senior Embedded Software Architect +32-16-286500 Essensium/Mind http://www.mind.be G.Geenslaan 9, 3001 Leuven, Belgium BE 872 984 063 RPR Leuven LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle GPG fingerprint: 7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] [PATCH v2] Fix build reproducibility in Make 3.82 2013-09-03 8:45 ` [Buildroot] [PATCH v2] " Jérôme Pouiller 2013-09-03 9:31 ` Arnout Vandecappelle @ 2013-09-07 6:06 ` Peter Korsgaard 1 sibling, 0 replies; 22+ messages in thread From: Peter Korsgaard @ 2013-09-07 6:06 UTC (permalink / raw) To: buildroot >>>>> "J?r?me" == J?r?me Pouiller <jezz@sysmic.org> writes: J?r?me> Make 3.82 do not sort anymore result of wildcards (see J?r?me> http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260). This may break J?r?me> build reproducibility. J?r?me> This patch sort results of wildcards to ensure reproducibility. J?r?me> Signed-off-by: J?r?me Pouiller <jezz@sysmic.org> J?r?me> --- J?r?me> V2: J?r?me> Add forgotten file support/dependencies/dependencies.mk. Even if it J?r?me> does not make difference for the build result, I prefer to be J?r?me> consistent. Committed, thanks. -- Bye, Peter Korsgaard ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-08-30 8:31 [Buildroot] Build reproducibility Jérôme Pouiller 2013-08-30 8:31 ` [Buildroot] [PATCH] Fix build reproducibility in Make 3.82 Jérôme Pouiller @ 2013-08-30 11:59 ` Thomas De Schampheleire 2013-08-30 12:44 ` Jérôme Pouiller 1 sibling, 1 reply; 22+ messages in thread From: Thomas De Schampheleire @ 2013-08-30 11:59 UTC (permalink / raw) To: buildroot Op 30-aug.-2013 10:32 schreef "J?r?me Pouiller" <jezz@sysmic.org> het volgende: > > Hello, > > I noticed a problem with reproducibility of build. If I list targets > (using make show-targets) on Fedora and on Ubuntu, targets are on > sorted on same orders. Indeed, result of $(wildcard *.mk) are not > the same on these systems. > > It seems Make does not sort anymore result of wildcard since 3.82: > http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260. > > Following patch should fix problem. In one of the previous buildroot developer days we discussed this. IIRC, the conclusion was that this change in order shouldn't matter, because all dependencies should be expressed in make. Hence, the end result should be the same, even though build order was not. Have you come across a scenario where there actually was a problem? Best regards, Thomas > > _______________________________________________ > buildroot mailing list > buildroot at busybox.net > http://lists.busybox.net/mailman/listinfo/buildroot -------------- next part -------------- An HTML attachment was scrubbed... URL: <http://lists.busybox.net/pipermail/buildroot/attachments/20130830/1296e9e7/attachment.html> ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-08-30 11:59 ` [Buildroot] Build reproducibility Thomas De Schampheleire @ 2013-08-30 12:44 ` Jérôme Pouiller 2013-08-30 12:52 ` Thomas Petazzoni 0 siblings, 1 reply; 22+ messages in thread From: Jérôme Pouiller @ 2013-08-30 12:44 UTC (permalink / raw) To: buildroot On 2013-08-30 13:59, Thomas De Schampheleire wrote: > Op 30-aug.-2013 10:32 schreef "J?r?me Pouiller" <jezz@sysmic.org> het > volgende: > > > > Hello, > > > > I noticed a problem with reproducibility of build. If I list > targets > > (using make show-targets) on Fedora and on Ubuntu, targets are on > > sorted on same orders. Indeed, result of $(wildcard *.mk) are not > > the same on these systems. > > > > It seems Make does not sort anymore result of wildcard since 3.82: > > http://comments.gmane.org/gmane.comp.gnu.make.bugs/4260 [1]. > > > > Following patch should fix problem. > > In one of the previous buildroot developer days we discussed this. > IIRC, the conclusion was that this change in order shouldn't matter, > because all dependencies should be expressed in make. Hence, the end > result should be the same, even though build order was not. > > Have you come across a scenario where there actually was a problem? Some packages may detect installed library during ./configure run. Buildroot should be aware of this and fix dependencies with that library. Although, it is not always the case and Buildroot should at guarantee reproducibility of build. Without that, we cannot guarantee to reproduce a build done by Autobuilder. -- J?r?me Pouiller, Sysmic Embedded Linux specialist http://www.sysmic.fr ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-08-30 12:44 ` Jérôme Pouiller @ 2013-08-30 12:52 ` Thomas Petazzoni 2013-09-02 8:44 ` Thomas De Schampheleire 0 siblings, 1 reply; 22+ messages in thread From: Thomas Petazzoni @ 2013-08-30 12:52 UTC (permalink / raw) To: buildroot Dear J?r?me Pouiller, On Fri, 30 Aug 2013 14:44:01 +0200, J?r?me Pouiller wrote: > > In one of the previous buildroot developer days we discussed this. > > IIRC, the conclusion was that this change in order shouldn't matter, > > because all dependencies should be expressed in make. Hence, the end > > result should be the same, even though build order was not. > > > > Have you come across a scenario where there actually was a problem? > > Some packages may detect installed library during ./configure run. > Buildroot > should be aware of this and fix dependencies with that library. > Although, it > is not always the case and Buildroot should at guarantee > reproducibility of > build. > > Without that, we cannot guarantee to reproduce a build done by > Autobuilder. I agree with this. Thomas, I'm not sure that what you say what a conclusion of a developer day. I believe we always said that it is hardly possible to guarantee that a package .mk file will contain *all* the possible dependencies of the package. Whenever someone bumps a package, we rarely look in detail at whether the software has gained some optional dependencies, and make sure they are handled in the corresponding package .mk file. Having the packages always built in the same order guarantees that, in the absence of expressed dependencies, the build result will be reproducible. Thomas -- Thomas Petazzoni, Free Electrons Kernel, drivers, real-time and embedded Linux development, consulting, training and support. http://free-electrons.com ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-08-30 12:52 ` Thomas Petazzoni @ 2013-09-02 8:44 ` Thomas De Schampheleire 2013-09-02 8:53 ` Thomas Petazzoni 2013-09-02 16:11 ` Arnout Vandecappelle 0 siblings, 2 replies; 22+ messages in thread From: Thomas De Schampheleire @ 2013-09-02 8:44 UTC (permalink / raw) To: buildroot Hi Thomas, On Fri, Aug 30, 2013 at 2:52 PM, Thomas Petazzoni <thomas.petazzoni@free-electrons.com> wrote: > Dear J?r?me Pouiller, > > On Fri, 30 Aug 2013 14:44:01 +0200, J?r?me Pouiller wrote: > >> > In one of the previous buildroot developer days we discussed this. >> > IIRC, the conclusion was that this change in order shouldn't matter, >> > because all dependencies should be expressed in make. Hence, the end >> > result should be the same, even though build order was not. >> > >> > Have you come across a scenario where there actually was a problem? >> >> Some packages may detect installed library during ./configure run. >> Buildroot >> should be aware of this and fix dependencies with that library. >> Although, it >> is not always the case and Buildroot should at guarantee >> reproducibility of >> build. >> >> Without that, we cannot guarantee to reproduce a build done by >> Autobuilder. > > I agree with this. Thomas, I'm not sure that what you say what a > conclusion of a developer day. I believe we always said that it is > hardly possible to guarantee that a package .mk file will contain *all* > the possible dependencies of the package. Whenever someone bumps a > package, we rarely look in detail at whether the software has gained > some optional dependencies, and make sure they are handled in the > corresponding package .mk file. > > Having the packages always built in the same order guarantees that, in > the absence of expressed dependencies, the build result will be > reproducible. I may be wrong about the conclusion. Regardless: it's true that it's hard to guarantee that all dependencies are expressed properly in the .mk files. However, by 'fixing' the wildcard behavior into a sorted one such as with previous versions of make, we just hide the problem. We will never notice that some dependencies are missing, as long as the dependency comes before the dependant (or whichever word) in alphabetical order. With the random behavior of wildcards in newer versions of make, we would still see issues in autobuilds, and get the opportunity to fix them. It may take time, and may be a never-ending story as packages are bumped and new packages are added, but at least there can be improvement. So, my viewpoint is to keep the current behavior and instead focus on fixing any missing dependency when we spot it. Best regards, Thomas ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-02 8:44 ` Thomas De Schampheleire @ 2013-09-02 8:53 ` Thomas Petazzoni 2013-09-02 13:18 ` Thomas De Schampheleire 2013-09-02 16:11 ` Arnout Vandecappelle 1 sibling, 1 reply; 22+ messages in thread From: Thomas Petazzoni @ 2013-09-02 8:53 UTC (permalink / raw) To: buildroot Dear Thomas De Schampheleire, On Mon, 2 Sep 2013 10:44:17 +0200, Thomas De Schampheleire wrote: > > I agree with this. Thomas, I'm not sure that what you say what a > > conclusion of a developer day. I believe we always said that it is > > hardly possible to guarantee that a package .mk file will contain *all* > > the possible dependencies of the package. Whenever someone bumps a > > package, we rarely look in detail at whether the software has gained > > some optional dependencies, and make sure they are handled in the > > corresponding package .mk file. > > > > Having the packages always built in the same order guarantees that, in > > the absence of expressed dependencies, the build result will be > > reproducible. > > I may be wrong about the conclusion. > Regardless: it's true that it's hard to guarantee that all > dependencies are expressed properly in the .mk files. However, by > 'fixing' the wildcard behavior into a sorted one such as with previous > versions of make, we just hide the problem. We will never notice that > some dependencies are missing, as long as the dependency comes before > the dependant (or whichever word) in alphabetical order. > With the random behavior of wildcards in newer versions of make, we > would still see issues in autobuilds, and get the opportunity to fix > them. It may take time, and may be a never-ending story as packages > are bumped and new packages are added, but at least there can be > improvement. > So, my viewpoint is to keep the current behavior and instead focus on > fixing any missing dependency when we spot it. I obviously disagree, because in the mean time, our users are having non-reproducible builds. An user within a company uses Buildroot to create a system, adds some packages, creates a configuration for his project. Then he passes this Buildroot to another colleague: the date/times of the various Buildroot files will be different, maybe affecting the order in which the wildcards are resolved by make 3.82. This colleague will attempt the build, and maybe get a failure, or a different build result than the first colleague who has created the Buildroot configuration. This is really damaging for Buildroot's reputation and the user experience. We clearly to not want this to happen. Of course, if within the Buildroot project we are interested in fixing such missing dependencies, then we can find a way of adding randomness into the build order in our autobuilders. But clearly, we do want to expose this randomness to our users. In fact, I had already thought about adding such randomness in the autobuilders. But I've refrained from doing so because it also means that the builds that the autobuilders are doing cannot be reproduced. So when you'll get an autobuilder failure that you can't reproduce locally, you'll never know if it's due to the random order of package building, or due to some difference in the build environment. Best regards, Thomas -- Thomas Petazzoni, Free Electrons Kernel, drivers, real-time and embedded Linux development, consulting, training and support. http://free-electrons.com ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-02 8:53 ` Thomas Petazzoni @ 2013-09-02 13:18 ` Thomas De Schampheleire 2013-09-03 17:13 ` Thomas Petazzoni 0 siblings, 1 reply; 22+ messages in thread From: Thomas De Schampheleire @ 2013-09-02 13:18 UTC (permalink / raw) To: buildroot Hi Thomas, On Mon, Sep 2, 2013 at 10:53 AM, Thomas Petazzoni <thomas.petazzoni@free-electrons.com> wrote: > Dear Thomas De Schampheleire, > > On Mon, 2 Sep 2013 10:44:17 +0200, Thomas De Schampheleire wrote: > >> > I agree with this. Thomas, I'm not sure that what you say what a >> > conclusion of a developer day. I believe we always said that it is >> > hardly possible to guarantee that a package .mk file will contain *all* >> > the possible dependencies of the package. Whenever someone bumps a >> > package, we rarely look in detail at whether the software has gained >> > some optional dependencies, and make sure they are handled in the >> > corresponding package .mk file. >> > >> > Having the packages always built in the same order guarantees that, in >> > the absence of expressed dependencies, the build result will be >> > reproducible. >> >> I may be wrong about the conclusion. >> Regardless: it's true that it's hard to guarantee that all >> dependencies are expressed properly in the .mk files. However, by >> 'fixing' the wildcard behavior into a sorted one such as with previous >> versions of make, we just hide the problem. We will never notice that >> some dependencies are missing, as long as the dependency comes before >> the dependant (or whichever word) in alphabetical order. >> With the random behavior of wildcards in newer versions of make, we >> would still see issues in autobuilds, and get the opportunity to fix >> them. It may take time, and may be a never-ending story as packages >> are bumped and new packages are added, but at least there can be >> improvement. >> So, my viewpoint is to keep the current behavior and instead focus on >> fixing any missing dependency when we spot it. > > I obviously disagree, because in the mean time, our users are having > non-reproducible builds. An user within a company uses Buildroot to > create a system, adds some packages, creates a configuration for his > project. Then he passes this Buildroot to another colleague: the > date/times of the various Buildroot files will be different, maybe > affecting the order in which the wildcards are resolved by make 3.82. > This colleague will attempt the build, and maybe get a failure, or a > different build result than the first colleague who has created the > Buildroot configuration. This is really damaging for Buildroot's > reputation and the user experience. We clearly to not want this to > happen. Ok, fair enough, I can follow this argument. > > Of course, if within the Buildroot project we are interested in fixing > such missing dependencies, then we can find a way of adding randomness > into the build order in our autobuilders. But clearly, we do want to > expose this randomness to our users. I think indeed we should try to set the dependencies right some way or another. If we assume that a package does not have any configurable options that would change its dependencies, a simple way to check if all dependencies are properly expressed is through: make clean toolchain foo Correct? Although it would take many more builds, each of them would be relatively short. If we even can find a way to 'clean-all-but-the-toolchain', the cleaning time will be much shorter. For this type of simple package test, only one toolchain needs to be used because the dependencies should not depend on the toolchain. Also, it's not necessary that each package gets build every night: once the dependencies are correct, they will stay correct until a version bump. This means we can spread out the execution of this type of tests over time, interleaving them with the already existing autobuilds with random configurations. Of course, when a package is more complex and the its Config.in file allows to enable/disable certain options which are based on another library that would need to be added to the dependencies, then we'd still need a way to test the different configurations. This could be achieved with the regular autobuilds, but disabling the wildcard sorting, which brings us to... > > In fact, I had already thought about adding such randomness in the > autobuilders. But I've refrained from doing so because it also means > that the builds that the autobuilders are doing cannot be reproduced. > So when you'll get an autobuilder failure that you can't reproduce > locally, you'll never know if it's due to the random order of package > building, or due to some difference in the build environment. The make targets of buildroot itself are executed sequentially. Suppose that we keep a list of all targets executed, something like: python-source python-extract python-patch python-configure python-build python-install-target pyfoo-source pyfoo-extract pyfoo-build ... To reproduce a build, we can explicitly pass this list on the make command-line, roughly like: cat <target-list> | xargs make clean toolchain Regardless of this possibility (or not, I may be overlooking something), not all autobuilds need to be randomized. We could randomize some, and clearly indicate on the autobuild page that this was the case. A full build log would be useful in this case. Best regards, Thomas ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-02 13:18 ` Thomas De Schampheleire @ 2013-09-03 17:13 ` Thomas Petazzoni 2013-09-05 19:56 ` Thomas De Schampheleire 0 siblings, 1 reply; 22+ messages in thread From: Thomas Petazzoni @ 2013-09-03 17:13 UTC (permalink / raw) To: buildroot Dear Thomas De Schampheleire, On Mon, 2 Sep 2013 15:18:09 +0200, Thomas De Schampheleire wrote: > > Of course, if within the Buildroot project we are interested in > > fixing such missing dependencies, then we can find a way of adding > > randomness into the build order in our autobuilders. But clearly, > > we do want to expose this randomness to our users. > > I think indeed we should try to set the dependencies right some way > or another. > > If we assume that a package does not have any configurable options > that would change its dependencies, a simple way to check if all > dependencies are properly expressed is through: > make clean toolchain foo This is already done by the autobuilders. Thanks to the randomness of the configuration, if a package fails to express a mandatory dependency, sooner or later the autobuilders will generate a configuration that has the package enabled but not one of its unknown dependencies. The autobuilders have triggered such cases very quickly in the past when a new package was added, so I'm pretty confident we have good coverage on this one. However, I think this suggestion misses the point of the current discussion: we are talking about *optional* dependencies, i.e functionality of a given package that are enabled if OpenSSL is available, or disabled if OpenSSL is not available. Those optional dependencies cannot be checked/discovered by a build such as "make clean toolchain foo". > Also, it's not necessary that each package gets build every night: > once the dependencies are correct, they will stay correct until a > version bump. This means we can spread out the execution of this type > of tests over time, interleaving them with the already existing > autobuilds with random configurations. As stated above, for the mandatory dependencies, the autobuilders are already finding them very quickly. Introduce a new package that lacks a mandatory dependency expressed in its .mk file, and you'll see it very quickly in the autobuilders. > The make targets of buildroot itself are executed sequentially. > Suppose that we keep a list of all targets executed, something like: > python-source > python-extract > python-patch > python-configure > python-build > python-install-target > pyfoo-source > pyfoo-extract > pyfoo-build > ... > > To reproduce a build, we can explicitly pass this list on the make > command-line, roughly like: > cat <target-list> | xargs make clean toolchain Could be doable. I've lost track. Are we talking about all of this to use "include package/*/*.mk" in make 3.82 which doesn't guarantee sorting, or to be able to do top-level parallel build? Seeing how simple it is to get make 3.82 to behave like make 3.81 by sorting the "include package/*/*.mk" inclusions, I don't think it's worth doing anything but the fix that J?r?me has suggested. Only the top-level parallel build would be a good enough to worry about reproducibility of more complicated builds (and therefore the need to list in which order the targets were built). Best regards, Thomas -- Thomas Petazzoni, Free Electrons Kernel, drivers, real-time and embedded Linux development, consulting, training and support. http://free-electrons.com ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-03 17:13 ` Thomas Petazzoni @ 2013-09-05 19:56 ` Thomas De Schampheleire 2013-09-05 20:49 ` Jérôme Pouiller 0 siblings, 1 reply; 22+ messages in thread From: Thomas De Schampheleire @ 2013-09-05 19:56 UTC (permalink / raw) To: buildroot Hi Thomas, On Tue, Sep 3, 2013 at 7:13 PM, Thomas Petazzoni <thomas.petazzoni@free-electrons.com> wrote: > Dear Thomas De Schampheleire, > > On Mon, 2 Sep 2013 15:18:09 +0200, Thomas De Schampheleire wrote: >> > Of course, if within the Buildroot project we are interested in >> > fixing such missing dependencies, then we can find a way of adding >> > randomness into the build order in our autobuilders. But clearly, >> > we do want to expose this randomness to our users. >> >> I think indeed we should try to set the dependencies right some way >> or another. >> >> If we assume that a package does not have any configurable options >> that would change its dependencies, a simple way to check if all >> dependencies are properly expressed is through: >> make clean toolchain foo > > This is already done by the autobuilders. Thanks to the randomness of > the configuration, if a package fails to express a mandatory > dependency, sooner or later the autobuilders will generate a > configuration that has the package enabled but not one of its unknown > dependencies. The autobuilders have triggered such cases very quickly > in the past when a new package was added, so I'm pretty confident we > have good coverage on this one. I'm currently running 'make clean toolchain <pkg>' for each package, and preliminary results have already found several dependency problems. I will analyze the results when it's done and submit patches for them. The advantage of such a non-random test is that it will find problems faster than the autobuilders. > > However, I think this suggestion misses the point of the current > discussion: we are talking about *optional* dependencies, i.e > functionality of a given package that are enabled if OpenSSL is > available, or disabled if OpenSSL is not available. Those optional > dependencies cannot be checked/discovered by a build such as "make > clean toolchain foo". Correct. When I wrote my mail, I hadn't though about such optional dependencies. > >> Also, it's not necessary that each package gets build every night: >> once the dependencies are correct, they will stay correct until a >> version bump. This means we can spread out the execution of this type >> of tests over time, interleaving them with the already existing >> autobuilds with random configurations. > > As stated above, for the mandatory dependencies, the autobuilders are > already finding them very quickly. Introduce a new package that lacks a > mandatory dependency expressed in its .mk file, and you'll see it very > quickly in the autobuilders. > >> The make targets of buildroot itself are executed sequentially. >> Suppose that we keep a list of all targets executed, something like: >> python-source >> python-extract >> python-patch >> python-configure >> python-build >> python-install-target >> pyfoo-source >> pyfoo-extract >> pyfoo-build >> ... >> >> To reproduce a build, we can explicitly pass this list on the make >> command-line, roughly like: >> cat <target-list> | xargs make clean toolchain > > Could be doable. > > I've lost track. Are we talking about all of this to use "include > package/*/*.mk" in make 3.82 which doesn't guarantee sorting, or to be > able to do top-level parallel build? The original mail was about the wildcard-make-3.82-thingy. You were the one that dragged in the parallel build situation :-) > > Seeing how simple it is to get make 3.82 to behave like make 3.81 by > sorting the "include package/*/*.mk" inclusions, I don't think it's > worth doing anything but the fix that J?r?me has suggested. Agreed. > > Only the top-level parallel build would be a good enough to worry about > reproducibility of more complicated builds (and therefore the need to > list in which order the targets were built). Although such a list could help in reproducing a parallel build, in the context of making buildroot work successfully with top-level parallel build, it is just a side-problem I think. Do you think we should be doing something about it right now? Best regards, Thomas ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-05 19:56 ` Thomas De Schampheleire @ 2013-09-05 20:49 ` Jérôme Pouiller 0 siblings, 0 replies; 22+ messages in thread From: Jérôme Pouiller @ 2013-09-05 20:49 UTC (permalink / raw) To: buildroot Hello Thomas, On Thursday 05 September 2013 21:56:01 Thomas De Schampheleire wrote: > On Tue, Sep 3, 2013 at 7:13 PM, Thomas Petazzoni > <thomas.petazzoni@free-electrons.com> wrote: > > On Mon, 2 Sep 2013 15:18:09 +0200, Thomas De Schampheleire wrote: > >> > Of course, if within the Buildroot project we are interested in > >> > fixing such missing dependencies, then we can find a way of adding > >> > randomness into the build order in our autobuilders. But clearly, > >> > we do want to expose this randomness to our users. > >> > >> I think indeed we should try to set the dependencies right some way > >> or another. > >> > >> If we assume that a package does not have any configurable options > >> that would change its dependencies, a simple way to check if all > >> dependencies are properly expressed is through: > >> make clean toolchain foo > > > > This is already done by the autobuilders. Thanks to the randomness of > > the configuration, if a package fails to express a mandatory > > dependency, sooner or later the autobuilders will generate a > > configuration that has the package enabled but not one of its unknown > > dependencies. The autobuilders have triggered such cases very quickly > > in the past when a new package was added, so I'm pretty confident we > > have good coverage on this one. > > I'm currently running 'make clean toolchain <pkg>' for each package, > and preliminary results have already found several dependency > problems. I will analyze the results when it's done and submit patches > for them. > The advantage of such a non-random test is that it will find problems > faster than the autobuilders. Just to avoid duplicate work, I am currently testing an automated system to detect dependencies of packages (based on inotify as I explain in my previous mail). First results can be found there: http://www.sysmic.org/~jezz/dependencies.cooked -- J?r?me Pouiller, Sysmic Embedded Linux specialist http://www.sysmic.fr ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-02 8:44 ` Thomas De Schampheleire 2013-09-02 8:53 ` Thomas Petazzoni @ 2013-09-02 16:11 ` Arnout Vandecappelle 2013-09-03 6:26 ` Peter Korsgaard 1 sibling, 1 reply; 22+ messages in thread From: Arnout Vandecappelle @ 2013-09-02 16:11 UTC (permalink / raw) To: buildroot On 09/02/13 10:44, Thomas De Schampheleire wrote: > Hi Thomas, > > On Fri, Aug 30, 2013 at 2:52 PM, Thomas Petazzoni > <thomas.petazzoni@free-electrons.com> wrote: >> Dear J?r?me Pouiller, >> >> On Fri, 30 Aug 2013 14:44:01 +0200, J?r?me Pouiller wrote: >> >>>> In one of the previous buildroot developer days we discussed this. >>>> IIRC, the conclusion was that this change in order shouldn't matter, >>>> because all dependencies should be expressed in make. Hence, the end >>>> result should be the same, even though build order was not. >>>> >>>> Have you come across a scenario where there actually was a problem? >>> >>> Some packages may detect installed library during ./configure run. >>> Buildroot >>> should be aware of this and fix dependencies with that library. >>> Although, it >>> is not always the case and Buildroot should at guarantee >>> reproducibility of >>> build. >>> >>> Without that, we cannot guarantee to reproduce a build done by >>> Autobuilder. >> >> I agree with this. Thomas, I'm not sure that what you say what a >> conclusion of a developer day. I believe we always said that it is >> hardly possible to guarantee that a package .mk file will contain *all* >> the possible dependencies of the package. Whenever someone bumps a >> package, we rarely look in detail at whether the software has gained >> some optional dependencies, and make sure they are handled in the >> corresponding package .mk file. >> >> Having the packages always built in the same order guarantees that, in >> the absence of expressed dependencies, the build result will be >> reproducible. > > I may be wrong about the conclusion. > Regardless: it's true that it's hard to guarantee that all > dependencies are expressed properly in the .mk files. However, by > 'fixing' the wildcard behavior into a sorted one such as with previous > versions of make, we just hide the problem. We will never notice that > some dependencies are missing, as long as the dependency comes before > the dependant (or whichever word) in alphabetical order. > With the random behavior of wildcards in newer versions of make, we > would still see issues in autobuilds, and get the opportunity to fix > them. It may take time, and may be a never-ending story as packages > are bumped and new packages are added, but at least there can be > improvement. > So, my viewpoint is to keep the current behavior and instead focus on > fixing any missing dependency when we spot it. I think this scenario is rather unlikely in reality. If it's a dependency which is not mentioned at all in .mk or Config.in, then at some point there will be a random config that has the package enabled but not the dependency. So this scenario will only occur if the dependency is mentioned in Config.in, but not in the .mk. With the amount of review we do, that is not very likely to happen. And even if it does, it is not correct but buildroot will still function correctly in the normal use case ('make' instead of 'make pkg'). What is much more likely to happen is that there is some optional dependency in the package's configure or build system that is not expressed in Config.in or pkg.mk. Most reviewers do not run a 'configure --help', and even then it is easy to miss something. Since the dependency is optional, the build will not fail either way. Only, when user A builds it, TLS support is included, but when user B builds it, it is not... That's the kind of lack of reproducability we really need to avoid. Note that doing more randomized build order in the autobuilder also will not capture the latter scenario. You would have to compare the build result - but binary differences are likely because of changing timestamps or changing optimizations depending on memory randomness. Regards, Arnout > > Best regards, > Thomas > _______________________________________________ > buildroot mailing list > buildroot at busybox.net > http://lists.busybox.net/mailman/listinfo/buildroot > > -- Arnout Vandecappelle arnout at mind be Senior Embedded Software Architect +32-16-286500 Essensium/Mind http://www.mind.be G.Geenslaan 9, 3001 Leuven, Belgium BE 872 984 063 RPR Leuven LinkedIn profile: http://www.linkedin.com/in/arnoutvandecappelle GPG fingerprint: 7CB5 E4CC 6C2E EFD4 6E3D A754 F963 ECAB 2450 2F1F ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-02 16:11 ` Arnout Vandecappelle @ 2013-09-03 6:26 ` Peter Korsgaard 2013-09-03 7:16 ` Thomas Petazzoni 2013-09-03 8:15 ` Jérôme Pouiller 0 siblings, 2 replies; 22+ messages in thread From: Peter Korsgaard @ 2013-09-03 6:26 UTC (permalink / raw) To: buildroot >>>>> "Arnout" == Arnout Vandecappelle <arnout@mind.be> writes: Hi, Arnout> What is much more likely to happen is that there is some optional Arnout> dependency in the package's configure or build system that is not Arnout> expressed in Config.in or pkg.mk. Most reviewers do not run a Arnout> configure --help', and even then it is easy to miss something. Since Arnout> the dependency is optional, the build will not fail either way. Only, Arnout> when user A builds it, TLS support is included, but when user B builds Arnout> it, it is not... That's the kind of lack of reproducability we really Arnout> need to avoid. Indeed. Arnout> Note that doing more randomized build order in the autobuilder also Arnout> will not capture the latter scenario. You would have to compare the Arnout> build result - but binary differences are likely because of changing Arnout> timestamps or changing optimizations depending on memory randomness. Exactly. I don't have any good ideas about how to detect this (besides building all packages in clean staging dirs, E.G. only populated with its explicit dependencies like afaik OE lite can do, but that would require quite some work), anyone? -- Bye, Peter Korsgaard ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-03 6:26 ` Peter Korsgaard @ 2013-09-03 7:16 ` Thomas Petazzoni 2013-09-03 7:47 ` Peter Korsgaard 2013-09-03 8:15 ` Jérôme Pouiller 1 sibling, 1 reply; 22+ messages in thread From: Thomas Petazzoni @ 2013-09-03 7:16 UTC (permalink / raw) To: buildroot Dear Peter Korsgaard, On Tue, 03 Sep 2013 08:26:07 +0200, Peter Korsgaard wrote: > Arnout> Note that doing more randomized build order in the autobuilder also > Arnout> will not capture the latter scenario. You would have to compare the > Arnout> build result - but binary differences are likely because of changing > Arnout> timestamps or changing optimizations depending on memory randomness. > > Exactly. I don't have any good ideas about how to detect this (besides > building all packages in clean staging dirs, E.G. only populated with > its explicit dependencies like afaik OE lite can do, but that would > require quite some work), anyone? Doing a per-package sysroot that is generated for each package before it gets built, with only the explicitly listed dependencies, is indeed the only way to ensure that a package is not seeing/using something that isn't declared as a dependency. This would certainly be nice to have (as it also helps top-level parallel build, as was discussed with Fabio Porcedda some time ago), but: 1) I'm worried about the additional complexity inside Buildroot. 2) I'm worried about the additional build time required to generate a per-package sysroot for each package. When building large stacks like X.org that has many small packages, but each have a lot of dependencies, the cost of creating a sysroot before building each package could be huge. 3) We still need to provide the user a global sysroot with all libraries installed, so that he can use the toolchain generated by Buildroot to build his own libraries/applications. This would mean we would need to have two sysroots: the global sysroot, that gets incrementally populated with what all packages are installing, but that isn't used for building packages inside Buildroot, and a separate temporary sysroot, used when building the current package. Since the compiler would default to the 'global sysroot', we would have to pass --sysroot $(TMPSYSROOT) all the time, or have a separate wrapper, or something. Not impossible, but see (1). Best regards, Thomas -- Thomas Petazzoni, Free Electrons Kernel, drivers, real-time and embedded Linux development, consulting, training and support. http://free-electrons.com ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-03 7:16 ` Thomas Petazzoni @ 2013-09-03 7:47 ` Peter Korsgaard 2013-09-03 16:48 ` Thomas Petazzoni 0 siblings, 1 reply; 22+ messages in thread From: Peter Korsgaard @ 2013-09-03 7:47 UTC (permalink / raw) To: buildroot >>>>> "Thomas" == Thomas Petazzoni <thomas.petazzoni@free-electrons.com> writes: Hi, Thomas> This would certainly be nice to have (as it also helps top-level Thomas> parallel build, as was discussed with Fabio Porcedda some time ago), Thomas> but: [snip] Indeed. For the record, I also don't think we should do this, I was just stating what was needed if we wanted to do so. As long as the dependencies are correct, I don't think seperate staging dirs are needed for toplevel parallel builds. -- Bye, Peter Korsgaard ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-03 7:47 ` Peter Korsgaard @ 2013-09-03 16:48 ` Thomas Petazzoni 0 siblings, 0 replies; 22+ messages in thread From: Thomas Petazzoni @ 2013-09-03 16:48 UTC (permalink / raw) To: buildroot Dear Peter Korsgaard, (Adding Fabio to the CC list) On Tue, 03 Sep 2013 09:47:14 +0200, Peter Korsgaard wrote: > Thomas> This would certainly be nice to have (as it also helps > Thomas> top-level parallel build, as was discussed with Fabio > Thomas> Porcedda some time ago), but: > > [snip] > > Indeed. For the record, I also don't think we should do this, I was > just stating what was needed if we wanted to do so. > > As long as the dependencies are correct, I don't think seperate > staging dirs are needed for toplevel parallel builds. I disagree on this one. Since there is no way to be sure that all the possible optional dependencies have been taken into account, I believe separate staging dirs are a requirement to ensure that toplevel parallel builds are reproducible. Whenever we bump a package, some additional optional dependencies may have been added by the upstream developers, and we rarely go deep into the configure.ac script or other build system files to verify that the addition or removal of optional dependencies. Having to do such review work would be very time-consuming and terribly annoying. So the only way to be sure that the optional dependencies are all properly taken into account when doing top-level parallel build is by ensuring that the staging dir used when building a package only contains the dependencies that have been explicitly listed by the package .mk file. Best regards, Thomas -- Thomas Petazzoni, Free Electrons Kernel, drivers, real-time and embedded Linux development, consulting, training and support. http://free-electrons.com ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-03 6:26 ` Peter Korsgaard 2013-09-03 7:16 ` Thomas Petazzoni @ 2013-09-03 8:15 ` Jérôme Pouiller 2013-09-03 16:54 ` Thomas Petazzoni 1 sibling, 1 reply; 22+ messages in thread From: Jérôme Pouiller @ 2013-09-03 8:15 UTC (permalink / raw) To: buildroot On 2013-09-03 08:26, Peter Korsgaard wrote: >>>>>> "Arnout" == Arnout Vandecappelle <arnout@mind.be> writes: > > Hi, > > Arnout> What is much more likely to happen is that there is some > optional > Arnout> dependency in the package's configure or build system that > is not > Arnout> expressed in Config.in or pkg.mk. Most reviewers do not run > a > Arnout> configure --help', and even then it is easy to miss > something. Since > Arnout> the dependency is optional, the build will not fail either > way. Only, > Arnout> when user A builds it, TLS support is included, but when > user B builds > Arnout> it, it is not... That's the kind of lack of reproducability > we really > Arnout> need to avoid. > > Indeed. > > Arnout> Note that doing more randomized build order in the > autobuilder also > Arnout> will not capture the latter scenario. You would have to > compare the > Arnout> build result - but binary differences are likely because of > changing > Arnout> timestamps or changing optimizations depending on memory > randomness. > > Exactly. I don't have any good ideas about how to detect this > (besides > building all packages in clean staging dirs, E.G. only populated with > its explicit dependencies like afaik OE lite can do, but that would > require quite some work), anyone? I may have an idea to detect cases where a package has a non declared dependency. Inotify may help us to know which files are accessed during build of one package. Especially, we can know which files in staging/ are accessed. If we knew which package own which file under staging and we may find what are dependencies of a package I already made a system to monitor which files are installed by each package (http://comments.gmane.org/gmane.comp.lib.uclibc.buildroot/53094). It may be adapted to detect owner of staging/ files So, process would be: make allyespackageconfig make # And monitor which package create which file in staging/ for PKG in $PACKAGES[@]; do make $PKG-dirclean make $PKG # And monitor which files are accessed in staging/ # process result to get $PKG dependencies done It would not fix automatically the problem, but an autobuilder may send notifications. Inotify instrumentation may stay in a separate patch or branch to not be intrusive in Buildroot. Do you think it would be useful? (I will not have enough time to implement this until end of October, at least) -- J?r?me Pouiller, Sysmic Embedded Linux specialist http://www.sysmic.fr ^ permalink raw reply [flat|nested] 22+ messages in thread
* [Buildroot] Build reproducibility 2013-09-03 8:15 ` Jérôme Pouiller @ 2013-09-03 16:54 ` Thomas Petazzoni 0 siblings, 0 replies; 22+ messages in thread From: Thomas Petazzoni @ 2013-09-03 16:54 UTC (permalink / raw) To: buildroot Dear J?r?me Pouiller, (Your e-mail client has wrapped the e-mail in a somewhat weird fashion, leaving only one word one every two lines. Ah, yes a Roundcube Webmail!) On Tue, 03 Sep 2013 10:15:55 +0200, J?r?me Pouiller wrote: > I may have an idea to detect cases where a package has a non declared > dependency. > > Inotify may help us to know which files are accessed during build of > one package. > Especially, we can know which files in staging/ are accessed. If we > knew which > package own which file under staging and we may find what are > dependencies > of a package > > I already made a system to monitor which files are installed by each > package > (http://comments.gmane.org/gmane.comp.lib.uclibc.buildroot/53094). It > may be adapted to detect owner of staging/ files This indeed looks interesting. I'm not sure it's really easy to do (on one side, we need to get the list of files that are installed/owned by each package, and then whenever a file is accessed, check whether this file is owned by a package which is part of the dependencies of the currently built package), but if it's possible, it would quite certainly raise a number of interesting warnings/issues. > make allyespackageconfig Note that an allyespackageconfig doesn't build all packages: they are some choices in the Buildroot Config.in options, and some packages depend on the value of those choices. Hence, in a given configuration, it is not possible to really enable and test all packages. > make # And monitor which package create which file in > staging/ for PKG in $PACKAGES[@]; do > make $PKG-dirclean > make $PKG # And monitor which files are accessed in staging/ > # process result to get $PKG dependencies I don't think we should try to infer automatically the dependencies, but at least show a report of files that were accessed even though they don't belong to packages that are part of the dependencies. > done > > It would not fix automatically the problem, but an autobuilder may > send notifications. Inotify instrumentation may stay in a separate > patch or branch to not be intrusive in Buildroot. > > Do you think it would be useful? It might certainly be useful to do some experiments around this idea and see if it brings a reasonable result (both in terms of code to merge, and in terms of warnings/issues that we are able to discover and fix). Thanks, Thomas -- Thomas Petazzoni, Free Electrons Kernel, drivers, real-time and embedded Linux development, consulting, training and support. http://free-electrons.com ^ permalink raw reply [flat|nested] 22+ messages in thread
end of thread, other threads:[~2013-09-07 6:06 UTC | newest] Thread overview: 22+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2013-08-30 8:31 [Buildroot] Build reproducibility Jérôme Pouiller 2013-08-30 8:31 ` [Buildroot] [PATCH] Fix build reproducibility in Make 3.82 Jérôme Pouiller 2013-09-03 6:13 ` Arnout Vandecappelle 2013-09-03 8:45 ` [Buildroot] [PATCH v2] " Jérôme Pouiller 2013-09-03 9:31 ` Arnout Vandecappelle 2013-09-07 6:06 ` Peter Korsgaard 2013-08-30 11:59 ` [Buildroot] Build reproducibility Thomas De Schampheleire 2013-08-30 12:44 ` Jérôme Pouiller 2013-08-30 12:52 ` Thomas Petazzoni 2013-09-02 8:44 ` Thomas De Schampheleire 2013-09-02 8:53 ` Thomas Petazzoni 2013-09-02 13:18 ` Thomas De Schampheleire 2013-09-03 17:13 ` Thomas Petazzoni 2013-09-05 19:56 ` Thomas De Schampheleire 2013-09-05 20:49 ` Jérôme Pouiller 2013-09-02 16:11 ` Arnout Vandecappelle 2013-09-03 6:26 ` Peter Korsgaard 2013-09-03 7:16 ` Thomas Petazzoni 2013-09-03 7:47 ` Peter Korsgaard 2013-09-03 16:48 ` Thomas Petazzoni 2013-09-03 8:15 ` Jérôme Pouiller 2013-09-03 16:54 ` Thomas Petazzoni
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox