[Buildroot] [PATCH 1/4] linux: create a scmversion file based on the custom repo variables

[Buildroot] [PATCH 1/4] linux: create a scmversion file based on the custom repo variables

[ckhardin]

From: Charles Hardin <ckhardin@gmail.com>

When using the custom repositories, the archive creation will
omit the source history to get reproducible hashes. This has
the side effect of making the setlocalversion scripts within the
builds to not create the tracking information as expected. So,
just generate a .scmversion so that kernels will be labelled
with where buildroot referenced them.

For example (line wrapped)

    #define UTS_RELEASE "5.4.70-repo-version-" \
                        "92c23d10d0031e0a13244a1a97e7530d3b53fc6e"

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 linux/linux.mk | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/linux/linux.mk b/linux/linux.mk
index 7645b5f507..0b27c8dee2 100644
--- a/linux/linux.mk
+++ b/linux/linux.mk
@@ -596,6 +596,20 @@ endif
 ifeq ($(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL)),)
 $(error No custom repo URL set. Check your BR2_LINUX_KERNEL_CUSTOM_REPO_URL setting)
 endif
+
+# Create a custom scm version file to reflect the source version since the
+# archive will omit source directories like .git to maintain reproducible
+# hashes for the archives
+LINUX_CUSTOM_REPO_SCMVERSION = \
+	"-repo-version-$(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_VERSION))"
+define LINUX_CUSTOM_REPO_SCMVERSION_HOOK
+	(cd $(@D); \
+		if [ ! -f .scmversion ]; then \
+			echo $(LINUX_CUSTOM_REPO_SCMVERSION) > .scmversion; \
+		fi)
+endef
+
+LINUX_POST_EXTRACT_HOOKS += LINUX_CUSTOM_REPO_SCMVERSION_HOOK
 endif
 
 ifeq ($(BR_BUILDING),y)
-- 
2.37.1 (Apple Git-137.1)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH 2/4] boot/uboot: create a scmversion file based on the custom repo variables

[ckhardin]

From: Charles Hardin <ckhardin@gmail.com>

When using the custom repositories, the archive creation will
omit the source history to get reproducible hashes. This has
the side effect of making the setlocalversion scripts within the
builds to not create the tracking information as expected. So,
just generate a .scmversion so that kernels will be labelled
with where buildroot referenced them.

Example (line wrapped)

    #define PLAIN_VERSION "2018.03-repo-version-" \
                          "d72b9d0908364a64718b652b75e4cac9b85cf0c3"

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 boot/uboot/uboot.mk | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/boot/uboot/uboot.mk b/boot/uboot/uboot.mk
index 4eae8e95c3..d735b47326 100644
--- a/boot/uboot/uboot.mk
+++ b/boot/uboot/uboot.mk
@@ -526,12 +526,25 @@ endif # BR2_TARGET_UBOOT_CUSTOM_TARBALL
 ifeq ($(BR2_TARGET_UBOOT_CUSTOM_GIT)$(BR2_TARGET_UBOOT_CUSTOM_HG),y)
 ifeq ($(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL)),)
 $(error No custom U-Boot repository URL specified. Check your BR2_TARGET_UBOOT_CUSTOM_REPO_URL setting)
-endif # qstrip BR2_TARGET_UBOOT_CUSTOM_CUSTOM_REPO_URL
+endif # qstrip BR2_TARGET_UBOOT_CUSTOM_REPO_URL
 ifeq ($(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_VERSION)),)
 $(error No custom U-Boot repository version specified. Check your BR2_TARGET_UBOOT_CUSTOM_REPO_VERSION setting)
-endif # qstrip BR2_TARGET_UBOOT_CUSTOM_CUSTOM_REPO_VERSION
+endif # qstrip BR2_TARGET_UBOOT_CUSTOM_REPO_VERSION
 endif # BR2_TARGET_UBOOT_CUSTOM_GIT || BR2_TARGET_UBOOT_CUSTOM_HG
 
+# Create a custom scm version file to reflect the source version since the
+# archive will omit source directories like .git to maintain reproducible
+# hashes for the archives
+UBOOT_CUSTOM_REPO_SCMVERSION = \
+	"-repo-version-$(call qstrip,$(BR2_UBOOT_TARGET_CUSTOM_REPO_VERSION))"
+define UBOOT_CUSTOM_REPO_SCMVERSION_HOOK
+	(cd $(@D); \
+		if [ ! -f .scmversion ]; then \
+			echo $(UBOOT_CUSTOM_REPO_SCMVERSION) > .scmversion; \
+		fi)
+endef
+
+UBOOT_POST_EXTRACT_HOOKS += UBOOT_CUSTOM_REPO_SCMVERSION_HOOK
 endif # BR2_TARGET_UBOOT && BR_BUILDING
 
 ifeq ($(BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY),y)
-- 
2.37.1 (Apple Git-137.1)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH 3/4] boot/uboot: add a setlocalversion rsync hook when overriding srcdir

[ckhardin]

From: Charles Hardin <ckhardin@gmail.com>

During development the override srcdir is often used with a local
git repo and branch. To help track the builds used in those flows
it is useful to have the scm version available as part of the info.
With this hook, the auto generated version include will include the
example.

    #define PLAIN_VERSION "2018.03-01358-gbda75d6a8b"
    #define U_BOOT_VERSION "U-Boot " PLAIN_VERSION

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 boot/uboot/uboot.mk | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/boot/uboot/uboot.mk b/boot/uboot/uboot.mk
index d735b47326..3f8479e623 100644
--- a/boot/uboot/uboot.mk
+++ b/boot/uboot/uboot.mk
@@ -40,6 +40,18 @@ UBOOT_SITE = https://ftp.denx.de/pub/u-boot
 UBOOT_SOURCE = u-boot-$(UBOOT_VERSION).tar.bz2
 endif
 
+ifneq ($(UBOOT_OVERRIDE_SRCDIR),)
+define UBOOT_OVERRIDE_SRCDIR_SETLOCALVERSION
+	(cd $(@D); \
+		if [ -x ./scripts/setlocalversion ]; then \
+			./scripts/setlocalversion --save-scmversion \
+				$(abspath $(UBOOT_OVERRIDE_SRCDIR)) ||:; \
+		fi)
+endef
+
+UBOOT_POST_RSYNC_HOOKS += UBOOT_OVERRIDE_SRCDIR_SETLOCALVERSION
+endif
+
 ifeq ($(BR2_TARGET_UBOOT)$(BR2_TARGET_UBOOT_LATEST_VERSION),y)
 BR_NO_CHECK_HASH_FOR += $(UBOOT_SOURCE)
 endif
-- 
2.37.1 (Apple Git-137.1)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH 4/4] linux: add a setlocalversion rsync hook when overriding srcdir

[ckhardin]

From: Charles Hardin <ckhardin@gmail.com>

During development the override srcdir is often used with a local
git repo and branch. To help track the builds used in those flows
it is useful to have the scm version available as part of the info.
With this hook, the auto generated version include will include the
example.

    #define UTS_RELEASE "5.4.70-00031-g92c23d10d003"

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 linux/linux.mk | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/linux/linux.mk b/linux/linux.mk
index 0b27c8dee2..3ef8e555c5 100644
--- a/linux/linux.mk
+++ b/linux/linux.mk
@@ -46,6 +46,18 @@ LINUX_SITE = $(BR2_KERNEL_MIRROR)/linux/kernel/v$(firstword $(subst ., ,$(LINUX_
 endif
 endif
 
+ifneq ($(LINUX_OVERRIDE_SRCDIR),)
+define LINUX_OVERRIDE_SRCDIR_SETLOCALVERSION
+	(cd $(@D); \
+		if [ -x ./scripts/setlocalversion ]; then \
+			./scripts/setlocalversion --save-scmversion \
+				$(abspath $(LINUX_OVERRIDE_SRCDIR)) ||:; \
+		fi)
+endef
+
+LINUX_POST_RSYNC_HOOKS += LINUX_OVERRIDE_SRCDIR_SETLOCALVERSION
+endif
+
 ifeq ($(BR2_LINUX_KERNEL)$(BR2_LINUX_KERNEL_LATEST_VERSION),y)
 BR_NO_CHECK_HASH_FOR += $(LINUX_SOURCE)
 endif
-- 
2.37.1 (Apple Git-137.1)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/4] linux: create a scmversion file based on the custom repo variables

[Arnout Vandecappelle]

  Hi Charles,

  First of all, in patchwork [1] there are about twelve patches from you about 
this subject, but I think all except these four are superseded. Is that correct? 
Could you update patchwork and mark the old ones as superseded?


On 10/02/2023 00:15, ckhardin@gmail.com wrote:
> From: Charles Hardin <ckhardin@gmail.com>
> 
> When using the custom repositories, the archive creation will
> omit the source history to get reproducible hashes. This has
> the side effect of making the setlocalversion scripts within the
> builds to not create the tracking information as expected. So,
> just generate a .scmversion so that kernels will be labelled
> with where buildroot referenced them.

  This is less than ideal because it's completely different than the normal git 
describe output. In particular, if you first develop something in an 
OVERRIDE_SRCDIR (after applying patch 4) and then change the config to point to 
the exact same commit with BR2_LINUX_KERNEL_CUSTOM_REPO_VERSION, it will create 
a completely different UTS_RELEASE string...

  I think a better solution would be to extend the git download wrapper with a 
hook that gets called in the git tree itself. E.g.:

LINUX_SCMVERSION_HOOK = ./scripts/setlocalversion --save-scmversion

  The same hook could then also be used for rsync. And it would be easy to add 
the same feature to barebox, tf-a, ...

  Regards,
  Arnout


> 
> For example (line wrapped)
> 
>      #define UTS_RELEASE "5.4.70-repo-version-" \
>                          "92c23d10d0031e0a13244a1a97e7530d3b53fc6e"
> 
> Signed-off-by: Charles Hardin <ckhardin@gmail.com>
> ---
>   linux/linux.mk | 14 ++++++++++++++
>   1 file changed, 14 insertions(+)
> 
> diff --git a/linux/linux.mk b/linux/linux.mk
> index 7645b5f507..0b27c8dee2 100644
> --- a/linux/linux.mk
> +++ b/linux/linux.mk
> @@ -596,6 +596,20 @@ endif
>   ifeq ($(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL)),)
>   $(error No custom repo URL set. Check your BR2_LINUX_KERNEL_CUSTOM_REPO_URL setting)
>   endif
> +
> +# Create a custom scm version file to reflect the source version since the
> +# archive will omit source directories like .git to maintain reproducible
> +# hashes for the archives
> +LINUX_CUSTOM_REPO_SCMVERSION = \
> +	"-repo-version-$(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_VERSION))"
> +define LINUX_CUSTOM_REPO_SCMVERSION_HOOK
> +	(cd $(@D); \
> +		if [ ! -f .scmversion ]; then \
> +			echo $(LINUX_CUSTOM_REPO_SCMVERSION) > .scmversion; \
> +		fi)
> +endef
> +
> +LINUX_POST_EXTRACT_HOOKS += LINUX_CUSTOM_REPO_SCMVERSION_HOOK
>   endif
>   
>   ifeq ($(BR_BUILDING),y)
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/4] linux: create a scmversion file based on the custom repo variables

[Charles Hardin]

[-- Attachment #1.1: Type: text/plain, Size: 4468 bytes --]



> On Feb 14, 2023, at 1:46 PM, Arnout Vandecappelle <arnout@mind.be> wrote:
> 
> Hi Charles,
> 
> First of all, in patchwork [1] there are about twelve patches from you about this subject, but I think all except these four are superseded. Is that correct? Could you update patchwork and mark the old ones as superseded?
> 

Yeah, sorry - had some mail server changes that kept saying an error but apparently the mail went thru.

> 
> On 10/02/2023 00:15, ckhardin@gmail.com wrote:
>> From: Charles Hardin <ckhardin@gmail.com>
>> When using the custom repositories, the archive creation will
>> omit the source history to get reproducible hashes. This has
>> the side effect of making the setlocalversion scripts within the
>> builds to not create the tracking information as expected. So,
>> just generate a .scmversion so that kernels will be labelled
>> with where buildroot referenced them.
> 
> This is less than ideal because it's completely different than the normal git describe output. In particular, if you first develop something in an OVERRIDE_SRCDIR (after applying patch 4) and then change the config to point to the exact same commit with BR2_LINUX_KERNEL_CUSTOM_REPO_VERSION, it will create a completely different UTS_RELEASE string...

Agreed -I would argue that the OVERRIDE_SRCDIR patch is functionally correct and what is desired particularly for local dev but the CUSTOM_REPO is a bit hackish and broken. Trying to figure out a way to do that.

> 
> I think a better solution would be to extend the git download wrapper with a hook that gets called in the git tree itself. E.g.:
> 
> LINUX_SCMVERSION_HOOK = ./scripts/setlocalversion --save-scmversion
> 
> The same hook could then also be used for rsync. And it would be easy to add the same feature to barebox, tf-a, ...

That is a good idea - but, from what I can tell the git download script does a pull and extract all at once with no hooks being run

from support/download/git
```
#!/usr/bin/env bash

# NOTE: if the output of this backend has to change (e.g. we change what gets
# included in the archive (e.g. LFS), or we change the format of the archive
# (e.g. tar options, compression ratio or method)), we MUST update the format
# version in the variable BR_FMT_VERSION_git, in package/pkg-download.mk.

# We want to catch any unexpected failure, and exit immediately
set -E

… snip snip …

Download from git repo in here

… snip snip …

# Generate the archive.
# We do not want the .git dir; we keep other .git files, in case they are the
# only files in their directory.
# The .git dir would generate non reproducible tarballs as it depends on
# the state of the remote server. It also would generate large tarballs
# (gigabytes for some linux trees) when a full clone took place.
mk_tar_gz "${git_cache}" "${basename}" "${date}" "${output}" ".git/*”
```

So from the above I don’t see where to put a HOOK in from the make scripts - is there an example for this because we cann’t run the hook from the tarball because there is no “.git” history in the tarball and the hook would need to be done at the creation? right?

> 
> Regards,
> Arnout
> 
> 
>> For example (line wrapped)
>>     #define UTS_RELEASE "5.4.70-repo-version-" \
>>                         "92c23d10d0031e0a13244a1a97e7530d3b53fc6e"
>> Signed-off-by: Charles Hardin <ckhardin@gmail.com>
>> ---
>>  linux/linux.mk | 14 ++++++++++++++
>>  1 file changed, 14 insertions(+)
>> diff --git a/linux/linux.mk b/linux/linux.mk
>> index 7645b5f507..0b27c8dee2 100644
>> --- a/linux/linux.mk
>> +++ b/linux/linux.mk
>> @@ -596,6 +596,20 @@ endif
>>  ifeq ($(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL)),)
>>  $(error No custom repo URL set. Check your BR2_LINUX_KERNEL_CUSTOM_REPO_URL setting)
>>  endif
>> +
>> +# Create a custom scm version file to reflect the source version since the
>> +# archive will omit source directories like .git to maintain reproducible
>> +# hashes for the archives
>> +LINUX_CUSTOM_REPO_SCMVERSION = \
>> +	"-repo-version-$(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_VERSION))"
>> +define LINUX_CUSTOM_REPO_SCMVERSION_HOOK
>> +	(cd $(@D); \
>> +		if [ ! -f .scmversion ]; then \
>> +			echo $(LINUX_CUSTOM_REPO_SCMVERSION) > .scmversion; \
>> +		fi)
>> +endef
>> +
>> +LINUX_POST_EXTRACT_HOOKS += LINUX_CUSTOM_REPO_SCMVERSION_HOOK
>>  endif
>>    ifeq ($(BR_BUILDING),y)


[-- Attachment #1.2: Type: text/html, Size: 10757 bytes --]

[-- Attachment #2: Type: text/plain, Size: 150 bytes --]

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH v2 0/3] RFC source control tracking for linux/uboot

[Charles Hardin]

Adding source control tracking into the linux and uboot packages
specifically.

The generic problem is that buildroot needs to make reproducible tarballs
for verification which means the ".git" directories have to exlcuded from
the archive since they can change even thought the contents from the git
reference has not changed.

This leads to the problem that common setlocalversion script will not
work in the build because after the extract there is no access to the
history information given the reasons above.

To mitigate this problem and allow change control tracking for local
development and remote references a somewhat intruisive change needs to
be introduced at the download step for the archive to add an scmversion
during the creation when both the revision history in the source tree
exists and the archive is being created.

Charles Hardin (3):
  support/download: add a helper scipt to generate scmversions
  boot/uboot: add a scmversion rsync hook when overriding srcdir
  linux: add a scmversion rsync hook when overriding srcdir

 boot/uboot/uboot.mk         | 12 +++++++
 linux/linux.mk              | 12 +++++++
 package/pkg-download.mk     |  1 +
 support/download/dl-wrapper |  5 +--
 support/download/git        | 10 ++++++
 support/download/hg         | 10 ++++++
 support/download/scmversion | 63 +++++++++++++++++++++++++++++++++++++
 support/download/svn        | 10 ++++++
 8 files changed, 121 insertions(+), 2 deletions(-)
 create mode 100755 support/download/scmversion

-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH v2 1/3] support/download: add a helper scipt to generate scmversions

[Charles Hardin]

Primarily focused on uboot and linux, getting the scmversion from
the custom repository references is required for change control
tracking off of different builds and pipelines. So, extend the
download framework to generate these files while the locks are
being held for the generation to avoid the git information from
changes during the download process.

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 package/pkg-download.mk     |  1 +
 support/download/dl-wrapper |  5 +--
 support/download/git        | 10 ++++++
 support/download/hg         | 10 ++++++
 support/download/scmversion | 63 +++++++++++++++++++++++++++++++++++++
 support/download/svn        | 10 ++++++
 6 files changed, 97 insertions(+), 2 deletions(-)
 create mode 100755 support/download/scmversion

diff --git a/package/pkg-download.mk b/package/pkg-download.mk
index 0718f21aad..333a53ce7e 100644
--- a/package/pkg-download.mk
+++ b/package/pkg-download.mk
@@ -119,6 +119,7 @@ define DOWNLOAD
 		-n '$($(2)_BASENAME_RAW)' \
 		-N '$($(2)_RAWNAME)' \
 		-o '$($(2)_DL_DIR)/$(notdir $(1))' \
+		$(if $($(2)_SCMVERSION),-s) \
 		$(if $($(2)_GIT_SUBMODULES),-r) \
 		$(if $($(2)_GIT_LFS),-l) \
 		$(foreach uri,$(call DOWNLOAD_URIS,$(1),$(2)),-u $(uri)) \
diff --git a/support/download/dl-wrapper b/support/download/dl-wrapper
index 1e8d6058f6..dfb02f7fe8 100755
--- a/support/download/dl-wrapper
+++ b/support/download/dl-wrapper
@@ -25,7 +25,7 @@ main() {
     local -a uris
 
     # Parse our options; anything after '--' is for the backend
-    while getopts ":c:d:D:o:n:N:H:lrf:u:qp:" OPT; do
+    while getopts ":c:d:D:o:n:N:H:slrf:u:qp:" OPT; do
         case "${OPT}" in
         c)  cset="${OPTARG}";;
         d)  dl_dir="${OPTARG}";;
@@ -34,6 +34,7 @@ main() {
         n)  raw_base_name="${OPTARG}";;
         N)  base_name="${OPTARG}";;
         H)  hfile="${OPTARG}";;
+        s)  scmversion="-s";;
         l)  large_file="-l";;
         r)  recurse="-r";;
         f)  filename="${OPTARG}";;
@@ -129,7 +130,7 @@ main() {
                 -f "${filename}" \
                 -u "${uri}" \
                 -o "${tmpf}" \
-                ${quiet} ${large_file} ${recurse} -- "${@}"
+                ${quiet} ${scmversion} ${large_file} ${recurse} -- "${@}"
         then
             # cd back to keep path coherence
             cd "${OLDPWD}"
diff --git a/support/download/git b/support/download/git
index 1a1c315f73..0a640bbec9 100755
--- a/support/download/git
+++ b/support/download/git
@@ -12,6 +12,7 @@ set -E
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -r          Clone and archive sub-modules.
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Clone from repository at URI.
@@ -51,11 +52,13 @@ _on_error() {
 }
 
 quiet=
+scmversion=0
 large_file=0
 recurse=0
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q; exec >/dev/null;;
+    s)  scmversion=1;;
     l)  large_file=1;;
     r)  recurse=1;;
     o)  output="${OPTARG}";;
@@ -229,3 +232,10 @@ popd >/dev/null
 # the state of the remote server. It also would generate large tarballs
 # (gigabytes for some linux trees) when a full clone took place.
 mk_tar_gz "${git_cache}" "${basename}" "${date}" "${output}" ".git/*"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${base_name}" "${output}"
+    support/download/scmversion "${git_cache}" "${base_name}/.scmversion"
+    post_process_repack "$(pwd)" "${base_name}" "${output}"
+fi
diff --git a/support/download/hg b/support/download/hg
index 768a27e06f..6bf058f0a7 100755
--- a/support/download/hg
+++ b/support/download/hg
@@ -7,6 +7,7 @@ set -e
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Clone from repository at URI.
 #   -c CSET     Use changeset (or revision) CSET.
@@ -16,9 +17,11 @@ set -e
 #   HG       : the hg command to call
 
 quiet=
+scmversion=
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q;;
+    s)  scmversion=1;;
     o)  output="${OPTARG}";;
     u)  uri="${OPTARG}";;
     c)  cset="${OPTARG}";;
@@ -48,3 +51,10 @@ _hg clone ${quiet} "${@}" --noupdate "'${uri}'" "'${basename}'"
 _plain_hg archive ${quiet} --repository "'${basename}'" --type tgz \
             --prefix "'${basename}'" --rev "'${cset}'" \
             - >"${output}"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${base_name}" "${output}"
+    support/download/scmversion "${git_cache}" "${base_name}/.scmversion"
+    post_process_repack "$(pwd)" "${base_name}" "${output}"
+fi
diff --git a/support/download/scmversion b/support/download/scmversion
new file mode 100755
index 0000000000..16868a11e5
--- /dev/null
+++ b/support/download/scmversion
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+# Helper to generate an scmversion in a dowload script before
+# the tarball is created. Because the tarball has to exclude
+# repository directories like .git and .hg to make reproducible
+# archives on subsequent downloads the version information is
+# not available during the builds for a script like setlocalversion
+# to run.
+#
+# So, this is hook to call right before the make tarball gz that
+# will generate a .scmversion file that will be included in an
+# archive and then available during the build steps.
+#
+# Call it with:
+#   $1: the path to the srctree (see mk_tar_gz in helpers)
+#   $2: output file
+#
+# Because this can fail for a variety of reason, there is no exit
+# code to avoid build breakage. It is assumed the scripts will be
+# reproducible because they are based on the contents that get generated
+
+while getopts :q OPT; do
+    case "${OPT}" in
+    q)  exec >/dev/null;;
+    \?) exit 1;;
+    esac
+done
+shift $((OPTIND-1))
+
+srctree="${1}"
+scmversion="${2}"
+
+# Bail early if no srctree or scmversion
+if [ -z "${srctree}" -o -z "${scmversion}" ]; then
+    exit 0
+fi
+
+# Does the scmversion exist and readonly
+if [ -f "${scmversion}" -a ! -w "${scmversion}" ]; then
+    printf "WARNING: scmversion %s is readonly\n" "${scmversion}" >&2
+    exit 0
+fi
+
+#
+# Generate the scmversion from some expected locations in srctree
+# and fallback to the buildroot version of setlocalversion another
+# script is not used
+#
+if [ -x "${srctree}/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./setlocalversion)
+elif [ -x "${srctree}/scripts/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./scripts/setlocalversion)
+elif [ -x "${srctree}/tools/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./tools/setlocalversion)
+fi
+if [ -z "${res}" ]; then
+    res=$("${0%/*}/../scripts/setlocalversion" "${srctree}")
+fi
+
+[ -n "${res}" ] && (echo "${res}" > "${scmversion}")
+exit 0
diff --git a/support/download/svn b/support/download/svn
index b23b7773d3..89da736172 100755
--- a/support/download/svn
+++ b/support/download/svn
@@ -12,6 +12,7 @@ set -e
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Checkout from repository at URI.
 #   -c REV      Use revision REV.
@@ -23,9 +24,11 @@ set -e
 . "${0%/*}/helpers"
 
 quiet=
+scmversion=0
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q;;
+    s)  scmversion=1;;
     o)  output="${OPTARG}";;
     u)  uri="${OPTARG}";;
     c)  rev="${OPTARG}";;
@@ -65,3 +68,10 @@ date="$( _plain_svn info "'${uri}@${rev}'" \
 # We did a 'svn export' above, so it's not a working copy (there is no .svn
 # directory or file to ignore).
 mk_tar_gz "${basename}" "${basename}" "${date}" "${output}"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${base_name}" "${output}"
+    support/download/scmversion "${git_cache}" "${base_name}/.scmversion"
+    post_process_repack "$(pwd)" "${base_name}" "${output}"
+fi
-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH v2 2/3] boot/uboot: add a scmversion rsync hook when overriding srcdir

[Charles Hardin]

During development the override srcdir is often used with a local
git repo and branch. To help track the builds used in those flows
it is useful to have the scm version available as part of the info.
With this hook, the auto generated version include will include the
example.

    #define PLAIN_VERSION "2018.03-01358-gbda75d6a8b"
    #define U_BOOT_VERSION "U-Boot " PLAIN_VERSION

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 boot/uboot/uboot.mk | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/boot/uboot/uboot.mk b/boot/uboot/uboot.mk
index 4eae8e95c3..edb6c646c7 100644
--- a/boot/uboot/uboot.mk
+++ b/boot/uboot/uboot.mk
@@ -28,18 +28,30 @@ UBOOT_SOURCE = $(notdir $(UBOOT_TARBALL))
 else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_GIT),y)
 UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
 UBOOT_SITE_METHOD = git
+UBOOT_SCMVERSION = YES
 else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_HG),y)
 UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
 UBOOT_SITE_METHOD = hg
+UBOOT_SCMVERSION = YES
 else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_SVN),y)
 UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
 UBOOT_SITE_METHOD = svn
+UBOOT_SCMVERSION = YES
 else
 # Handle stable official U-Boot versions
 UBOOT_SITE = https://ftp.denx.de/pub/u-boot
 UBOOT_SOURCE = u-boot-$(UBOOT_VERSION).tar.bz2
 endif
 
+ifneq ($(UBOOT_OVERRIDE_SRCDIR),)
+define UBOOT_SCMVERSION_HOOK
+	$(TOPDIR)/support/download/scmversion \
+		$(abspath $(UBOOT_OVERRIDE_SRCDIR)) $(@D)/.scmversion
+endef
+
+UBOOT_POST_RSYNC_HOOKS += UBOOT_SCMVERSION_HOOK
+endif
+
 ifeq ($(BR2_TARGET_UBOOT)$(BR2_TARGET_UBOOT_LATEST_VERSION),y)
 BR_NO_CHECK_HASH_FOR += $(UBOOT_SOURCE)
 endif
-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH v2 3/3] linux: add a scmversion rsync hook when overriding srcdir

[Charles Hardin]

During development the override srcdir is often used with a local
git repo and branch. To help track the builds used in those flows
it is useful to have the scm version available as part of the info.
With this hook, the auto generated version include will include the
example.

    #define UTS_RELEASE "5.4.70-00031-g92c23d10d003"

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 linux/linux.mk | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/linux/linux.mk b/linux/linux.mk
index 7645b5f507..b72318e3f2 100644
--- a/linux/linux.mk
+++ b/linux/linux.mk
@@ -24,12 +24,15 @@ LINUX_SOURCE = $(notdir $(LINUX_TARBALL))
 else ifeq ($(BR2_LINUX_KERNEL_CUSTOM_GIT),y)
 LINUX_SITE = $(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL))
 LINUX_SITE_METHOD = git
+LINUX_SCMVERSION = YES
 else ifeq ($(BR2_LINUX_KERNEL_CUSTOM_HG),y)
 LINUX_SITE = $(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL))
 LINUX_SITE_METHOD = hg
+LINUX_SCMVERSION = YES
 else ifeq ($(BR2_LINUX_KERNEL_CUSTOM_SVN),y)
 LINUX_SITE = $(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL))
 LINUX_SITE_METHOD = svn
+LINUX_SCMVERSION = YES
 else ifeq ($(BR2_LINUX_KERNEL_LATEST_CIP_VERSION)$(BR2_LINUX_KERNEL_LATEST_CIP_RT_VERSION),y)
 LINUX_SOURCE = linux-cip-$(LINUX_VERSION).tar.gz
 LINUX_SITE = https://git.kernel.org/pub/scm/linux/kernel/git/cip/linux-cip.git/snapshot
@@ -46,6 +49,15 @@ LINUX_SITE = $(BR2_KERNEL_MIRROR)/linux/kernel/v$(firstword $(subst ., ,$(LINUX_
 endif
 endif
 
+ifneq ($(LINUX_OVERRIDE_SRCDIR),)
+define LINUX_SCMVERSION_HOOK
+	$(TOPDIR)/support/download/scmversion \
+		$(abspath $(LINUX_OVERRIDE_SRCDIR)) $(@D)/.scmversion
+endef
+
+LINUX_POST_RSYNC_HOOKS += LINUX_SCMVERSION_HOOK
+endif
+
 ifeq ($(BR2_LINUX_KERNEL)$(BR2_LINUX_KERNEL_LATEST_VERSION),y)
 BR_NO_CHECK_HASH_FOR += $(LINUX_SOURCE)
 endif
-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] RFC [PATCH v2 0/3] scmversion tracking for linux and uboot

[Charles Hardin]

Adding source control tracking into the linux and uboot packages
specifically.

The generic problem is that buildroot needs to make reproducible tarballs
for verification which means the ".git" directories have to be excluded
from the archive since they can change even though the contents from the
git reference has not changed.

This leads to the problem that a setlocalversion script will not
work in the build because after the extract there is no access to the
history information given the reasons above.

To mitigate this problem and allow change control tracking for local
development and remote references a somewhat intruisive change needs to
be introduced at the download step for the archive to add an scmversion
during the creation when both the revision history in the source tree
exists and the archive is being created.

Charles Hardin (3):
  support/download: add a helper scipt to generate scmversions
  boot/uboot: add a scmversion rsync hook when overriding srcdir
  linux: add a scmversion rsync hook when overriding srcdir

 boot/uboot/uboot.mk         | 12 +++++++
 linux/linux.mk              | 12 +++++++
 package/pkg-download.mk     |  1 +
 support/download/dl-wrapper |  5 +--
 support/download/git        | 10 ++++++
 support/download/hg         | 10 ++++++
 support/download/scmversion | 63 +++++++++++++++++++++++++++++++++++++
 support/download/svn        | 10 ++++++
 8 files changed, 121 insertions(+), 2 deletions(-)
 create mode 100755 support/download/scmversion

-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH v2 1/3] support/download: add a helper scipt to generate scmversions

[Charles Hardin]

Primarily focused on uboot and linux, getting the scmversion from
the custom repository references is required for change control
tracking off of different builds and pipelines. So, extend the
download framework to generate these files while the locks are
being held for the generation to avoid the git information from
changes during the download process.

Signed-off-by: Charles Hardin <ckhardin@gmail.com>

wip scmversion

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 package/pkg-download.mk     |  1 +
 support/download/dl-wrapper |  5 +--
 support/download/git        | 10 ++++++
 support/download/hg         | 10 ++++++
 support/download/scmversion | 63 +++++++++++++++++++++++++++++++++++++
 support/download/svn        | 10 ++++++
 6 files changed, 97 insertions(+), 2 deletions(-)
 create mode 100755 support/download/scmversion

diff --git a/package/pkg-download.mk b/package/pkg-download.mk
index 0718f21aad..333a53ce7e 100644
--- a/package/pkg-download.mk
+++ b/package/pkg-download.mk
@@ -119,6 +119,7 @@ define DOWNLOAD
 		-n '$($(2)_BASENAME_RAW)' \
 		-N '$($(2)_RAWNAME)' \
 		-o '$($(2)_DL_DIR)/$(notdir $(1))' \
+		$(if $($(2)_SCMVERSION),-s) \
 		$(if $($(2)_GIT_SUBMODULES),-r) \
 		$(if $($(2)_GIT_LFS),-l) \
 		$(foreach uri,$(call DOWNLOAD_URIS,$(1),$(2)),-u $(uri)) \
diff --git a/support/download/dl-wrapper b/support/download/dl-wrapper
index 1e8d6058f6..dfb02f7fe8 100755
--- a/support/download/dl-wrapper
+++ b/support/download/dl-wrapper
@@ -25,7 +25,7 @@ main() {
     local -a uris
 
     # Parse our options; anything after '--' is for the backend
-    while getopts ":c:d:D:o:n:N:H:lrf:u:qp:" OPT; do
+    while getopts ":c:d:D:o:n:N:H:slrf:u:qp:" OPT; do
         case "${OPT}" in
         c)  cset="${OPTARG}";;
         d)  dl_dir="${OPTARG}";;
@@ -34,6 +34,7 @@ main() {
         n)  raw_base_name="${OPTARG}";;
         N)  base_name="${OPTARG}";;
         H)  hfile="${OPTARG}";;
+        s)  scmversion="-s";;
         l)  large_file="-l";;
         r)  recurse="-r";;
         f)  filename="${OPTARG}";;
@@ -129,7 +130,7 @@ main() {
                 -f "${filename}" \
                 -u "${uri}" \
                 -o "${tmpf}" \
-                ${quiet} ${large_file} ${recurse} -- "${@}"
+                ${quiet} ${scmversion} ${large_file} ${recurse} -- "${@}"
         then
             # cd back to keep path coherence
             cd "${OLDPWD}"
diff --git a/support/download/git b/support/download/git
index 1a1c315f73..277c412e26 100755
--- a/support/download/git
+++ b/support/download/git
@@ -12,6 +12,7 @@ set -E
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -r          Clone and archive sub-modules.
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Clone from repository at URI.
@@ -51,11 +52,13 @@ _on_error() {
 }
 
 quiet=
+scmversion=0
 large_file=0
 recurse=0
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q; exec >/dev/null;;
+    s)  scmversion=1;;
     l)  large_file=1;;
     r)  recurse=1;;
     o)  output="${OPTARG}";;
@@ -229,3 +232,10 @@ popd >/dev/null
 # the state of the remote server. It also would generate large tarballs
 # (gigabytes for some linux trees) when a full clone took place.
 mk_tar_gz "${git_cache}" "${basename}" "${date}" "${output}" ".git/*"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${basename}" "${output}"
+    support/download/scmversion "${git_cache}" "${basename}/.scmversion"
+    post_process_repack "$(pwd)" "${basename}" "${output}"
+fi
diff --git a/support/download/hg b/support/download/hg
index 768a27e06f..dcf36e6e1f 100755
--- a/support/download/hg
+++ b/support/download/hg
@@ -7,6 +7,7 @@ set -e
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Clone from repository at URI.
 #   -c CSET     Use changeset (or revision) CSET.
@@ -16,9 +17,11 @@ set -e
 #   HG       : the hg command to call
 
 quiet=
+scmversion=
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q;;
+    s)  scmversion=1;;
     o)  output="${OPTARG}";;
     u)  uri="${OPTARG}";;
     c)  cset="${OPTARG}";;
@@ -48,3 +51,10 @@ _hg clone ${quiet} "${@}" --noupdate "'${uri}'" "'${basename}'"
 _plain_hg archive ${quiet} --repository "'${basename}'" --type tgz \
             --prefix "'${basename}'" --rev "'${cset}'" \
             - >"${output}"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${basename}" "${output}"
+    support/download/scmversion "${@}" "${basename}/.scmversion"
+    post_process_repack "$(pwd)" "${basename}" "${output}"
+fi
diff --git a/support/download/scmversion b/support/download/scmversion
new file mode 100755
index 0000000000..16868a11e5
--- /dev/null
+++ b/support/download/scmversion
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+# Helper to generate an scmversion in a dowload script before
+# the tarball is created. Because the tarball has to exclude
+# repository directories like .git and .hg to make reproducible
+# archives on subsequent downloads the version information is
+# not available during the builds for a script like setlocalversion
+# to run.
+#
+# So, this is hook to call right before the make tarball gz that
+# will generate a .scmversion file that will be included in an
+# archive and then available during the build steps.
+#
+# Call it with:
+#   $1: the path to the srctree (see mk_tar_gz in helpers)
+#   $2: output file
+#
+# Because this can fail for a variety of reason, there is no exit
+# code to avoid build breakage. It is assumed the scripts will be
+# reproducible because they are based on the contents that get generated
+
+while getopts :q OPT; do
+    case "${OPT}" in
+    q)  exec >/dev/null;;
+    \?) exit 1;;
+    esac
+done
+shift $((OPTIND-1))
+
+srctree="${1}"
+scmversion="${2}"
+
+# Bail early if no srctree or scmversion
+if [ -z "${srctree}" -o -z "${scmversion}" ]; then
+    exit 0
+fi
+
+# Does the scmversion exist and readonly
+if [ -f "${scmversion}" -a ! -w "${scmversion}" ]; then
+    printf "WARNING: scmversion %s is readonly\n" "${scmversion}" >&2
+    exit 0
+fi
+
+#
+# Generate the scmversion from some expected locations in srctree
+# and fallback to the buildroot version of setlocalversion another
+# script is not used
+#
+if [ -x "${srctree}/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./setlocalversion)
+elif [ -x "${srctree}/scripts/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./scripts/setlocalversion)
+elif [ -x "${srctree}/tools/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./tools/setlocalversion)
+fi
+if [ -z "${res}" ]; then
+    res=$("${0%/*}/../scripts/setlocalversion" "${srctree}")
+fi
+
+[ -n "${res}" ] && (echo "${res}" > "${scmversion}")
+exit 0
diff --git a/support/download/svn b/support/download/svn
index b23b7773d3..b0e9f69a6b 100755
--- a/support/download/svn
+++ b/support/download/svn
@@ -12,6 +12,7 @@ set -e
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Checkout from repository at URI.
 #   -c REV      Use revision REV.
@@ -23,9 +24,11 @@ set -e
 . "${0%/*}/helpers"
 
 quiet=
+scmversion=0
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q;;
+    s)  scmversion=1;;
     o)  output="${OPTARG}";;
     u)  uri="${OPTARG}";;
     c)  rev="${OPTARG}";;
@@ -65,3 +68,10 @@ date="$( _plain_svn info "'${uri}@${rev}'" \
 # We did a 'svn export' above, so it's not a working copy (there is no .svn
 # directory or file to ignore).
 mk_tar_gz "${basename}" "${basename}" "${date}" "${output}"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${basename}" "${output}"
+    support/download/scmversion "${basename}" "${basename}/.scmversion"
+    post_process_repack "$(pwd)" "${basename}" "${output}"
+fi
-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH v2 1/3] support/download: add a helper scipt to generate scmversions

[Arnout Vandecappelle via buildroot]

  Hi Charles,

  Sorry that it took so long to get back to this. I can imagine that you're no 
longer motivated to work on this, but perhaps someone else is willing to pick it up.

  We discussed this in the BR hackaton and we think there should be a better way 
to achieve this.

On 22/02/2023 23:30, Charles Hardin wrote:
> Primarily focused on uboot and linux, getting the scmversion from
> the custom repository references is required for change control
> tracking off of different builds and pipelines. So, extend the
> download framework to generate these files while the locks are
> being held for the generation to avoid the git information from
> changes during the download process.
> 
> Signed-off-by: Charles Hardin <ckhardin@gmail.com>
> 
> wip scmversion
> 
> Signed-off-by: Charles Hardin <ckhardin@gmail.com>
> ---
>   package/pkg-download.mk     |  1 +
>   support/download/dl-wrapper |  5 +--
>   support/download/git        | 10 ++++++
>   support/download/hg         | 10 ++++++
>   support/download/scmversion | 63 +++++++++++++++++++++++++++++++++++++
>   support/download/svn        | 10 ++++++
>   6 files changed, 97 insertions(+), 2 deletions(-)
>   create mode 100755 support/download/scmversion
> 
> diff --git a/package/pkg-download.mk b/package/pkg-download.mk
> index 0718f21aad..333a53ce7e 100644
> --- a/package/pkg-download.mk
> +++ b/package/pkg-download.mk
> @@ -119,6 +119,7 @@ define DOWNLOAD
>   		-n '$($(2)_BASENAME_RAW)' \
>   		-N '$($(2)_RAWNAME)' \
>   		-o '$($(2)_DL_DIR)/$(notdir $(1))' \
> +		$(if $($(2)_SCMVERSION),-s) \

  It's a pity that we need all this forwarding of arguments, but that's how the 
infra is built :-(

>   		$(if $($(2)_GIT_SUBMODULES),-r) \
>   		$(if $($(2)_GIT_LFS),-l) \
>   		$(foreach uri,$(call DOWNLOAD_URIS,$(1),$(2)),-u $(uri)) \


[snip]

> diff --git a/support/download/git b/support/download/git
> index 1a1c315f73..277c412e26 100755
> --- a/support/download/git
> +++ b/support/download/git
> @@ -12,6 +12,7 @@ set -E
>   #
>   # Options:
>   #   -q          Be quiet.
> +#   -s          Generate an scmversion file
>   #   -r          Clone and archive sub-modules.
>   #   -o FILE     Generate archive in FILE.
>   #   -u URI      Clone from repository at URI.
> @@ -51,11 +52,13 @@ _on_error() {
>   }
>   
>   quiet=
> +scmversion=0
>   large_file=0
>   recurse=0
>   while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
>       case "${OPT}" in
>       q)  quiet=-q; exec >/dev/null;;
> +    s)  scmversion=1;;
>       l)  large_file=1;;
>       r)  recurse=1;;
>       o)  output="${OPTARG}";;
> @@ -229,3 +232,10 @@ popd >/dev/null
>   # the state of the remote server. It also would generate large tarballs
>   # (gigabytes for some linux trees) when a full clone took place.
>   mk_tar_gz "${git_cache}" "${basename}" "${date}" "${output}" ".git/*"
> +
> +# If an scmversion is needed then generate the version information
> +if [ ${scmversion} -eq 1 ]; then
> +    post_process_unpack "${basename}" "${output}"
> +    support/download/scmversion "${git_cache}" "${basename}/.scmversion"
> +    post_process_repack "$(pwd)" "${basename}" "${output}"
> +fi

  AFAICS, there's no need to unpack like this. You can simply do it before the 
tarball is created (i.e. before the mk_tar_gz call).


[snip]

> diff --git a/support/download/scmversion b/support/download/scmversion
> new file mode 100755
> index 0000000000..16868a11e5
> --- /dev/null
> +++ b/support/download/scmversion
> @@ -0,0 +1,63 @@
> +#!/usr/bin/env bash
> +set -x
> +set -e
> +
> +# Helper to generate an scmversion in a dowload script before
> +# the tarball is created. Because the tarball has to exclude
> +# repository directories like .git and .hg to make reproducible
> +# archives on subsequent downloads the version information is
> +# not available during the builds for a script like setlocalversion
> +# to run.
> +#
> +# So, this is hook to call right before the make tarball gz that
> +# will generate a .scmversion file that will be included in an
> +# archive and then available during the build steps.
> +#
> +# Call it with:
> +#   $1: the path to the srctree (see mk_tar_gz in helpers)
> +#   $2: output file
> +#
> +# Because this can fail for a variety of reason, there is no exit
> +# code to avoid build breakage. It is assumed the scripts will be
> +# reproducible because they are based on the contents that get generated
> +
> +while getopts :q OPT; do
> +    case "${OPT}" in
> +    q)  exec >/dev/null;;
> +    \?) exit 1;;
> +    esac
> +done
> +shift $((OPTIND-1))
> +
> +srctree="${1}"
> +scmversion="${2}"
> +
> +# Bail early if no srctree or scmversion
> +if [ -z "${srctree}" -o -z "${scmversion}" ]; then
> +    exit 0
> +fi
> +
> +# Does the scmversion exist and readonly
> +if [ -f "${scmversion}" -a ! -w "${scmversion}" ]; then
> +    printf "WARNING: scmversion %s is readonly\n" "${scmversion}" >&2
> +    exit 0
> +fi
> +
> +#
> +# Generate the scmversion from some expected locations in srctree
> +# and fallback to the buildroot version of setlocalversion another
> +# script is not used
> +#
> +if [ -x "${srctree}/setlocalversion" ]; then
> +    res=$(cd "${srctree}"; ./setlocalversion)
> +elif [ -x "${srctree}/scripts/setlocalversion" ]; then
> +    res=$(cd "${srctree}"; ./scripts/setlocalversion)
> +elif [ -x "${srctree}/tools/setlocalversion" ]; then
> +    res=$(cd "${srctree}"; ./tools/setlocalversion)
> +fi
> +if [ -z "${res}" ]; then
> +    res=$("${0%/*}/../scripts/setlocalversion" "${srctree}")
> +fi

  This is where it gets hairy. We add all this complexity, and in the end we 
call a script that needs to do some heuristics to find out which script to call 
exactly.

  What we actually want is for the script that needs to be called to be 
specified in the .mk file, because there you know exactly what it should be 
called. Something like:

define UBOOT_SET_SCMVERSION
	./tools/setlocalversion > .scmversion
endef


  The trouble is how to pass that through the download infrastructure to make 
sure the git download helper can use it. You'll still need a new option for 
that. Then you can pass it trough (properly quoted) and finally in the git 
download helper:

if [ -n "${setscmversion}" ]; then
     eval "${setscmversion}"
fi


  Regards,
  Arnout
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH v2 2/3] boot/uboot: add a scmversion rsync hook when overriding srcdir

[Charles Hardin]

During development the override srcdir is often used with a local
git repo and branch. To help track the builds used in those flows
it is useful to have the scm version available as part of the info.
With this hook, the auto generated version include will include the
example.

    #define PLAIN_VERSION "2018.03-01358-gbda75d6a8b"
    #define U_BOOT_VERSION "U-Boot " PLAIN_VERSION

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 boot/uboot/uboot.mk | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/boot/uboot/uboot.mk b/boot/uboot/uboot.mk
index 4eae8e95c3..edb6c646c7 100644
--- a/boot/uboot/uboot.mk
+++ b/boot/uboot/uboot.mk
@@ -28,18 +28,30 @@ UBOOT_SOURCE = $(notdir $(UBOOT_TARBALL))
 else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_GIT),y)
 UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
 UBOOT_SITE_METHOD = git
+UBOOT_SCMVERSION = YES
 else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_HG),y)
 UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
 UBOOT_SITE_METHOD = hg
+UBOOT_SCMVERSION = YES
 else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_SVN),y)
 UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
 UBOOT_SITE_METHOD = svn
+UBOOT_SCMVERSION = YES
 else
 # Handle stable official U-Boot versions
 UBOOT_SITE = https://ftp.denx.de/pub/u-boot
 UBOOT_SOURCE = u-boot-$(UBOOT_VERSION).tar.bz2
 endif
 
+ifneq ($(UBOOT_OVERRIDE_SRCDIR),)
+define UBOOT_SCMVERSION_HOOK
+	$(TOPDIR)/support/download/scmversion \
+		$(abspath $(UBOOT_OVERRIDE_SRCDIR)) $(@D)/.scmversion
+endef
+
+UBOOT_POST_RSYNC_HOOKS += UBOOT_SCMVERSION_HOOK
+endif
+
 ifeq ($(BR2_TARGET_UBOOT)$(BR2_TARGET_UBOOT_LATEST_VERSION),y)
 BR_NO_CHECK_HASH_FOR += $(UBOOT_SOURCE)
 endif
-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH v2 2/3] boot/uboot: add a scmversion rsync hook when overriding srcdir

[Arnout Vandecappelle via buildroot]

On 22/02/2023 23:30, Charles Hardin wrote:
> During development the override srcdir is often used with a local
> git repo and branch. To help track the builds used in those flows
> it is useful to have the scm version available as part of the info.
> With this hook, the auto generated version include will include the
> example.
> 
>      #define PLAIN_VERSION "2018.03-01358-gbda75d6a8b"
>      #define U_BOOT_VERSION "U-Boot " PLAIN_VERSION
> 
> Signed-off-by: Charles Hardin <ckhardin@gmail.com>
> ---
>   boot/uboot/uboot.mk | 12 ++++++++++++
>   1 file changed, 12 insertions(+)
> 
> diff --git a/boot/uboot/uboot.mk b/boot/uboot/uboot.mk
> index 4eae8e95c3..edb6c646c7 100644
> --- a/boot/uboot/uboot.mk
> +++ b/boot/uboot/uboot.mk
> @@ -28,18 +28,30 @@ UBOOT_SOURCE = $(notdir $(UBOOT_TARBALL))
>   else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_GIT),y)
>   UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
>   UBOOT_SITE_METHOD = git
> +UBOOT_SCMVERSION = YES
>   else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_HG),y)
>   UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
>   UBOOT_SITE_METHOD = hg
> +UBOOT_SCMVERSION = YES
>   else ifeq ($(BR2_TARGET_UBOOT_CUSTOM_SVN),y)
>   UBOOT_SITE = $(call qstrip,$(BR2_TARGET_UBOOT_CUSTOM_REPO_URL))
>   UBOOT_SITE_METHOD = svn
> +UBOOT_SCMVERSION = YES
>   else
>   # Handle stable official U-Boot versions
>   UBOOT_SITE = https://ftp.denx.de/pub/u-boot
>   UBOOT_SOURCE = u-boot-$(UBOOT_VERSION).tar.bz2
>   endif
>   
> +ifneq ($(UBOOT_OVERRIDE_SRCDIR),)
> +define UBOOT_SCMVERSION_HOOK
> +	$(TOPDIR)/support/download/scmversion \
> +		$(abspath $(UBOOT_OVERRIDE_SRCDIR)) $(@D)/.scmversion
> +endef
> +
> +UBOOT_POST_RSYNC_HOOKS += UBOOT_SCMVERSION_HOOK

  I think this override srcdir handling can actually be done directly in the 
generic infrastructure. Maybe even directly in the rsync target:

$(BUILD_DIR)/%/.stamp_rsynced:
	...
	rsync ...
         $(if $($(PKG)_SET_SCMVERSION),\
		@cd $(SRCIDR); $($(PKG)_SET_SCMVERSION))
	...

  The only problem is that here, the output has to go to the output directory 
while in the git case it has to go the current directory...

  Regards,
  Arnout

> +endif
> +
>   ifeq ($(BR2_TARGET_UBOOT)$(BR2_TARGET_UBOOT_LATEST_VERSION),y)
>   BR_NO_CHECK_HASH_FOR += $(UBOOT_SOURCE)
>   endif
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

[Buildroot] [PATCH v2 3/3] linux: add a scmversion rsync hook when overriding srcdir

[Charles Hardin]

During development the override srcdir is often used with a local
git repo and branch. To help track the builds used in those flows
it is useful to have the scm version available as part of the info.
With this hook, the auto generated version include will include the
example.

    #define UTS_RELEASE "5.4.70-00031-g92c23d10d003"

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 linux/linux.mk | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/linux/linux.mk b/linux/linux.mk
index 7645b5f507..b72318e3f2 100644
--- a/linux/linux.mk
+++ b/linux/linux.mk
@@ -24,12 +24,15 @@ LINUX_SOURCE = $(notdir $(LINUX_TARBALL))
 else ifeq ($(BR2_LINUX_KERNEL_CUSTOM_GIT),y)
 LINUX_SITE = $(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL))
 LINUX_SITE_METHOD = git
+LINUX_SCMVERSION = YES
 else ifeq ($(BR2_LINUX_KERNEL_CUSTOM_HG),y)
 LINUX_SITE = $(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL))
 LINUX_SITE_METHOD = hg
+LINUX_SCMVERSION = YES
 else ifeq ($(BR2_LINUX_KERNEL_CUSTOM_SVN),y)
 LINUX_SITE = $(call qstrip,$(BR2_LINUX_KERNEL_CUSTOM_REPO_URL))
 LINUX_SITE_METHOD = svn
+LINUX_SCMVERSION = YES
 else ifeq ($(BR2_LINUX_KERNEL_LATEST_CIP_VERSION)$(BR2_LINUX_KERNEL_LATEST_CIP_RT_VERSION),y)
 LINUX_SOURCE = linux-cip-$(LINUX_VERSION).tar.gz
 LINUX_SITE = https://git.kernel.org/pub/scm/linux/kernel/git/cip/linux-cip.git/snapshot
@@ -46,6 +49,15 @@ LINUX_SITE = $(BR2_KERNEL_MIRROR)/linux/kernel/v$(firstword $(subst ., ,$(LINUX_
 endif
 endif
 
+ifneq ($(LINUX_OVERRIDE_SRCDIR),)
+define LINUX_SCMVERSION_HOOK
+	$(TOPDIR)/support/download/scmversion \
+		$(abspath $(LINUX_OVERRIDE_SRCDIR)) $(@D)/.scmversion
+endef
+
+LINUX_POST_RSYNC_HOOKS += LINUX_SCMVERSION_HOOK
+endif
+
 ifeq ($(BR2_LINUX_KERNEL)$(BR2_LINUX_KERNEL_LATEST_VERSION),y)
 BR_NO_CHECK_HASH_FOR += $(LINUX_SOURCE)
 endif
-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot