[Buildroot] [PATCH v2 1/3] support/download: add a helper scipt to generate scmversions

[Charles Hardin <ckhardin@gmail.com>]

Primarily focused on uboot and linux, getting the scmversion from
the custom repository references is required for change control
tracking off of different builds and pipelines. So, extend the
download framework to generate these files while the locks are
being held for the generation to avoid the git information from
changes during the download process.

Signed-off-by: Charles Hardin <ckhardin@gmail.com>
---
 package/pkg-download.mk     |  1 +
 support/download/dl-wrapper |  5 +--
 support/download/git        | 10 ++++++
 support/download/hg         | 10 ++++++
 support/download/scmversion | 63 +++++++++++++++++++++++++++++++++++++
 support/download/svn        | 10 ++++++
 6 files changed, 97 insertions(+), 2 deletions(-)
 create mode 100755 support/download/scmversion

diff --git a/package/pkg-download.mk b/package/pkg-download.mk
index 0718f21aad..333a53ce7e 100644
--- a/package/pkg-download.mk
+++ b/package/pkg-download.mk
@@ -119,6 +119,7 @@ define DOWNLOAD
 		-n '$($(2)_BASENAME_RAW)' \
 		-N '$($(2)_RAWNAME)' \
 		-o '$($(2)_DL_DIR)/$(notdir $(1))' \
+		$(if $($(2)_SCMVERSION),-s) \
 		$(if $($(2)_GIT_SUBMODULES),-r) \
 		$(if $($(2)_GIT_LFS),-l) \
 		$(foreach uri,$(call DOWNLOAD_URIS,$(1),$(2)),-u $(uri)) \
diff --git a/support/download/dl-wrapper b/support/download/dl-wrapper
index 1e8d6058f6..dfb02f7fe8 100755
--- a/support/download/dl-wrapper
+++ b/support/download/dl-wrapper
@@ -25,7 +25,7 @@ main() {
     local -a uris
 
     # Parse our options; anything after '--' is for the backend
-    while getopts ":c:d:D:o:n:N:H:lrf:u:qp:" OPT; do
+    while getopts ":c:d:D:o:n:N:H:slrf:u:qp:" OPT; do
         case "${OPT}" in
         c)  cset="${OPTARG}";;
         d)  dl_dir="${OPTARG}";;
@@ -34,6 +34,7 @@ main() {
         n)  raw_base_name="${OPTARG}";;
         N)  base_name="${OPTARG}";;
         H)  hfile="${OPTARG}";;
+        s)  scmversion="-s";;
         l)  large_file="-l";;
         r)  recurse="-r";;
         f)  filename="${OPTARG}";;
@@ -129,7 +130,7 @@ main() {
                 -f "${filename}" \
                 -u "${uri}" \
                 -o "${tmpf}" \
-                ${quiet} ${large_file} ${recurse} -- "${@}"
+                ${quiet} ${scmversion} ${large_file} ${recurse} -- "${@}"
         then
             # cd back to keep path coherence
             cd "${OLDPWD}"
diff --git a/support/download/git b/support/download/git
index 1a1c315f73..0a640bbec9 100755
--- a/support/download/git
+++ b/support/download/git
@@ -12,6 +12,7 @@ set -E
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -r          Clone and archive sub-modules.
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Clone from repository at URI.
@@ -51,11 +52,13 @@ _on_error() {
 }
 
 quiet=
+scmversion=0
 large_file=0
 recurse=0
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q; exec >/dev/null;;
+    s)  scmversion=1;;
     l)  large_file=1;;
     r)  recurse=1;;
     o)  output="${OPTARG}";;
@@ -229,3 +232,10 @@ popd >/dev/null
 # the state of the remote server. It also would generate large tarballs
 # (gigabytes for some linux trees) when a full clone took place.
 mk_tar_gz "${git_cache}" "${basename}" "${date}" "${output}" ".git/*"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${base_name}" "${output}"
+    support/download/scmversion "${git_cache}" "${base_name}/.scmversion"
+    post_process_repack "$(pwd)" "${base_name}" "${output}"
+fi
diff --git a/support/download/hg b/support/download/hg
index 768a27e06f..6bf058f0a7 100755
--- a/support/download/hg
+++ b/support/download/hg
@@ -7,6 +7,7 @@ set -e
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Clone from repository at URI.
 #   -c CSET     Use changeset (or revision) CSET.
@@ -16,9 +17,11 @@ set -e
 #   HG       : the hg command to call
 
 quiet=
+scmversion=
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q;;
+    s)  scmversion=1;;
     o)  output="${OPTARG}";;
     u)  uri="${OPTARG}";;
     c)  cset="${OPTARG}";;
@@ -48,3 +51,10 @@ _hg clone ${quiet} "${@}" --noupdate "'${uri}'" "'${basename}'"
 _plain_hg archive ${quiet} --repository "'${basename}'" --type tgz \
             --prefix "'${basename}'" --rev "'${cset}'" \
             - >"${output}"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${base_name}" "${output}"
+    support/download/scmversion "${git_cache}" "${base_name}/.scmversion"
+    post_process_repack "$(pwd)" "${base_name}" "${output}"
+fi
diff --git a/support/download/scmversion b/support/download/scmversion
new file mode 100755
index 0000000000..16868a11e5
--- /dev/null
+++ b/support/download/scmversion
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+# Helper to generate an scmversion in a dowload script before
+# the tarball is created. Because the tarball has to exclude
+# repository directories like .git and .hg to make reproducible
+# archives on subsequent downloads the version information is
+# not available during the builds for a script like setlocalversion
+# to run.
+#
+# So, this is hook to call right before the make tarball gz that
+# will generate a .scmversion file that will be included in an
+# archive and then available during the build steps.
+#
+# Call it with:
+#   $1: the path to the srctree (see mk_tar_gz in helpers)
+#   $2: output file
+#
+# Because this can fail for a variety of reason, there is no exit
+# code to avoid build breakage. It is assumed the scripts will be
+# reproducible because they are based on the contents that get generated
+
+while getopts :q OPT; do
+    case "${OPT}" in
+    q)  exec >/dev/null;;
+    \?) exit 1;;
+    esac
+done
+shift $((OPTIND-1))
+
+srctree="${1}"
+scmversion="${2}"
+
+# Bail early if no srctree or scmversion
+if [ -z "${srctree}" -o -z "${scmversion}" ]; then
+    exit 0
+fi
+
+# Does the scmversion exist and readonly
+if [ -f "${scmversion}" -a ! -w "${scmversion}" ]; then
+    printf "WARNING: scmversion %s is readonly\n" "${scmversion}" >&2
+    exit 0
+fi
+
+#
+# Generate the scmversion from some expected locations in srctree
+# and fallback to the buildroot version of setlocalversion another
+# script is not used
+#
+if [ -x "${srctree}/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./setlocalversion)
+elif [ -x "${srctree}/scripts/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./scripts/setlocalversion)
+elif [ -x "${srctree}/tools/setlocalversion" ]; then
+    res=$(cd "${srctree}"; ./tools/setlocalversion)
+fi
+if [ -z "${res}" ]; then
+    res=$("${0%/*}/../scripts/setlocalversion" "${srctree}")
+fi
+
+[ -n "${res}" ] && (echo "${res}" > "${scmversion}")
+exit 0
diff --git a/support/download/svn b/support/download/svn
index b23b7773d3..89da736172 100755
--- a/support/download/svn
+++ b/support/download/svn
@@ -12,6 +12,7 @@ set -e
 #
 # Options:
 #   -q          Be quiet.
+#   -s          Generate an scmversion file
 #   -o FILE     Generate archive in FILE.
 #   -u URI      Checkout from repository at URI.
 #   -c REV      Use revision REV.
@@ -23,9 +24,11 @@ set -e
 . "${0%/*}/helpers"
 
 quiet=
+scmversion=0
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  quiet=-q;;
+    s)  scmversion=1;;
     o)  output="${OPTARG}";;
     u)  uri="${OPTARG}";;
     c)  rev="${OPTARG}";;
@@ -65,3 +68,10 @@ date="$( _plain_svn info "'${uri}@${rev}'" \
 # We did a 'svn export' above, so it's not a working copy (there is no .svn
 # directory or file to ignore).
 mk_tar_gz "${basename}" "${basename}" "${date}" "${output}"
+
+# If an scmversion is needed then generate the version information
+if [ ${scmversion} -eq 1 ]; then
+    post_process_unpack "${base_name}" "${output}"
+    support/download/scmversion "${git_cache}" "${base_name}/.scmversion"
+    post_process_repack "$(pwd)" "${base_name}" "${output}"
+fi
-- 
2.24.3 (Apple Git-128)

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot