[Buildroot] [PATCH 1/1] utils/update-rust: add new util for updating rust/rust-bin

* [Buildroot] [PATCH 1/1] utils/update-rust: add new util for updating rust/rust-bin
@ 2022-09-26 20:45 James Hilliard
  2023-08-26 21:51 ` Thomas Petazzoni via buildroot
  0 siblings, 1 reply; 3+ messages in thread
From: James Hilliard @ 2022-09-26 20:45 UTC (permalink / raw)
  To: buildroot; +Cc: James Hilliard

Manually updating the rust package is tedious and slow as we have to
update and validate hashes for all supported rust-bin arch specific
toolchains.

To simplify this process add a python script which will update and
validate hashes and signatures for the new desired rust version.

This script is additionally capable of resuming an update if
interrupted which may be useful on slower network connections
as validating gpg signatures requires fully downloading each rust
toolchain distribution file.

This script has no external dependencies other than the optional
python-gnupg library which is needed for gpg signature validation.

Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
 utils/update-rust | 226 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 226 insertions(+)
 create mode 100755 utils/update-rust

diff --git a/utils/update-rust b/utils/update-rust
new file mode 100755
index 0000000000..2aad3fffa3
--- /dev/null
+++ b/utils/update-rust
@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""
+
+Utility for updating rust
+
+"""
+import argparse
+import errno
+import urllib.request
+import tempfile
+import shutil
+import io
+import os
+import hashlib
+from pathlib import Path
+from os import fdopen
+
+try:
+    # Requires the python-gnupg library
+    from gnupg import GPG
+except ImportError:
+    print("Unable to verify signatures, python-gnupg required")
+    GPG = None
+
+RUST_KEY = None
+RUST_KEY_URL = "https://static.rust-lang.org/rust-key.gpg.ascii"
+
+TOPDIR = Path(__file__).resolve().parent.parent
+PACKAGE_DIR = TOPDIR.joinpath("package")
+RUST_DIR = PACKAGE_DIR.joinpath("rust")
+RUST_BIN_DIR = PACKAGE_DIR.joinpath("rust-bin")
+RUST_MK_FILE = RUST_DIR.joinpath("rust.mk")
+RUST_HASH_FILE = RUST_DIR.joinpath("rust.hash")
+RUST_BIN_MK_FILE = RUST_BIN_DIR.joinpath("rust-bin.mk")
+RUST_BIN_HASH_FILE = RUST_BIN_DIR.joinpath("rust-bin.hash")
+
+
+def url_data(url):
+    with urllib.request.urlopen(url) as f:
+        return f.read()
+
+
+def rust_verify(sha256, sig_url):
+    global RUST_KEY
+    file_data = url_data(sig_url.rpartition(".")[0])
+    file_data_sha256 = hashlib.sha256(file_data).hexdigest()
+    valid_sig = True
+    if file_data_sha256 != sha256:
+        valid_sig = False
+    if GPG is not None:
+        try:
+            tmp_dir = tempfile.mkdtemp()
+            gpg = GPG(gnupghome=tmp_dir)
+            if RUST_KEY is None:
+                RUST_KEY = url_data(RUST_KEY_URL)
+            import_result = gpg.import_keys(RUST_KEY)
+            gpg.trust_keys(import_result.fingerprints, "TRUST_ULTIMATE")
+            sig_data = url_data(sig_url)
+            with tempfile.NamedTemporaryFile() as tmp_file:
+                tmp_file.write(file_data)
+                verified = gpg.verify_file(io.BytesIO(sig_data), tmp_file.name)
+                if verified.trust_level is None:
+                    print("Signature validation for %s failed." % sig_url)
+                    valid_sig = False
+                if verified.trust_level < verified.TRUST_ULTIMATE:
+                    print(
+                        "Signature validation for %s failed: %s"
+                        % (sig_url, verified.trust_text)
+                    )
+                    valid_sig = False
+        finally:
+            try:
+                shutil.rmtree(tmp_dir)
+            except OSError as exc:
+                if exc.errno != errno.ENOENT:
+                    valid_sig = False
+                    raise exc
+    else:
+        print("Skipping validation for %s" % sig_url)
+    return valid_sig
+
+
+def update_hash_file_entry(hash_file, old_version, new_version):
+    tmpfd, tmpfpath = tempfile.mkstemp()
+    updated = False
+    with fdopen(tmpfd, "w") as new_file:
+        with hash_file.open("r") as old_file:
+            line = old_file.readline()
+            while line:
+                words = line.split()
+                sha256_url_line = None
+                sig_url_line = None
+                sha256_hash_line = None
+                has_section_start = (
+                    len(words) == 3 and words[0] == "#" and words[1] == "From"
+                )
+                is_old_version = old_version in words[2]
+                is_new_version = new_version in words[2]
+                needs_version_update = is_old_version and not is_new_version
+                if updated is False and has_section_start and needs_version_update:
+                    if is_old_version:
+                        old_sha256_url = words[2]
+                        new_sha256_url = old_sha256_url.replace(
+                            old_version, new_version
+                        )
+                        sha256_url_line = line.replace(old_version, new_version)
+                        new_sha256_data = url_data(new_sha256_url)
+                        print(
+                            "processing: %s" % new_sha256_data.strip().decode("utf-8")
+                        )
+                        line = old_file.readline()
+                        words = line.split()
+                        if (
+                            len(words) == 4
+                            and words[0] == "#"
+                            and words[1] == "Verified"
+                            and words[2] == "using"
+                        ):
+                            old_sig_url = words[3]
+                            if old_version in old_sig_url:
+                                new_sig_url = old_sig_url.replace(
+                                    old_version, new_version
+                                )
+                                sig_url_line = line.replace(old_version, new_version)
+                                new_sha256_hash = new_sha256_data.split()[0].decode(
+                                    "ascii"
+                                )
+                                new_sha256_archive = new_sha256_data.split()[1].decode(
+                                    "ascii"
+                                )
+                                if (
+                                    rust_verify(new_sha256_hash, new_sig_url)
+                                    is not True
+                                ):
+                                    raise Exception("signature verification failed")
+                                line = old_file.readline()
+                                words = line.split()
+                                if len(words) == 3 and words[0] == "sha256":
+                                    old_sha256_hash = words[1]
+                                    old_archive_name = words[2]
+                                    if old_version in old_archive_name:
+                                        sha256_hash_line = line.replace(
+                                            old_version, new_version
+                                        ).replace(old_sha256_hash, new_sha256_hash)
+                                        if (
+                                            new_sha256_archive
+                                            == sha256_hash_line.split()[2]
+                                        ):
+                                            updated = True
+                                        else:
+                                            raise Exception(
+                                                "archive name mismatch in sha256 file"
+                                            )
+                    if updated is True:
+                        new_file.write(sha256_url_line)
+                        new_file.write(sig_url_line)
+                        new_file.write(sha256_hash_line)
+                else:
+                    new_file.write(line)
+                line = old_file.readline()
+        if updated:
+            shutil.copymode(hash_file, tmpfpath)
+            shutil.move(tmpfpath, hash_file)
+        else:
+            os.remove(tmpfpath)
+
+    return updated
+
+
+def update_hash_file(hash_file, old_version, new_version):
+    updated = True
+    while updated:
+        updated = update_hash_file_entry(hash_file, old_version, new_version)
+
+
+def update_mk_file(mk_file, old_version, new_version):
+    tmpfd, tmpfpath = tempfile.mkstemp()
+    updated = False
+    with fdopen(tmpfd, "w") as new_file:
+        with mk_file.open() as old_file:
+            version_var = mk_file.stem.upper().replace("-", "_") + "_VERSION"
+            for line in old_file.readlines():
+                words = line.split()
+                if (
+                    len(words) != 0
+                    and words[0] == version_var
+                    and words[1] == "="
+                    and words[2] == old_version
+                ):
+                    updated = True
+                    new_file.write(line.replace(old_version, new_version))
+                else:
+                    new_file.write(line)
+
+        if updated:
+            shutil.copymode(mk_file, tmpfpath)
+            shutil.move(tmpfpath, mk_file)
+        else:
+            os.remove(tmpfpath)
+
+    return updated
+
+
+def get_old_version(mk_file):
+    with mk_file.open() as f:
+        version_var = mk_file.stem.upper().replace("-", "_") + "_VERSION"
+        for line in f.readlines():
+            words = line.split()
+            if len(words) != 0 and words[0] == version_var:
+                return words[-1]
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Update rust")
+    parser.add_argument("version", help="Rust version to update to", type=str)
+
+    args = parser.parse_args()
+    new_version = args.version
+    update_hash_file(RUST_HASH_FILE, get_old_version(RUST_MK_FILE), new_version)
+    update_mk_file(RUST_MK_FILE, get_old_version(RUST_MK_FILE), new_version)
+    update_hash_file(RUST_BIN_HASH_FILE, get_old_version(RUST_BIN_MK_FILE), new_version)
+    update_mk_file(RUST_BIN_MK_FILE, get_old_version(RUST_BIN_MK_FILE), new_version)
+
+
+if __name__ == "__main__":
+    main()
-- 
2.34.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply related	[flat|nested] 3+ messages in thread