[meta-zephyr][PATCH 1/4] zephyr-core/scripts: Introduce script to generate new versions

[Peter Hoyes <peter.hoyes@arm.com>]

From: Peter Hoyes <Peter.Hoyes@arm.com>

Add a Python script which can be used to automatically generate
configuration for new Zephyr versions. This script:
 * Takes the Zephyr version as a single argument
 * Uses the Github API to find the version tag's SHA1
 * Uses West as a library to parse the version's manifest file
 * Uses a Jinja template to generate a .inc file for the version
 * Outputs the .inc file directly into the zephyr-kernel directory

The generated .inc file includes:
 * SRCREVs for all modules
 * Separate SRC_URI_x variables for each module, to make it easier to
   swap out a specific URL for a fork or mirror
 * A version-specific SRC_URI, containing only the modules defined in
   the release
 * A list of the ZEPHYR_MODULES

Signed-off-by: Peter Hoyes <Peter.Hoyes@arm.com>
---
 README.txt                                    | 17 +++++
 meta-zephyr-core/scripts/generate-version.py  | 73 +++++++++++++++++++
 .../scripts/zephyr-kernel-src.inc.jinja       | 35 +++++++++
 3 files changed, 125 insertions(+)
 create mode 100755 meta-zephyr-core/scripts/generate-version.py
 create mode 100644 meta-zephyr-core/scripts/zephyr-kernel-src.inc.jinja

diff --git a/README.txt b/README.txt
index 4776a8a..ea129ba 100644
--- a/README.txt
+++ b/README.txt
@@ -125,6 +125,23 @@ you will need to run the above, copy the conf files from the deploy dir to the
 machine conf directory and then run your build. This shouldn't need to happen 
 often.
 
+Generating new Zephyr recipe versions
+=====================================
+The script meta-zephyr-core/scripts/generate-version.py is used to generate
+Yocto configuration for a Zephyr version from the West configuration in the
+Zephyr repository. It requires the west and jinja2 Python packages to be
+installed on the host. Run it as follows:
+
+    $ ./meta-zephyr-core/scripts/generate-version.py x.x.x
+
+where x.x.x is the Zephyr version.
+
+The patch files added to SRC_URI in the generated file should be validated and
+modified if required.
+
+The new version should be committed and submitted to the mailing list as
+described in "Contributing".
+
 Contributing
 ============
 
diff --git a/meta-zephyr-core/scripts/generate-version.py b/meta-zephyr-core/scripts/generate-version.py
new file mode 100755
index 0000000..550f8df
--- /dev/null
+++ b/meta-zephyr-core/scripts/generate-version.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+import json
+import pathlib
+import re
+import sys
+import urllib.parse
+import urllib.request
+
+# These non-standard modules must be installed on the host
+import jinja2
+import west.manifest
+
+# This script takes one argument - the Zephyr version in the form x.y.z
+version = sys.argv[1]
+if not re.match(r'\d+.\d+.\d+', version):
+    raise ValueError("Please provide a valid Zephyr version")
+
+# Convert the version (x.y.z) into the Git commit SHA using the Github API
+# This is a two-step process - first obtain the tag SHA
+ref_url = f'https://api.github.com/repos/zephyrproject-rtos/zephyr/git/refs/tags/v{version}'
+with urllib.request.urlopen(ref_url) as f:
+    ref_data = json.load(f)
+    ref_sha = ref_data['object']['sha']
+
+# Secondly, obtain the commit SHA of the tag SHA
+tag_url = f'https://api.github.com/repos/zephyrproject-rtos/zephyr/git/tags/{ref_sha}'
+with urllib.request.urlopen(tag_url) as f:
+    tag_data = json.load(f)
+    tag_sha = tag_data['object']['sha']
+
+# Obtain the West manifest and decode using west as a library
+manifest_url = f'https://raw.githubusercontent.com/zephyrproject-rtos/zephyr/v{version}/west.yml'
+with urllib.request.urlopen(manifest_url) as f:
+    source_data = f.read().decode()
+    manifest = west.manifest.Manifest(source_data=source_data,
+        import_flags=west.manifest.ImportFlag.IGNORE)
+    projects = manifest.get_projects([])
+
+# projects contains a 'manifest' project for 'self' which we don't want to use
+projects = list(filter(lambda project: project.name != 'manifest', projects))
+template_params = dict(version=version, tag_sha=tag_sha, projects=projects)
+
+def git_url_to_bitbake(url):
+    """
+    A template helper function which converts an URL for a Git repository into
+    a Bitbake-style URL with a 'protocol' suffix
+    """
+    parts = urllib.parse.urlparse(url)
+    original_sceme = parts.scheme
+    parts = parts._replace(scheme='git')
+    return parts.geturl() + ';protocol=' + original_sceme
+
+def bitbake_var(name):
+    """
+    Returns a string suitable for use in a Bitbake variable name
+    """
+    return name.upper().replace('-', '_')
+
+# Set up the Jinja environment
+template_dir = pathlib.Path(__file__).parent
+env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
+env.filters['git_url_to_bitbake'] = git_url_to_bitbake
+env.filters['bitbake_var'] = bitbake_var
+template = env.get_template('zephyr-kernel-src.inc.jinja')
+
+# Output directly to the zephyr-kernel directory
+dest_path = pathlib.Path(__file__).parents[1] / 'recipes-kernel' /\
+    'zephyr-kernel' / f'zephyr-kernel-src-{version}.inc'
+
+# Generate the Bitbake include file
+with open(dest_path, 'w') as f:
+    f.write(template.render(**template_params))
diff --git a/meta-zephyr-core/scripts/zephyr-kernel-src.inc.jinja b/meta-zephyr-core/scripts/zephyr-kernel-src.inc.jinja
new file mode 100644
index 0000000..e7981ed
--- /dev/null
+++ b/meta-zephyr-core/scripts/zephyr-kernel-src.inc.jinja
@@ -0,0 +1,35 @@
+# Auto-generated from zephyr-kernel-src.inc.jinja
+{%- set short_version = '.'.join(version.split('.')[0:2]) %}
+
+SRCREV_FORMAT = "default"
+
+SRCREV_default = "{{ tag_sha }}"
+{% for project in projects -%}
+SRCREV_{{ project.name }} = "{{ project.revision }}"
+{% endfor %}
+SRC_URI_ZEPHYR ?= "git://github.com/zephyrproject-rtos/zephyr.git;protocol=https"
+{%- for project in projects %}
+SRC_URI_{{ project.name | bitbake_var }} ?= "{{ project.url | git_url_to_bitbake }}"
+{%- endfor %}
+
+SRC_URI_PATCHES ?= "\
+    file://0001-{{ short_version }}-cmake-add-yocto-toolchain.patch;patchdir=zephyr \
+    file://0001-{{ short_version }}-x86-fix-efi-binary-generation-issue-in-cross-compila.patch;patchdir=zephyr \
+"
+
+SRC_URI = "\
+    ${SRC_URI_ZEPHYR};branch=${ZEPHYR_BRANCH};name=default;destsuffix=git/zephyr \
+{%- for project in projects %}
+    ${SRC_URI_{{ project.name | bitbake_var }}};name={{ project.name }};nobranch=1;destsuffix=git/{{ project.path }} \
+{%- endfor %}
+    ${SRC_URI_PATCHES} \
+"
+
+ZEPHYR_MODULES = "\{% for project in projects %}
+${S}/{{ project.path }}\;\
+{%- endfor %}
+"
+
+ZEPHYR_BRANCH = "v{{ short_version }}-branch"
+PV = "{{ version }}+git${SRCPV}"
+
-- 
2.25.1