[OE-core][PATCH 01/22] scripts/pull-spdx-licenses.py: Add exceptions

[Joshua Watt <jpewhacker@gmail.com>]

Adds exceptions to the script that updates the SPDX license files

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
 scripts/pull-spdx-licenses.py | 124 ++++++++++++++++++++++++----------
 1 file changed, 87 insertions(+), 37 deletions(-)

diff --git a/scripts/pull-spdx-licenses.py b/scripts/pull-spdx-licenses.py
index 597a62133f..7d1715ff24 100755
--- a/scripts/pull-spdx-licenses.py
+++ b/scripts/pull-spdx-licenses.py
@@ -13,6 +13,88 @@ from pathlib import Path
 TOP_DIR = Path(__file__).parent.parent
 
 
+def update_licenses(dest, version, overwrite, deprecated):
+    print(f"Pulling SPDX license list version {version}")
+    req = urllib.request.Request(
+        f"https://raw.githubusercontent.com/spdx/license-list-data/{version}/json/licenses.json"
+    )
+    with urllib.request.urlopen(req) as response:
+        spdx_licenses = json.load(response)
+
+    with (TOP_DIR / "meta" / "files" / "spdx-licenses.json").open("w") as f:
+        json.dump(spdx_licenses, f, sort_keys=True, indent=2)
+
+    total_count = len(spdx_licenses["licenses"])
+    updated = 0
+    for idx, lic in enumerate(spdx_licenses["licenses"]):
+        lic_id = lic["licenseId"]
+
+        print(f"[{idx + 1} of {total_count}] ", end="")
+
+        dest_license_file = dest / lic_id
+        if dest_license_file.is_file() and not overwrite:
+            print(f"Skipping {lic_id} since it already exists")
+            continue
+
+        print(f"Fetching {lic_id}... ", end="", flush=True)
+
+        req = urllib.request.Request(lic["detailsUrl"])
+        with urllib.request.urlopen(req) as response:
+            lic_data = json.load(response)
+
+        if lic_data["isDeprecatedLicenseId"] and not deprecated:
+            print("Skipping (deprecated)")
+            continue
+
+        with dest_license_file.open("w") as f:
+            f.write(lic_data["licenseText"])
+        updated += 1
+        print("done")
+
+    return updated
+
+
+def update_exceptions(dest, version, overwrite, deprecated):
+    print(f"Pulling SPDX license exceptions list version {version}")
+    req = urllib.request.Request(
+        f"https://raw.githubusercontent.com/spdx/license-list-data/{version}/json/exceptions.json"
+    )
+    with urllib.request.urlopen(req) as response:
+        spdx_exceptions = json.load(response)
+
+    with (TOP_DIR / "meta" / "files" / "spdx-exceptions.json").open("w") as f:
+        json.dump(spdx_exceptions, f, sort_keys=True, indent=2)
+
+    total_count = len(spdx_exceptions["exceptions"])
+    updated = 0
+    for idx, lic in enumerate(spdx_exceptions["exceptions"]):
+        lic_id = lic["licenseExceptionId"]
+
+        print(f"[{idx + 1} of {total_count}] ", end="")
+
+        dest_license_file = dest / lic_id
+        if dest_license_file.is_file() and not overwrite:
+            print(f"Skipping {lic_id} since it already exists")
+            continue
+
+        print(f"Fetching {lic_id}... ", end="", flush=True)
+
+        req = urllib.request.Request(lic["detailsUrl"])
+        with urllib.request.urlopen(req) as response:
+            lic_data = json.load(response)
+
+        if lic_data["isDeprecatedLicenseId"] and not deprecated:
+            print("Skipping (deprecated)")
+            continue
+
+        with dest_license_file.open("w") as f:
+            f.write(lic_data["licenseExceptionText"])
+        updated += 1
+        print("done")
+
+    return updated
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="Update SPDX License files from upstream"
@@ -55,44 +137,12 @@ def main():
             data = json.load(response)
             version = data["tag_name"]
 
-    print(f"Pulling SPDX license list version {version}")
-    req = urllib.request.Request(
-        f"https://raw.githubusercontent.com/spdx/license-list-data/{version}/json/licenses.json"
+    license_count = update_licenses(args.dest, version, args.overwrite, args.deprecated)
+    exception_count = update_exceptions(
+        args.dest, version, args.overwrite, args.deprecated
     )
-    with urllib.request.urlopen(req) as response:
-        spdx_licenses = json.load(response)
-
-    with (TOP_DIR / "meta" / "files" / "spdx-licenses.json").open("w") as f:
-        json.dump(spdx_licenses, f, sort_keys=True, indent=2)
-
-    total_count = len(spdx_licenses["licenses"])
-    updated = 0
-    for idx, lic in enumerate(spdx_licenses["licenses"]):
-        lic_id = lic["licenseId"]
-
-        print(f"[{idx + 1} of {total_count}] ", end="")
-
-        dest_license_file = args.dest / lic_id
-        if dest_license_file.is_file() and not args.overwrite:
-            print(f"Skipping {lic_id} since it already exists")
-            continue
-
-        print(f"Fetching {lic_id}... ", end="", flush=True)
-
-        req = urllib.request.Request(lic["detailsUrl"])
-        with urllib.request.urlopen(req) as response:
-            lic_data = json.load(response)
-
-        if lic_data["isDeprecatedLicenseId"] and not args.deprecated:
-            print("Skipping (deprecated)")
-            continue
-
-        with dest_license_file.open("w") as f:
-            f.write(lic_data["licenseText"])
-        updated += 1
-        print("done")
-
-    print(f"Updated {updated} licenses")
+    print(f"Updated {license_count} licenses")
+    print(f"Updated {exception_count} exceptions")
 
     return 0
 
-- 
2.54.0