From: Joshua Watt <jpewhacker@gmail.com>
To: openembedded-core@lists.openembedded.org
Cc: Joshua Watt <JPEWhacker@gmail.com>
Subject: [OE-core][PATCH 07/22] convert-spdx-licenses: Convert to valid SPDX expressions
Date: Tue, 14 Jul 2026 12:31:17 -0600 [thread overview]
Message-ID: <20260714190405.3328796-8-JPEWhacker@gmail.com> (raw)
In-Reply-To: <20260714190405.3328796-1-JPEWhacker@gmail.com>
Reworks the script to re-write LICENSE lines as valid SPDX license
expressions in addition to remapping the license names
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
scripts/contrib/convert-spdx-licenses.py | 254 +++++++++++++++--------
1 file changed, 162 insertions(+), 92 deletions(-)
diff --git a/scripts/contrib/convert-spdx-licenses.py b/scripts/contrib/convert-spdx-licenses.py
index 13cf12a33f..39cbbb28e5 100755
--- a/scripts/contrib/convert-spdx-licenses.py
+++ b/scripts/contrib/convert-spdx-licenses.py
@@ -7,113 +7,175 @@
# SPDX-License-Identifier: GPL-2.0-only
#
-import re
+
+from abc import abstractmethod, ABC
+from enum import Enum
+from pathlib import Path
+import json
+import mimetypes
import os
+import re
+import shutil
import sys
import tempfile
-import shutil
-import mimetypes
+
+THIS_DIR = Path(__file__).parent
+
+sys.path.insert(0, str(THIS_DIR.parent.parent / "meta" / "lib"))
+import oe.spdx_license
if len(sys.argv) < 2:
print("Please specify a directory to run the conversion script against.")
sys.exit(1)
-license_map = {
-"AGPL-3" : "AGPL-3.0-only",
-"AGPL-3+" : "AGPL-3.0-or-later",
-"AGPLv3" : "AGPL-3.0-only",
-"AGPLv3+" : "AGPL-3.0-or-later",
-"AGPLv3.0" : "AGPL-3.0-only",
-"AGPLv3.0+" : "AGPL-3.0-or-later",
-"AGPL-3.0" : "AGPL-3.0-only",
-"AGPL-3.0+" : "AGPL-3.0-or-later",
-"BSD-0-Clause" : "0BSD",
-"GPL-1" : "GPL-1.0-only",
-"GPL-1+" : "GPL-1.0-or-later",
-"GPLv1" : "GPL-1.0-only",
-"GPLv1+" : "GPL-1.0-or-later",
-"GPLv1.0" : "GPL-1.0-only",
-"GPLv1.0+" : "GPL-1.0-or-later",
-"GPL-1.0" : "GPL-1.0-only",
-"GPL-1.0+" : "GPL-1.0-or-later",
-"GPL-2" : "GPL-2.0-only",
-"GPL-2+" : "GPL-2.0-or-later",
-"GPLv2" : "GPL-2.0-only",
-"GPLv2+" : "GPL-2.0-or-later",
-"GPLv2.0" : "GPL-2.0-only",
-"GPLv2.0+" : "GPL-2.0-or-later",
-"GPL-2.0" : "GPL-2.0-only",
-"GPL-2.0+" : "GPL-2.0-or-later",
-"GPL-3" : "GPL-3.0-only",
-"GPL-3+" : "GPL-3.0-or-later",
-"GPLv3" : "GPL-3.0-only",
-"GPLv3+" : "GPL-3.0-or-later",
-"GPLv3.0" : "GPL-3.0-only",
-"GPLv3.0+" : "GPL-3.0-or-later",
-"GPL-3.0" : "GPL-3.0-only",
-"GPL-3.0+" : "GPL-3.0-or-later",
-"LGPLv2" : "LGPL-2.0-only",
-"LGPLv2+" : "LGPL-2.0-or-later",
-"LGPLv2.0" : "LGPL-2.0-only",
-"LGPLv2.0+" : "LGPL-2.0-or-later",
-"LGPL-2.0" : "LGPL-2.0-only",
-"LGPL-2.0+" : "LGPL-2.0-or-later",
-"LGPL2.1" : "LGPL-2.1-only",
-"LGPL2.1+" : "LGPL-2.1-or-later",
-"LGPLv2.1" : "LGPL-2.1-only",
-"LGPLv2.1+" : "LGPL-2.1-or-later",
-"LGPL-2.1" : "LGPL-2.1-only",
-"LGPL-2.1+" : "LGPL-2.1-or-later",
-"LGPLv3" : "LGPL-3.0-only",
-"LGPLv3+" : "LGPL-3.0-or-later",
-"LGPL-3.0" : "LGPL-3.0-only",
-"LGPL-3.0+" : "LGPL-3.0-or-later",
-"MPL-1" : "MPL-1.0",
-"MPLv1" : "MPL-1.0",
-"MPLv1.1" : "MPL-1.1",
-"MPLv2" : "MPL-2.0",
-"MIT-X" : "MIT",
-"MIT-style" : "MIT",
-"openssl" : "OpenSSL",
-"PSF" : "PSF-2.0",
-"PSFv2" : "PSF-2.0",
-"Python-2" : "Python-2.0",
-"Apachev2" : "Apache-2.0",
-"Apache-2" : "Apache-2.0",
-"Artisticv1" : "Artistic-1.0",
-"Artistic-1" : "Artistic-1.0",
-"AFL-2" : "AFL-2.0",
-"AFL-1" : "AFL-1.2",
-"AFLv2" : "AFL-2.0",
-"AFLv1" : "AFL-1.2",
-"CDDLv1" : "CDDL-1.0",
-"CDDL-1" : "CDDL-1.0",
-"EPLv1.0" : "EPL-1.0",
-"FreeType" : "FTL",
-"Nauman" : "Naumen",
-"tcl" : "TCL",
-"vim" : "Vim",
-"SGIv1" : "SGI-OpenGL",
+LICENSE_MAP = {
+ "AGPL-3": "AGPL-3.0-only",
+ "AGPL-3+": "AGPL-3.0-or-later",
+ "AGPLv3": "AGPL-3.0-only",
+ "AGPLv3+": "AGPL-3.0-or-later",
+ "AGPLv3.0": "AGPL-3.0-only",
+ "AGPLv3.0+": "AGPL-3.0-or-later",
+ "AGPL-3.0": "AGPL-3.0-only",
+ "AGPL-3.0+": "AGPL-3.0-or-later",
+ "BSD-0-Clause": "0BSD",
+ "GPL-1": "GPL-1.0-only",
+ "GPL-1+": "GPL-1.0-or-later",
+ "GPLv1": "GPL-1.0-only",
+ "GPLv1+": "GPL-1.0-or-later",
+ "GPLv1.0": "GPL-1.0-only",
+ "GPLv1.0+": "GPL-1.0-or-later",
+ "GPL-1.0": "GPL-1.0-only",
+ "GPL-1.0+": "GPL-1.0-or-later",
+ "GPL-2": "GPL-2.0-only",
+ "GPL-2+": "GPL-2.0-or-later",
+ "GPLv2": "GPL-2.0-only",
+ "GPLv2+": "GPL-2.0-or-later",
+ "GPLv2.0": "GPL-2.0-only",
+ "GPLv2.0+": "GPL-2.0-or-later",
+ "GPL-2.0": "GPL-2.0-only",
+ "GPL-2.0+": "GPL-2.0-or-later",
+ "GPL-3": "GPL-3.0-only",
+ "GPL-3+": "GPL-3.0-or-later",
+ "GPLv3": "GPL-3.0-only",
+ "GPLv3+": "GPL-3.0-or-later",
+ "GPLv3.0": "GPL-3.0-only",
+ "GPLv3.0+": "GPL-3.0-or-later",
+ "GPL-3.0": "GPL-3.0-only",
+ "GPL-3.0+": "GPL-3.0-or-later",
+ "LGPLv2": "LGPL-2.0-only",
+ "LGPLv2+": "LGPL-2.0-or-later",
+ "LGPLv2.0": "LGPL-2.0-only",
+ "LGPLv2.0+": "LGPL-2.0-or-later",
+ "LGPL-2.0": "LGPL-2.0-only",
+ "LGPL-2.0+": "LGPL-2.0-or-later",
+ "LGPL2.1": "LGPL-2.1-only",
+ "LGPL2.1+": "LGPL-2.1-or-later",
+ "LGPLv2.1": "LGPL-2.1-only",
+ "LGPLv2.1+": "LGPL-2.1-or-later",
+ "LGPL-2.1": "LGPL-2.1-only",
+ "LGPL-2.1+": "LGPL-2.1-or-later",
+ "LGPLv3": "LGPL-3.0-only",
+ "LGPLv3+": "LGPL-3.0-or-later",
+ "LGPL-3.0": "LGPL-3.0-only",
+ "LGPL-3.0+": "LGPL-3.0-or-later",
+ "MPL-1": "MPL-1.0",
+ "MPLv1": "MPL-1.0",
+ "MPLv1.1": "MPL-1.1",
+ "MPLv2": "MPL-2.0",
+ "MIT-X": "MIT",
+ "MIT-style": "MIT",
+ "openssl": "OpenSSL",
+ "PSF": "PSF-2.0",
+ "PSFv2": "PSF-2.0",
+ "Python-2": "Python-2.0",
+ "Apachev2": "Apache-2.0",
+ "Apache-2": "Apache-2.0",
+ "Artisticv1": "Artistic-1.0",
+ "Artistic-1": "Artistic-1.0",
+ "AFL-2": "AFL-2.0",
+ "AFL-1": "AFL-1.2",
+ "AFLv2": "AFL-2.0",
+ "AFLv1": "AFL-1.2",
+ "CDDLv1": "CDDL-1.0",
+ "CDDL-1": "CDDL-1.0",
+ "EPLv1.0": "EPL-1.0",
+ "FreeType": "FTL",
+ "Nauman": "Naumen",
+ "tcl": "TCL",
+ "vim": "Vim",
+ "SGIv1": "SGI-OpenGL",
+ "Apache-2.0-with-LLVM-exception": "Apache-2.0 WITH LLVM-exception",
+ "GPL-3-with-bison-exception": "GPL-3.0-or-later WITH Bison-exception-2.2",
+ "GPL-2.0-with-Linux-syscall-note": "GPL-2.0-only WITH Linux-syscall-note",
}
+
+LINE_RE = re.compile(r'^(?P<var>LICENSE[\s:].*=.*)"(?P<expression>.*)"')
+
+
+def convert_unknown(node):
+ node.children = [convert_unknown(c) for c in node.children]
+
+ if not isinstance(node, oe.spdx_license.UnknownId):
+ return node
+
+ if node.ident.startswith("$"):
+ return node
+
+ if node.ident in LICENSE_MAP:
+ return oe.spdx_license.parse(LICENSE_MAP[node.ident])
+
+ ident = re.sub(r"[^a-zA-Z0-9\.\-]", "-", node.ident)
+
+ return oe.spdx_license.LicenseRef("LicenseRef-" + ident, ident, token=node.token)
+
+
def processfile(fn):
- print("processing file '%s'" % fn)
+ # print("processing file '%s'" % fn)
try:
fh, abs_path = tempfile.mkstemp()
+ if os.path.basename(fn) == "bitbake.conf":
+ return
+
modified = False
- with os.fdopen(fh, 'w') as new_file:
+ with os.fdopen(fh, "w") as new_file:
with open(fn, "r") as old_file:
- for line in old_file:
- if not line.startswith("LICENSE"):
+ for lineno, line in enumerate(old_file):
+ m = LINE_RE.match(line)
+ if m is None:
+ new_file.write(line)
+ continue
+
+ if "[doc]" in m.group("var"):
+ new_file.write(line)
+ continue
+
+ expression = (
+ m.group("expression").replace("|", " OR ").replace("&", " AND ")
+ ).strip()
+ if not expression or expression == "CLOSED":
new_file.write(line)
continue
- orig = line
- for license in sorted(license_map, key=len, reverse=True):
- for ending in ['"', "'", " ", ")"]:
- line = line.replace(license + ending, license_map[license] + ending)
- if orig != line:
- modified = True
- new_file.write(line)
+
+ try:
+ t = oe.spdx_license.parse(expression, allow_unknown=True)
+ if t is None:
+ new_file.write(line)
+ continue
+ t = convert_unknown(t)
+ except oe.spdx_license.ParseError as e:
+ print(f"Cannot convert {fn}:{lineno + 1}")
+ print(e.format())
+ new_file.write(line)
+ continue
+
+ if t.to_string() == expression:
+ new_file.write(line)
+ continue
+
+ new_line = m.group("var") + '"' + t.sort().to_string() + '"\n'
+ new_file.write(new_line)
+ modified = True
new_file.close()
if modified:
shutil.copymode(fn, abs_path)
@@ -122,8 +184,9 @@ def processfile(fn):
except UnicodeDecodeError:
pass
+
ourname = os.path.basename(sys.argv[0])
-ourversion = "0.01"
+ourversion = "0.02"
if os.path.isfile(sys.argv[1]):
processfile(sys.argv[1])
@@ -138,7 +201,14 @@ for targetdir in sys.argv[1:]:
fn = os.path.join(root, name)
if os.path.islink(fn):
continue
- if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff") or fn.endswith(".orig"):
+ if (
+ "/.git/" in fn
+ or fn.endswith(".html")
+ or fn.endswith(".patch")
+ or fn.endswith(".m4")
+ or fn.endswith(".diff")
+ or fn.endswith(".orig")
+ ):
continue
processfile(fn)
--
2.54.0
next prev parent reply other threads:[~2026-07-14 19:04 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-07-14 18:31 [OE-core][PATCH 00/22] Rework LICENSE to be SPDX License Expressions Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 01/22] scripts/pull-spdx-licenses.py: Add exceptions Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 02/22] Add SPDX License Exceptions Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 03/22] Add SPDX license library Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 04/22] Parse LICENSE as SPDX Expression Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 05/22] linux-firmware: Convert to SPDX license strings Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 06/22] meta-selftest: Add NO_GENERIC_LICENSE Joshua Watt
2026-07-14 18:31 ` Joshua Watt [this message]
2026-07-14 18:31 ` [OE-core][PATCH 08/22] gcc-common.inc: Remove LICENSE Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 09/22] gcc: Update license Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 10/22] xz: Replace deprecated SPDX identifier Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 11/22] autoconf-archive: " Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 12/22] gnu-config: " Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 13/22] libglvnd: " Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 14/22] flac: " Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 15/22] libgit2: " Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 16/22] Fix up licenses Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 17/22] lib: oe: license: Rework package licenses matching Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 18/22] classes/go-mod-update-modules: Switch to SPDX license Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 19/22] license: Remove tidy_licenses() Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 20/22] meta-selftest: Change license on test recipes Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 21/22] Convert licenses to SPDX expressions Joshua Watt
2026-07-14 18:31 ` [OE-core][PATCH 22/22] meta-selftest: libxpm: Fix license Joshua Watt
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260714190405.3328796-8-JPEWhacker@gmail.com \
--to=jpewhacker@gmail.com \
--cc=openembedded-core@lists.openembedded.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox