[Buildroot] [PATCH] package/micropython: make use of recent micropython-lib

* [Buildroot] [PATCH] package/micropython: make use of recent micropython-lib
@ 2024-01-28 18:37 Abilio Marques
  2024-01-28 20:03 ` Yann E. MORIN
  0 siblings, 1 reply; 8+ messages in thread
From: Abilio Marques @ 2024-01-28 18:37 UTC (permalink / raw)
  To: buildroot; +Cc: Chris Packham, Abilio Marques

Until now, micropython-lib was a package that installed v1.9.3,
which is more than 6 years old. This was acceptable since micropython
never made any other official release of the library until v1.20.

Meanwhile, the libraries underwent a reorganization, and they are now
available in a directory structure that cannot be copied directly into
the target. This is my theory on why v1.9.3 is still present in the
current day buildroot (which comes with micropython v1.22).

This commit introduces an auxiliary script to collect those libraries
and reorder them into a structure that can then be copied into
/usr/lib/micropython. The script utilizes a module from the tools
directory of the micropython repo.

As part of the changes made by the micropython project, the libraries
are now released together with the interpreter. They are cloned as a
submodule into the lib/micropython-lib directory.

These are the 2 main reasons why the old buildroot micropython-lib
package is being removed.

With this commit, micropython-lib is installed (optionally) as part
of micropython. The original config variable name was retained for
backwards compatibility.

This commit also ensures that the libraries in micropython-lib will
be updated together with newer versions of micropython.

Signed-off-by: Abilio Marques <abiliojr@gmail.com>
---
 DEVELOPERS                                    |  1 -
 package/Config.in                             |  1 -
 package/micropython-lib/Config.in             |  8 --
 package/micropython-lib/micropython-lib.hash  |  3 -
 package/micropython-lib/micropython-lib.mk    | 18 ----
 package/micropython/Config.in                 | 10 +++
 .../micropython/collect_micropython_lib.py    | 90 +++++++++++++++++++
 package/micropython/micropython.mk            | 12 +++
 8 files changed, 112 insertions(+), 31 deletions(-)
 delete mode 100644 package/micropython-lib/Config.in
 delete mode 100644 package/micropython-lib/micropython-lib.hash
 delete mode 100644 package/micropython-lib/micropython-lib.mk
 create mode 100755 package/micropython/collect_micropython_lib.py

diff --git a/DEVELOPERS b/DEVELOPERS
index 9528837dd0..cc9bc5b5cc 100644
--- a/DEVELOPERS
+++ b/DEVELOPERS
@@ -572,7 +572,6 @@ F:	package/coremark/
 F:	package/coremark-pro/
 F:	package/gstreamer1/gst1-shark/
 F:	package/micropython/
-F:	package/micropython-lib/
 F:	package/syslog-ng/
 
 N:	Christian Kellermann <christian.kellermann@solectrix.de>
diff --git a/package/Config.in b/package/Config.in
index 5b8b15fa54..426bd7d090 100644
--- a/package/Config.in
+++ b/package/Config.in
@@ -798,7 +798,6 @@ menu "Lua libraries/modules"
 endmenu
 endif
 	source "package/micropython/Config.in"
-	source "package/micropython-lib/Config.in"
 	source "package/moarvm/Config.in"
 	source "package/mono/Config.in"
 if BR2_PACKAGE_MONO
diff --git a/package/micropython-lib/Config.in b/package/micropython-lib/Config.in
deleted file mode 100644
index 76557b220b..0000000000
--- a/package/micropython-lib/Config.in
+++ /dev/null
@@ -1,8 +0,0 @@
-config BR2_PACKAGE_MICROPYTHON_LIB
-	bool "micropython-lib"
-	depends on BR2_PACKAGE_MICROPYTHON
-	select BR2_PACKAGE_PCRE # runtime
-	help
-	  Core Python libraries ported to MicroPython.
-
-	  http://micropython.org
diff --git a/package/micropython-lib/micropython-lib.hash b/package/micropython-lib/micropython-lib.hash
deleted file mode 100644
index cbdda23844..0000000000
--- a/package/micropython-lib/micropython-lib.hash
+++ /dev/null
@@ -1,3 +0,0 @@
-# Locally computed
-sha256  66e15380eb109613263beb6825b8eecb9191088270c1a59e8c7d922dd57183c7  micropython-lib-1.9.3.tar.gz
-sha256  baed4196a4310c576c2010f0a49f987a49e63856df7cd45af11cb3571df4bf74  LICENSE
diff --git a/package/micropython-lib/micropython-lib.mk b/package/micropython-lib/micropython-lib.mk
deleted file mode 100644
index 78ac0d3b35..0000000000
--- a/package/micropython-lib/micropython-lib.mk
+++ /dev/null
@@ -1,18 +0,0 @@
-################################################################################
-#
-# micropython-lib
-#
-################################################################################
-
-MICROPYTHON_LIB_VERSION = 1.9.3
-MICROPYTHON_LIB_SITE = $(call github,micropython,micropython-lib,v$(MICROPYTHON_LIB_VERSION))
-MICROPYTHON_LIB_LICENSE = Python-2.0 (some modules), MIT (everything else)
-MICROPYTHON_LIB_LICENSE_FILES = LICENSE
-
-define MICROPYTHON_LIB_INSTALL_TARGET_CMDS
-	$(TARGET_MAKE_ENV) $(MAKE) -C $(@D) \
-		PREFIX=$(TARGET_DIR)/usr/lib/micropython \
-		install
-endef
-
-$(eval $(generic-package))
diff --git a/package/micropython/Config.in b/package/micropython/Config.in
index 30161c8b70..2c2ae7a4df 100644
--- a/package/micropython/Config.in
+++ b/package/micropython/Config.in
@@ -9,5 +9,15 @@ config BR2_PACKAGE_MICROPYTHON
 
 	  http://micropython.org
 
+config BR2_PACKAGE_MICROPYTHON_LIB
+	bool "micropython-lib"
+	default y
+	depends on BR2_PACKAGE_MICROPYTHON
+	select BR2_PACKAGE_PCRE # runtime
+	help
+	  Core Python libraries ported to MicroPython.
+
+	  http://micropython.org
+
 comment "micropython needs a toolchain w/ threads, dynamic library"
 	depends on !BR2_TOOLCHAIN_HAS_THREADS || BR2_STATIC_LIBS
diff --git a/package/micropython/collect_micropython_lib.py b/package/micropython/collect_micropython_lib.py
new file mode 100755
index 0000000000..778a1c4c7d
--- /dev/null
+++ b/package/micropython/collect_micropython_lib.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+"""
+Sometime after v1.9.3, micropython-lib's directory structure was cleaned up and
+enhanced with manifest files.
+
+Though cleaner, it means it cannot be directly copied into the target.
+This script is during build process to perform that conversion.
+
+It makes use of manifestfile.py, which is normally located in micropython's
+tool directory.
+
+It also depends on the micropython-lib that is cloned in lib/micropython-lib
+during build.
+"""
+
+import argparse
+import importlib
+import os
+import shutil
+import sys
+
+
+def get_library_name_type(manifest_path: str) -> tuple[str, bool]:
+    split = manifest_path.split("/")
+    return (split[-2], "unix-ffi" in split)  # -1: "manifest.py", -2: library name
+
+
+def get_all_libraries(mpy_lib_dir: str):
+    # reuse manifestfile module capabilities to scan the micropython-lib directory
+
+    collected_list = manifestfile.ManifestFile(
+        manifestfile.MODE_FREEZE, {"MPY_LIB_DIR": mpy_lib_dir}
+    )
+    collected_list.freeze(mpy_lib_dir)
+
+    for file in collected_list.files():
+        if file.target_path.endswith("manifest.py"):
+            yield get_library_name_type(file.full_path)
+
+
+def copy_file(src: str, target: str, destination: str):
+    s = target.split("/")
+    s.pop()
+    d = f"{destination}/{'/'.join(s)}"
+    if not os.path.exists(d):
+        os.makedirs(d)
+    shutil.copy(src, f"{destination}/{target}")
+
+
+def copy_libraries(manifest, destination: str):
+    for f in manifest.files():
+        copy_file(f.full_path, f.target_path, destination)
+
+
+def process_cmdline_args():
+    parser = argparse.ArgumentParser(
+        description="Prepare micropython-lib to be installed"
+    )
+    parser.add_argument("micropython", help="Path to micropython source directory")
+    parser.add_argument("destination", help="Destination directory")
+
+    args = parser.parse_args()
+    return os.path.abspath(args.micropython), os.path.abspath(args.destination)
+
+
+def load_manifestfile_module(micropython_dir: str):
+    global manifestfile
+    sys.path.append(f"{micropython_dir}/tools")
+    manifestfile = importlib.import_module("manifestfile")
+
+
+def main():
+    micropython_dir, destination_dir = process_cmdline_args()
+    mpy_lib_dir = f"{micropython_dir}/lib/micropython-lib"
+
+    # load manifest file after knowing where it is
+    load_manifestfile_module(micropython_dir)
+
+    manifest = manifestfile.ManifestFile(
+        manifestfile.MODE_FREEZE, {"MPY_LIB_DIR": mpy_lib_dir}
+    )
+
+    for library, is_ffi in get_all_libraries(mpy_lib_dir):
+        manifest.require(library, unix_ffi=is_ffi)
+
+    copy_libraries(manifest, destination_dir)
+
+
+main()
diff --git a/package/micropython/micropython.mk b/package/micropython/micropython.mk
index 37c148da94..a9757e266c 100644
--- a/package/micropython/micropython.mk
+++ b/package/micropython/micropython.mk
@@ -47,6 +47,8 @@ define MICROPYTHON_BUILD_CMDS
 	$(TARGET_MAKE_ENV) $(MAKE) -C $(@D)/mpy-cross
 	$(TARGET_MAKE_ENV) $(MAKE) -C $(@D)/ports/unix \
 		$(MICROPYTHON_MAKE_OPTS)
+	$(EXTRA_ENV) package/micropython/collect_micropython_lib.py \
+		$(@D) $(@D)/.built_pylib
 endef
 
 define MICROPYTHON_INSTALL_TARGET_CMDS
@@ -57,4 +59,14 @@ define MICROPYTHON_INSTALL_TARGET_CMDS
 		install
 endef
 
+define MICROPYTHON_INSTALL_LIBS
+	$(INSTALL) -d -m 0755 $(TARGET_DIR)/usr/lib/micropython
+	cp -a $(@D)/.built_pylib/* $(TARGET_DIR)/usr/lib/micropython
+endef
+
+ifeq ($(BR2_PACKAGE_MICROPYTHON_LIB),y)
+MICROPYTHON_POST_INSTALL_TARGET_HOOKS += MICROPYTHON_INSTALL_LIBS
+endif
+
+
 $(eval $(generic-package))
-- 
2.43.0

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply related	[flat|nested] 8+ messages in thread