Re: [OE-core] [PATCH v2 4/4] scripts:recipetool:create_buildsys_python: add PEP517 support

[Alexandre Belloni <alexandre.belloni@bootlin.com>]

On 19/10/2023 20:20:33+0200, Julien Stephan wrote:
> Le jeu. 19 oct. 2023 � 15:49, Alexandre Belloni
> <alexandre.belloni@bootlin.com> a �crit :
> >
> > Hello,
> >
> > On 19/10/2023 09:36:53+0200, Julien Stephan wrote:
> > > add support for PEP517 [1]
> > >
> > > if a pyproject.toml file is found, use it to create the recipe,
> > > otherwise fallback to the old setup.py method.
> > >
> > > [YOCTO #14737]
> > >
> > > [1]: https://peps.python.org/pep-0517/
> > >
> > > Signed-off-by: Julien Stephan <jstephan@baylibre.com>
> > > ---
> > >  .../lib/recipetool/create_buildsys_python.py  | 234 +++++++++++++++++-
> > >  1 file changed, 233 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
> > > index 69f6f5ca511..0b601d50a4b 100644
> > > --- a/scripts/lib/recipetool/create_buildsys_python.py
> > > +++ b/scripts/lib/recipetool/create_buildsys_python.py
> > > @@ -18,6 +18,7 @@ import os
> > >  import re
> > >  import sys
> > >  import subprocess
> > > +import toml
> >
> > This fails on the autobuilders because we don't have the toml module installed so I guess you need to add a dependency.
> >
> 
> Hello,
> 
> Sure I 'll do it. Just to confirm, I should add it here:
> https://docs.yoctoproject.org/ref-manual/system-requirements.html#required-packages-for-the-build-host
> ?

I guess the preferred way would be to depend on python3-toml-native
instead of requiring installation on the host.

> 
> Cheers
> Julien
> 
> > >  from recipetool.create import RecipeHandler
> > >
> > >  logger = logging.getLogger('recipetool')
> > > @@ -656,6 +657,235 @@ class PythonSetupPyRecipeHandler(PythonRecipeHandler):
> > >
> > >          handled.append('buildsystem')
> > >
> > > +class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler):
> > > +    """Base class to support PEP517 and PEP518
> > > +
> > > +    PEP517 https://peps.python.org/pep-0517/#source-trees
> > > +    PEP518 https://peps.python.org/pep-0518/#build-system-table
> > > +    """
> > > +
> > > +    # PEP621: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/
> > > +    # add only the ones that map to a BB var
> > > +    # potentially missing: optional-dependencies
> > > +    bbvar_map = {
> > > +        "name": "PN",
> > > +        "version": "PV",
> > > +        "Homepage": "HOMEPAGE",
> > > +        "description": "SUMMARY",
> > > +        "license": "LICENSE",
> > > +        "dependencies": "RDEPENDS:${PN}",
> > > +        "requires": "DEPENDS",
> > > +    }
> > > +
> > > +    replacements = [
> > > +        ("license", r" +$", ""),
> > > +        ("license", r"^ +", ""),
> > > +        ("license", r" ", "-"),
> > > +        ("license", r"^GNU-", ""),
> > > +        ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""),
> > > +        ("license", r"^UNKNOWN$", ""),
> > > +        # Remove currently unhandled version numbers from these variables
> > > +        ("requires", r"\[[^\]]+\]$", ""),
> > > +        ("requires", r"^([^><= ]+).*", r"\1"),
> > > +        ("dependencies", r"\[[^\]]+\]$", ""),
> > > +        ("dependencies", r"^([^><= ]+).*", r"\1"),
> > > +    ]
> > > +
> > > +    build_backend_map = {
> > > +        "setuptools.build_meta": "python_setuptools_build_meta",
> > > +        "poetry.core.masonry.api": "python_poetry_core",
> > > +        "flit_core.buildapi": "python_flit_core",
> > > +    }
> > > +
> > > +    excluded_native_pkgdeps = [
> > > +        # already provided by python_setuptools_build_meta.bbclass
> > > +        "python3-setuptools-native",
> > > +        "python3-wheel-native",
> > > +        # already provided by python_poetry_core.bbclass
> > > +        "python3-poetry-core-native",
> > > +        # already provided by python_flit_core.bbclass
> > > +        "python3-flit-core-native",
> > > +    ]
> > > +
> > > +    # add here a list of known and often used packages and the corresponding bitbake package
> > > +    known_deps_map = {
> > > +        "setuptools": "python3-setuptools",
> > > +        "wheel": "python3-wheel",
> > > +        "poetry-core": "python3-poetry-core",
> > > +        "flit_core": "python3-flit-core",
> > > +        "setuptools-scm": "python3-setuptools-scm",
> > > +    }
> > > +
> > > +    def __init__(self):
> > > +        pass
> > > +
> > > +    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
> > > +        info = {}
> > > +
> > > +        if 'buildsystem' in handled:
> > > +            return False
> > > +
> > > +        # Check for non-zero size setup.py files
> > > +        setupfiles = RecipeHandler.checkfiles(srctree, ["pyproject.toml"])
> > > +        for fn in setupfiles:
> > > +            if os.path.getsize(fn):
> > > +                break
> > > +        else:
> > > +            return False
> > > +
> > > +        setupscript = os.path.join(srctree, "pyproject.toml")
> > > +
> > > +        try:
> > > +            config = self.parse_pyproject_toml(setupscript)
> > > +            build_backend = config["build-system"]["build-backend"]
> > > +            if build_backend in self.build_backend_map:
> > > +                classes.append(self.build_backend_map[build_backend])
> > > +            else:
> > > +                logger.error(
> > > +                    "Unsupported build-backend: %s, cannot use pyproject.toml. Will try to use legacy setup.py"
> > > +                    % build_backend
> > > +                )
> > > +                return False
> > > +
> > > +            licfile = ""
> > > +            if "project" in config:
> > > +                for field, values in config["project"].items():
> > > +                    if field == "license":
> > > +                        value = values.get("text", "")
> > > +                        if not value:
> > > +                            licfile = values.get("file", "")
> > > +                    elif isinstance(values, dict):
> > > +                        for k, v in values.items():
> > > +                            info[k] = v
> > > +                        continue
> > > +                    else:
> > > +                        value = values
> > > +
> > > +                    info[field] = value
> > > +
> > > +            # Grab the license value before applying replacements
> > > +            license_str = info.get("license", "").strip()
> > > +
> > > +            if license_str:
> > > +                for i, line in enumerate(lines_before):
> > > +                    if line.startswith("##LICENSE_PLACEHOLDER##"):
> > > +                        lines_before.insert(
> > > +                            i, "# NOTE: License in pyproject.toml is: %s" % license_str
> > > +                        )
> > > +                        break
> > > +
> > > +            info["requires"] = config["build-system"]["requires"]
> > > +
> > > +            self.apply_info_replacements(info)
> > > +
> > > +            if "classifiers" in info:
> > > +                license = self.handle_classifier_license(
> > > +                    info["classifiers"], info.get("license", "")
> > > +                )
> > > +                if license:
> > > +                    if licfile:
> > > +                        lines = []
> > > +                        md5value = bb.utils.md5_file(os.path.join(srctree, licfile))
> > > +                        lines.append('LICENSE = "%s"' % license)
> > > +                        lines.append(
> > > +                            'LIC_FILES_CHKSUM = "file://%s;md5=%s"'
> > > +                            % (licfile, md5value)
> > > +                        )
> > > +                        lines.append("")
> > > +
> > > +                        # Replace the placeholder so we get the values in the right place in the recipe file
> > > +                        try:
> > > +                            pos = lines_before.index("##LICENSE_PLACEHOLDER##")
> > > +                        except ValueError:
> > > +                            pos = -1
> > > +                        if pos == -1:
> > > +                            lines_before.extend(lines)
> > > +                        else:
> > > +                            lines_before[pos : pos + 1] = lines
> > > +
> > > +                        handled.append(("license", [license, licfile, md5value]))
> > > +                    else:
> > > +                        info["license"] = license
> > > +
> > > +            provided_packages = self.parse_pkgdata_for_python_packages()
> > > +            provided_packages.update(self.known_deps_map)
> > > +            native_mapped_deps, native_unmapped_deps = set(), set()
> > > +            mapped_deps, unmapped_deps = set(), set()
> > > +
> > > +            if "requires" in info:
> > > +                for require in info["requires"]:
> > > +                    mapped = provided_packages.get(require)
> > > +
> > > +                    if mapped:
> > > +                        logger.error("Mapped %s to %s" % (require, mapped))
> > > +                        native_mapped_deps.add(mapped)
> > > +                    else:
> > > +                        logger.error("Could not map %s" % require)
> > > +                        native_unmapped_deps.add(require)
> > > +
> > > +                info.pop("requires")
> > > +
> > > +                if native_mapped_deps != set():
> > > +                    native_mapped_deps = {
> > > +                        item + "-native" for item in native_mapped_deps
> > > +                    }
> > > +                    native_mapped_deps -= set(self.excluded_native_pkgdeps)
> > > +                    if native_mapped_deps != set():
> > > +                        info["requires"] = " ".join(sorted(native_mapped_deps))
> > > +
> > > +                if native_unmapped_deps:
> > > +                    lines_after.append("")
> > > +                    lines_after.append(
> > > +                        "# WARNING: We were unable to map the following python package/module"
> > > +                    )
> > > +                    lines_after.append(
> > > +                        "# dependencies to the bitbake packages which include them:"
> > > +                    )
> > > +                    lines_after.extend(
> > > +                        "#    {}".format(d) for d in sorted(native_unmapped_deps)
> > > +                    )
> > > +
> > > +            if "dependencies" in info:
> > > +                for dependency in info["dependencies"]:
> > > +                    mapped = provided_packages.get(dependency)
> > > +                    if mapped:
> > > +                        logger.error("Mapped %s to %s" % (dependency, mapped))
> > > +                        mapped_deps.add(mapped)
> > > +                    else:
> > > +                        logger.error("Could not map %s" % dependency)
> > > +                        unmapped_deps.add(dependency)
> > > +
> > > +                info.pop("dependencies")
> > > +
> > > +                if mapped_deps != set():
> > > +                    if mapped_deps != set():
> > > +                        info["dependencies"] = " ".join(sorted(mapped_deps))
> > > +
> > > +                if unmapped_deps:
> > > +                    lines_after.append("")
> > > +                    lines_after.append(
> > > +                        "# WARNING: We were unable to map the following python package/module"
> > > +                    )
> > > +                    lines_after.append(
> > > +                        "# runtime dependencies to the bitbake packages which include them:"
> > > +                    )
> > > +                    lines_after.extend(
> > > +                        "#    {}".format(d) for d in sorted(unmapped_deps)
> > > +                    )
> > > +
> > > +            self.map_info_to_bbvar(info, extravalues)
> > > +
> > > +            handled.append("buildsystem")
> > > +        except Exception:
> > > +            logger.exception("Failed to parse pyproject.toml")
> > > +            return False
> > > +
> > > +    def parse_pyproject_toml(self, setupscript):
> > > +        with open(setupscript, "r") as f:
> > > +            config = toml.load(f)
> > > +        return config
> > > +
> > > +
> > >  def gather_setup_info(fileobj):
> > >      parsed = ast.parse(fileobj.read(), fileobj.name)
> > >      visitor = SetupScriptVisitor()
> > > @@ -769,5 +999,7 @@ def has_non_literals(value):
> > >
> > >
> > >  def register_recipe_handlers(handlers):
> > > -    # We need to make sure this is ahead of the makefile fallback handler
> > > +    # We need to make sure these are ahead of the makefile fallback handler
> > > +    # and the pyproject.toml handler ahead of the setup.py handler
> > > +    handlers.append((PythonPyprojectTomlRecipeHandler(), 75))
> > >      handlers.append((PythonSetupPyRecipeHandler(), 70))
> > > --
> > > 2.42.0
> > >
> >
> > >
> > > -=-=-=-=-=-=-=-=-=-=-=-
> > > Links: You receive all messages sent to this group.
> > > View/Reply Online (#189431): https://lists.openembedded.org/g/openembedded-core/message/189431
> > > Mute This Topic: https://lists.openembedded.org/mt/102055999/3617179
> > > Group Owner: openembedded-core+owner@lists.openembedded.org
> > > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alexandre.belloni@bootlin.com]
> > > -=-=-=-=-=-=-=-=-=-=-=-
> > >
> >
> >
> > --
> > Alexandre Belloni, co-owner and COO, Bootlin
> > Embedded Linux and Kernel engineering
> > https://bootlin.com

-- 
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com