[Buildroot] [PATCH 1/1] utils/scanpypi: add flit package support

[Buildroot] [PATCH 1/1] utils/scanpypi: add flit package support

[James Hilliard]

These packages don't have a setup.py so we instead need to parse their
pyproject.toml file.

Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
 utils/scanpypi | 126 ++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 105 insertions(+), 21 deletions(-)

diff --git a/utils/scanpypi b/utils/scanpypi
index 17d8a0017a..98426820c6 100755
--- a/utils/scanpypi
+++ b/utils/scanpypi
@@ -42,6 +42,48 @@ except ImportError:
           'pip install spdx_lookup')
     liclookup = None
 
+def toml_load(f):
+    with open(f, 'rb') as fh:
+        ex = None
+
+        # Try regular tomli first
+        try:
+            from tomli import load
+            return load(fh)
+        except ImportError as e:
+            ex = e
+
+        # Try pip's vendored tomli
+        try:
+            from pip._vendor.tomli import load
+            try:
+                return load(fh)
+            except TypeError:
+                # Fallback to handle older version
+                try:
+                    fh.seek(0)
+                    w = io.TextIOWrapper(fh, encoding="utf8", newline="")
+                    return load(w)
+                finally:
+                    w.detach()
+        except ImportError as e:
+            pass
+
+        # Try regular toml last
+        try:
+            from toml import load
+            fh.seek(0)
+            w = io.TextIOWrapper(fh, encoding="utf8", newline="")
+            try:
+                return load(w)
+            finally:
+                w.detach()
+        except ImportError:
+            pass
+
+        print('This package needs tomli')
+        raise ex
+
 
 def setup_decorator(func, method):
     """
@@ -296,20 +338,52 @@ class BuildrootPackage():
         current_dir = os.getcwd()
         os.chdir(self.tmp_extract)
         sys.path.insert(0, self.tmp_extract)
-        s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
-        imp.load_module('__main__', s_file, s_path, s_desc)
-        if self.metadata_name in self.setup_args:
-            pass
-        elif self.metadata_name.replace('_', '-') in self.setup_args:
-            self.metadata_name = self.metadata_name.replace('_', '-')
-        elif self.metadata_name.replace('-', '_') in self.setup_args:
-            self.metadata_name = self.metadata_name.replace('-', '_')
         try:
-            self.setup_metadata = self.setup_args[self.metadata_name]
-        except KeyError:
-            # This means setup was not called
-            print('ERROR: Could not determine package metadata for {pkg}.\n'
-                  .format(pkg=self.real_name))
+            s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
+            imp.load_module('__main__', s_file, s_path, s_desc)
+            if self.metadata_name in self.setup_args:
+                pass
+            elif self.metadata_name.replace('_', '-') in self.setup_args:
+                self.metadata_name = self.metadata_name.replace('_', '-')
+            elif self.metadata_name.replace('-', '_') in self.setup_args:
+                self.metadata_name = self.metadata_name.replace('-', '_')
+            try:
+                self.setup_metadata = self.setup_args[self.metadata_name]
+            except KeyError:
+                # This means setup was not called
+                print('ERROR: Could not determine package metadata for {pkg}.\n'
+                      .format(pkg=self.real_name))
+                raise
+        finally:
+            os.chdir(current_dir)
+            sys.path.remove(self.tmp_extract)
+
+    def load_pyproject(self):
+        """
+        Loads the corresponding pyproject.toml and store its metadata
+        """
+        current_dir = os.getcwd()
+        os.chdir(self.tmp_extract)
+        sys.path.insert(0, self.tmp_extract)
+        try:
+            pyproject_data = toml_load('pyproject.toml')
+            try:
+                self.setup_metadata = pyproject_data.get('project', {})
+                self.metadata_name = self.setup_metadata.get('name', self.real_name)
+                build_system = pyproject_data.get('build-system', {})
+                build_backend = build_system.get('build-backend', None)
+                if build_backend is not None and build_backend == 'flit_core.buildapi':
+                    self.setup_metadata['method'] = 'flit'
+                elif build_system.get('backend-path', None) is not None:
+                    self.setup_metadata['method'] = 'pep517'
+                else:
+                    self.setup_metadata['method'] = 'unknown'
+            except KeyError:
+                # This means setup was not called
+                print('ERROR: Could not determine package metadata for {pkg}.\n'
+                      .format(pkg=self.real_name))
+                raise
+        except FileNotFoundError:
             raise
         os.chdir(current_dir)
         sys.path.remove(self.tmp_extract)
@@ -609,7 +683,8 @@ class BuildrootPackage():
 
         lines.append('\thelp\n')
 
-        help_lines = textwrap.wrap(self.metadata['info']['summary'], 62,
+        md_info = self.metadata['info']
+        help_lines = textwrap.wrap(md_info['summary'], 62,
                                    initial_indent='\t  ',
                                    subsequent_indent='\t  ')
 
@@ -617,11 +692,17 @@ class BuildrootPackage():
         if help_lines[-1][-1] != '.':
             help_lines[-1] += '.'
 
-        # \t + two spaces is 3 char long
-        help_lines.append('')
-        help_lines.append('\t  ' + self.metadata['info']['home_page'])
-        help_lines = [x + '\n' for x in help_lines]
-        lines += help_lines
+        home_page = md_info.get('home_page', None)
+        if home_page is None or len(home_page) == 0:
+            project_urls = md_info.get('project_urls', {})
+            home_page = project_urls.get('Homepage', None)
+
+        if home_page is not None and len(home_page) != 0:
+            # \t + two spaces is 3 char long
+            help_lines.append('')
+            help_lines.append('\t  ' + home_page)
+            help_lines = [x + '\n' for x in help_lines]
+            lines += help_lines
 
         with open(path_to_config, 'w') as config_file:
             config_file.writelines(lines)
@@ -692,9 +773,12 @@ def main():
             except ImportError as err:
                 if 'buildutils' in str(err):
                     print('This package needs buildutils')
+                    continue
                 else:
-                    raise
-                continue
+                    try:
+                        package.load_pyproject()
+                    except Exception as e:
+                        raise
             except (AttributeError, KeyError) as error:
                 print('Error: Could not install package {pkg}: {error}'.format(
                     pkg=package.real_name, error=error))
-- 
2.25.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH 1/1] utils/scanpypi: add flit package support

[Yann E. MORIN]

James, All,

On 2022-03-13 08:37 -0600, James Hilliard spake thusly:
> These packages don't have a setup.py so we instead need to parse their
> pyproject.toml file.

Please, split this into at least two patches;

  - offload the loading to a separate function (i.e. load_pyproject())
  - introduce support for flit packages

Also, even though I did not look at the code too closely, it looks like
the handling of the homepage is not strictly tied to the flit support,
so perhaps it should be also separated (unless it is tied to the
introduction of load_pyproject()?)

Anyway, separate patches into semantically contained minimalist patches,
please.

Alos, please try to provide more detailed explanations in your commit
logs. Consider that reviewers are dumb in your area of expertise (I
certainly am!), and consider what information we will need to understand
the change, and also what we will need in two weeks, two months, two
years when we have an issue and we want to understand how the code came
to be...

Regards,
Yann E. MORIN.

> Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
> ---
>  utils/scanpypi | 126 ++++++++++++++++++++++++++++++++++++++++---------
>  1 file changed, 105 insertions(+), 21 deletions(-)
> 
> diff --git a/utils/scanpypi b/utils/scanpypi
> index 17d8a0017a..98426820c6 100755
> --- a/utils/scanpypi
> +++ b/utils/scanpypi
> @@ -42,6 +42,48 @@ except ImportError:
>            'pip install spdx_lookup')
>      liclookup = None
>  
> +def toml_load(f):
> +    with open(f, 'rb') as fh:
> +        ex = None
> +
> +        # Try regular tomli first
> +        try:
> +            from tomli import load
> +            return load(fh)
> +        except ImportError as e:
> +            ex = e
> +
> +        # Try pip's vendored tomli
> +        try:
> +            from pip._vendor.tomli import load
> +            try:
> +                return load(fh)
> +            except TypeError:
> +                # Fallback to handle older version
> +                try:
> +                    fh.seek(0)
> +                    w = io.TextIOWrapper(fh, encoding="utf8", newline="")
> +                    return load(w)
> +                finally:
> +                    w.detach()
> +        except ImportError as e:
> +            pass
> +
> +        # Try regular toml last
> +        try:
> +            from toml import load
> +            fh.seek(0)
> +            w = io.TextIOWrapper(fh, encoding="utf8", newline="")
> +            try:
> +                return load(w)
> +            finally:
> +                w.detach()
> +        except ImportError:
> +            pass
> +
> +        print('This package needs tomli')
> +        raise ex
> +
>  
>  def setup_decorator(func, method):
>      """
> @@ -296,20 +338,52 @@ class BuildrootPackage():
>          current_dir = os.getcwd()
>          os.chdir(self.tmp_extract)
>          sys.path.insert(0, self.tmp_extract)
> -        s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
> -        imp.load_module('__main__', s_file, s_path, s_desc)
> -        if self.metadata_name in self.setup_args:
> -            pass
> -        elif self.metadata_name.replace('_', '-') in self.setup_args:
> -            self.metadata_name = self.metadata_name.replace('_', '-')
> -        elif self.metadata_name.replace('-', '_') in self.setup_args:
> -            self.metadata_name = self.metadata_name.replace('-', '_')
>          try:
> -            self.setup_metadata = self.setup_args[self.metadata_name]
> -        except KeyError:
> -            # This means setup was not called
> -            print('ERROR: Could not determine package metadata for {pkg}.\n'
> -                  .format(pkg=self.real_name))
> +            s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
> +            imp.load_module('__main__', s_file, s_path, s_desc)
> +            if self.metadata_name in self.setup_args:
> +                pass
> +            elif self.metadata_name.replace('_', '-') in self.setup_args:
> +                self.metadata_name = self.metadata_name.replace('_', '-')
> +            elif self.metadata_name.replace('-', '_') in self.setup_args:
> +                self.metadata_name = self.metadata_name.replace('-', '_')
> +            try:
> +                self.setup_metadata = self.setup_args[self.metadata_name]
> +            except KeyError:
> +                # This means setup was not called
> +                print('ERROR: Could not determine package metadata for {pkg}.\n'
> +                      .format(pkg=self.real_name))
> +                raise
> +        finally:
> +            os.chdir(current_dir)
> +            sys.path.remove(self.tmp_extract)
> +
> +    def load_pyproject(self):
> +        """
> +        Loads the corresponding pyproject.toml and store its metadata
> +        """
> +        current_dir = os.getcwd()
> +        os.chdir(self.tmp_extract)
> +        sys.path.insert(0, self.tmp_extract)
> +        try:
> +            pyproject_data = toml_load('pyproject.toml')
> +            try:
> +                self.setup_metadata = pyproject_data.get('project', {})
> +                self.metadata_name = self.setup_metadata.get('name', self.real_name)
> +                build_system = pyproject_data.get('build-system', {})
> +                build_backend = build_system.get('build-backend', None)
> +                if build_backend is not None and build_backend == 'flit_core.buildapi':
> +                    self.setup_metadata['method'] = 'flit'
> +                elif build_system.get('backend-path', None) is not None:
> +                    self.setup_metadata['method'] = 'pep517'
> +                else:
> +                    self.setup_metadata['method'] = 'unknown'
> +            except KeyError:
> +                # This means setup was not called
> +                print('ERROR: Could not determine package metadata for {pkg}.\n'
> +                      .format(pkg=self.real_name))
> +                raise
> +        except FileNotFoundError:
>              raise
>          os.chdir(current_dir)
>          sys.path.remove(self.tmp_extract)
> @@ -609,7 +683,8 @@ class BuildrootPackage():
>  
>          lines.append('\thelp\n')
>  
> -        help_lines = textwrap.wrap(self.metadata['info']['summary'], 62,
> +        md_info = self.metadata['info']
> +        help_lines = textwrap.wrap(md_info['summary'], 62,
>                                     initial_indent='\t  ',
>                                     subsequent_indent='\t  ')
>  
> @@ -617,11 +692,17 @@ class BuildrootPackage():
>          if help_lines[-1][-1] != '.':
>              help_lines[-1] += '.'
>  
> -        # \t + two spaces is 3 char long
> -        help_lines.append('')
> -        help_lines.append('\t  ' + self.metadata['info']['home_page'])
> -        help_lines = [x + '\n' for x in help_lines]
> -        lines += help_lines
> +        home_page = md_info.get('home_page', None)
> +        if home_page is None or len(home_page) == 0:
> +            project_urls = md_info.get('project_urls', {})
> +            home_page = project_urls.get('Homepage', None)
> +
> +        if home_page is not None and len(home_page) != 0:
> +            # \t + two spaces is 3 char long
> +            help_lines.append('')
> +            help_lines.append('\t  ' + home_page)
> +            help_lines = [x + '\n' for x in help_lines]
> +            lines += help_lines
>  
>          with open(path_to_config, 'w') as config_file:
>              config_file.writelines(lines)
> @@ -692,9 +773,12 @@ def main():
>              except ImportError as err:
>                  if 'buildutils' in str(err):
>                      print('This package needs buildutils')
> +                    continue
>                  else:
> -                    raise
> -                continue
> +                    try:
> +                        package.load_pyproject()
> +                    except Exception as e:
> +                        raise
>              except (AttributeError, KeyError) as error:
>                  print('Error: Could not install package {pkg}: {error}'.format(
>                      pkg=package.real_name, error=error))
> -- 
> 2.25.1
> 
> _______________________________________________
> buildroot mailing list
> buildroot@buildroot.org
> https://lists.buildroot.org/mailman/listinfo/buildroot

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot