* [Buildroot] [PATCH 1/1] utils/scanpypi: add flit package support
@ 2022-03-13 14:37 James Hilliard
2022-03-13 16:31 ` Yann E. MORIN
0 siblings, 1 reply; 2+ messages in thread
From: James Hilliard @ 2022-03-13 14:37 UTC (permalink / raw)
To: buildroot; +Cc: James Hilliard
These packages don't have a setup.py so we instead need to parse their
pyproject.toml file.
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
utils/scanpypi | 126 ++++++++++++++++++++++++++++++++++++++++---------
1 file changed, 105 insertions(+), 21 deletions(-)
diff --git a/utils/scanpypi b/utils/scanpypi
index 17d8a0017a..98426820c6 100755
--- a/utils/scanpypi
+++ b/utils/scanpypi
@@ -42,6 +42,48 @@ except ImportError:
'pip install spdx_lookup')
liclookup = None
+def toml_load(f):
+ with open(f, 'rb') as fh:
+ ex = None
+
+ # Try regular tomli first
+ try:
+ from tomli import load
+ return load(fh)
+ except ImportError as e:
+ ex = e
+
+ # Try pip's vendored tomli
+ try:
+ from pip._vendor.tomli import load
+ try:
+ return load(fh)
+ except TypeError:
+ # Fallback to handle older version
+ try:
+ fh.seek(0)
+ w = io.TextIOWrapper(fh, encoding="utf8", newline="")
+ return load(w)
+ finally:
+ w.detach()
+ except ImportError as e:
+ pass
+
+ # Try regular toml last
+ try:
+ from toml import load
+ fh.seek(0)
+ w = io.TextIOWrapper(fh, encoding="utf8", newline="")
+ try:
+ return load(w)
+ finally:
+ w.detach()
+ except ImportError:
+ pass
+
+ print('This package needs tomli')
+ raise ex
+
def setup_decorator(func, method):
"""
@@ -296,20 +338,52 @@ class BuildrootPackage():
current_dir = os.getcwd()
os.chdir(self.tmp_extract)
sys.path.insert(0, self.tmp_extract)
- s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
- imp.load_module('__main__', s_file, s_path, s_desc)
- if self.metadata_name in self.setup_args:
- pass
- elif self.metadata_name.replace('_', '-') in self.setup_args:
- self.metadata_name = self.metadata_name.replace('_', '-')
- elif self.metadata_name.replace('-', '_') in self.setup_args:
- self.metadata_name = self.metadata_name.replace('-', '_')
try:
- self.setup_metadata = self.setup_args[self.metadata_name]
- except KeyError:
- # This means setup was not called
- print('ERROR: Could not determine package metadata for {pkg}.\n'
- .format(pkg=self.real_name))
+ s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
+ imp.load_module('__main__', s_file, s_path, s_desc)
+ if self.metadata_name in self.setup_args:
+ pass
+ elif self.metadata_name.replace('_', '-') in self.setup_args:
+ self.metadata_name = self.metadata_name.replace('_', '-')
+ elif self.metadata_name.replace('-', '_') in self.setup_args:
+ self.metadata_name = self.metadata_name.replace('-', '_')
+ try:
+ self.setup_metadata = self.setup_args[self.metadata_name]
+ except KeyError:
+ # This means setup was not called
+ print('ERROR: Could not determine package metadata for {pkg}.\n'
+ .format(pkg=self.real_name))
+ raise
+ finally:
+ os.chdir(current_dir)
+ sys.path.remove(self.tmp_extract)
+
+ def load_pyproject(self):
+ """
+ Loads the corresponding pyproject.toml and store its metadata
+ """
+ current_dir = os.getcwd()
+ os.chdir(self.tmp_extract)
+ sys.path.insert(0, self.tmp_extract)
+ try:
+ pyproject_data = toml_load('pyproject.toml')
+ try:
+ self.setup_metadata = pyproject_data.get('project', {})
+ self.metadata_name = self.setup_metadata.get('name', self.real_name)
+ build_system = pyproject_data.get('build-system', {})
+ build_backend = build_system.get('build-backend', None)
+ if build_backend is not None and build_backend == 'flit_core.buildapi':
+ self.setup_metadata['method'] = 'flit'
+ elif build_system.get('backend-path', None) is not None:
+ self.setup_metadata['method'] = 'pep517'
+ else:
+ self.setup_metadata['method'] = 'unknown'
+ except KeyError:
+ # This means setup was not called
+ print('ERROR: Could not determine package metadata for {pkg}.\n'
+ .format(pkg=self.real_name))
+ raise
+ except FileNotFoundError:
raise
os.chdir(current_dir)
sys.path.remove(self.tmp_extract)
@@ -609,7 +683,8 @@ class BuildrootPackage():
lines.append('\thelp\n')
- help_lines = textwrap.wrap(self.metadata['info']['summary'], 62,
+ md_info = self.metadata['info']
+ help_lines = textwrap.wrap(md_info['summary'], 62,
initial_indent='\t ',
subsequent_indent='\t ')
@@ -617,11 +692,17 @@ class BuildrootPackage():
if help_lines[-1][-1] != '.':
help_lines[-1] += '.'
- # \t + two spaces is 3 char long
- help_lines.append('')
- help_lines.append('\t ' + self.metadata['info']['home_page'])
- help_lines = [x + '\n' for x in help_lines]
- lines += help_lines
+ home_page = md_info.get('home_page', None)
+ if home_page is None or len(home_page) == 0:
+ project_urls = md_info.get('project_urls', {})
+ home_page = project_urls.get('Homepage', None)
+
+ if home_page is not None and len(home_page) != 0:
+ # \t + two spaces is 3 char long
+ help_lines.append('')
+ help_lines.append('\t ' + home_page)
+ help_lines = [x + '\n' for x in help_lines]
+ lines += help_lines
with open(path_to_config, 'w') as config_file:
config_file.writelines(lines)
@@ -692,9 +773,12 @@ def main():
except ImportError as err:
if 'buildutils' in str(err):
print('This package needs buildutils')
+ continue
else:
- raise
- continue
+ try:
+ package.load_pyproject()
+ except Exception as e:
+ raise
except (AttributeError, KeyError) as error:
print('Error: Could not install package {pkg}: {error}'.format(
pkg=package.real_name, error=error))
--
2.25.1
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [Buildroot] [PATCH 1/1] utils/scanpypi: add flit package support
2022-03-13 14:37 [Buildroot] [PATCH 1/1] utils/scanpypi: add flit package support James Hilliard
@ 2022-03-13 16:31 ` Yann E. MORIN
0 siblings, 0 replies; 2+ messages in thread
From: Yann E. MORIN @ 2022-03-13 16:31 UTC (permalink / raw)
To: James Hilliard; +Cc: buildroot
James, All,
On 2022-03-13 08:37 -0600, James Hilliard spake thusly:
> These packages don't have a setup.py so we instead need to parse their
> pyproject.toml file.
Please, split this into at least two patches;
- offload the loading to a separate function (i.e. load_pyproject())
- introduce support for flit packages
Also, even though I did not look at the code too closely, it looks like
the handling of the homepage is not strictly tied to the flit support,
so perhaps it should be also separated (unless it is tied to the
introduction of load_pyproject()?)
Anyway, separate patches into semantically contained minimalist patches,
please.
Alos, please try to provide more detailed explanations in your commit
logs. Consider that reviewers are dumb in your area of expertise (I
certainly am!), and consider what information we will need to understand
the change, and also what we will need in two weeks, two months, two
years when we have an issue and we want to understand how the code came
to be...
Regards,
Yann E. MORIN.
> Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
> ---
> utils/scanpypi | 126 ++++++++++++++++++++++++++++++++++++++++---------
> 1 file changed, 105 insertions(+), 21 deletions(-)
>
> diff --git a/utils/scanpypi b/utils/scanpypi
> index 17d8a0017a..98426820c6 100755
> --- a/utils/scanpypi
> +++ b/utils/scanpypi
> @@ -42,6 +42,48 @@ except ImportError:
> 'pip install spdx_lookup')
> liclookup = None
>
> +def toml_load(f):
> + with open(f, 'rb') as fh:
> + ex = None
> +
> + # Try regular tomli first
> + try:
> + from tomli import load
> + return load(fh)
> + except ImportError as e:
> + ex = e
> +
> + # Try pip's vendored tomli
> + try:
> + from pip._vendor.tomli import load
> + try:
> + return load(fh)
> + except TypeError:
> + # Fallback to handle older version
> + try:
> + fh.seek(0)
> + w = io.TextIOWrapper(fh, encoding="utf8", newline="")
> + return load(w)
> + finally:
> + w.detach()
> + except ImportError as e:
> + pass
> +
> + # Try regular toml last
> + try:
> + from toml import load
> + fh.seek(0)
> + w = io.TextIOWrapper(fh, encoding="utf8", newline="")
> + try:
> + return load(w)
> + finally:
> + w.detach()
> + except ImportError:
> + pass
> +
> + print('This package needs tomli')
> + raise ex
> +
>
> def setup_decorator(func, method):
> """
> @@ -296,20 +338,52 @@ class BuildrootPackage():
> current_dir = os.getcwd()
> os.chdir(self.tmp_extract)
> sys.path.insert(0, self.tmp_extract)
> - s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
> - imp.load_module('__main__', s_file, s_path, s_desc)
> - if self.metadata_name in self.setup_args:
> - pass
> - elif self.metadata_name.replace('_', '-') in self.setup_args:
> - self.metadata_name = self.metadata_name.replace('_', '-')
> - elif self.metadata_name.replace('-', '_') in self.setup_args:
> - self.metadata_name = self.metadata_name.replace('-', '_')
> try:
> - self.setup_metadata = self.setup_args[self.metadata_name]
> - except KeyError:
> - # This means setup was not called
> - print('ERROR: Could not determine package metadata for {pkg}.\n'
> - .format(pkg=self.real_name))
> + s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
> + imp.load_module('__main__', s_file, s_path, s_desc)
> + if self.metadata_name in self.setup_args:
> + pass
> + elif self.metadata_name.replace('_', '-') in self.setup_args:
> + self.metadata_name = self.metadata_name.replace('_', '-')
> + elif self.metadata_name.replace('-', '_') in self.setup_args:
> + self.metadata_name = self.metadata_name.replace('-', '_')
> + try:
> + self.setup_metadata = self.setup_args[self.metadata_name]
> + except KeyError:
> + # This means setup was not called
> + print('ERROR: Could not determine package metadata for {pkg}.\n'
> + .format(pkg=self.real_name))
> + raise
> + finally:
> + os.chdir(current_dir)
> + sys.path.remove(self.tmp_extract)
> +
> + def load_pyproject(self):
> + """
> + Loads the corresponding pyproject.toml and store its metadata
> + """
> + current_dir = os.getcwd()
> + os.chdir(self.tmp_extract)
> + sys.path.insert(0, self.tmp_extract)
> + try:
> + pyproject_data = toml_load('pyproject.toml')
> + try:
> + self.setup_metadata = pyproject_data.get('project', {})
> + self.metadata_name = self.setup_metadata.get('name', self.real_name)
> + build_system = pyproject_data.get('build-system', {})
> + build_backend = build_system.get('build-backend', None)
> + if build_backend is not None and build_backend == 'flit_core.buildapi':
> + self.setup_metadata['method'] = 'flit'
> + elif build_system.get('backend-path', None) is not None:
> + self.setup_metadata['method'] = 'pep517'
> + else:
> + self.setup_metadata['method'] = 'unknown'
> + except KeyError:
> + # This means setup was not called
> + print('ERROR: Could not determine package metadata for {pkg}.\n'
> + .format(pkg=self.real_name))
> + raise
> + except FileNotFoundError:
> raise
> os.chdir(current_dir)
> sys.path.remove(self.tmp_extract)
> @@ -609,7 +683,8 @@ class BuildrootPackage():
>
> lines.append('\thelp\n')
>
> - help_lines = textwrap.wrap(self.metadata['info']['summary'], 62,
> + md_info = self.metadata['info']
> + help_lines = textwrap.wrap(md_info['summary'], 62,
> initial_indent='\t ',
> subsequent_indent='\t ')
>
> @@ -617,11 +692,17 @@ class BuildrootPackage():
> if help_lines[-1][-1] != '.':
> help_lines[-1] += '.'
>
> - # \t + two spaces is 3 char long
> - help_lines.append('')
> - help_lines.append('\t ' + self.metadata['info']['home_page'])
> - help_lines = [x + '\n' for x in help_lines]
> - lines += help_lines
> + home_page = md_info.get('home_page', None)
> + if home_page is None or len(home_page) == 0:
> + project_urls = md_info.get('project_urls', {})
> + home_page = project_urls.get('Homepage', None)
> +
> + if home_page is not None and len(home_page) != 0:
> + # \t + two spaces is 3 char long
> + help_lines.append('')
> + help_lines.append('\t ' + home_page)
> + help_lines = [x + '\n' for x in help_lines]
> + lines += help_lines
>
> with open(path_to_config, 'w') as config_file:
> config_file.writelines(lines)
> @@ -692,9 +773,12 @@ def main():
> except ImportError as err:
> if 'buildutils' in str(err):
> print('This package needs buildutils')
> + continue
> else:
> - raise
> - continue
> + try:
> + package.load_pyproject()
> + except Exception as e:
> + raise
> except (AttributeError, KeyError) as error:
> print('Error: Could not install package {pkg}: {error}'.format(
> pkg=package.real_name, error=error))
> --
> 2.25.1
>
> _______________________________________________
> buildroot mailing list
> buildroot@buildroot.org
> https://lists.buildroot.org/mailman/listinfo/buildroot
--
.-----------------.--------------------.------------------.--------------------.
| Yann E. MORIN | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software Designer | \ / CAMPAIGN | ___ |
| +33 561 099 427 `------------.-------: X AGAINST | \e/ There is no |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL | v conspiracy. |
'------------------------------^-------^------------------^--------------------'
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2022-03-13 16:32 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-03-13 14:37 [Buildroot] [PATCH 1/1] utils/scanpypi: add flit package support James Hilliard
2022-03-13 16:31 ` Yann E. MORIN
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.