| Message ID | 20231125073418.854169-1-james.hilliard1@gmail.com |
|---|---|
| State | Accepted |
| Headers | show |
| Series | [v6,1/1] utils/scanpypi: add flit package support | expand |
James, All, On 2023-11-25 00:34 -0700, James Hilliard spake thusly: > These packages don't have a setup.py so we instead need to parse their > pyproject.toml file. > > Note that this currently doesn't handle flit package dependency > resolution. > > Signed-off-by: James Hilliard <james.hilliard1@gmail.com> > Reviewed-by: Yegor Yefremov <yegorslists@googlemail.com> > --- > Changes v5 -> v6: > - handle None project_urls > Changes v4 -> v5: > - remove unnecessary "as e" from tomlib exception path > Changes v3 -> v4: > - prefer/support python3.11 tomllib > - use bool interpretation for None > Changes v2 -> v3: > - minor cleanup > - rebase > - more detailed commit log > Changes v1 -> v2: > - remove homepage format fixes(sent as separate patch) > - remove load_setup fixes > --- > utils/scanpypi | 93 +++++++++++++++++++++++++++++++++++++++++++++++--- > 1 file changed, 89 insertions(+), 4 deletions(-) $ ./utils/docker-run make check-package utils/scanpypi:0: run 'flake8' and fix the warnings $ ./utils/docker-run python3 -m flake8 utils/scanpypi utils/scanpypi:45:1: E302 expected 2 blank lines, found 1 utils/scanpypi:76:9: F841 local variable 'e' is assigned to but never used utils/scanpypi:788:21: F841 local variable 'e' is assigned to but never used :-( > diff --git a/utils/scanpypi b/utils/scanpypi > index 3c98bb4bcc..eae9783a1f 100755 > --- a/utils/scanpypi > +++ b/utils/scanpypi > @@ -42,6 +42,55 @@ except ImportError: > 'pip install spdx_lookup') > liclookup = None > > +def toml_load(f): > + with open(f, 'rb') as fh: > + ex = None > + > + # Try standard library tomllib first > + try: > + from tomllib import load > + return load(fh) > + except ImportError: > + pass > + > + # Try regular tomli next > + try: > + from tomli import load > + return load(fh) > + except ImportError as e: > + ex = e > + > + # Try pip's vendored tomli > + try: > + from pip._vendor.tomli import load > + try: > + return load(fh) > + except TypeError: > + # Fallback to handle older version > + try: > + fh.seek(0) > + w = io.TextIOWrapper(fh, encoding="utf8", newline="") > + return load(w) > + finally: > + w.detach() > + except ImportError as e: > + pass It was not obvious to me why we would not assign ex here, and I had to think about it quite a bit. That's because the real solution to all those fallbacks is to have the user actualy install tomli, so we want to raise the first import-error. OK, makes sense. [--SNIP--] > @@ -620,8 +698,12 @@ class BuildrootPackage(): > if help_lines[-1][-1] != '.': > help_lines[-1] += '.' > > - home_page = md_info.get('home_page', None) or \ > - md_info.get('project_urls', {}).get('Homepage', None) # noqa: E127 > + home_page = md_info.get('home_page', None) > + > + if not home_page: > + project_urls = md_info.get('project_urls', None) > + if project_urls: > + home_page = project_urls.get('Homepage', None) I was not sure why this had to be done in this patch, so I assume that the issue is that flit packages may not carry such a config item, while the previously supported mechanisms always had that. I added a little blurb in the commit log to briefly explain that. > if home_page: > # \t + two spaces is 3 char long > @@ -699,9 +781,12 @@ def main(): > except ImportError as err: > if 'buildutils' in str(err): > print('This package needs buildutils') > + continue > else: > - raise > - continue > + try: > + package.load_pyproject() > + except Exception as e: > + raise I remember Thomas was not very happy about nesting all those try-except blocks, and I am not too happy either. However, this is currently a 2-deep nesting, which is not *too* ugly, and not nesting would probably require an uglier code, so meh as well. If/when we are going to add more mechanisms that would require deeper nesting, then we can refactor that code. Until then, it is good-enough (or is not bad-enough). Applied to next, thanks. Regards, Yann E. MORIN. > except (AttributeError, KeyError) as error: > print('Error: Could not install package {pkg}: {error}'.format( > pkg=package.real_name, error=error)) > -- > 2.34.1 > > _______________________________________________ > buildroot mailing list > buildroot@buildroot.org > https://lists.buildroot.org/mailman/listinfo/buildroot
diff --git a/utils/scanpypi b/utils/scanpypi index 3c98bb4bcc..eae9783a1f 100755 --- a/utils/scanpypi +++ b/utils/scanpypi @@ -42,6 +42,55 @@ except ImportError: 'pip install spdx_lookup') liclookup = None +def toml_load(f): + with open(f, 'rb') as fh: + ex = None + + # Try standard library tomllib first + try: + from tomllib import load + return load(fh) + except ImportError: + pass + + # Try regular tomli next + try: + from tomli import load + return load(fh) + except ImportError as e: + ex = e + + # Try pip's vendored tomli + try: + from pip._vendor.tomli import load + try: + return load(fh) + except TypeError: + # Fallback to handle older version + try: + fh.seek(0) + w = io.TextIOWrapper(fh, encoding="utf8", newline="") + return load(w) + finally: + w.detach() + except ImportError as e: + pass + + # Try regular toml last + try: + from toml import load + fh.seek(0) + w = io.TextIOWrapper(fh, encoding="utf8", newline="") + try: + return load(w) + finally: + w.detach() + except ImportError: + pass + + print('This package needs tomli') + raise ex + def setup_decorator(func, method): """ @@ -316,6 +365,35 @@ class BuildrootPackage(): os.chdir(current_dir) sys.path.remove(self.tmp_extract) + def load_pyproject(self): + """ + Loads the corresponding pyproject.toml and store its metadata + """ + current_dir = os.getcwd() + os.chdir(self.tmp_extract) + sys.path.insert(0, self.tmp_extract) + try: + pyproject_data = toml_load('pyproject.toml') + try: + self.setup_metadata = pyproject_data.get('project', {}) + self.metadata_name = self.setup_metadata.get('name', self.real_name) + build_system = pyproject_data.get('build-system', {}) + build_backend = build_system.get('build-backend', None) + if build_backend and build_backend == 'flit_core.buildapi': + self.setup_metadata['method'] = 'flit' + elif build_system.get('backend-path', None): + self.setup_metadata['method'] = 'pep517' + else: + self.setup_metadata['method'] = 'unknown' + except KeyError: + print('ERROR: Could not determine package metadata for {pkg}.\n' + .format(pkg=self.real_name)) + raise + except FileNotFoundError: + raise + os.chdir(current_dir) + sys.path.remove(self.tmp_extract) + def get_requirements(self, pkg_folder): """ Retrieve dependencies from the metadata found in the setup.py script of @@ -620,8 +698,12 @@ class BuildrootPackage(): if help_lines[-1][-1] != '.': help_lines[-1] += '.' - home_page = md_info.get('home_page', None) or \ - md_info.get('project_urls', {}).get('Homepage', None) # noqa: E127 + home_page = md_info.get('home_page', None) + + if not home_page: + project_urls = md_info.get('project_urls', None) + if project_urls: + home_page = project_urls.get('Homepage', None) if home_page: # \t + two spaces is 3 char long @@ -699,9 +781,12 @@ def main(): except ImportError as err: if 'buildutils' in str(err): print('This package needs buildutils') + continue else: - raise - continue + try: + package.load_pyproject() + except Exception as e: + raise except (AttributeError, KeyError) as error: print('Error: Could not install package {pkg}: {error}'.format( pkg=package.real_name, error=error))