| Message ID | 20221128183230.1915592-1-james.hilliard1@gmail.com |
|---|---|
| State | Superseded, archived |
| Headers | show |
| Series | [v5,1/1] utils/scanpypi: add flit package support | expand |
On Mon, Nov 28, 2022 at 7:32 PM James Hilliard <james.hilliard1@gmail.com> wrote: > > These packages don't have a setup.py so we instead need to parse their > pyproject.toml file. > > Note that this currently doesn't handle flit package dependency > resolution. > > Signed-off-by: James Hilliard <james.hilliard1@gmail.com> Reviewed-by: Yegor Yefremov <yegorslists@googlemail.com> > --- > Changes v4 -> v5: > - remove unnecessary "as e" from tomlib exception path > Changes v3 -> v4: > - prefer/support python3.11 tomllib > - use bool interpretation for None > Changes v2 -> v3: > - minor cleanup > - rebase > - more detailed commit log > Changes v1 -> v2: > - remove homepage format fixes(sent as separate patch) > - remove load_setup fixes > --- > utils/scanpypi | 85 ++++++++++++++++++++++++++++++++++++++++++++++++-- > 1 file changed, 83 insertions(+), 2 deletions(-) > > diff --git a/utils/scanpypi b/utils/scanpypi > index 3c98bb4bcc..79c5db8a5f 100755 > --- a/utils/scanpypi > +++ b/utils/scanpypi > @@ -42,6 +42,55 @@ except ImportError: > 'pip install spdx_lookup') > liclookup = None > > +def toml_load(f): > + with open(f, 'rb') as fh: > + ex = None > + > + # Try standard library tomllib first > + try: > + from tomllib import load > + return load(fh) > + except ImportError: > + pass > + > + # Try regular tomli next > + try: > + from tomli import load > + return load(fh) > + except ImportError as e: > + ex = e > + > + # Try pip's vendored tomli > + try: > + from pip._vendor.tomli import load > + try: > + return load(fh) > + except TypeError: > + # Fallback to handle older version > + try: > + fh.seek(0) > + w = io.TextIOWrapper(fh, encoding="utf8", newline="") > + return load(w) > + finally: > + w.detach() > + except ImportError as e: > + pass > + > + # Try regular toml last > + try: > + from toml import load > + fh.seek(0) > + w = io.TextIOWrapper(fh, encoding="utf8", newline="") > + try: > + return load(w) > + finally: > + w.detach() > + except ImportError: > + pass > + > + print('This package needs tomli') > + raise ex > + > > def setup_decorator(func, method): > """ > @@ -316,6 +365,35 @@ class BuildrootPackage(): > os.chdir(current_dir) > sys.path.remove(self.tmp_extract) > > + def load_pyproject(self): > + """ > + Loads the corresponding pyproject.toml and store its metadata > + """ > + current_dir = os.getcwd() > + os.chdir(self.tmp_extract) > + sys.path.insert(0, self.tmp_extract) > + try: > + pyproject_data = toml_load('pyproject.toml') > + try: > + self.setup_metadata = pyproject_data.get('project', {}) > + self.metadata_name = self.setup_metadata.get('name', self.real_name) > + build_system = pyproject_data.get('build-system', {}) > + build_backend = build_system.get('build-backend', None) > + if build_backend and build_backend == 'flit_core.buildapi': > + self.setup_metadata['method'] = 'flit' > + elif build_system.get('backend-path', None): > + self.setup_metadata['method'] = 'pep517' > + else: > + self.setup_metadata['method'] = 'unknown' > + except KeyError: > + print('ERROR: Could not determine package metadata for {pkg}.\n' > + .format(pkg=self.real_name)) > + raise > + except FileNotFoundError: > + raise > + os.chdir(current_dir) > + sys.path.remove(self.tmp_extract) > + > def get_requirements(self, pkg_folder): > """ > Retrieve dependencies from the metadata found in the setup.py script of > @@ -699,9 +777,12 @@ def main(): > except ImportError as err: > if 'buildutils' in str(err): > print('This package needs buildutils') > + continue > else: > - raise > - continue > + try: > + package.load_pyproject() > + except Exception as e: > + raise > except (AttributeError, KeyError) as error: > print('Error: Could not install package {pkg}: {error}'.format( > pkg=package.real_name, error=error)) > -- > 2.34.1 >
diff --git a/utils/scanpypi b/utils/scanpypi index 3c98bb4bcc..79c5db8a5f 100755 --- a/utils/scanpypi +++ b/utils/scanpypi @@ -42,6 +42,55 @@ except ImportError: 'pip install spdx_lookup') liclookup = None +def toml_load(f): + with open(f, 'rb') as fh: + ex = None + + # Try standard library tomllib first + try: + from tomllib import load + return load(fh) + except ImportError: + pass + + # Try regular tomli next + try: + from tomli import load + return load(fh) + except ImportError as e: + ex = e + + # Try pip's vendored tomli + try: + from pip._vendor.tomli import load + try: + return load(fh) + except TypeError: + # Fallback to handle older version + try: + fh.seek(0) + w = io.TextIOWrapper(fh, encoding="utf8", newline="") + return load(w) + finally: + w.detach() + except ImportError as e: + pass + + # Try regular toml last + try: + from toml import load + fh.seek(0) + w = io.TextIOWrapper(fh, encoding="utf8", newline="") + try: + return load(w) + finally: + w.detach() + except ImportError: + pass + + print('This package needs tomli') + raise ex + def setup_decorator(func, method): """ @@ -316,6 +365,35 @@ class BuildrootPackage(): os.chdir(current_dir) sys.path.remove(self.tmp_extract) + def load_pyproject(self): + """ + Loads the corresponding pyproject.toml and store its metadata + """ + current_dir = os.getcwd() + os.chdir(self.tmp_extract) + sys.path.insert(0, self.tmp_extract) + try: + pyproject_data = toml_load('pyproject.toml') + try: + self.setup_metadata = pyproject_data.get('project', {}) + self.metadata_name = self.setup_metadata.get('name', self.real_name) + build_system = pyproject_data.get('build-system', {}) + build_backend = build_system.get('build-backend', None) + if build_backend and build_backend == 'flit_core.buildapi': + self.setup_metadata['method'] = 'flit' + elif build_system.get('backend-path', None): + self.setup_metadata['method'] = 'pep517' + else: + self.setup_metadata['method'] = 'unknown' + except KeyError: + print('ERROR: Could not determine package metadata for {pkg}.\n' + .format(pkg=self.real_name)) + raise + except FileNotFoundError: + raise + os.chdir(current_dir) + sys.path.remove(self.tmp_extract) + def get_requirements(self, pkg_folder): """ Retrieve dependencies from the metadata found in the setup.py script of @@ -699,9 +777,12 @@ def main(): except ImportError as err: if 'buildutils' in str(err): print('This package needs buildutils') + continue else: - raise - continue + try: + package.load_pyproject() + except Exception as e: + raise except (AttributeError, KeyError) as error: print('Error: Could not install package {pkg}: {error}'.format( pkg=package.real_name, error=error))
These packages don't have a setup.py so we instead need to parse their pyproject.toml file. Note that this currently doesn't handle flit package dependency resolution. Signed-off-by: James Hilliard <james.hilliard1@gmail.com> --- Changes v4 -> v5: - remove unnecessary "as e" from tomlib exception path Changes v3 -> v4: - prefer/support python3.11 tomllib - use bool interpretation for None Changes v2 -> v3: - minor cleanup - rebase - more detailed commit log Changes v1 -> v2: - remove homepage format fixes(sent as separate patch) - remove load_setup fixes --- utils/scanpypi | 85 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 2 deletions(-)