@@ -18,8 +18,9 @@ import hashlib
import re
import textwrap
import tempfile
-import imp
-from functools import wraps
+import traceback
+import importlib
+import importlib.metadata
import six.moves.urllib.request
import six.moves.urllib.error
import six.moves.urllib.parse
@@ -93,32 +94,6 @@ def toml_load(f):
raise ex
-def setup_decorator(func, method):
- """
- Decorator for distutils.core.setup and setuptools.setup.
- Puts the arguments with which setup is called as a dict
- Add key 'method' which should be either 'setuptools' or 'distutils'.
-
- Keyword arguments:
- func -- either setuptools.setup or distutils.core.setup
- method -- either 'setuptools' or 'distutils'
- """
-
- @wraps(func)
- def closure(*args, **kwargs):
- # Any python packages calls its setup function to be installed.
- # Argument 'name' of this setup function is the package's name
- BuildrootPackage.setup_args[kwargs['name']] = kwargs
- BuildrootPackage.setup_args[kwargs['name']]['method'] = method
- return closure
-
-# monkey patch
-import setuptools # noqa E402
-setuptools.setup = setup_decorator(setuptools.setup, 'setuptools')
-import distutils # noqa E402
-distutils.core.setup = setup_decorator(setuptools.setup, 'distutils')
-
-
def find_file_upper_case(filenames, path='./'):
"""
List generator:
@@ -156,6 +131,44 @@ class DownloadFailed(Exception):
pass
+class BackendUnavailable(Exception):
+ """Raised if we cannot import the backend"""
+
+ def __init__(self, message, traceback=None):
+ super().__init__(message)
+ self.message = message
+ self.traceback = traceback
+
+
+class BackendPathFinder:
+ """Implements the MetaPathFinder interface to locate modules in ``backend-path``.
+
+ Since the environment provided by the frontend can contain all sorts of
+ MetaPathFinders, the only way to ensure the backend is loaded from the
+ right place is to prepend our own.
+ """
+
+ def __init__(self, backend_path, backend_module):
+ self.backend_path = backend_path
+ self.backend_module = backend_module
+ self.backend_parent, _, _ = backend_module.partition(".")
+
+ def find_spec(self, fullname, _path, _target=None):
+ if "." in fullname:
+ # Rely on importlib to find nested modules based on parent's path
+ return None
+
+ # Ignore other items in _path or sys.path and use backend_path instead:
+ spec = importlib.machinery.PathFinder.find_spec(fullname, path=self.backend_path)
+ if spec is None and fullname == self.backend_parent:
+ # According to the spec, the backend MUST be loaded from backend-path.
+ # Therefore, we can halt the import machinery and raise a clean error.
+ msg = f"Cannot find module {self.backend_module!r} in {self.backend_path!r}"
+ raise BackendUnavailable(msg)
+
+ return spec
+
+
class BuildrootPackage():
"""This class's methods are not meant to be used individually please
use them in the correct order:
@@ -191,6 +204,8 @@ class BuildrootPackage():
self.metadata_url = None
self.pkg_req = None
self.setup_metadata = None
+ self.backend_path = None
+ self.build_backend = None
self.tmp_extract = None
self.used_url = None
self.filename = None
@@ -339,32 +354,46 @@ class BuildrootPackage():
folder=tmp_pkg,
name=pkg_filename)
- def load_setup(self):
+ def load_metadata(self):
"""
Loads the corresponding setup and store its metadata
"""
current_dir = os.getcwd()
os.chdir(self.tmp_extract)
- sys.path.insert(0, self.tmp_extract)
try:
- s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
- imp.load_module('__main__', s_file, s_path, s_desc)
- if self.metadata_name in self.setup_args:
- pass
- elif self.metadata_name.replace('_', '-') in self.setup_args:
- self.metadata_name = self.metadata_name.replace('_', '-')
- elif self.metadata_name.replace('-', '_') in self.setup_args:
- self.metadata_name = self.metadata_name.replace('-', '_')
+ mod_path, _, obj_path = self.build_backend.partition(":")
+
+ path_finder = None
+ if self.backend_path:
+ path_finder = BackendPathFinder(self.backend_path, mod_path)
+ sys.meta_path.insert(0, path_finder)
+
+ try:
+ build_backend = importlib.import_module(self.build_backend)
+ except ImportError:
+ msg = f"Cannot import {mod_path!r}"
+ raise BackendUnavailable(msg, traceback.format_exc())
+
+ if obj_path:
+ for path_part in obj_path.split("."):
+ build_backend = getattr(build_backend, path_part)
+
+ if path_finder:
+ sys.meta_path.remove(path_finder)
+
+ prepare_metadata_for_build_wheel = getattr(
+ build_backend, 'prepare_metadata_for_build_wheel'
+ )
+ metadata = prepare_metadata_for_build_wheel(self.tmp_extract)
try:
- self.setup_metadata = self.setup_args[self.metadata_name]
- except KeyError:
- # This means setup was not called
- print('ERROR: Could not determine package metadata for {pkg}.\n'
- .format(pkg=self.real_name))
- raise
+ dist = importlib.metadata.Distribution.at(metadata)
+ self.metadata_name = dist.name
+ if dist.requires:
+ self.setup_metadata['install_requires'] = dist.requires
+ finally:
+ shutil.rmtree(metadata)
finally:
os.chdir(current_dir)
- sys.path.remove(self.tmp_extract)
def load_pyproject(self):
"""
@@ -372,28 +401,29 @@ class BuildrootPackage():
"""
current_dir = os.getcwd()
os.chdir(self.tmp_extract)
- sys.path.insert(0, self.tmp_extract)
try:
pyproject_data = toml_load('pyproject.toml')
- try:
- self.setup_metadata = pyproject_data.get('project', {})
- self.metadata_name = self.setup_metadata.get('name', self.real_name)
- build_system = pyproject_data.get('build-system', {})
- build_backend = build_system.get('build-backend', None)
- if build_backend and build_backend == 'flit_core.buildapi':
+ self.setup_metadata = pyproject_data.get('project', {})
+ self.metadata_name = self.setup_metadata.get('name', self.real_name)
+ build_system = pyproject_data.get('build-system', {})
+ build_backend = build_system.get('build-backend', None)
+ self.backend_path = build_system.get('backend-path', None)
+ if build_backend:
+ self.build_backend = build_backend
+ if build_backend == 'flit_core.buildapi':
self.setup_metadata['method'] = 'flit'
- elif build_system.get('backend-path', None):
- self.setup_metadata['method'] = 'pep517'
+ elif build_backend == 'setuptools.build_meta':
+ self.setup_metadata['method'] = 'setuptools'
else:
- self.setup_metadata['method'] = 'unknown'
- except KeyError:
- print('ERROR: Could not determine package metadata for {pkg}.\n'
- .format(pkg=self.real_name))
- raise
+ if self.backend_path:
+ self.setup_metadata['method'] = 'pep517'
+ else:
+ self.setup_metadata['method'] = 'unknown'
except FileNotFoundError:
- raise
- os.chdir(current_dir)
- sys.path.remove(self.tmp_extract)
+ self.build_backend = 'setuptools.build_meta'
+ self.setup_metadata = {'method': 'setuptools'}
+ finally:
+ os.chdir(current_dir)
def get_requirements(self, pkg_folder):
"""
@@ -406,7 +436,11 @@ class BuildrootPackage():
if 'install_requires' not in self.setup_metadata:
self.pkg_req = None
return set()
- self.pkg_req = self.setup_metadata['install_requires']
+ self.pkg_req = set()
+ extra_re = re.compile(r'''extra\s*==\s*("([^"]+)"|'([^']+)')''')
+ for req in self.setup_metadata['install_requires']:
+ if not extra_re.search(req):
+ self.pkg_req.add(req)
self.pkg_req = [re.sub(r'([-.\w]+).*', r'\1', req)
for req in self.pkg_req]
@@ -753,11 +787,6 @@ def main():
package.fetch_package_info()
except (six.moves.urllib.error.URLError, six.moves.urllib.error.HTTPError):
continue
- if package.metadata_name.lower() == 'setuptools':
- # setuptools imports itself, that does not work very well
- # with the monkey path at the begining
- print('Error: setuptools cannot be built using scanPyPI')
- continue
try:
package.download_package()
@@ -777,17 +806,15 @@ def main():
continue
# Loading the package install info from the package
+ package.load_pyproject()
try:
- package.load_setup()
+ package.load_metadata()
except ImportError as err:
if 'buildutils' in str(err):
print('This package needs buildutils')
continue
else:
- try:
- package.load_pyproject()
- except Exception:
- raise
+ raise
except (AttributeError, KeyError) as error:
print('Error: Could not install package {pkg}: {error}'.format(
pkg=package.real_name, error=error))
The imp module is deprecated as of python verison 3.12. Refactor setuptools handling to remove monkeypatching hack and instead do pep517 metadata generation and dependency resolution. This is effectively done by implementing the minimal neccesary pep517 frontend hooks needed for dependency resolution in scanpypi. Signed-off-by: James Hilliard <james.hilliard1@gmail.com> --- Changes v2 -> v3: - support pep517 dependency resolution for all build systems - replace setuptools specific load_setup with generic load_metadata - remove setuptools load_setup fallback exception hacks Changes v1 -> v2: - split out set comprehension changes --- utils/scanpypi | 173 ++++++++++++++++++++++++++++--------------------- 1 file changed, 100 insertions(+), 73 deletions(-)