| Message ID | 20220725064338.1326554-1-james.hilliard1@gmail.com |
|---|---|
| State | Superseded, archived |
| Headers | show |
| Series | [1/1] package/pkg-python: clean conflicting pep517 packages before install | expand |
On 25.07.22 08:43, James Hilliard wrote: > The python installer package isn't able to overwrite packages that > already exist, this causes problems when doing a rebuild or update > without a full clean. > > To fix this we can use functionality from importlib to identify and > remove any conflicting python package files before installation. > > Since we are adding this functionality to our pyinstaller.py script > we must also use pyinstaller.py for host python packages to avoid > hitting this issue there as well. > > Fixes: > Traceback (most recent call last): > File "/home/buildroot/buildroot/support/scripts/pyinstaller.py", line 69, in <module> > main() > File "/home/buildroot/buildroot/support/scripts/pyinstaller.py", line 61, in main > install( > File "/home/buildroot/buildroot/output/host/lib/python3.10/site-packages/installer/_core.py", line 109, in install > record = destination.write_file( > File "/home/buildroot/buildroot/output/host/lib/python3.10/site-packages/installer/destinations.py", line 207, in write_file > return self.write_to_fs(scheme, path_, stream, is_executable) > File "/home/buildroot/buildroot/output/host/lib/python3.10/site-packages/installer/destinations.py", line 167, in write_to_fs > raise FileExistsError(message) > FileExistsError: File already exists: /home/buildroot/buildroot/output/target/usr/lib/python3.10/site-packages/tinycss2/__init__.py > > Signed-off-by: James Hilliard <james.hilliard1@gmail.com> Tested-by: Marcus Hoffmann <marcus.hoffmann@othermo.de> > --- > package/pkg-python.mk | 12 ++++++++++-- > support/scripts/pyinstaller.py | 23 +++++++++++++++++++++++ > 2 files changed, 33 insertions(+), 2 deletions(-) > > diff --git a/package/pkg-python.mk b/package/pkg-python.mk > index 5794e3a195..b147827821 100644 > --- a/package/pkg-python.mk > +++ b/package/pkg-python.mk > @@ -139,6 +139,14 @@ PKG_PYTHON_PEP517_INSTALL_STAGING_OPTS = \ > HOST_PKG_PYTHON_PEP517_ENV = \ > $(HOST_PKG_PYTHON_ENV) > > +HOST_PKG_PYTHON_PEP517_INSTALL_OPTS = \ > + --interpreter=/bin/python \ > + --script-kind=posix \ > + --purelib=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages \ > + --headers=$(HOST_DIR)/include/python$(PYTHON3_VERSION_MAJOR) \ > + --scripts=$(HOST_DIR)/bin \ > + --data=$(HOST_DIR) > + > HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS = \ > --installdir=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages > > @@ -200,7 +208,7 @@ $(2)_BASE_INSTALL_STAGING_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* > else > $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV) > $(2)_BASE_BUILD_CMD = -m build -n -w > -$(2)_BASE_INSTALL_CMD = -m installer dist/* > +$(2)_BASE_INSTALL_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS) > endif > else ifeq ($$($(2)_SETUP_TYPE),flit-bootstrap) > ifeq ($(4),target) > @@ -208,7 +216,7 @@ $$(error flit-bootstrap setup type only supported for host packages) > else > $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV) > $(2)_BASE_BUILD_CMD = -m flit_core.wheel > -$(2)_BASE_INSTALL_CMD ?= -m installer dist/* > +$(2)_BASE_INSTALL_CMD ?= $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS) > endif > else > $$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'setuptools', 'pep517' or 'flit'.") > diff --git a/support/scripts/pyinstaller.py b/support/scripts/pyinstaller.py > index 6dd9242327..1ab1b83178 100755 > --- a/support/scripts/pyinstaller.py > +++ b/support/scripts/pyinstaller.py > @@ -2,12 +2,34 @@ > > import argparse > import glob > +import os > +import pathlib > + > +from importlib.machinery import PathFinder > +from importlib.metadata import DistributionFinder > > from installer import install > +from installer._core import _process_WHEEL_file > from installer.destinations import SchemeDictionaryDestination > from installer.sources import WheelFile > > > +def clean(source, destination): > + scheme = _process_WHEEL_file(source) > + scheme_path = destination.scheme_dict[scheme] > + context = DistributionFinder.Context( > + name=source.distribution, > + path=[scheme_path], > + ) > + for path in PathFinder.find_distributions(context=context): > + if not path.files: > + continue > + for file in path.files: > + file_path = pathlib.Path(file.locate()) > + if file_path.exists(): > + file_path.unlink() > + > + > def main(): > """Entry point for CLI.""" > ap = argparse.ArgumentParser("python pyinstaller.py") > @@ -58,6 +80,7 @@ def main(): > ) > > with WheelFile.open(glob.glob(args.wheel_file)[0]) as source: > + clean(source, destination) > install( > source=source, > destination=destination,
diff --git a/package/pkg-python.mk b/package/pkg-python.mk index 5794e3a195..b147827821 100644 --- a/package/pkg-python.mk +++ b/package/pkg-python.mk @@ -139,6 +139,14 @@ PKG_PYTHON_PEP517_INSTALL_STAGING_OPTS = \ HOST_PKG_PYTHON_PEP517_ENV = \ $(HOST_PKG_PYTHON_ENV) +HOST_PKG_PYTHON_PEP517_INSTALL_OPTS = \ + --interpreter=/bin/python \ + --script-kind=posix \ + --purelib=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages \ + --headers=$(HOST_DIR)/include/python$(PYTHON3_VERSION_MAJOR) \ + --scripts=$(HOST_DIR)/bin \ + --data=$(HOST_DIR) + HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS = \ --installdir=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages @@ -200,7 +208,7 @@ $(2)_BASE_INSTALL_STAGING_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* else $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV) $(2)_BASE_BUILD_CMD = -m build -n -w -$(2)_BASE_INSTALL_CMD = -m installer dist/* +$(2)_BASE_INSTALL_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS) endif else ifeq ($$($(2)_SETUP_TYPE),flit-bootstrap) ifeq ($(4),target) @@ -208,7 +216,7 @@ $$(error flit-bootstrap setup type only supported for host packages) else $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV) $(2)_BASE_BUILD_CMD = -m flit_core.wheel -$(2)_BASE_INSTALL_CMD ?= -m installer dist/* +$(2)_BASE_INSTALL_CMD ?= $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS) endif else $$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'setuptools', 'pep517' or 'flit'.") diff --git a/support/scripts/pyinstaller.py b/support/scripts/pyinstaller.py index 6dd9242327..1ab1b83178 100755 --- a/support/scripts/pyinstaller.py +++ b/support/scripts/pyinstaller.py @@ -2,12 +2,34 @@ import argparse import glob +import os +import pathlib + +from importlib.machinery import PathFinder +from importlib.metadata import DistributionFinder from installer import install +from installer._core import _process_WHEEL_file from installer.destinations import SchemeDictionaryDestination from installer.sources import WheelFile +def clean(source, destination): + scheme = _process_WHEEL_file(source) + scheme_path = destination.scheme_dict[scheme] + context = DistributionFinder.Context( + name=source.distribution, + path=[scheme_path], + ) + for path in PathFinder.find_distributions(context=context): + if not path.files: + continue + for file in path.files: + file_path = pathlib.Path(file.locate()) + if file_path.exists(): + file_path.unlink() + + def main(): """Entry point for CLI.""" ap = argparse.ArgumentParser("python pyinstaller.py") @@ -58,6 +80,7 @@ def main(): ) with WheelFile.open(glob.glob(args.wheel_file)[0]) as source: + clean(source, destination) install( source=source, destination=destination,
The python installer package isn't able to overwrite packages that already exist, this causes problems when doing a rebuild or update without a full clean. To fix this we can use functionality from importlib to identify and remove any conflicting python package files before installation. Since we are adding this functionality to our pyinstaller.py script we must also use pyinstaller.py for host python packages to avoid hitting this issue there as well. Fixes: Traceback (most recent call last): File "/home/buildroot/buildroot/support/scripts/pyinstaller.py", line 69, in <module> main() File "/home/buildroot/buildroot/support/scripts/pyinstaller.py", line 61, in main install( File "/home/buildroot/buildroot/output/host/lib/python3.10/site-packages/installer/_core.py", line 109, in install record = destination.write_file( File "/home/buildroot/buildroot/output/host/lib/python3.10/site-packages/installer/destinations.py", line 207, in write_file return self.write_to_fs(scheme, path_, stream, is_executable) File "/home/buildroot/buildroot/output/host/lib/python3.10/site-packages/installer/destinations.py", line 167, in write_to_fs raise FileExistsError(message) FileExistsError: File already exists: /home/buildroot/buildroot/output/target/usr/lib/python3.10/site-packages/tinycss2/__init__.py Signed-off-by: James Hilliard <james.hilliard1@gmail.com> --- package/pkg-python.mk | 12 ++++++++++-- support/scripts/pyinstaller.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-)