diff mbox series

[v4,1/5] package/pkg-python: add pyo3(setuptools-rust/maturin) infrastructure

Message ID 20220807003832.1177015-1-james.hilliard1@gmail.com
State Accepted
Headers show
Series [v4,1/5] package/pkg-python: add pyo3(setuptools-rust/maturin) infrastructure | expand

Commit Message

James Hilliard Aug. 7, 2022, 12:38 a.m. UTC 
The pyo3 package infrastructure provides two build backends, one
of which is based on setuptools while the other is a pep517 backend.

The setuptools-rust build backend is a setuptools extension that is
capable of building python rust extensions.

The maturin build backend is a pep517 build extension that is itself
written in rust, it is itself bootstrapped using setuptools-rust but
is not itself a setuptools extension and uses pep517 infrastructure.

As both backends use pyo3 we need to set up the pyo3 environment in
addition to cargo vendoring for each.

Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
Changes v3 -> v4:
  - generalize infrastructure to support both pyo3 build backends
---
 package/pkg-python.mk | 92 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 87 insertions(+), 5 deletions(-)

Comments

Arnout Vandecappelle Sept. 29, 2023, 8:04 p.m. UTC | #1 
On 07/08/2022 02:38, James Hilliard wrote:
> The pyo3 package infrastructure provides two build backends, one
> of which is based on setuptools while the other is a pep517 backend.
> 
> The setuptools-rust build backend is a setuptools extension that is
> capable of building python rust extensions.
> 
> The maturin build backend is a pep517 build extension that is itself
> written in rust, it is itself bootstrapped using setuptools-rust but
> is not itself a setuptools extension and uses pep517 infrastructure.
> 
> As both backends use pyo3 we need to set up the pyo3 environment in
> addition to cargo vendoring for each.
> 
> Signed-off-by: James Hilliard <james.hilliard1@gmail.com>

  Series _finally_ applied to master, thanks!

> ---
> Changes v3 -> v4:
>    - generalize infrastructure to support both pyo3 build backends
> ---
>   package/pkg-python.mk | 92 ++++++++++++++++++++++++++++++++++++++++---
>   1 file changed, 87 insertions(+), 5 deletions(-)
> 
> diff --git a/package/pkg-python.mk b/package/pkg-python.mk
> index 5794e3a195..0dcd2622c5 100644
> --- a/package/pkg-python.mk
> +++ b/package/pkg-python.mk
> @@ -113,6 +113,24 @@ HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS = \
>   	--root=/ \
>   	--single-version-externally-managed
>   
> +PKG_PYTHON_PYO3_ENV = \
> +	$(PKG_CARGO_ENV) \
> +	PYO3_CROSS_LIB_DIR="$(STAGING_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)"

  This _PYO3_ENV variables aren't contributing much and make the code even more 
difficult to read, so I removed them.

> +
> +HOST_PKG_PYTHON_PYO3_ENV = \
> +	$(HOST_PKG_CARGO_ENV) \
> +	PYO3_CROSS_LIB_DIR="$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)"
> +
> +# Target setuptools-rust-based packages
> +PKG_PYTHON_SETUPTOOLS_RUST_ENV = \
> +	$(PKG_PYTHON_SETUPTOOLS_ENV) \
> +	$(PKG_PYTHON_PYO3_ENV)
> +
> +# Host setuptools-rust-based packages
> +HOST_PKG_PYTHON_SETUPTOOLS_RUST_ENV = \
> +	$(HOST_PKG_PYTHON_SETUPTOOLS_ENV) \
> +	$(HOST_PKG_PYTHON_PYO3_ENV)
> +
>   # Target pep517-based packages
>   PKG_PYTHON_PEP517_ENV = \
>   	$(PKG_PYTHON_ENV)
> @@ -142,6 +160,16 @@ HOST_PKG_PYTHON_PEP517_ENV = \
>   HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS = \
>   	--installdir=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages
>   
> +# Target maturin packages
> +PKG_PYTHON_MATURIN_ENV = \
> +	$(PKG_PYTHON_PEP517_ENV) \
> +	$(PKG_PYTHON_PYO3_ENV)
> +
> +# Host maturin packages
> +HOST_PKG_PYTHON_MATURIN_ENV = \
> +	$(HOST_PKG_PYTHON_PEP517_ENV) \
> +	$(HOST_PKG_PYTHON_PYO3_ENV)
> +
>   ################################################################################
>   # inner-python-package -- defines how the configuration, compilation
>   # and installation of a Python package should be done, implements a
> @@ -180,25 +208,41 @@ $(2)_BASE_BUILD_CMD   = setup.py build
>   $(2)_BASE_INSTALL_CMD = setup.py install $$(HOST_PKG_PYTHON_DISTUTILS_INSTALL_OPTS)
>   endif
>   # Setuptools
> -else ifeq ($$($(2)_SETUP_TYPE),setuptools)
> +else ifneq ($$(filter setuptools setuptools-rust,$$($(2)_SETUP_TYPE)),)

  This stuff is becoming increasingly difficult to read with all the different 
setup types. I'm going to try to refactor it a little.


  Regards,
  Arnout

[snip]
diff mbox series

Patch

diff --git a/package/pkg-python.mk b/package/pkg-python.mk
index 5794e3a195..0dcd2622c5 100644
--- a/package/pkg-python.mk
+++ b/package/pkg-python.mk
@@ -113,6 +113,24 @@  HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS = \
 	--root=/ \
 	--single-version-externally-managed
 
+PKG_PYTHON_PYO3_ENV = \
+	$(PKG_CARGO_ENV) \
+	PYO3_CROSS_LIB_DIR="$(STAGING_DIR)/usr/lib/python$(PYTHON3_VERSION_MAJOR)"
+
+HOST_PKG_PYTHON_PYO3_ENV = \
+	$(HOST_PKG_CARGO_ENV) \
+	PYO3_CROSS_LIB_DIR="$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)"
+
+# Target setuptools-rust-based packages
+PKG_PYTHON_SETUPTOOLS_RUST_ENV = \
+	$(PKG_PYTHON_SETUPTOOLS_ENV) \
+	$(PKG_PYTHON_PYO3_ENV)
+
+# Host setuptools-rust-based packages
+HOST_PKG_PYTHON_SETUPTOOLS_RUST_ENV = \
+	$(HOST_PKG_PYTHON_SETUPTOOLS_ENV) \
+	$(HOST_PKG_PYTHON_PYO3_ENV)
+
 # Target pep517-based packages
 PKG_PYTHON_PEP517_ENV = \
 	$(PKG_PYTHON_ENV)
@@ -142,6 +160,16 @@  HOST_PKG_PYTHON_PEP517_ENV = \
 HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS = \
 	--installdir=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages
 
+# Target maturin packages
+PKG_PYTHON_MATURIN_ENV = \
+	$(PKG_PYTHON_PEP517_ENV) \
+	$(PKG_PYTHON_PYO3_ENV)
+
+# Host maturin packages
+HOST_PKG_PYTHON_MATURIN_ENV = \
+	$(HOST_PKG_PYTHON_PEP517_ENV) \
+	$(HOST_PKG_PYTHON_PYO3_ENV)
+
 ################################################################################
 # inner-python-package -- defines how the configuration, compilation
 # and installation of a Python package should be done, implements a
@@ -180,25 +208,41 @@  $(2)_BASE_BUILD_CMD   = setup.py build
 $(2)_BASE_INSTALL_CMD = setup.py install $$(HOST_PKG_PYTHON_DISTUTILS_INSTALL_OPTS)
 endif
 # Setuptools
-else ifeq ($$($(2)_SETUP_TYPE),setuptools)
+else ifneq ($$(filter setuptools setuptools-rust,$$($(2)_SETUP_TYPE)),)
 ifeq ($(4),target)
+ifeq ($$($(2)_SETUP_TYPE),setuptools-rust)
+$(2)_BASE_ENV = $$(PKG_PYTHON_SETUPTOOLS_RUST_ENV)
+else
 $(2)_BASE_ENV = $$(PKG_PYTHON_SETUPTOOLS_ENV)
+endif
 $(2)_BASE_BUILD_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) build
 $(2)_BASE_INSTALL_TARGET_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) install --no-compile $$(PKG_PYTHON_SETUPTOOLS_INSTALL_TARGET_OPTS)
 $(2)_BASE_INSTALL_STAGING_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) install $$(PKG_PYTHON_SETUPTOOLS_INSTALL_STAGING_OPTS)
 else
+ifeq ($$($(2)_SETUP_TYPE),setuptools-rust)
+$(2)_BASE_ENV = $$(HOST_PKG_PYTHON_SETUPTOOLS_RUST_ENV)
+else
 $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_SETUPTOOLS_ENV)
+endif
 $(2)_BASE_BUILD_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) build
 $(2)_BASE_INSTALL_CMD = $$(PKG_PYTHON_SETUPTOOLS_CMD) install $$(HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS)
 endif
-else ifneq ($$(filter flit pep517,$$($(2)_SETUP_TYPE)),)
+else ifneq ($$(filter flit maturin pep517,$$($(2)_SETUP_TYPE)),)
 ifeq ($(4),target)
+ifeq ($$($(2)_SETUP_TYPE),maturin)
+$(2)_BASE_ENV = $$(PKG_PYTHON_MATURIN_ENV)
+else
 $(2)_BASE_ENV = $$(PKG_PYTHON_PEP517_ENV)
+endif
 $(2)_BASE_BUILD_CMD = -m build -n -w
 $(2)_BASE_INSTALL_TARGET_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(PKG_PYTHON_PEP517_INSTALL_TARGET_OPTS)
 $(2)_BASE_INSTALL_STAGING_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(PKG_PYTHON_PEP517_INSTALL_STAGING_OPTS)
 else
+ifeq ($$($(2)_SETUP_TYPE),maturin)
+$(2)_BASE_ENV = $$(HOST_PKG_PYTHON_MATURIN_ENV)
+else
 $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV)
+endif
 $(2)_BASE_BUILD_CMD = -m build -n -w
 $(2)_BASE_INSTALL_CMD = -m installer dist/*
 endif
@@ -211,7 +255,29 @@  $(2)_BASE_BUILD_CMD = -m flit_core.wheel
 $(2)_BASE_INSTALL_CMD ?= -m installer dist/*
 endif
 else
-$$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'setuptools', 'pep517' or 'flit'.")
+$$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'maturin', 'setuptools', 'setuptools-rust', 'pep517' or 'flit'.")
+endif
+
+# We need to vendor the Cargo crates at download time for pyo3 based
+# packages.
+#
+ifneq ($$(filter maturin setuptools-rust,$$($(2)_SETUP_TYPE)),)
+ifeq ($(4),target)
+$(2)_DL_ENV = $$(PKG_CARGO_ENV)
+else
+$(2)_DL_ENV = $$(HOST_PKG_CARGO_ENV)
+endif
+ifndef $(2)_CARGO_MANIFEST_PATH
+ifdef $(3)_CARGO_MANIFEST_PATH
+$(2)_DL_ENV += BR_CARGO_MANIFEST_PATH=$$($(3)_CARGO_MANIFEST_PATH)
+else
+ifneq ($$($(2)_SUBDIR),)
+$(2)_DL_ENV += BR_CARGO_MANIFEST_PATH=$$($(2)_SUBDIR)/Cargo.toml
+endif
+endif
+else
+$(2)_DL_ENV += BR_CARGO_MANIFEST_PATH=$$($(2)_CARGO_MANIFEST_PATH)
+endif
 endif
 
 # Target packages need both the python interpreter on the target (for
@@ -228,19 +294,35 @@  endif # ($(4),target)
 # Setuptools based packages will need setuptools for the host Python
 # interpreter (both host and target).
 #
-ifeq ($$($(2)_SETUP_TYPE),setuptools)
+ifneq ($$(filter setuptools setuptools-rust,$$($(2)_SETUP_TYPE)),)
 $(2)_DEPENDENCIES += $$(if $$(filter host-python-setuptools,$(1)),,host-python-setuptools)
-else ifneq ($$(filter flit pep517,$$($(2)_SETUP_TYPE)),)
+ifeq ($$($(2)_SETUP_TYPE),setuptools-rust)
+$(2)_DEPENDENCIES += host-python-setuptools-rust
+endif
+else ifneq ($$(filter flit maturin pep517,$$($(2)_SETUP_TYPE)),)
 $(2)_DEPENDENCIES += host-python-pypa-build host-python-installer
 ifeq ($$($(2)_SETUP_TYPE),flit)
 $(2)_DEPENDENCIES += host-python-flit-core
 endif
+ifeq ($$($(2)_SETUP_TYPE),maturin)
+$(2)_DEPENDENCIES += host-python-maturin
+endif
 else ifeq ($$($(2)_SETUP_TYPE),flit-bootstrap)
 # Don't add dependency on host-python-installer for
 # host-python-installer itself, and its dependencies.
 ifeq ($$(filter host-python-flit-core host-python-installer,$(1)),)
 $(2)_DEPENDENCIES += host-python-installer
 endif
+endif
+
+# Pyo3 based packages(setuptools-rust and maturin) will need rust
+# toolchain dependencies for the host Python interpreter (both host
+# and target).
+#
+ifneq ($$(filter maturin setuptools-rust,$$($(2)_SETUP_TYPE)),)
+$(2)_DEPENDENCIES += host-rustc
+$(2)_DOWNLOAD_POST_PROCESS = cargo
+$(2)_DOWNLOAD_DEPENDENCIES = host-rustc
 endif # SETUP_TYPE
 
 # Python interpreter to use for building the package.