Patchwork [1/2] python-pip: new package, update python-setuptools, enable openssl for host python

login
register
mail settings
Submitter Rohan Fletcher
Date July 13, 2013, 9:19 a.m.
Message ID <1373707158-31756-1-git-send-email-rohfledev@gmail.com>
Download mbox | patch
Permalink /patch/258826/
State Superseded
Headers show

Comments

Rohan Fletcher - July 13, 2013, 9:19 a.m.
Added python-pip - a package manager for python. easy way of adding python packages to buildroot.
Updated python-setuptools to version 0.8 - distribute and setuptools have now merged.
Enabled ssl in host python config so pip can download packages over https.

Signed-off-by: Rohan Fletcher <rohfledev@gmail.com>
---
 package/Config.in                              |  2 +
 package/python-pip/Config.in                   | 20 +++++++++
 package/python-pip/python-pip.mk               | 58 ++++++++++++++++++++++++++
 package/python-setuptools/python-setuptools.mk | 10 ++---
 package/python/python.mk                       |  5 +--
 5 files changed, 87 insertions(+), 8 deletions(-)
 create mode 100644 package/python-pip/Config.in
 create mode 100644 package/python-pip/python-pip.mk
Thomas Petazzoni - July 13, 2013, 2:24 p.m.
Dear Rohan Fletcher,

Thanks for your contribution! A few comments below, to help improve
your patches and make them ready for merging in Buildroot.

On Sat, 13 Jul 2013 21:19:17 +1200, Rohan Fletcher wrote:
> Added python-pip - a package manager for python. easy way of adding python packages to buildroot.
> Updated python-setuptools to version 0.8 - distribute and setuptools have now merged.
> Enabled ssl in host python config so pip can download packages over https.

In this commit description, you're mentioning three totally separate
things. This is a very good indication that those three separate things
should be in separate patches: adding ssl in host python, bumping
setuptools, and finally adding python-pip.

However, your second patch that adds the license to python-pip should
be part of the patch adding python-pip itself.

To re-organize your patches, I recommend you to have a look about "git
rebase -i", it's really the git killer feature to make your patch
series look nice.


> diff --git a/package/Config.in b/package/Config.in
> index 6d5ff01..d6678a3 100644
> --- a/package/Config.in
> +++ b/package/Config.in
> @@ -359,6 +359,8 @@ source "package/python-pyparsing/Config.in"
>  source "package/python-serial/Config.in"
>  source "package/python-setuptools/Config.in"
>  source "package/python-thrift/Config.in"
> +comment "custom python modules"

Not sure why we need this comment here. Could you explain?

> +source "package/python-pip/Config.in"
>  endmenu
>  endif
>  source "package/python3/Config.in"
> diff --git a/package/python-pip/Config.in b/package/python-pip/Config.in
> new file mode 100644
> index 0000000..48f3b04
> --- /dev/null
> +++ b/package/python-pip/Config.in
> @@ -0,0 +1,20 @@
> +config BR2_PACKAGE_PYTHON_PIP
> +    bool "python-pip"
> +    help

Indentation should be one tab, as you did later.

> +      A tool for installing and managing Python packages.

And here indentation should be one tab + two spaces. Check the
Buildroot manual for examples/coding style details.

> +
> +      http://www.pip-installer.org
> +
> +if BR2_PACKAGE_PYTHON_PIP
> +
> +config BR2_PACKAGE_PYTHON_PIP_MODULES_ADDITIONAL
> +	string "Additional modules"
> +	help
> +	  List of space-separated python modules to install via pip.
> +	  See 'pip help install' for available installation methods. 
> +	  For repeatable builds, download and save tgz files or clone 
> +	  git repos for the components you care about.
> +
> +	  Example: module-name module-name==1.3.4 /my/module/mymodule.tgz git://github.com/someuser/somemodule.git#v1.2

This last line should be wrapped.

Don't certain pip modules have native dependencies, that should be
handled? Like a Python module that uses a separate native library?

Generally, we don't really like this kind of package that builds other
packages, because it completely breaks the download and license
infrastructure of Buildroot. But maybe for languages like Python, Perl
and Ruby, it makes sense to have support for those things, even if it's
not perfect.

> +endif
> diff --git a/package/python-pip/python-pip.mk b/package/python-pip/python-pip.mk
> new file mode 100644
> index 0000000..7cf8ab4
> --- /dev/null
> +++ b/package/python-pip/python-pip.mk
> @@ -0,0 +1,58 @@
> +
> +#############################################################
> +#
> +# python-pip
> +#
> +#############################################################
> +
> +PYTHON_PIP_VERSION = 1.3.1
> +PYTHON_PIP_SOURCE = pip-$(PYTHON_PIP_VERSION).tar.gz
> +PYTHON_PIP_SITE = https://pypi.python.org/packages/source/p/pip
> +PYTHON_PIP_DEPENDENCIES = python python-setuptools host-python-setuptools host-python-pip
> +
> +# README.rst refers to the file "LICENSE" but it's not included
> +
> +define PYTHON_PIP_BUILD_CMDS
> +	(cd $(@D); \
> +	PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \
> +	$(HOST_DIR)/usr/bin/python setup.py build --executable=/usr/bin/python)
> +endef
> +
> +define HOST_PYTHON_PIP_INSTALL_CMDS
> +	(cd $(@D); PYTHONPATH=$(HOST_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \
> +	$(HOST_DIR)/usr/bin/python setup.py install --prefix=$(HOST_DIR)/usr)
> +endef
> +
> +PYTHON_PIP_MODULES_LIST=$(call qstrip, $(BR2_PACKAGE_PYTHON_PIP_MODULES_ADDITIONAL))
> +
> +ifneq ($(PYTHON_PIP_MODULES_LIST),)
> +define PYTHON_PIP_INSTALL_MODULES
> +	# Explanation of environment variables
> +	# PATH: the staging dir is required here so that xslt-config can be found
> +	# 	when trying to install the python lxml package

Putting STAGING_DIR in the PATH is really bad. Isn't there a way to
specifically pass the path to xslt-config?

> +	# PIP_DOWNLOAD_CACHE: all downloads go into the buildroot download folder
> +	# PIP_TARGET: this is where the packages end up
> +	# PIP_BUILD: where the packages are built - a subdirectory of the pip folder
> +	($(TARGET_CONFIGURE_OPTS) \
> +	PATH=$(STAGING_DIR)/usr/bin:$(PATH) \
> +	PIP_DOWNLOAD_CACHE=$(BR2_DL_DIR) \
> +	PIP_TARGET=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \
> +	PIP_BUILD=$(BUILD_DIR)/python-pip-$(PYTHON_PIP_VERSION)/packages \
> +	CC="$(TARGET_CC)"		\
> +	CFLAGS="$(TARGET_CFLAGS)" 	\
> +	LDSHARED="$(TARGET_CC) -shared" \
> +	LDFLAGS="$(TARGET_LDFLAGS)" 	\
> +	$(HOST_DIR)/usr/bin/pip install \
> +	$(PYTHON_PIP_MODULES_LIST))
> +endef
> +endif
> +
> +define PYTHON_PIP_INSTALL_TARGET_CMDS
> +	(cd $(@D); PYTHONPATH=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \
> +	$(HOST_DIR)/usr/bin/python setup.py install --prefix=$(TARGET_DIR)/usr)

So this is installing pip itself on the target.

> +	$(PYTHON_PIP_INSTALL_MODULES)

And this is installing the modules.

Isn't there a way of installing pip only on the host, use it during the
build to install other Python modules to the target, but keep pip out
of the target, since it's generally not used at runtime?


>  define PYTHON_SETUPTOOLS_BUILD_CMDS
>  	(cd $(@D); \
> -	PYTHONPATH="/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \
> +	PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \

Gustavo will tell here, but isn't the PYTHONPATH going to leak into the
target, and therefore contain invalid paths?

> diff --git a/package/python/python.mk b/package/python/python.mk
> index ecea638..45a0563 100644
> --- a/package/python/python.mk
> +++ b/package/python/python.mk
> @@ -31,8 +31,7 @@ HOST_PYTHON_CONF_OPT += 	\
>  	--disable-gdbm		\
>  	--disable-bsddb		\
>  	--disable-test-modules	\
> -	--disable-bz2		\
> -	--disable-ssl
> +	--disable-bz2		
>  
>  HOST_PYTHON_MAKE_ENV = \
>  	PYTHON_MODULES_INCLUDE=$(HOST_DIR)/usr/include \
> @@ -50,7 +49,7 @@ HOST_PYTHON_MAKE = $(MAKE1)
>  
>  PYTHON_DEPENDENCIES  = host-python libffi
>  
> -HOST_PYTHON_DEPENDENCIES = host-expat host-zlib
> +HOST_PYTHON_DEPENDENCIES = host-expat host-zlib host-openssl

The annoying thing here is that host-python will now *always* trigger
the build of openssl, which is quite huge, even though in many cases,
SSL support will not be needed in host-python.

I guess the only solution is an hidden
BR2_HOST_PYTHON_NEEDS_SSL_SUPPORT option, which is selected by
python-pip.

But definitely, on all this Python stuff, Gustavo is the one who will
have the most accurate and detailed review.

Best regards,

Thomas
Rohan Fletcher - July 15, 2013, 1:29 a.m.
Hello Thomas,

Thanks for your contribution! A few comments below, to help improve
> your patches and make them ready for merging in Buildroot.


You are welcome. Ok, I will make changes to the patches based upon your
comments and submit again. This is the first time I have submitted anything
to an open source project.


Don't certain pip modules have native dependencies, that should be
> handled? Like a Python module that uses a separate native library?
>

Yes some python modules have native dependencies - pip does not
automatically select and compile the dependencies in buildroot - they need
to be preselected.

For instance, the python-lxml module requires the libxml2 and libxslt
libraries. When these libraries have been preselected in buildroot, the
python-lxml module will successfully compile and install to the target.

So far I have installed gevent, gevent-socketio, flask, jinja2, psutil,
daemon, passlib, requests, werkzeug, pillow, and feedparser successfully on
the target using this method.

I have conducted basic tests of the installed modules on the target python
interpreter, with mostly success.

There are tiny issues I am coming across installing modules that require
libraries with pip:
 - Pillow (a fork of Python Image Library) won't compile with jpeg support
because it expects libjpeg headers to be located at /usr/include instead of
$BASE_DIR/output/staging/usr/include.
 - The python-magic package can't find libmagic because it relies on
ldconfig cache output to find the library.
 - The requests package gives an error
SSL3_GET_SERVER_CERTIFICATE:certificate verify failed, even though I can
connect fine to the same website using httplib. Still investigating this
one.
 - The lxml package must be able to find xslt-config on $PATH.

Some of these python modules that compile against libraries might need to
be made into a buildroot package with patches to work 100%. However, pip
will cover the rest.


Generally, we don't really like this kind of package that builds other
> packages, because it completely breaks the download and license
> infrastructure of Buildroot. But maybe for languages like Python, Perl
> and Ruby, it makes sense to have support for those things, even if it's
> not perfect.
>

I can understand that. Its definitely a package addition that will bend the
rules of the game.

My inspiration for this package is found under nodejs -> module selection
in menuconfig. There is an option to provide a list of additional modules
that will be automatically installed by npm, the nodejs package manager.

The reasoning for adding pip to my setup was:
1. Quick and easy - If I want to add another python module, I can just add
a name to a list in menuconfig and it will be included.
2. Dependency resolution - By using pip, the (python) dependencies of each
python module are automatically retrieved and installed during the build
process.

What are your main concerns about breaking the download infrastructure?

In regards to licensing infrastructure, is it possible to create a script
that downloads the license metadata from https://pypi.python.org/ and makes
it available to "make legal-info"?


> +     # Explanation of environment variables
> > +     # PATH: the staging dir is required here so that xslt-config can
> be found
> > +     #       when trying to install the python lxml package
>


Putting STAGING_DIR in the PATH is really bad. Isn't there a way to
> specifically pass the path to xslt-config?


Yes, I was wrong. The staging bin directory has binaries for the target
platform in it. Oops.

The path modification is now removed. I have tried giving pip an argument
which contained --with-xslt-config (supported by lxml), but other modules I
install at the same time trigger errors because they don't recognise this
option.

Maybe I can create a directory which contains symlinks to only the
"-config" scripts from staging, and then stick that directory of symlinks
on PATH temporarily. I'm not sure where you stand on such a strategy.


> +comment "custom python modules"
> Not sure why we need this comment here. Could you explain?


I'm not sure why either. Comment deleted.


> @@ -31,8 +31,7 @@ HOST_PYTHON_CONF_OPT +=     \
> >       --disable-gdbm          \
> >       --disable-bsddb         \
> >       --disable-test-modules  \
> > -     --disable-bz2           \
> > -     --disable-ssl
> > +     --disable-bz2
> >
> [...]
> >
> > -HOST_PYTHON_DEPENDENCIES = host-expat host-zlib
> > +HOST_PYTHON_DEPENDENCIES = host-expat host-zlib host-openssl
>
> The annoying thing here is that host-python will now *always* trigger
> the build of openssl, which is quite huge, even though in many cases,
> SSL support will not be needed in host-python.


> I guess the only solution is an hidden
> BR2_HOST_PYTHON_NEEDS_SSL_SUPPORT option, which is selected by
> python-pip.
>

I will add a patch for this.


Isn't there a way of installing pip only on the host, use it during the
> build to install other Python modules to the target, but keep pip out
> of the target, since it's generally not used at runtime?
>

From your suggestion, I have removed the installation of pip to the target
with no resulting errors.


 >  define PYTHON_SETUPTOOLS_BUILD_CMDS
> >       (cd $(@D); \
> > -     PYTHONPATH="/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \
> > +
> PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages"
> \
>
> Gustavo will tell here, but isn't the PYTHONPATH going to leak into the
> target, and therefore contain invalid paths?
>

Ok I have tested this again, and I have found I no longer need to change
the PYTHONPATH in order to get setuptools to work. So these changes have
been reverted as well.


Thanks for the feedback. I will send updated patches through shortly.

Regards,
Rohan Fletcher

Patch

diff --git a/package/Config.in b/package/Config.in
index 6d5ff01..d6678a3 100644
--- a/package/Config.in
+++ b/package/Config.in
@@ -359,6 +359,8 @@  source "package/python-pyparsing/Config.in"
 source "package/python-serial/Config.in"
 source "package/python-setuptools/Config.in"
 source "package/python-thrift/Config.in"
+comment "custom python modules"
+source "package/python-pip/Config.in"
 endmenu
 endif
 source "package/python3/Config.in"
diff --git a/package/python-pip/Config.in b/package/python-pip/Config.in
new file mode 100644
index 0000000..48f3b04
--- /dev/null
+++ b/package/python-pip/Config.in
@@ -0,0 +1,20 @@ 
+config BR2_PACKAGE_PYTHON_PIP
+    bool "python-pip"
+    help
+      A tool for installing and managing Python packages.
+
+      http://www.pip-installer.org
+
+if BR2_PACKAGE_PYTHON_PIP
+
+config BR2_PACKAGE_PYTHON_PIP_MODULES_ADDITIONAL
+	string "Additional modules"
+	help
+	  List of space-separated python modules to install via pip.
+	  See 'pip help install' for available installation methods. 
+	  For repeatable builds, download and save tgz files or clone 
+	  git repos for the components you care about.
+
+	  Example: module-name module-name==1.3.4 /my/module/mymodule.tgz git://github.com/someuser/somemodule.git#v1.2
+
+endif
diff --git a/package/python-pip/python-pip.mk b/package/python-pip/python-pip.mk
new file mode 100644
index 0000000..7cf8ab4
--- /dev/null
+++ b/package/python-pip/python-pip.mk
@@ -0,0 +1,58 @@ 
+
+#############################################################
+#
+# python-pip
+#
+#############################################################
+
+PYTHON_PIP_VERSION = 1.3.1
+PYTHON_PIP_SOURCE = pip-$(PYTHON_PIP_VERSION).tar.gz
+PYTHON_PIP_SITE = https://pypi.python.org/packages/source/p/pip
+PYTHON_PIP_DEPENDENCIES = python python-setuptools host-python-setuptools host-python-pip
+
+# README.rst refers to the file "LICENSE" but it's not included
+
+define PYTHON_PIP_BUILD_CMDS
+	(cd $(@D); \
+	PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \
+	$(HOST_DIR)/usr/bin/python setup.py build --executable=/usr/bin/python)
+endef
+
+define HOST_PYTHON_PIP_INSTALL_CMDS
+	(cd $(@D); PYTHONPATH=$(HOST_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \
+	$(HOST_DIR)/usr/bin/python setup.py install --prefix=$(HOST_DIR)/usr)
+endef
+
+PYTHON_PIP_MODULES_LIST=$(call qstrip, $(BR2_PACKAGE_PYTHON_PIP_MODULES_ADDITIONAL))
+
+ifneq ($(PYTHON_PIP_MODULES_LIST),)
+define PYTHON_PIP_INSTALL_MODULES
+	# Explanation of environment variables
+	# PATH: the staging dir is required here so that xslt-config can be found
+	# 	when trying to install the python lxml package
+	# PIP_DOWNLOAD_CACHE: all downloads go into the buildroot download folder
+	# PIP_TARGET: this is where the packages end up
+	# PIP_BUILD: where the packages are built - a subdirectory of the pip folder
+	($(TARGET_CONFIGURE_OPTS) \
+	PATH=$(STAGING_DIR)/usr/bin:$(PATH) \
+	PIP_DOWNLOAD_CACHE=$(BR2_DL_DIR) \
+	PIP_TARGET=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \
+	PIP_BUILD=$(BUILD_DIR)/python-pip-$(PYTHON_PIP_VERSION)/packages \
+	CC="$(TARGET_CC)"		\
+	CFLAGS="$(TARGET_CFLAGS)" 	\
+	LDSHARED="$(TARGET_CC) -shared" \
+	LDFLAGS="$(TARGET_LDFLAGS)" 	\
+	$(HOST_DIR)/usr/bin/pip install \
+	$(PYTHON_PIP_MODULES_LIST))
+endef
+endif
+
+define PYTHON_PIP_INSTALL_TARGET_CMDS
+	(cd $(@D); PYTHONPATH=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \
+	$(HOST_DIR)/usr/bin/python setup.py install --prefix=$(TARGET_DIR)/usr)
+	$(PYTHON_PIP_INSTALL_MODULES)
+endef
+
+
+$(eval $(generic-package))
+$(eval $(host-generic-package))
diff --git a/package/python-setuptools/python-setuptools.mk b/package/python-setuptools/python-setuptools.mk
index cd5ee4d..416abfb 100644
--- a/package/python-setuptools/python-setuptools.mk
+++ b/package/python-setuptools/python-setuptools.mk
@@ -9,9 +9,9 @@ 
 # switch back to it.
 # See http://pypi.python.org/packages/source/s/setuptools
 
-PYTHON_SETUPTOOLS_VERSION = 0.6.36
-PYTHON_SETUPTOOLS_SOURCE  = distribute-$(PYTHON_SETUPTOOLS_VERSION).tar.gz
-PYTHON_SETUPTOOLS_SITE    = http://pypi.python.org/packages/source/d/distribute
+PYTHON_SETUPTOOLS_VERSION = 0.8
+PYTHON_SETUPTOOLS_SOURCE  = setuptools-$(PYTHON_SETUPTOOLS_VERSION).tar.gz
+PYTHON_SETUPTOOLS_SITE    = http://pypi.python.org/packages/source/s/setuptools
 PYTHON_SETUPTOOLS_DEPENDENCIES = python
 
 define HOST_PYTHON_SETUPTOOLS_BUILD_CMDS
@@ -20,7 +20,7 @@  endef
 
 define PYTHON_SETUPTOOLS_BUILD_CMDS
 	(cd $(@D); \
-	PYTHONPATH="/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \
+	PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \
 	$(HOST_DIR)/usr/bin/python setup.py build)
 endef
 
@@ -32,7 +32,7 @@  endef
 
 define PYTHON_SETUPTOOLS_INSTALL_TARGET_CMDS
 	(cd $(@D); \
-	PYTHONPATH="/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \
+	PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \
 	$(HOST_DIR)/usr/bin/python setup.py install --executable=/usr/bin/python \
 	--single-version-externally-managed --root=/ --prefix=$(TARGET_DIR)/usr)
 endef
diff --git a/package/python/python.mk b/package/python/python.mk
index ecea638..45a0563 100644
--- a/package/python/python.mk
+++ b/package/python/python.mk
@@ -31,8 +31,7 @@  HOST_PYTHON_CONF_OPT += 	\
 	--disable-gdbm		\
 	--disable-bsddb		\
 	--disable-test-modules	\
-	--disable-bz2		\
-	--disable-ssl
+	--disable-bz2		
 
 HOST_PYTHON_MAKE_ENV = \
 	PYTHON_MODULES_INCLUDE=$(HOST_DIR)/usr/include \
@@ -50,7 +49,7 @@  HOST_PYTHON_MAKE = $(MAKE1)
 
 PYTHON_DEPENDENCIES  = host-python libffi
 
-HOST_PYTHON_DEPENDENCIES = host-expat host-zlib
+HOST_PYTHON_DEPENDENCIES = host-expat host-zlib host-openssl
 
 PYTHON_INSTALL_STAGING = YES