| Message ID | 1373707158-31756-1-git-send-email-rohfledev@gmail.com |
|---|---|
| State | Superseded |
| Headers | show |
Dear Rohan Fletcher, Thanks for your contribution! A few comments below, to help improve your patches and make them ready for merging in Buildroot. On Sat, 13 Jul 2013 21:19:17 +1200, Rohan Fletcher wrote: > Added python-pip - a package manager for python. easy way of adding python packages to buildroot. > Updated python-setuptools to version 0.8 - distribute and setuptools have now merged. > Enabled ssl in host python config so pip can download packages over https. In this commit description, you're mentioning three totally separate things. This is a very good indication that those three separate things should be in separate patches: adding ssl in host python, bumping setuptools, and finally adding python-pip. However, your second patch that adds the license to python-pip should be part of the patch adding python-pip itself. To re-organize your patches, I recommend you to have a look about "git rebase -i", it's really the git killer feature to make your patch series look nice. > diff --git a/package/Config.in b/package/Config.in > index 6d5ff01..d6678a3 100644 > --- a/package/Config.in > +++ b/package/Config.in > @@ -359,6 +359,8 @@ source "package/python-pyparsing/Config.in" > source "package/python-serial/Config.in" > source "package/python-setuptools/Config.in" > source "package/python-thrift/Config.in" > +comment "custom python modules" Not sure why we need this comment here. Could you explain? > +source "package/python-pip/Config.in" > endmenu > endif > source "package/python3/Config.in" > diff --git a/package/python-pip/Config.in b/package/python-pip/Config.in > new file mode 100644 > index 0000000..48f3b04 > --- /dev/null > +++ b/package/python-pip/Config.in > @@ -0,0 +1,20 @@ > +config BR2_PACKAGE_PYTHON_PIP > + bool "python-pip" > + help Indentation should be one tab, as you did later. > + A tool for installing and managing Python packages. And here indentation should be one tab + two spaces. Check the Buildroot manual for examples/coding style details. > + > + http://www.pip-installer.org > + > +if BR2_PACKAGE_PYTHON_PIP > + > +config BR2_PACKAGE_PYTHON_PIP_MODULES_ADDITIONAL > + string "Additional modules" > + help > + List of space-separated python modules to install via pip. > + See 'pip help install' for available installation methods. > + For repeatable builds, download and save tgz files or clone > + git repos for the components you care about. > + > + Example: module-name module-name==1.3.4 /my/module/mymodule.tgz git://github.com/someuser/somemodule.git#v1.2 This last line should be wrapped. Don't certain pip modules have native dependencies, that should be handled? Like a Python module that uses a separate native library? Generally, we don't really like this kind of package that builds other packages, because it completely breaks the download and license infrastructure of Buildroot. But maybe for languages like Python, Perl and Ruby, it makes sense to have support for those things, even if it's not perfect. > +endif > diff --git a/package/python-pip/python-pip.mk b/package/python-pip/python-pip.mk > new file mode 100644 > index 0000000..7cf8ab4 > --- /dev/null > +++ b/package/python-pip/python-pip.mk > @@ -0,0 +1,58 @@ > + > +############################################################# > +# > +# python-pip > +# > +############################################################# > + > +PYTHON_PIP_VERSION = 1.3.1 > +PYTHON_PIP_SOURCE = pip-$(PYTHON_PIP_VERSION).tar.gz > +PYTHON_PIP_SITE = https://pypi.python.org/packages/source/p/pip > +PYTHON_PIP_DEPENDENCIES = python python-setuptools host-python-setuptools host-python-pip > + > +# README.rst refers to the file "LICENSE" but it's not included > + > +define PYTHON_PIP_BUILD_CMDS > + (cd $(@D); \ > + PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ > + $(HOST_DIR)/usr/bin/python setup.py build --executable=/usr/bin/python) > +endef > + > +define HOST_PYTHON_PIP_INSTALL_CMDS > + (cd $(@D); PYTHONPATH=$(HOST_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \ > + $(HOST_DIR)/usr/bin/python setup.py install --prefix=$(HOST_DIR)/usr) > +endef > + > +PYTHON_PIP_MODULES_LIST=$(call qstrip, $(BR2_PACKAGE_PYTHON_PIP_MODULES_ADDITIONAL)) > + > +ifneq ($(PYTHON_PIP_MODULES_LIST),) > +define PYTHON_PIP_INSTALL_MODULES > + # Explanation of environment variables > + # PATH: the staging dir is required here so that xslt-config can be found > + # when trying to install the python lxml package Putting STAGING_DIR in the PATH is really bad. Isn't there a way to specifically pass the path to xslt-config? > + # PIP_DOWNLOAD_CACHE: all downloads go into the buildroot download folder > + # PIP_TARGET: this is where the packages end up > + # PIP_BUILD: where the packages are built - a subdirectory of the pip folder > + ($(TARGET_CONFIGURE_OPTS) \ > + PATH=$(STAGING_DIR)/usr/bin:$(PATH) \ > + PIP_DOWNLOAD_CACHE=$(BR2_DL_DIR) \ > + PIP_TARGET=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \ > + PIP_BUILD=$(BUILD_DIR)/python-pip-$(PYTHON_PIP_VERSION)/packages \ > + CC="$(TARGET_CC)" \ > + CFLAGS="$(TARGET_CFLAGS)" \ > + LDSHARED="$(TARGET_CC) -shared" \ > + LDFLAGS="$(TARGET_LDFLAGS)" \ > + $(HOST_DIR)/usr/bin/pip install \ > + $(PYTHON_PIP_MODULES_LIST)) > +endef > +endif > + > +define PYTHON_PIP_INSTALL_TARGET_CMDS > + (cd $(@D); PYTHONPATH=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \ > + $(HOST_DIR)/usr/bin/python setup.py install --prefix=$(TARGET_DIR)/usr) So this is installing pip itself on the target. > + $(PYTHON_PIP_INSTALL_MODULES) And this is installing the modules. Isn't there a way of installing pip only on the host, use it during the build to install other Python modules to the target, but keep pip out of the target, since it's generally not used at runtime? > define PYTHON_SETUPTOOLS_BUILD_CMDS > (cd $(@D); \ > - PYTHONPATH="/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ > + PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ Gustavo will tell here, but isn't the PYTHONPATH going to leak into the target, and therefore contain invalid paths? > diff --git a/package/python/python.mk b/package/python/python.mk > index ecea638..45a0563 100644 > --- a/package/python/python.mk > +++ b/package/python/python.mk > @@ -31,8 +31,7 @@ HOST_PYTHON_CONF_OPT += \ > --disable-gdbm \ > --disable-bsddb \ > --disable-test-modules \ > - --disable-bz2 \ > - --disable-ssl > + --disable-bz2 > > HOST_PYTHON_MAKE_ENV = \ > PYTHON_MODULES_INCLUDE=$(HOST_DIR)/usr/include \ > @@ -50,7 +49,7 @@ HOST_PYTHON_MAKE = $(MAKE1) > > PYTHON_DEPENDENCIES = host-python libffi > > -HOST_PYTHON_DEPENDENCIES = host-expat host-zlib > +HOST_PYTHON_DEPENDENCIES = host-expat host-zlib host-openssl The annoying thing here is that host-python will now *always* trigger the build of openssl, which is quite huge, even though in many cases, SSL support will not be needed in host-python. I guess the only solution is an hidden BR2_HOST_PYTHON_NEEDS_SSL_SUPPORT option, which is selected by python-pip. But definitely, on all this Python stuff, Gustavo is the one who will have the most accurate and detailed review. Best regards, Thomas
Hello Thomas, Thanks for your contribution! A few comments below, to help improve > your patches and make them ready for merging in Buildroot. You are welcome. Ok, I will make changes to the patches based upon your comments and submit again. This is the first time I have submitted anything to an open source project. Don't certain pip modules have native dependencies, that should be > handled? Like a Python module that uses a separate native library? > Yes some python modules have native dependencies - pip does not automatically select and compile the dependencies in buildroot - they need to be preselected. For instance, the python-lxml module requires the libxml2 and libxslt libraries. When these libraries have been preselected in buildroot, the python-lxml module will successfully compile and install to the target. So far I have installed gevent, gevent-socketio, flask, jinja2, psutil, daemon, passlib, requests, werkzeug, pillow, and feedparser successfully on the target using this method. I have conducted basic tests of the installed modules on the target python interpreter, with mostly success. There are tiny issues I am coming across installing modules that require libraries with pip: - Pillow (a fork of Python Image Library) won't compile with jpeg support because it expects libjpeg headers to be located at /usr/include instead of $BASE_DIR/output/staging/usr/include. - The python-magic package can't find libmagic because it relies on ldconfig cache output to find the library. - The requests package gives an error SSL3_GET_SERVER_CERTIFICATE:certificate verify failed, even though I can connect fine to the same website using httplib. Still investigating this one. - The lxml package must be able to find xslt-config on $PATH. Some of these python modules that compile against libraries might need to be made into a buildroot package with patches to work 100%. However, pip will cover the rest. Generally, we don't really like this kind of package that builds other > packages, because it completely breaks the download and license > infrastructure of Buildroot. But maybe for languages like Python, Perl > and Ruby, it makes sense to have support for those things, even if it's > not perfect. > I can understand that. Its definitely a package addition that will bend the rules of the game. My inspiration for this package is found under nodejs -> module selection in menuconfig. There is an option to provide a list of additional modules that will be automatically installed by npm, the nodejs package manager. The reasoning for adding pip to my setup was: 1. Quick and easy - If I want to add another python module, I can just add a name to a list in menuconfig and it will be included. 2. Dependency resolution - By using pip, the (python) dependencies of each python module are automatically retrieved and installed during the build process. What are your main concerns about breaking the download infrastructure? In regards to licensing infrastructure, is it possible to create a script that downloads the license metadata from https://pypi.python.org/ and makes it available to "make legal-info"? > + # Explanation of environment variables > > + # PATH: the staging dir is required here so that xslt-config can > be found > > + # when trying to install the python lxml package > Putting STAGING_DIR in the PATH is really bad. Isn't there a way to > specifically pass the path to xslt-config? Yes, I was wrong. The staging bin directory has binaries for the target platform in it. Oops. The path modification is now removed. I have tried giving pip an argument which contained --with-xslt-config (supported by lxml), but other modules I install at the same time trigger errors because they don't recognise this option. Maybe I can create a directory which contains symlinks to only the "-config" scripts from staging, and then stick that directory of symlinks on PATH temporarily. I'm not sure where you stand on such a strategy. > +comment "custom python modules" > Not sure why we need this comment here. Could you explain? I'm not sure why either. Comment deleted. > @@ -31,8 +31,7 @@ HOST_PYTHON_CONF_OPT += \ > > --disable-gdbm \ > > --disable-bsddb \ > > --disable-test-modules \ > > - --disable-bz2 \ > > - --disable-ssl > > + --disable-bz2 > > > [...] > > > > -HOST_PYTHON_DEPENDENCIES = host-expat host-zlib > > +HOST_PYTHON_DEPENDENCIES = host-expat host-zlib host-openssl > > The annoying thing here is that host-python will now *always* trigger > the build of openssl, which is quite huge, even though in many cases, > SSL support will not be needed in host-python. > I guess the only solution is an hidden > BR2_HOST_PYTHON_NEEDS_SSL_SUPPORT option, which is selected by > python-pip. > I will add a patch for this. Isn't there a way of installing pip only on the host, use it during the > build to install other Python modules to the target, but keep pip out > of the target, since it's generally not used at runtime? > From your suggestion, I have removed the installation of pip to the target with no resulting errors. > define PYTHON_SETUPTOOLS_BUILD_CMDS > > (cd $(@D); \ > > - PYTHONPATH="/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ > > + > PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" > \ > > Gustavo will tell here, but isn't the PYTHONPATH going to leak into the > target, and therefore contain invalid paths? > Ok I have tested this again, and I have found I no longer need to change the PYTHONPATH in order to get setuptools to work. So these changes have been reverted as well. Thanks for the feedback. I will send updated patches through shortly. Regards, Rohan Fletcher
diff --git a/package/Config.in b/package/Config.in index 6d5ff01..d6678a3 100644 --- a/package/Config.in +++ b/package/Config.in @@ -359,6 +359,8 @@ source "package/python-pyparsing/Config.in" source "package/python-serial/Config.in" source "package/python-setuptools/Config.in" source "package/python-thrift/Config.in" +comment "custom python modules" +source "package/python-pip/Config.in" endmenu endif source "package/python3/Config.in" diff --git a/package/python-pip/Config.in b/package/python-pip/Config.in new file mode 100644 index 0000000..48f3b04 --- /dev/null +++ b/package/python-pip/Config.in @@ -0,0 +1,20 @@ +config BR2_PACKAGE_PYTHON_PIP + bool "python-pip" + help + A tool for installing and managing Python packages. + + http://www.pip-installer.org + +if BR2_PACKAGE_PYTHON_PIP + +config BR2_PACKAGE_PYTHON_PIP_MODULES_ADDITIONAL + string "Additional modules" + help + List of space-separated python modules to install via pip. + See 'pip help install' for available installation methods. + For repeatable builds, download and save tgz files or clone + git repos for the components you care about. + + Example: module-name module-name==1.3.4 /my/module/mymodule.tgz git://github.com/someuser/somemodule.git#v1.2 + +endif diff --git a/package/python-pip/python-pip.mk b/package/python-pip/python-pip.mk new file mode 100644 index 0000000..7cf8ab4 --- /dev/null +++ b/package/python-pip/python-pip.mk @@ -0,0 +1,58 @@ + +############################################################# +# +# python-pip +# +############################################################# + +PYTHON_PIP_VERSION = 1.3.1 +PYTHON_PIP_SOURCE = pip-$(PYTHON_PIP_VERSION).tar.gz +PYTHON_PIP_SITE = https://pypi.python.org/packages/source/p/pip +PYTHON_PIP_DEPENDENCIES = python python-setuptools host-python-setuptools host-python-pip + +# README.rst refers to the file "LICENSE" but it's not included + +define PYTHON_PIP_BUILD_CMDS + (cd $(@D); \ + PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ + $(HOST_DIR)/usr/bin/python setup.py build --executable=/usr/bin/python) +endef + +define HOST_PYTHON_PIP_INSTALL_CMDS + (cd $(@D); PYTHONPATH=$(HOST_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \ + $(HOST_DIR)/usr/bin/python setup.py install --prefix=$(HOST_DIR)/usr) +endef + +PYTHON_PIP_MODULES_LIST=$(call qstrip, $(BR2_PACKAGE_PYTHON_PIP_MODULES_ADDITIONAL)) + +ifneq ($(PYTHON_PIP_MODULES_LIST),) +define PYTHON_PIP_INSTALL_MODULES + # Explanation of environment variables + # PATH: the staging dir is required here so that xslt-config can be found + # when trying to install the python lxml package + # PIP_DOWNLOAD_CACHE: all downloads go into the buildroot download folder + # PIP_TARGET: this is where the packages end up + # PIP_BUILD: where the packages are built - a subdirectory of the pip folder + ($(TARGET_CONFIGURE_OPTS) \ + PATH=$(STAGING_DIR)/usr/bin:$(PATH) \ + PIP_DOWNLOAD_CACHE=$(BR2_DL_DIR) \ + PIP_TARGET=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \ + PIP_BUILD=$(BUILD_DIR)/python-pip-$(PYTHON_PIP_VERSION)/packages \ + CC="$(TARGET_CC)" \ + CFLAGS="$(TARGET_CFLAGS)" \ + LDSHARED="$(TARGET_CC) -shared" \ + LDFLAGS="$(TARGET_LDFLAGS)" \ + $(HOST_DIR)/usr/bin/pip install \ + $(PYTHON_PIP_MODULES_LIST)) +endef +endif + +define PYTHON_PIP_INSTALL_TARGET_CMDS + (cd $(@D); PYTHONPATH=$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages \ + $(HOST_DIR)/usr/bin/python setup.py install --prefix=$(TARGET_DIR)/usr) + $(PYTHON_PIP_INSTALL_MODULES) +endef + + +$(eval $(generic-package)) +$(eval $(host-generic-package)) diff --git a/package/python-setuptools/python-setuptools.mk b/package/python-setuptools/python-setuptools.mk index cd5ee4d..416abfb 100644 --- a/package/python-setuptools/python-setuptools.mk +++ b/package/python-setuptools/python-setuptools.mk @@ -9,9 +9,9 @@ # switch back to it. # See http://pypi.python.org/packages/source/s/setuptools -PYTHON_SETUPTOOLS_VERSION = 0.6.36 -PYTHON_SETUPTOOLS_SOURCE = distribute-$(PYTHON_SETUPTOOLS_VERSION).tar.gz -PYTHON_SETUPTOOLS_SITE = http://pypi.python.org/packages/source/d/distribute +PYTHON_SETUPTOOLS_VERSION = 0.8 +PYTHON_SETUPTOOLS_SOURCE = setuptools-$(PYTHON_SETUPTOOLS_VERSION).tar.gz +PYTHON_SETUPTOOLS_SITE = http://pypi.python.org/packages/source/s/setuptools PYTHON_SETUPTOOLS_DEPENDENCIES = python define HOST_PYTHON_SETUPTOOLS_BUILD_CMDS @@ -20,7 +20,7 @@ endef define PYTHON_SETUPTOOLS_BUILD_CMDS (cd $(@D); \ - PYTHONPATH="/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ + PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ $(HOST_DIR)/usr/bin/python setup.py build) endef @@ -32,7 +32,7 @@ endef define PYTHON_SETUPTOOLS_INSTALL_TARGET_CMDS (cd $(@D); \ - PYTHONPATH="/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ + PYTHONPATH="$(TARGET_DIR)/usr/lib/python$(PYTHON_VERSION_MAJOR)/site-packages" \ $(HOST_DIR)/usr/bin/python setup.py install --executable=/usr/bin/python \ --single-version-externally-managed --root=/ --prefix=$(TARGET_DIR)/usr) endef diff --git a/package/python/python.mk b/package/python/python.mk index ecea638..45a0563 100644 --- a/package/python/python.mk +++ b/package/python/python.mk @@ -31,8 +31,7 @@ HOST_PYTHON_CONF_OPT += \ --disable-gdbm \ --disable-bsddb \ --disable-test-modules \ - --disable-bz2 \ - --disable-ssl + --disable-bz2 HOST_PYTHON_MAKE_ENV = \ PYTHON_MODULES_INCLUDE=$(HOST_DIR)/usr/include \ @@ -50,7 +49,7 @@ HOST_PYTHON_MAKE = $(MAKE1) PYTHON_DEPENDENCIES = host-python libffi -HOST_PYTHON_DEPENDENCIES = host-expat host-zlib +HOST_PYTHON_DEPENDENCIES = host-expat host-zlib host-openssl PYTHON_INSTALL_STAGING = YES
Added python-pip - a package manager for python. easy way of adding python packages to buildroot. Updated python-setuptools to version 0.8 - distribute and setuptools have now merged. Enabled ssl in host python config so pip can download packages over https. Signed-off-by: Rohan Fletcher <rohfledev@gmail.com> --- package/Config.in | 2 + package/python-pip/Config.in | 20 +++++++++ package/python-pip/python-pip.mk | 58 ++++++++++++++++++++++++++ package/python-setuptools/python-setuptools.mk | 10 ++--- package/python/python.mk | 5 +-- 5 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 package/python-pip/Config.in create mode 100644 package/python-pip/python-pip.mk