From patchwork Wed May 14 09:23:40 2014 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacob Kjaergaard X-Patchwork-Id: 348668 X-Patchwork-Delegate: esben@haabendal.dk Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from hugin.dotsrc.org (hugin.dotsrc.org [IPv6:2001:878:346::102]) by ozlabs.org (Postfix) with ESMTP id 5DA2E1400B0 for ; Wed, 14 May 2014 19:23:04 +1000 (EST) Received: from hugin.dotsrc.org (localhost [127.0.0.1]) by hugin.dotsrc.org (Postfix) with ESMTP id 888243FA64 for ; Wed, 14 May 2014 11:23:02 +0200 (CEST) X-Original-To: dev@oe-lite.org Delivered-To: dev@oe-lite.org Received: from mail01.prevas.se (mail01.prevas.se [62.95.78.3]) by hugin.dotsrc.org (Postfix) with ESMTPS id 20E0D3FA64 for ; Wed, 14 May 2014 11:23:01 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=prevas.dk; i=@prevas.dk; l=5963; q=dns/txt; s=ironport1; t=1400059381; x=1431595381; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version; bh=/V+CL4BZGES+9JT7MhH6zQZA1yXE9LOOBXAvm3MJHH4=; b=N+UT4J1/Z4ms/oYGHovw5H/fJaO/ZFfAriUMk2Qzk8rNaPQ5lIAPDA4M HhflIXNiJzVVSlDiYAcN+kLURgKdsRYBDTRvvcbuherQiIQ1QWeS0CSnX yBUOZjcjeK5nTuN4hI4mLyAqC5EI9eRF1v6NovrkoBqcN5QrIaW86w522 8=; X-IronPort-AV: E=Sophos;i="4.97,1051,1389740400"; d="scan'208";a="4707124" Received: from vmprevas3.prevas.se (HELO smtp.prevas.se) ([172.16.8.103]) by ironport1.prevas.se with ESMTP/TLS/AES128-SHA; 14 May 2014 11:23:00 +0200 Received: from arh116.prevas.se (172.16.11.15) by smtp.prevas.se (172.16.8.105) with Microsoft SMTP Server (TLS) id 14.2.347.0; Wed, 14 May 2014 11:23:00 +0200 From: To: Subject: [core v2] prebake: allow packages to be fetched from web instead of local dirs only Date: Wed, 14 May 2014 11:23:40 +0200 Message-ID: <1400059420-26426-1-git-send-email-jabk@prevas.dk> X-Mailer: git-send-email 1.7.9.5 In-Reply-To: <871tvwpwbf.fsf@xpc.ulstruplund.dk> References: <871tvwpwbf.fsf@xpc.ulstruplund.dk> MIME-Version: 1.0 X-Originating-IP: [172.16.11.15] X-BeenThere: dev@oe-lite.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: OE-lite development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: dev-bounces@oe-lite.org Errors-To: dev-bounces@oe-lite.org From: Jacob Kjaergaard set PREBAKE_URL to point to your prebaked packages url prefix e.g: PREBAKE_CACHE_DIR can be set to pin point where the fetched prebaked packages are located http://myserver.com/packages/ the directory structure under "packages" should match the structure under tmp/packages/ --- conf/oe-lite.conf | 1 + conf/oelayout.conf | 3 +++ doc/manual.txt | 2 ++ doc/prebake.txt | 29 +++++++++++++++++++++++++++++ lib/oelite/baker.py | 27 +++++++++++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 doc/prebake.txt diff --git a/conf/oe-lite.conf b/conf/oe-lite.conf index b1e23b2..4468b37 100644 --- a/conf/oe-lite.conf +++ b/conf/oe-lite.conf @@ -41,6 +41,7 @@ DEFAULT_RELAX[nohash] = "1" PREBAKE_PATH[nohash] = "1" PARALLEL_MAKE[nohash] = True +PREBAKE_URL[nohash] = True export PATH export LD_LIBRARY_PATH diff --git a/conf/oelayout.conf b/conf/oelayout.conf index 535cb6f..405e4a2 100644 --- a/conf/oelayout.conf +++ b/conf/oelayout.conf @@ -57,6 +57,9 @@ IMAGE_DEPLOY_DIR[nohash] = True PACKAGE_DEPLOY_DIR = "${TMPDIR}/packages" PACKAGE_DEPLOY_DIR[nohash] = True +PREBAKE_CACHE_DIR = "${TMPDIR}/prebake" +PREBAKE_CACHE_DIR[nohash] = True + # Recipe directory layout FILESPATHBASE = "${FILE_DIRNAME}" FILESPATHBASE[emit] = "" diff --git a/doc/manual.txt b/doc/manual.txt index 2bcd230..e0fe093 100644 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -40,6 +40,8 @@ include::quickstart.txt[] // Overview //include::overview.txt[] +include::prebake.txt[] + // Build Configuration //include::buildcfg.txt[] diff --git a/doc/prebake.txt b/doc/prebake.txt new file mode 100644 index 0000000..b821e3d --- /dev/null +++ b/doc/prebake.txt @@ -0,0 +1,29 @@ +Using prebaked packages +======================= + +This section describes how to use packages build on another machine +available via http. + + +First you need to setup a http server that serves the prebaked packages +from another machine. The baked prebakes will be in `${TMPDIR}/packages` +so setup the server to make files available online as you prefer. + +Secondly, you need to set PREBAKE_URL to point to your prebaked packages url +prefix e.g: PREBAKE_CACHE_DIR can be set to pin point where the fetched +prebaked packages are located + +---- +#cat conf/site.conf +PREBAKE_URL="http://myserver.com/packages/" +PREBAKE_CACHE_DIR="" +---- + +Note, that the directory structure under "packages" should match the +structure under `${TMPDIR}/packages`. + +Presently, 32-bit machines should bake prebaked packages that should be +used on 32-bit machines as well (and likewise for 64-bit). + +The feature has been tested on Ubuntu 12.04 so please report a bug if you find +problems using this feature. diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py index 20cf146..476b22a 100644 --- a/lib/oelite/baker.py +++ b/lib/oelite/baker.py @@ -12,6 +12,10 @@ import oelite.item from oelite.parse import * from oelite.cookbook import CookBook +import oelite.fetch + +import bb.utils + import sys import os import glob @@ -437,6 +441,9 @@ class OEliteBaker: # check for availability of prebaked packages, and set package # filename for all packages. depend_packages = self.runq.get_depend_packages() + url_prefix = self.config.get("PREBAKE_URL") + if url_prefix is not None: + info("Trying to use prebakes from url: %s"%(url_prefix)) for package in depend_packages: # FIXME: skip this package if it is to be rebuild prebake = self.find_prebaked_package(package) @@ -608,6 +615,7 @@ class OEliteBaker: def find_prebaked_package(self, package): """return full-path filename string or None""" package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR") + prebake_url_cache_dir = self.config.get("PREBAKE_CACHE_DIR") if not package_deploy_dir: die("PACKAGE_DEPLOY_DIR not defined") if self.options.prebake: @@ -615,6 +623,7 @@ class OEliteBaker: if prebake_path: prebake_path = prebake_path.split(":") prebake_path.insert(0, package_deploy_dir) + prebake_path.insert(0, prebake_url_cache_dir) else: prebake_path = [package_deploy_dir] debug("package=%s"%(repr(package))) @@ -628,6 +637,7 @@ class OEliteBaker: raise NoSuchPackage() filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash) debug("prebake_path=%s"%(prebake_path)) + #test local paths first for base_dir in prebake_path: path = os.path.join( base_dir, @@ -638,6 +648,23 @@ class OEliteBaker: if os.path.exists(path): debug("found prebake: %s"%(path)) return path + #then test URLs from PREBAKE_URL + url_prefix = self.config.get("PREBAKE_URL") + if url_prefix is not None: + package_path =os.path.join( + package.type, + package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""), + filename) + downloaded_file =os.path.join( + prebake_url_cache_dir, + package_path) + url = os.path.join(url_prefix, package_path) + if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1): + if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0: + debug("using prebake from web: %s"%(url)) + return downloaded_file + else: + os.unlink(downloaded_file) return None