From patchwork Thu Jan 23 10:02:02 2014 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jacob Kjaergaard X-Patchwork-Id: 313493 X-Patchwork-Delegate: esben@haabendal.dk Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from hugin.dotsrc.org (hugin.dotsrc.org [IPv6:2001:878:346::102]) by ozlabs.org (Postfix) with ESMTP id 483E52C0081 for ; Thu, 23 Jan 2014 21:02:15 +1100 (EST) Received: from hugin.dotsrc.org (localhost [127.0.0.1]) by hugin.dotsrc.org (Postfix) with ESMTP id 8398C3FDB8 for ; Thu, 23 Jan 2014 11:02:14 +0100 (CET) X-Original-To: dev@oe-lite.org Delivered-To: dev@oe-lite.org Received: from mail01.prevas.se (mail01.prevas.se [62.95.78.3]) by hugin.dotsrc.org (Postfix) with ESMTPS id 9AD653F81D for ; Thu, 23 Jan 2014 11:02:10 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=prevas.dk; i=@prevas.dk; l=4369; q=dns/txt; s=ironport1; t=1390471330; x=1422007330; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version; bh=D5R3zL3K4I2CXoz6QIEMPSA1N6Z5ifTShhI2MVgGDv0=; b=sGLcnXWN8H5xtm0HCt9PnGckS7JHsxrieJHPifIPiPkMIT8HG7bYXBAC xVw1iuwtIlau3jaoN8k53vvXvPl2Fye7bSMwKF9EchpEwdEn5fllkKhgA 5Bu2J2w79EiUbWoF2J2ogDavvSAxmy0JdIOVHrgodiCwseopyCRUSyow+ E=; X-IronPort-AV: E=Sophos;i="4.95,705,1384297200"; d="scan'208";a="4329571" Received: from vmprevas3.prevas.se (HELO smtp.prevas.se) ([172.16.8.103]) by ironport1.prevas.se with ESMTP/TLS/AES128-SHA; 23 Jan 2014 11:02:10 +0100 Received: from arh116.prevas.se (172.16.11.15) by smtp.prevas.se (172.16.8.105) with Microsoft SMTP Server (TLS) id 14.2.347.0; Thu, 23 Jan 2014 11:02:10 +0100 From: To: Subject: [core 3/3] prebake: allow packages to be fetched from web instead of local dirs only Date: Thu, 23 Jan 2014 11:02:02 +0100 Message-ID: <1390471322-24372-4-git-send-email-jabk@prevas.dk> X-Mailer: git-send-email 1.7.9.5 In-Reply-To: <1390471322-24372-1-git-send-email-jabk@prevas.dk> References: <1390471322-24372-1-git-send-email-jabk@prevas.dk> MIME-Version: 1.0 X-Originating-IP: [172.16.11.15] X-BeenThere: dev@oe-lite.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: OE-lite development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: dev-bounces@oe-lite.org Errors-To: dev-bounces@oe-lite.org From: Jacob Kjaergaard set PREBAKE_URL to point to your prebaked packages url prefix e.g: PREBAKE_URL_CACHE_DIR can be set to pin point where the fetched prebaked packages are located http://myserver.com/packages/ the directory structure under "packages" should ressemble the structure under tmp/packages/ --- conf/oe-lite.conf | 3 +++ conf/oelayout.conf | 3 +++ lib/oelite/baker.py | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/conf/oe-lite.conf b/conf/oe-lite.conf index b1e23b2..a3fa033 100644 --- a/conf/oe-lite.conf +++ b/conf/oe-lite.conf @@ -41,6 +41,9 @@ DEFAULT_RELAX[nohash] = "1" PREBAKE_PATH[nohash] = "1" PARALLEL_MAKE[nohash] = True +PREBAKE_URL[nohash] = True +PREBAKE_URL_CACHE[nohash] = True + export PATH export LD_LIBRARY_PATH diff --git a/conf/oelayout.conf b/conf/oelayout.conf index 535cb6f..5726410 100644 --- a/conf/oelayout.conf +++ b/conf/oelayout.conf @@ -57,6 +57,9 @@ IMAGE_DEPLOY_DIR[nohash] = True PACKAGE_DEPLOY_DIR = "${TMPDIR}/packages" PACKAGE_DEPLOY_DIR[nohash] = True +PREBAKE_URL_CACHE_DIR = "${TMPDIR}/packages_url_cache" +PREBAKE_URL_CACHE_DIR[nohash] = True + # Recipe directory layout FILESPATHBASE = "${FILE_DIRNAME}" FILESPATHBASE[emit] = "" diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py index e3efb7b..8698ee7 100644 --- a/lib/oelite/baker.py +++ b/lib/oelite/baker.py @@ -12,6 +12,8 @@ import oelite.item from oelite.parse import * from oelite.cookbook import CookBook +import oelite.fetch + import bb.utils import sys @@ -440,6 +442,9 @@ class OEliteBaker: # check for availability of prebaked packages, and set package # filename for all packages. depend_packages = self.runq.get_depend_packages() + url_prefix = self.config.get("PREBAKE_URL") + if url_prefix is not None: + info("Trying to use prebakes from url: %s"%(url_prefix)) for package in depend_packages: # FIXME: skip this package if it is to be rebuild prebake = self.find_prebaked_package(package) @@ -611,6 +616,7 @@ class OEliteBaker: def find_prebaked_package(self, package): """return full-path filename string or None""" package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR") + prebake_url_cache_dir = self.config.get("PREBAKE_URL_CACHE_DIR") if not package_deploy_dir: die("PACKAGE_DEPLOY_DIR not defined") if self.options.prebake: @@ -618,6 +624,7 @@ class OEliteBaker: if prebake_path: prebake_path = prebake_path.split(":") prebake_path.insert(0, package_deploy_dir) + prebake_path.insert(0, prebake_url_cache_dir) else: prebake_path = [package_deploy_dir] debug("package=%s"%(repr(package))) @@ -631,6 +638,7 @@ class OEliteBaker: raise NoSuchPackage() filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash) debug("prebake_path=%s"%(prebake_path)) + #test local paths first for base_dir in prebake_path: path = os.path.join( base_dir, @@ -641,6 +649,23 @@ class OEliteBaker: if os.path.exists(path): debug("found prebake: %s"%(path)) return path + #then test URLs from PREBAKE_URL + url_prefix = self.config.get("PREBAKE_URL") + if url_prefix is not None: + package_path =os.path.join( + package.type, + package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""), + filename) + downloaded_file =os.path.join( + prebake_url_cache_dir, + package_path) + url = os.path.join(url_prefix, package_path) + if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1): + if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0: + debug("using prebake from web: %s"%(url)) + return downloaded_file + else: + os.unlink(downloaded_file) return None