Patchwork [core,3/3] prebake: allow packages to be fetched from web instead of local dirs only

login
register
mail settings
Submitter Jacob Kjaergaard
Date Jan. 23, 2014, 10:02 a.m.
Message ID <1390471322-24372-4-git-send-email-jabk@prevas.dk>
Download mbox | patch
Permalink /patch/313493/
State New
Delegated to: Esben Haabendal
Headers show

Comments

Jacob Kjaergaard - Jan. 23, 2014, 10:02 a.m.
From: Jacob Kjaergaard <jacob.kjaergaard@prevas.dk>

set PREBAKE_URL to point to your prebaked packages url prefix e.g:
PREBAKE_URL_CACHE_DIR can be set to pin point where the fetched prebaked
packages are located

http://myserver.com/packages/

the directory structure under "packages" should ressemble the structure
under tmp/packages/
---
 conf/oe-lite.conf   |    3 +++
 conf/oelayout.conf  |    3 +++
 lib/oelite/baker.py |   25 +++++++++++++++++++++++++
 3 files changed, 31 insertions(+)

Patch

diff --git a/conf/oe-lite.conf b/conf/oe-lite.conf
index b1e23b2..a3fa033 100644
--- a/conf/oe-lite.conf
+++ b/conf/oe-lite.conf
@@ -41,6 +41,9 @@  DEFAULT_RELAX[nohash] = "1"
 PREBAKE_PATH[nohash] = "1"
 
 PARALLEL_MAKE[nohash] = True
+PREBAKE_URL[nohash] = True
+PREBAKE_URL_CACHE[nohash] = True
+
 export PATH
 
 export LD_LIBRARY_PATH
diff --git a/conf/oelayout.conf b/conf/oelayout.conf
index 535cb6f..5726410 100644
--- a/conf/oelayout.conf
+++ b/conf/oelayout.conf
@@ -57,6 +57,9 @@  IMAGE_DEPLOY_DIR[nohash] = True
 PACKAGE_DEPLOY_DIR 	 = "${TMPDIR}/packages"
 PACKAGE_DEPLOY_DIR[nohash] = True
 
+PREBAKE_URL_CACHE_DIR 	      = "${TMPDIR}/packages_url_cache"
+PREBAKE_URL_CACHE_DIR[nohash] = True
+
 # Recipe directory layout
 FILESPATHBASE	= "${FILE_DIRNAME}"
 FILESPATHBASE[emit] = ""
diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py
index e3efb7b..8698ee7 100644
--- a/lib/oelite/baker.py
+++ b/lib/oelite/baker.py
@@ -12,6 +12,8 @@  import oelite.item
 from oelite.parse import *
 from oelite.cookbook import CookBook
 
+import oelite.fetch
+
 import bb.utils
 
 import sys
@@ -440,6 +442,9 @@  class OEliteBaker:
         # check for availability of prebaked packages, and set package
         # filename for all packages.
         depend_packages = self.runq.get_depend_packages()
+        url_prefix = self.config.get("PREBAKE_URL")
+        if url_prefix is not None:
+            info("Trying to use prebakes from url: %s"%(url_prefix))
         for package in depend_packages:
             # FIXME: skip this package if it is to be rebuild
             prebake = self.find_prebaked_package(package)
@@ -611,6 +616,7 @@  class OEliteBaker:
     def find_prebaked_package(self, package):
         """return full-path filename string or None"""
         package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
+        prebake_url_cache_dir = self.config.get("PREBAKE_URL_CACHE_DIR")
         if not package_deploy_dir:
             die("PACKAGE_DEPLOY_DIR not defined")
         if self.options.prebake:
@@ -618,6 +624,7 @@  class OEliteBaker:
             if prebake_path:
                 prebake_path = prebake_path.split(":")
             prebake_path.insert(0, package_deploy_dir)
+            prebake_path.insert(0, prebake_url_cache_dir)
         else:
             prebake_path = [package_deploy_dir]
         debug("package=%s"%(repr(package)))
@@ -631,6 +638,7 @@  class OEliteBaker:
             raise NoSuchPackage()
         filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash)
         debug("prebake_path=%s"%(prebake_path))
+        #test local paths first
         for base_dir in prebake_path:
             path = os.path.join(
                 base_dir,
@@ -641,6 +649,23 @@  class OEliteBaker:
             if os.path.exists(path):
                 debug("found prebake: %s"%(path))
                 return path
+        #then test URLs from PREBAKE_URL
+        url_prefix = self.config.get("PREBAKE_URL")
+        if url_prefix is not None:
+            package_path =os.path.join(
+                    package.type,
+                    package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
+                    filename)
+            downloaded_file =os.path.join(
+                    prebake_url_cache_dir,
+                    package_path)
+            url = os.path.join(url_prefix, package_path)
+            if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
+                if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0:
+                    debug("using prebake from web: %s"%(url))
+                    return downloaded_file
+                else:
+                    os.unlink(downloaded_file)
         return None