Patchwork [core,3/3] prebake: allow packages to be fetched from web instead of local dirs only

login
register
mail settings
Submitter Jacob Kjaergaard
Date Jan. 23, 2014, 10:02 a.m.
Message ID <1390471322-24372-4-git-send-email-jabk@prevas.dk>
Download mbox | patch
Permalink /patch/313493/
State Changes Requested
Delegated to: Esben Haabendal
Headers show

Comments

Jacob Kjaergaard - Jan. 23, 2014, 10:02 a.m.
From: Jacob Kjaergaard <jacob.kjaergaard@prevas.dk>

set PREBAKE_URL to point to your prebaked packages url prefix e.g:
PREBAKE_URL_CACHE_DIR can be set to pin point where the fetched prebaked
packages are located

http://myserver.com/packages/

the directory structure under "packages" should ressemble the structure
under tmp/packages/
---
 conf/oe-lite.conf   |    3 +++
 conf/oelayout.conf  |    3 +++
 lib/oelite/baker.py |   25 +++++++++++++++++++++++++
 3 files changed, 31 insertions(+)
Esben Haabendal - May 13, 2014, 1:11 p.m.
<jabk@prevas.dk> writes:

> From: Jacob Kjaergaard <jacob.kjaergaard@prevas.dk>
>
> set PREBAKE_URL to point to your prebaked packages url prefix e.g:
> PREBAKE_URL_CACHE_DIR can be set to pin point where the fetched prebaked
> packages are located
>
> http://myserver.com/packages/
>
> the directory structure under "packages" should ressemble the structure
> under tmp/packages/
> ---
>  conf/oe-lite.conf   |    3 +++
>  conf/oelayout.conf  |    3 +++
>  lib/oelite/baker.py |   25 +++++++++++++++++++++++++
>  3 files changed, 31 insertions(+)
>
> diff --git a/conf/oe-lite.conf b/conf/oe-lite.conf
> index b1e23b2..a3fa033 100644
> --- a/conf/oe-lite.conf
> +++ b/conf/oe-lite.conf
> @@ -41,6 +41,9 @@ DEFAULT_RELAX[nohash] = "1"
>  PREBAKE_PATH[nohash] = "1"
>  
>  PARALLEL_MAKE[nohash] = True
> +PREBAKE_URL[nohash] = True
> +PREBAKE_URL_CACHE[nohash] = True

What is the purpose of the PREBAKE_URL_CACHE variable?  I don't see any
references to it in the code.

> +
>  export PATH
>  
>  export LD_LIBRARY_PATH
> diff --git a/conf/oelayout.conf b/conf/oelayout.conf
> index 535cb6f..5726410 100644
> --- a/conf/oelayout.conf
> +++ b/conf/oelayout.conf
> @@ -57,6 +57,9 @@ IMAGE_DEPLOY_DIR[nohash] = True
>  PACKAGE_DEPLOY_DIR 	 = "${TMPDIR}/packages"
>  PACKAGE_DEPLOY_DIR[nohash] = True
>  
> +PREBAKE_URL_CACHE_DIR 	      = "${TMPDIR}/packages_url_cache"
> +PREBAKE_URL_CACHE_DIR[nohash] = True

Could we perhaps shorten it a bit, fx.

PREBAKE_CACHE_DIR = "${TMPDIR}/prebake"

?

> +
>  # Recipe directory layout
>  FILESPATHBASE	= "${FILE_DIRNAME}"
>  FILESPATHBASE[emit] = ""
> diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py
> index e3efb7b..8698ee7 100644
> --- a/lib/oelite/baker.py
> +++ b/lib/oelite/baker.py
> @@ -12,6 +12,8 @@ import oelite.item
>  from oelite.parse import *
>  from oelite.cookbook import CookBook
>  
> +import oelite.fetch
> +
>  import bb.utils
>  
>  import sys
> @@ -440,6 +442,9 @@ class OEliteBaker:
>          # check for availability of prebaked packages, and set package
>          # filename for all packages.
>          depend_packages = self.runq.get_depend_packages()
> +        url_prefix = self.config.get("PREBAKE_URL")
> +        if url_prefix is not None:
> +            info("Trying to use prebakes from url: %s"%(url_prefix))
>          for package in depend_packages:
>              # FIXME: skip this package if it is to be rebuild
>              prebake = self.find_prebaked_package(package)
> @@ -611,6 +616,7 @@ class OEliteBaker:
>      def find_prebaked_package(self, package):
>          """return full-path filename string or None"""
>          package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
> +        prebake_url_cache_dir = self.config.get("PREBAKE_URL_CACHE_DIR")
>          if not package_deploy_dir:
>              die("PACKAGE_DEPLOY_DIR not defined")
>          if self.options.prebake:
> @@ -618,6 +624,7 @@ class OEliteBaker:
>              if prebake_path:
>                  prebake_path = prebake_path.split(":")
>              prebake_path.insert(0, package_deploy_dir)
> +            prebake_path.insert(0, prebake_url_cache_dir)
>          else:
>              prebake_path = [package_deploy_dir]
>          debug("package=%s"%(repr(package)))
> @@ -631,6 +638,7 @@ class OEliteBaker:
>              raise NoSuchPackage()
>          filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash)
>          debug("prebake_path=%s"%(prebake_path))
> +        #test local paths first
>          for base_dir in prebake_path:
>              path = os.path.join(
>                  base_dir,
> @@ -641,6 +649,23 @@ class OEliteBaker:
>              if os.path.exists(path):
>                  debug("found prebake: %s"%(path))
>                  return path
> +        #then test URLs from PREBAKE_URL
> +        url_prefix = self.config.get("PREBAKE_URL")
> +        if url_prefix is not None:
> +            package_path =os.path.join(
> +                    package.type,
> +                    package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
> +                    filename)
> +            downloaded_file =os.path.join(
> +                    prebake_url_cache_dir,
> +                    package_path)
> +            url = os.path.join(url_prefix, package_path)
> +            if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
> +                if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0:
> +                    debug("using prebake from web: %s"%(url))
> +                    return downloaded_file
> +                else:
> +                    os.unlink(downloaded_file)
>          return None
>  
>
Esben Haabendal - May 13, 2014, 1:19 p.m.
Hi again

Could you also add a section to the OE-lite/core reference manual on how
to use this?

/Esben

Patch

diff --git a/conf/oe-lite.conf b/conf/oe-lite.conf
index b1e23b2..a3fa033 100644
--- a/conf/oe-lite.conf
+++ b/conf/oe-lite.conf
@@ -41,6 +41,9 @@  DEFAULT_RELAX[nohash] = "1"
 PREBAKE_PATH[nohash] = "1"
 
 PARALLEL_MAKE[nohash] = True
+PREBAKE_URL[nohash] = True
+PREBAKE_URL_CACHE[nohash] = True
+
 export PATH
 
 export LD_LIBRARY_PATH
diff --git a/conf/oelayout.conf b/conf/oelayout.conf
index 535cb6f..5726410 100644
--- a/conf/oelayout.conf
+++ b/conf/oelayout.conf
@@ -57,6 +57,9 @@  IMAGE_DEPLOY_DIR[nohash] = True
 PACKAGE_DEPLOY_DIR 	 = "${TMPDIR}/packages"
 PACKAGE_DEPLOY_DIR[nohash] = True
 
+PREBAKE_URL_CACHE_DIR 	      = "${TMPDIR}/packages_url_cache"
+PREBAKE_URL_CACHE_DIR[nohash] = True
+
 # Recipe directory layout
 FILESPATHBASE	= "${FILE_DIRNAME}"
 FILESPATHBASE[emit] = ""
diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py
index e3efb7b..8698ee7 100644
--- a/lib/oelite/baker.py
+++ b/lib/oelite/baker.py
@@ -12,6 +12,8 @@  import oelite.item
 from oelite.parse import *
 from oelite.cookbook import CookBook
 
+import oelite.fetch
+
 import bb.utils
 
 import sys
@@ -440,6 +442,9 @@  class OEliteBaker:
         # check for availability of prebaked packages, and set package
         # filename for all packages.
         depend_packages = self.runq.get_depend_packages()
+        url_prefix = self.config.get("PREBAKE_URL")
+        if url_prefix is not None:
+            info("Trying to use prebakes from url: %s"%(url_prefix))
         for package in depend_packages:
             # FIXME: skip this package if it is to be rebuild
             prebake = self.find_prebaked_package(package)
@@ -611,6 +616,7 @@  class OEliteBaker:
     def find_prebaked_package(self, package):
         """return full-path filename string or None"""
         package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
+        prebake_url_cache_dir = self.config.get("PREBAKE_URL_CACHE_DIR")
         if not package_deploy_dir:
             die("PACKAGE_DEPLOY_DIR not defined")
         if self.options.prebake:
@@ -618,6 +624,7 @@  class OEliteBaker:
             if prebake_path:
                 prebake_path = prebake_path.split(":")
             prebake_path.insert(0, package_deploy_dir)
+            prebake_path.insert(0, prebake_url_cache_dir)
         else:
             prebake_path = [package_deploy_dir]
         debug("package=%s"%(repr(package)))
@@ -631,6 +638,7 @@  class OEliteBaker:
             raise NoSuchPackage()
         filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash)
         debug("prebake_path=%s"%(prebake_path))
+        #test local paths first
         for base_dir in prebake_path:
             path = os.path.join(
                 base_dir,
@@ -641,6 +649,23 @@  class OEliteBaker:
             if os.path.exists(path):
                 debug("found prebake: %s"%(path))
                 return path
+        #then test URLs from PREBAKE_URL
+        url_prefix = self.config.get("PREBAKE_URL")
+        if url_prefix is not None:
+            package_path =os.path.join(
+                    package.type,
+                    package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
+                    filename)
+            downloaded_file =os.path.join(
+                    prebake_url_cache_dir,
+                    package_path)
+            url = os.path.join(url_prefix, package_path)
+            if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
+                if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0:
+                    debug("using prebake from web: %s"%(url))
+                    return downloaded_file
+                else:
+                    os.unlink(downloaded_file)
         return None