[core,3/3] prebake: allow packages to be fetched from web instead of local dirs only

From: Jacob Kjaergaard <jacob.kjaergaard@prevas.dk>

set PREBAKE_URL to point to your prebaked packages url prefix e.g:
PREBAKE_URL_CACHE_DIR can be set to pin point where the fetched prebaked
packages are located

http://myserver.com/packages/

the directory structure under "packages" should ressemble the structure
under tmp/packages/
---
 conf/oe-lite.conf   |    3 +++
 conf/oelayout.conf  |    3 +++
 lib/oelite/baker.py |   25 +++++++++++++++++++++++++
 3 files changed, 31 insertions(+)

<jabk@prevas.dk> writes:

> From: Jacob Kjaergaard <jacob.kjaergaard@prevas.dk>
>
> set PREBAKE_URL to point to your prebaked packages url prefix e.g:
> PREBAKE_URL_CACHE_DIR can be set to pin point where the fetched prebaked
> packages are located
>
> http://myserver.com/packages/
>
> the directory structure under "packages" should ressemble the structure
> under tmp/packages/
> ---
>  conf/oe-lite.conf   |    3 +++
>  conf/oelayout.conf  |    3 +++
>  lib/oelite/baker.py |   25 +++++++++++++++++++++++++
>  3 files changed, 31 insertions(+)
>
> diff --git a/conf/oe-lite.conf b/conf/oe-lite.conf
> index b1e23b2..a3fa033 100644
> --- a/conf/oe-lite.conf
> +++ b/conf/oe-lite.conf
> @@ -41,6 +41,9 @@ DEFAULT_RELAX[nohash] = "1"
>  PREBAKE_PATH[nohash] = "1"
>  
>  PARALLEL_MAKE[nohash] = True
> +PREBAKE_URL[nohash] = True
> +PREBAKE_URL_CACHE[nohash] = True

What is the purpose of the PREBAKE_URL_CACHE variable?  I don't see any
references to it in the code.

> +
>  export PATH
>  
>  export LD_LIBRARY_PATH
> diff --git a/conf/oelayout.conf b/conf/oelayout.conf
> index 535cb6f..5726410 100644
> --- a/conf/oelayout.conf
> +++ b/conf/oelayout.conf
> @@ -57,6 +57,9 @@ IMAGE_DEPLOY_DIR[nohash] = True
>  PACKAGE_DEPLOY_DIR 	 = "${TMPDIR}/packages"
>  PACKAGE_DEPLOY_DIR[nohash] = True
>  
> +PREBAKE_URL_CACHE_DIR 	      = "${TMPDIR}/packages_url_cache"
> +PREBAKE_URL_CACHE_DIR[nohash] = True

Could we perhaps shorten it a bit, fx.

PREBAKE_CACHE_DIR = "${TMPDIR}/prebake"

?

> +
>  # Recipe directory layout
>  FILESPATHBASE	= "${FILE_DIRNAME}"
>  FILESPATHBASE[emit] = ""
> diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py
> index e3efb7b..8698ee7 100644
> --- a/lib/oelite/baker.py
> +++ b/lib/oelite/baker.py
> @@ -12,6 +12,8 @@ import oelite.item
>  from oelite.parse import *
>  from oelite.cookbook import CookBook
>  
> +import oelite.fetch
> +
>  import bb.utils
>  
>  import sys
> @@ -440,6 +442,9 @@ class OEliteBaker:
>          # check for availability of prebaked packages, and set package
>          # filename for all packages.
>          depend_packages = self.runq.get_depend_packages()
> +        url_prefix = self.config.get("PREBAKE_URL")
> +        if url_prefix is not None:
> +            info("Trying to use prebakes from url: %s"%(url_prefix))
>          for package in depend_packages:
>              # FIXME: skip this package if it is to be rebuild
>              prebake = self.find_prebaked_package(package)
> @@ -611,6 +616,7 @@ class OEliteBaker:
>      def find_prebaked_package(self, package):
>          """return full-path filename string or None"""
>          package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
> +        prebake_url_cache_dir = self.config.get("PREBAKE_URL_CACHE_DIR")
>          if not package_deploy_dir:
>              die("PACKAGE_DEPLOY_DIR not defined")
>          if self.options.prebake:
> @@ -618,6 +624,7 @@ class OEliteBaker:
>              if prebake_path:
>                  prebake_path = prebake_path.split(":")
>              prebake_path.insert(0, package_deploy_dir)
> +            prebake_path.insert(0, prebake_url_cache_dir)
>          else:
>              prebake_path = [package_deploy_dir]
>          debug("package=%s"%(repr(package)))
> @@ -631,6 +638,7 @@ class OEliteBaker:
>              raise NoSuchPackage()
>          filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash)
>          debug("prebake_path=%s"%(prebake_path))
> +        #test local paths first
>          for base_dir in prebake_path:
>              path = os.path.join(
>                  base_dir,
> @@ -641,6 +649,23 @@ class OEliteBaker:
>              if os.path.exists(path):
>                  debug("found prebake: %s"%(path))
>                  return path
> +        #then test URLs from PREBAKE_URL
> +        url_prefix = self.config.get("PREBAKE_URL")
> +        if url_prefix is not None:
> +            package_path =os.path.join(
> +                    package.type,
> +                    package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
> +                    filename)
> +            downloaded_file =os.path.join(
> +                    prebake_url_cache_dir,
> +                    package_path)
> +            url = os.path.join(url_prefix, package_path)
> +            if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
> +                if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0:
> +                    debug("using prebake from web: %s"%(url))
> +                    return downloaded_file
> +                else:
> +                    os.unlink(downloaded_file)
>          return None
>  
>

Hi again

Could you also add a section to the OE-lite/core reference manual on how
to use this?

/Esben