diff mbox series

[meta-swupdate] Introduce functionality for automatically fetching package versions

Message ID 20210511121524.20266-1-thomas.haemmerle@leica-geosystems.com
State Changes Requested
Headers show
Series [meta-swupdate] Introduce functionality for automatically fetching package versions | expand

Commit Message

Thomas Haemmerle May 11, 2021, 12:15 p.m. UTC 
Versions of the packages included in update file needs to be entered
manually in sw-description file or provided in as a bitbake variable.

This patch searches for a group[1] in sw-description file containing the
tag `"SWU_AUTO_VERSION"`.
If found, the version of the corresponding package is fetched from it's
package-data-file (`PV`) and the tag is replaced with the version.
Possible trailing strings in `PV` starting with `+` (e.g.
"+gitAUTOINC+...") are dropped to keep semantic versions.

Since the filenames not always belong to the package name (deployed
with another name or the file is a container for the real package)
support providing the correct package name by appending the tag with
`:<package-name>`.

[1]: http://www.hyperrealm.com/libconfig/libconfig_manual.html#Groups

Signed-off-by: Thomas Haemmerle <thomas.haemmerle@leica-geosystems.com>
---
 classes/swupdate-common.bbclass | 50 +++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

Comments

Thomas Haemmerle May 25, 2021, 5:21 p.m. UTC | #1 
On 11.05.21 14:15, Thomas Haemmerle wrote:
> Versions of the packages included in update file needs to be entered
> manually in sw-description file or provided in as a bitbake variable.
> 
> This patch searches for a group[1] in sw-description file containing the
> tag `"SWU_AUTO_VERSION"`.
> If found, the version of the corresponding package is fetched from it's
> package-data-file (`PV`) and the tag is replaced with the version.
> Possible trailing strings in `PV` starting with `+` (e.g.
> "+gitAUTOINC+...") are dropped to keep semantic versions.
> 
> Since the filenames not always belong to the package name (deployed
> with another name or the file is a container for the real package)
> support providing the correct package name by appending the tag with
> `:<package-name>`.
> 
> [1]: http://www.hyperrealm.com/libconfig/libconfig_manual.html#Groups
> 
> Signed-off-by: Thomas Haemmerle <thomas.haemmerle@leica-geosystems.com>
> ---
>   classes/swupdate-common.bbclass | 50 +++++++++++++++++++++++++++++++++
>   1 file changed, 50 insertions(+)
> 
> diff --git a/classes/swupdate-common.bbclass b/classes/swupdate-common.bbclass
> index 578f305..c973307 100644
> --- a/classes/swupdate-common.bbclass
> +++ b/classes/swupdate-common.bbclass
> @@ -88,10 +88,60 @@ def swupdate_expand_bitbake_variables(d, s):
>           for line in write_lines:
>               f.write(line)
>   
> +def swupdate_expand_auto_versions(d, s, list_for_cpio):
> +    import re
> +    import oe.packagedata
> +    AUTO_VERSION_TAG = "SWU_AUTO_VERSION"
> +
> +    with open(os.path.join(s, "sw-description"), 'r') as f:
> +        data = f.read()
> +
> +    def get_package_name(group, file_list):
> +        m = re.search(r"%s:(?P<package>.+?(?=\"))" % (AUTO_VERSION_TAG), group)
> +        if m:
> +            package = m.group('package')
> +            return (package, True)
> +
> +        for filename in file_list:
> +            if filename in group:
> +                package = filename
> +
> +        if not package:
> +            bb.fatal("Failed to find %s in group with \"%s\"" % (filename, AUTO_VERSION_TAG))
> +
> +        return (package, False)
> +
> +    regexp = re.compile(r"\{[^\{]*%s.[^\}]*\}" % (AUTO_VERSION_TAG))
> +    while True:
> +        m = regexp.search(data)
> +        if not m:
> +            break
> +
> +        group = data[m.start():m.end()]
> +
> +        (package, pkg_name_defined) = get_package_name(group, list_for_cpio)
> +
> +        pkg_info = os.path.join(d.getVar('PKGDATA_DIR'), 'runtime-reverse', package)
> +        pkgdata = oe.packagedata.read_pkgdatafile(pkg_info)
> +
> +        if not "PV" in pkgdata.keys():
> +            bb.fatal("Failed to find version for package %s" % (package))
> +
> +        replace_str = AUTO_VERSION_TAG
> +        if pkg_name_defined:
> +            replace_str = replace_str + ":" + package
> +
> +        group = group.replace(replace_str, pkgdata['PV'].split('+')[0])
> +        data = data[:m.start()] + group + data[m.end():]
> +
> +    with open(os.path.join(s, "sw-description"), 'w+') as f:
> +        f.write(data)
> +
>   def prepare_sw_description(d, s, list_for_cpio):
>       import shutil
>   
>       swupdate_expand_bitbake_variables(d, s)
> +    swupdate_expand_auto_versions(d, s, list_for_cpio)
>   
>       for file in list_for_cpio:
>           if file != 'sw-description' and swupdate_is_hash_needed(s, file):
> 

Gentle ping.

Thomas
Stefano Babic May 26, 2021, 3:08 p.m. UTC | #2 
Hi Thomas,

sorry for late review:

On 11.05.21 14:15, 'Thomas Haemmerle' via swupdate wrote:
> Versions of the packages included in update file needs to be entered
> manually in sw-description file or provided in as a bitbake variable.
> 
> This patch searches for a group[1] in sw-description file containing the
> tag `"SWU_AUTO_VERSION"`.

This is for meta-swupdate, but it should be documented somewhere. As I 
found that documenting in meta-swupdate is misleading, there is a 
chapter regarding Yocto in SWUpdate's documentation. I am expecting 
that, together with this one, you send also a patch for SWUpdate to 
extend doc/source/building-with-yocto.rst.

> If found, the version of the corresponding package is fetched from it's
> package-data-file (`PV`) and the tag is replaced with the version.
> Possible trailing strings in `PV` starting with `+` (e.g.
> "+gitAUTOINC+...") are dropped to keep semantic versions.

It looks like there is no escape at all to signal that this is a 
keyword. At the moment we have:

@<filename>: replace with sha256sum of filename
@@<varname>@@ : replace with OE variable

As '@' in meta-swupdate for filenames is not allowed, this gives a 
(simple) way to get the object to be replaced without parsing with 
libconfig (or json). In your case, the keyword is not surrounded by 
anything. In the (even remote) case SWU_AUTO_VERSION is used as 
filename, a replace can take place.

I supposed you use this together with version comparison - is is 
something as :

	version = "SWU_AUTO_VERSION";

then "version" could be also part of the regexp.

> 
> Since the filenames not always belong to the package name (deployed
> with another name or the file is a container for the real package)
> support providing the correct package name by appending the tag with
> `:<package-name>`.

This will be found just in git history - there is the need to have this 
well documented, not only what it does, but even a (short) example for 
sw-description, like the ones in several parts of documentation (for 
example in handler.rst), like:

{
	filename = "dummy";
	version = "SWU_AUTO_VERSION";
	....



> 
> [1]: http://www.hyperrealm.com/libconfig/libconfig_manual.html#Groups

I am not getting why this is important.

> 
> Signed-off-by: Thomas Haemmerle <thomas.haemmerle@leica-geosystems.com> > ---
>   classes/swupdate-common.bbclass | 50 +++++++++++++++++++++++++++++++++
>   1 file changed, 50 insertions(+)
> 
> diff --git a/classes/swupdate-common.bbclass b/classes/swupdate-common.bbclass
> index 578f305..c973307 100644
> --- a/classes/swupdate-common.bbclass
> +++ b/classes/swupdate-common.bbclass
> @@ -88,10 +88,60 @@ def swupdate_expand_bitbake_variables(d, s):
>           for line in write_lines:
>               f.write(line)
>   
> +def swupdate_expand_auto_versions(d, s, list_for_cpio):
> +    import re
> +    import oe.packagedata
> +    AUTO_VERSION_TAG = "SWU_AUTO_VERSION"
> +
> +    with open(os.path.join(s, "sw-description"), 'r') as f:
> +        data = f.read()
> +
> +    def get_package_name(group, file_list):
> +        m = re.search(r"%s:(?P<package>.+?(?=\"))" % (AUTO_VERSION_TAG), group)
> +        if m:
> +            package = m.group('package')
> +            return (package, True)
> +
> +        for filename in file_list:
> +            if filename in group:
> +                package = filename
> +
> +        if not package:
> +            bb.fatal("Failed to find %s in group with \"%s\"" % (filename, AUTO_VERSION_TAG))
> +
> +        return (package, False)
> +
> +    regexp = re.compile(r"\{[^\{]*%s.[^\}]*\}" % (AUTO_VERSION_TAG))

See my comments above.

> +    while True:
> +        m = regexp.search(data)
> +        if not m:
> +            break
> +
> +        group = data[m.start():m.end()]
> +
> +        (package, pkg_name_defined) = get_package_name(group, list_for_cpio)
> +
> +        pkg_info = os.path.join(d.getVar('PKGDATA_DIR'), 'runtime-reverse', package)
> +        pkgdata = oe.packagedata.read_pkgdatafile(pkg_info)
> +
> +        if not "PV" in pkgdata.keys():
> +            bb.fatal("Failed to find version for package %s" % (package))

When does it happen ? If no PV is set in recipe. a 1.0 should be added.

> +
> +        replace_str = AUTO_VERSION_TAG
> +        if pkg_name_defined:
> +            replace_str = replace_str + ":" + package
> +
> +        group = group.replace(replace_str, pkgdata['PV'].split('+')[0])
> +        data = data[:m.start()] + group + data[m.end():]
> +
> +    with open(os.path.join(s, "sw-description"), 'w+') as f:
> +        f.write(data)
> +
>   def prepare_sw_description(d, s, list_for_cpio):
>       import shutil
>   
>       swupdate_expand_bitbake_variables(d, s)
> +    swupdate_expand_auto_versions(d, s, list_for_cpio)
>   
>       for file in list_for_cpio:
>           if file != 'sw-description' and swupdate_is_hash_needed(s, file):
> 

Best regards,
Stefano Babic
diff mbox series

Patch

diff --git a/classes/swupdate-common.bbclass b/classes/swupdate-common.bbclass
index 578f305..c973307 100644
--- a/classes/swupdate-common.bbclass
+++ b/classes/swupdate-common.bbclass
@@ -88,10 +88,60 @@  def swupdate_expand_bitbake_variables(d, s):
         for line in write_lines:
             f.write(line)
 
+def swupdate_expand_auto_versions(d, s, list_for_cpio):
+    import re
+    import oe.packagedata
+    AUTO_VERSION_TAG = "SWU_AUTO_VERSION"
+
+    with open(os.path.join(s, "sw-description"), 'r') as f:
+        data = f.read()
+
+    def get_package_name(group, file_list):
+        m = re.search(r"%s:(?P<package>.+?(?=\"))" % (AUTO_VERSION_TAG), group)
+        if m:
+            package = m.group('package')
+            return (package, True)
+
+        for filename in file_list:
+            if filename in group:
+                package = filename
+
+        if not package:
+            bb.fatal("Failed to find %s in group with \"%s\"" % (filename, AUTO_VERSION_TAG))
+
+        return (package, False)
+
+    regexp = re.compile(r"\{[^\{]*%s.[^\}]*\}" % (AUTO_VERSION_TAG))
+    while True:
+        m = regexp.search(data)
+        if not m:
+            break
+
+        group = data[m.start():m.end()]
+
+        (package, pkg_name_defined) = get_package_name(group, list_for_cpio)
+
+        pkg_info = os.path.join(d.getVar('PKGDATA_DIR'), 'runtime-reverse', package)
+        pkgdata = oe.packagedata.read_pkgdatafile(pkg_info)
+
+        if not "PV" in pkgdata.keys():
+            bb.fatal("Failed to find version for package %s" % (package))
+
+        replace_str = AUTO_VERSION_TAG
+        if pkg_name_defined:
+            replace_str = replace_str + ":" + package
+
+        group = group.replace(replace_str, pkgdata['PV'].split('+')[0])
+        data = data[:m.start()] + group + data[m.end():]
+
+    with open(os.path.join(s, "sw-description"), 'w+') as f:
+        f.write(data)
+
 def prepare_sw_description(d, s, list_for_cpio):
     import shutil
 
     swupdate_expand_bitbake_variables(d, s)
+    swupdate_expand_auto_versions(d, s, list_for_cpio)
 
     for file in list_for_cpio:
         if file != 'sw-description' and swupdate_is_hash_needed(s, file):