add support for generating multiple UBI images

We have a need to be able to support multiple NAND devices with distinct flash
parameters.  For example, one device uses an erase block size of 128KiB while
another uses 256KiB.  Depending on supply-chain availability, we may get boards
with one device or the other.  Because of this, we'd like the ability to create
multiple UBI images with a single "make" command so that we can be sure all
images are always up-to-date.

To accomplish this, I modified the build environment as per attached patch.  The
patch is relative to version 2017.02.5 and I see that, in the meantime, ubifs.mk
and ubi.mk have been split into separate directories.  Thus, the attached patch
won't apply directly.  However, I thought I'd share it as a basis for discussion
in case anyone else has similar needs.  I'm sure there are better ways of
accomplishing the same, but the patch works for our purposes and, if this is of
interest to others, I'd be happy on rework it for the latest buildroot version
and incorporate other feedback.

With the patch applied, the NAND flash parameters can be specified with a single
config option BR2_TARGET_ROOTFS_UBIFS_PARAMS.  It expects a blank-separated list
of triplets which specify the Logical Erase Block size, the minimum I/O size,
and the Physical Erase Block size in bytes (as hex strings).  For example, with:

 BR2_TARGET_ROOTFS_UBIFS_PARAMS="0x1f000-0x800-0x20000 0x3e000-0x1000-0x40000"

You'd get two images:

 rootfs.ubi-eb128k-pg2k and rootfs.ubi-eb256k-pg4

with:

 Logical Erase Block size: 0x1f000 and 0x3e000, respectively
 Minimum I/O size: 0x800 and 0x1000, respectively
 Physical Erase Block size: 0x20000 and 0x40000, respectively

(The patch assumes the subpage size is constant across the images.)

Best regards,

  --david

Sorry, the attachement got scrubbed.  Here is the patch.

  --david

commit 03d07d07e9e686fe388da362d8e14db369474ac4
Author: David Mosberger-Tang <davidm@egauge.net>
Date:   Thu Aug 18 10:51:50 2022 -0600

    fs: ubifs: support generating multiple UBI images
    
    This is needed in order to be able to handle NAND flash chips with
    different parameters.

diff --git a/fs/ubifs/Config.in b/fs/ubifs/Config.in
index ff604c5c38..23a3f61977 100644
--- a/fs/ubifs/Config.in
+++ b/fs/ubifs/Config.in
@@ -5,19 +5,16 @@ config BR2_TARGET_ROOTFS_UBIFS
 
 if BR2_TARGET_ROOTFS_UBIFS
 
-config BR2_TARGET_ROOTFS_UBIFS_LEBSIZE
-	hex "logical eraseblock size"
-	default 0x1f800
-	help
-	  Logical eraseblock (LEB) size. The value provided here is
-	  passed to the -e/--leb-size option of mkfs.ubifs.
-
-config BR2_TARGET_ROOTFS_UBIFS_MINIOSIZE
-	hex "minimum I/O unit size"
-	default 0x800
-	help
-	  Minimum I/O unit size. The value provided here is passed
-	  to the -m/--min-io-size option of mkfs.ubifs/ubinize.
+config BR2_TARGET_ROOTFS_UBIFS_PARAMS
+       string "List of UBIFS parameters for which images should be generated."
+       default "0x1f000-0x800-0x20000"
+       help
+	  This list contains one or more UBIFS parameter sets for which to
+	  generate images for.  Each parameter set consists of three
+	  hex strings separated by dashes.  The first hex string specifies
+	  the logical eraseblock (LEB) size in bytes.  The second the
+	  minimum I/O size in bytes (also known as page size or write size).
+	  The third the physical eraseblock (PEB) size in bytes.
 
 config BR2_TARGET_ROOTFS_UBIFS_MAXLEBCNT
 	int "maximum logical eraseblock count"
@@ -102,14 +99,6 @@ config BR2_TARGET_ROOTFS_UBI
 
 if BR2_TARGET_ROOTFS_UBI
 
-config BR2_TARGET_ROOTFS_UBI_PEBSIZE
-	hex "physical eraseblock size"
-	default 0x20000
-	help
-	  Tells ubinize the physical eraseblock (PEB) size of the
-	  flash chip the ubi image is created for. The value provided
-	  here is passed to the -p/--peb-size option of ubinize.
-
 config BR2_TARGET_ROOTFS_UBI_SUBSIZE
 	int "sub-page size"
 	default 512
diff --git a/fs/ubifs/ubi-common.mk b/fs/ubifs/ubi-common.mk
new file mode 100644
index 0000000000..824232b040
--- /dev/null
+++ b/fs/ubifs/ubi-common.mk
@@ -0,0 +1,21 @@
+#
+# Invoke this macro with $(call ROOTFS_DO_UBI_PARAMS CMD PARAMS) to
+# split the UBIFS parameters PARAMS and pass them to macro CMD.
+# PARAMS must be of the form "LEB-MINIO-PEB" where LEB is the logical
+# erase-block size in bytes, MINIO is the page size in bytes, and PEB
+# is the physical erase-block size in bytes (all three must be
+# specified as hex strings).
+#
+# CMD is called with parameters LEB, MINIO, PEB, and TAG.  The first
+# three are identical to the respective parameters passed in PARAMS.
+# TAG is a filename tag of the form "eb${leb}k-pg${minio}k" where
+# ${leb} is LEB and ${minio} is MINIO expressed in KiB (as a decimal
+# number).
+#
+export ROOTFS_DO_UBI_PARAMS =						  \
+	params="$(subst -, ,$2)";					  \
+	leb=$$(echo $$params | cut -f1 -d' ');				  \
+	minio=$$(echo $$params | cut -f2 -d' ');			  \
+	peb=$$(echo $$params | cut -f3 -d' ');				  \
+	tag=$$(printf "eb%dk-pg%dk" $$(($$peb/1024)) $$(($$minio/1024))); \
+	$(call $1,$$leb,$$minio,$$peb,$$tag)
diff --git a/fs/ubifs/ubi.mk b/fs/ubifs/ubi.mk
index e488cb5f8a..be2107c2a6 100644
--- a/fs/ubifs/ubi.mk
+++ b/fs/ubifs/ubi.mk
@@ -4,8 +4,7 @@
 #
################################################################################
 
-UBI_UBINIZE_OPTS := -m $(BR2_TARGET_ROOTFS_UBIFS_MINIOSIZE)
-UBI_UBINIZE_OPTS += -p $(BR2_TARGET_ROOTFS_UBI_PEBSIZE)
+UBI_UBINIZE_OPTS :=
 ifneq ($(BR2_TARGET_ROOTFS_UBI_SUBSIZE),0)
 UBI_UBINIZE_OPTS += -s $(BR2_TARGET_ROOTFS_UBI_SUBSIZE)
 endif
@@ -20,13 +19,20 @@ else
 UBINIZE_CONFIG_FILE_PATH = fs/ubifs/ubinize.cfg
 endif
 
+include $(dir $(lastword $(MAKEFILE_LIST)))/ubi-common.mk
+
 # don't use sed -i as it misbehaves on systems with SELinux enabled when this
is
 # executed through fakeroot (see #9386)
+mkubi
=									\
+	sed "s;BR2_ROOTFS_UBIFS_PATH;$@fs-$4;"				\
+		$(UBINIZE_CONFIG_FILE_PATH) >
$(BUILD_DIR)/ubinize.cfg;	\
+	$(HOST_DIR)/usr/sbin/ubinize -o $@-$4 $(UBI_UBINIZE_OPTS)	\
+		-m $2 -p $3 $(BUILD_DIR)/ubinize.cfg;			\
+	rm $(BUILD_DIR)/ubinize.cfg;
+
 define ROOTFS_UBI_CMD
-	sed 's;BR2_ROOTFS_UBIFS_PATH;$@fs;' \
-		$(UBINIZE_CONFIG_FILE_PATH) > $(BUILD_DIR)/ubinize.cfg
-	$(HOST_DIR)/usr/sbin/ubinize -o $@ $(UBI_UBINIZE_OPTS)
$(BUILD_DIR)/ubinize.cfg
-	rm $(BUILD_DIR)/ubinize.cfg
+	$(foreach p,$(call qstrip,$(BR2_TARGET_ROOTFS_UBIFS_PARAMS)),\
+		$(call ROOTFS_DO_UBI_PARAMS,mkubi,$p))
 endef
 
 $(eval $(call ROOTFS_TARGET,ubi))
diff --git a/fs/ubifs/ubifs.mk b/fs/ubifs/ubifs.mk
index f1e4126e86..a6779f49e8 100644
--- a/fs/ubifs/ubifs.mk
+++ b/fs/ubifs/ubifs.mk
@@ -4,7 +4,7 @@
 #
################################################################################
 
-UBIFS_OPTS := -e $(BR2_TARGET_ROOTFS_UBIFS_LEBSIZE) -c
$(BR2_TARGET_ROOTFS_UBIFS_MAXLEBCNT) -m $(BR2_TARGET_ROOTFS_UBIFS_MINIOSIZE)
+UBIFS_OPTS := -c $(BR2_TARGET_ROOTFS_UBIFS_MAXLEBCNT)
 
 ifeq ($(BR2_TARGET_ROOTFS_UBIFS_RT_ZLIB),y)
 UBIFS_OPTS += -x zlib
@@ -20,8 +20,14 @@ UBIFS_OPTS += $(call qstrip,$(BR2_TARGET_ROOTFS_UBIFS_OPTS))
 
 ROOTFS_UBIFS_DEPENDENCIES = host-mtd
 
+include $(dir $(lastword $(MAKEFILE_LIST)))/ubi-common.mk
+
+mkubifs = $(HOST_DIR)/usr/sbin/mkfs.ubifs -d $(TARGET_DIR)	\
+		-e $1 -m $2 $(UBIFS_OPTS) -o $@-$4;
+
 define ROOTFS_UBIFS_CMD
-	$(HOST_DIR)/usr/sbin/mkfs.ubifs -d $(TARGET_DIR) $(UBIFS_OPTS) -o $@
+	$(foreach p,$(call qstrip,$(BR2_TARGET_ROOTFS_UBIFS_PARAMS)),\
+		$(call ROOTFS_DO_UBI_PARAMS,mkubifs,$p))
 endef
 
 $(eval $(call ROOTFS_TARGET,ubifs))

On 18/08/2022 21:02, David Mosberger-Tang via buildroot wrote:
> We have a need to be able to support multiple NAND devices with distinct flash
> parameters.  For example, one device uses an erase block size of 128KiB while
> another uses 256KiB.  Depending on supply-chain availability, we may get boards
> with one device or the other.  Because of this, we'd like the ability to create
> multiple UBI images with a single "make" command so that we can be sure all
> images are always up-to-date.

  That situation sucks pretty hard... It makes upgrades a bit tricky because you 
have to choose a different upgrade image too depending on the hardware...

  My advise would be not to use ubifs. Use squashfs, which doesn't care about 
the LEB size. Writeable rootfs is generally not a good idea anyway. With 
squashfs, you still have the problem for ubinize, but ubinize is anyway just an 
add-on for the simplest case; you can easily add a post-build script that runs 
ubinize twice.

  Also, as an aside: if you don't want to patch buildroot itself, you can also 
add fs options to BR2_EXTERNAL.


> To accomplish this, I modified the build environment as per attached patch.  The
> patch is relative to version 2017.02.5 and I see that, in the meantime, ubifs.mk
> and ubi.mk have been split into separate directories.  Thus, the attached patch
> won't apply directly.  However, I thought I'd share it as a basis for discussion
> in case anyone else has similar needs.  I'm sure there are better ways of
> accomplishing the same, but the patch works for our purposes and, if this is of
> interest to others, I'd be happy on rework it for the latest buildroot version
> and incorporate other feedback.

  I think for upstream Buildroot, the patch is a bit too complicated and the use 
case too rare and specific. In addition, in its current form, it makes the usual 
case less user-friendly.


  Regards,
  Arnout

> With the patch applied, the NAND flash parameters can be specified with a single
> config option BR2_TARGET_ROOTFS_UBIFS_PARAMS.  It expects a blank-separated list
> of triplets which specify the Logical Erase Block size, the minimum I/O size,
> and the Physical Erase Block size in bytes (as hex strings).  For example, with:
> 
>   BR2_TARGET_ROOTFS_UBIFS_PARAMS="0x1f000-0x800-0x20000 0x3e000-0x1000-0x40000"
> 
> You'd get two images:
> 
>   rootfs.ubi-eb128k-pg2k and rootfs.ubi-eb256k-pg4
> 
> with:
> 
>   Logical Erase Block size: 0x1f000 and 0x3e000, respectively
>   Minimum I/O size: 0x800 and 0x1000, respectively
>   Physical Erase Block size: 0x20000 and 0x40000, respectively
> 
> (The patch assumes the subpage size is constant across the images.)
> 
> Best regards,
> 
>    --david
> 
> 
> _______________________________________________
> buildroot mailing list
> buildroot@buildroot.org
> https://lists.buildroot.org/mailman/listinfo/buildroot

Arnout,

Thanks for the quick response.

On Thu, 2022-08-18 at 23:10 +0200, Arnout Vandecappelle wrote:
> 
> On 18/08/2022 21:02, David Mosberger-Tang via buildroot wrote:
> > We have a need to be able to support multiple NAND devices with distinct flash
> > parameters.  For example, one device uses an erase block size of 128KiB while
> > another uses 256KiB.  Depending on supply-chain availability, we may get boards
> > with one device or the other.  Because of this, we'd like the ability to create
> > multiple UBI images with a single "make" command so that we can be sure all
> > images are always up-to-date.
> 
>   That situation sucks pretty hard... It makes upgrades a bit tricky because you 
> have to choose a different upgrade image too depending on the hardware...

That part we can handle fine, fortunately.

>   My advise would be not to use ubifs. Use squashfs, which doesn't care about 
> the LEB size.

Ah, that would solve the problem.  I don't think it's a viable option for us at
this point though.

>  Writeable rootfs is generally not a good idea anyway. With 
> squashfs, you still have the problem for ubinize, but ubinize is anyway just an 
> add-on for the simplest case; you can easily add a post-build script that runs 
> ubinize twice.
> 
>   Also, as an aside: if you don't want to patch buildroot itself, you can also 
> add fs options to BR2_EXTERNAL.

Got it, thanks for the suggestion!

> > To accomplish this, I modified the build environment as per attached patch.  The
> > patch is relative to version 2017.02.5 and I see that, in the meantime, ubifs.mk
> > and ubi.mk have been split into separate directories.  Thus, the attached patch
> > won't apply directly.  However, I thought I'd share it as a basis for discussion
> > in case anyone else has similar needs.  I'm sure there are better ways of
> > accomplishing the same, but the patch works for our purposes and, if this is of
> > interest to others, I'd be happy on rework it for the latest buildroot version
> > and incorporate other feedback.
> 
>   I think for upstream Buildroot, the patch is a bit too complicated and the use 
> case too rare and specific. In addition, in its current form, it makes the usual 
> case less user-friendly.

Makes sense.  And if someone does have the need for it, hopefully they'll find
the patch in the mailing list archive.

Thanks and best regards,

  --david

> 
> 
>   Regards,
>   Arnout
> 
> > With the patch applied, the NAND flash parameters can be specified with a single
> > config option BR2_TARGET_ROOTFS_UBIFS_PARAMS.  It expects a blank-separated list
> > of triplets which specify the Logical Erase Block size, the minimum I/O size,
> > and the Physical Erase Block size in bytes (as hex strings).  For example, with:
> > 
> >   BR2_TARGET_ROOTFS_UBIFS_PARAMS="0x1f000-0x800-0x20000 0x3e000-0x1000-0x40000"
> > 
> > You'd get two images:
> > 
> >   rootfs.ubi-eb128k-pg2k and rootfs.ubi-eb256k-pg4
> > 
> > with:
> > 
> >   Logical Erase Block size: 0x1f000 and 0x3e000, respectively
> >   Minimum I/O size: 0x800 and 0x1000, respectively
> >   Physical Erase Block size: 0x20000 and 0x40000, respectively
> > 
> > (The patch assumes the subpage size is constant across the images.)
> > 
> > Best regards,
> > 
> >    --david
> > 
> > 
> > _______________________________________________
> > buildroot mailing list
> > buildroot@buildroot.org
> > https://lists.buildroot.org/mailman/listinfo/buildroot