| Message ID | 20260102150757.1480511-1-fabien.lehoussel@smile.fr |
|---|---|
| State | New |
| Headers | show |
| Series | [1/5] utils/generate-cyclonedx: update package filtering for SBOM generation | expand |
In reply of: > - Filter packages to include only those with external download sources (i.e., not bundled within Buildroot). > - Remove virtual package handling from dependency analysis. > > Signed-off-by: Fabien Lehoussel <fabien.lehoussel@smile.fr> Hi Fabien, Thanks for your submissions. Regarding this change could you be more specific in the commit message on why this is needed. What are the packages that aren't in the final SBOM when applying this change and why we don't want them in the final SBOM ? I think you want to remove the 'skeleton' packages from the final SBOM right ? You can provide a analysis over the entire Buildroot packages by changing a single line in `Makefile`: ``` diff --git a/Makefile b/Makefile index d2257ed3a3..0735f39d68 100644 --- a/Makefile +++ b/Makefile @@ -920,7 +920,7 @@ show-info: @: $(info $(call clean-json, \ { $(foreach p, \ - $(sort $(foreach i,$(PACKAGES) $(TARGETS_ROOTFS), \ + $(sort $(foreach i,$(PACKAGES_ALL) $(TARGETS_ROOTFS), \ $(i) \ $($(call UPPERCASE,$(i))_FINAL_RECURSIVE_DEPENDENCIES) \ ) \ ``` Regards, PERALE Thomas > --- > utils/generate-cyclonedx | 23 +++++++++-------------- > 1 file changed, 9 insertions(+), 14 deletions(-) > > diff --git a/utils/generate-cyclonedx b/utils/generate-cyclonedx > index a103b7b707..29c73f3616 100755 > --- a/utils/generate-cyclonedx > +++ b/utils/generate-cyclonedx > @@ -280,7 +280,7 @@ def cyclonedx_component(name, comp): > **({ > "version": comp["version"], > **(cyclonedx_licenses(comp["licenses"]) if "licenses" in comp else {}), > - } if not comp["virtual"] else {}), > + } if "version" in comp else {}), > **({ > "cpe": comp["cpe-id"], > } if "cpe-id" in comp else {}), > @@ -337,12 +337,10 @@ def cyclonedx_vulnerabilities(show_info_dict): > } for cve, components in cves.items()] > > > -def br2_parse_deps_recursively(ref, show_info_dict, virtual=False, deps=[]): > +def br2_parse_deps_recursively(ref, show_info_dict, deps=[]): > """Parse dependencies from the show-info output. This function will > recursively collect all dependencies, and return a list where each dependency > is stated at most once. > - The dependency on virtual package will collect the final dependency without > - including the virtual one. > > Args: > ref (str): The identifier of the package for which the dependencies have > @@ -359,9 +357,8 @@ def br2_parse_deps_recursively(ref, show_info_dict, virtual=False, deps=[]): > """ > for dep in show_info_dict.get(ref, {}).get("dependencies", []): > if dep not in deps: > - if virtual or show_info_dict.get(dep, {}).get("virtual") is False: > - deps.append(dep) > - br2_parse_deps_recursively(dep, show_info_dict, virtual, deps) > + deps.append(dep) > + br2_parse_deps_recursively(dep, show_info_dict, deps) > > return deps > > @@ -376,8 +373,6 @@ def main(): > default=(None if sys.stdin.isatty() else sys.stdin)) > parser.add_argument("-o", "--out-file", nargs="?", type=argparse.FileType("w"), > default=sys.stdout) > - parser.add_argument("--virtual", default=False, action='store_true', > - help="This option includes virtual packages to the CycloneDX output") > parser.add_argument("--project-name", type=str, default="buildroot", > help="Specify the project name to use in the SBOM metadata (default:'buildroot')") > parser.add_argument("--project-version", type=str, default=f"{BR2_VERSION_FULL}", > @@ -391,10 +386,10 @@ def main(): > > show_info_dict = json.load(args.in_file) > > - # Remove rootfs and virtual packages if not explicitly included > - # from the cli arguments > + # Keep only packages with at least one download source defined > + # (packages with _SITE variable in Buildroot makefile). > filtered_show_info_dict = {k: v for k, v in show_info_dict.items() > - if ("rootfs" not in v["type"]) and (args.virtual or v["virtual"] is False)} > + if (len(v.get('downloads', [])) > 0)} > > cyclonedx_dict = { > "bomFormat": "CycloneDX", > @@ -429,10 +424,10 @@ def main(): > ], > "dependencies": [ > cyclonedx_dependency("buildroot", list(filtered_show_info_dict)), > - *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref, show_info_dict, args.virtual)) > + *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref, filtered_show_info_dict)) > for ref in filtered_show_info_dict], > ], > - "vulnerabilities": cyclonedx_vulnerabilities(show_info_dict), > + "vulnerabilities": cyclonedx_vulnerabilities(filtered_show_info_dict), > } > > args.out_file.write(json.dumps(cyclonedx_dict, indent=2)) > -- > 2.43.0 > > _______________________________________________ > buildroot mailing list > buildroot@buildroot.org > https://lists.buildroot.org/mailman/listinfo/buildroot
Le ven. 2 janv. 2026 à 16:48, Thomas Perale <thomas.perale@mind.be> a écrit : > In reply of: > > - Filter packages to include only those with external download sources > (i.e., not bundled within Buildroot). > > - Remove virtual package handling from dependency analysis. > > > > Signed-off-by: Fabien Lehoussel <fabien.lehoussel@smile.fr> > > Hi Fabien, > > Thanks for your submissions. > > Regarding this change could you be more specific in the commit message on > why > this is needed. What are the packages that aren't in the final SBOM when > applying this change and why we don't want them in the final SBOM ? > > I think you want to remove the 'skeleton' packages from the final SBOM > right ? > > Hi Thomas, I added this filter to remove non "real packages", packages without real sources and therefore not tracked with CPE or PURL. For example, using the log function added in the next commit, this new filter removes the following packages from my custom configuration: ============================================================ Difference between original and filtered dictionary ============================================================ 📊 Statistics: - Total original: 121 packages - Kept: 101 packages - Removed: 20 packages ❌ Removed packages (20): - host-gettext - host-makedevs - host-mkpasswd - host-openssl - host-rustc - host-skeleton - host-zlib - ifupdown-scripts - initscripts - openssl - rootfs-common - rootfs-cpio - rootfs-ext2 - skeleton - skeleton-init-common - skeleton-init-sysv - toolchain - toolchain-external - udev - zlib Test with entire Buildroot packages : ============================================================ Difference between original and filtered dictionary ============================================================ Statistics: - Total original: 3688 packages - Kept: 3626 packages - Removed: 62 packages Removed packages (62): - at91bootstrap3 - b43-firmware - cgroupfs-v2-mount - flutter-engine - getent - gettext - host-environment-setup - host-fakedate - host-gettext - host-go - host-luainterpreter - host-makedevs - host-mkpasswd - host-mkpimage - host-nodejs - host-openssl - host-rustc - host-skeleton - host-zlib - hwclock-initscript - ifupdown-scripts - initscripts - jpeg - libegl - libgbm - libgl - libgles - libopencl - libopenmax - libopenvg - linux-tools - luainterpreter - mcookie - nodejs - opensbi - openssl - optee-client - optee-examples - optee-test - powervr - rootfs-common - rootfs-cpio - rootfs-ext2 - skeleton - skeleton-custom - skeleton-init-common - skeleton-init-none - skeleton-init-openrc - skeleton-init-systemd - skeleton-init-sysv - tinyinit - toolchain - toolchain-bare-metal-buildroot - toolchain-buildroot - toolchain-external - toolchain-external-bootlin - tz - udev - urandom-scripts - xilinx-embeddedsw - xilinx-prebuilt - zlib ============================================================ Fabien You can provide a analysis over the entire Buildroot packages by changing a > single line in `Makefile`: > > ``` > diff --git a/Makefile b/Makefile > index d2257ed3a3..0735f39d68 100644 > --- a/Makefile > +++ b/Makefile > @@ -920,7 +920,7 @@ show-info: > @: > $(info $(call clean-json, \ > { $(foreach p, \ > - $(sort $(foreach i,$(PACKAGES) > $(TARGETS_ROOTFS), \ > + $(sort $(foreach i,$(PACKAGES_ALL) > $(TARGETS_ROOTFS), \ > $(i) \ > $($(call > UPPERCASE,$(i))_FINAL_RECURSIVE_DEPENDENCIES) \ > ) \ > ``` > > Regards, > PERALE Thomas > > > --- > > utils/generate-cyclonedx | 23 +++++++++-------------- > > 1 file changed, 9 insertions(+), 14 deletions(-) > > > > diff --git a/utils/generate-cyclonedx b/utils/generate-cyclonedx > > index a103b7b707..29c73f3616 100755 > > --- a/utils/generate-cyclonedx > > +++ b/utils/generate-cyclonedx > > @@ -280,7 +280,7 @@ def cyclonedx_component(name, comp): > > **({ > > "version": comp["version"], > > **(cyclonedx_licenses(comp["licenses"]) if "licenses" in > comp else {}), > > - } if not comp["virtual"] else {}), > > + } if "version" in comp else {}), > > **({ > > "cpe": comp["cpe-id"], > > } if "cpe-id" in comp else {}), > > @@ -337,12 +337,10 @@ def cyclonedx_vulnerabilities(show_info_dict): > > } for cve, components in cves.items()] > > > > > > -def br2_parse_deps_recursively(ref, show_info_dict, virtual=False, > deps=[]): > > +def br2_parse_deps_recursively(ref, show_info_dict, deps=[]): > > """Parse dependencies from the show-info output. This function will > > recursively collect all dependencies, and return a list where each > dependency > > is stated at most once. > > - The dependency on virtual package will collect the final dependency > without > > - including the virtual one. > > > > Args: > > ref (str): The identifier of the package for which the > dependencies have > > @@ -359,9 +357,8 @@ def br2_parse_deps_recursively(ref, show_info_dict, > virtual=False, deps=[]): > > """ > > for dep in show_info_dict.get(ref, {}).get("dependencies", []): > > if dep not in deps: > > - if virtual or show_info_dict.get(dep, {}).get("virtual") is > False: > > - deps.append(dep) > > - br2_parse_deps_recursively(dep, show_info_dict, virtual, > deps) > > + deps.append(dep) > > + br2_parse_deps_recursively(dep, show_info_dict, deps) > > > > return deps > > > > @@ -376,8 +373,6 @@ def main(): > > default=(None if sys.stdin.isatty() else > sys.stdin)) > > parser.add_argument("-o", "--out-file", nargs="?", > type=argparse.FileType("w"), > > default=sys.stdout) > > - parser.add_argument("--virtual", default=False, action='store_true', > > - help="This option includes virtual packages to > the CycloneDX output") > > parser.add_argument("--project-name", type=str, default="buildroot", > > help="Specify the project name to use in the > SBOM metadata (default:'buildroot')") > > parser.add_argument("--project-version", type=str, > default=f"{BR2_VERSION_FULL}", > > @@ -391,10 +386,10 @@ def main(): > > > > show_info_dict = json.load(args.in_file) > > > > - # Remove rootfs and virtual packages if not explicitly included > > - # from the cli arguments > > + # Keep only packages with at least one download source defined > > + # (packages with _SITE variable in Buildroot makefile). > > filtered_show_info_dict = {k: v for k, v in show_info_dict.items() > > - if ("rootfs" not in v["type"]) and > (args.virtual or v["virtual"] is False)} > > + if (len(v.get('downloads', [])) > 0)} > > > > cyclonedx_dict = { > > "bomFormat": "CycloneDX", > > @@ -429,10 +424,10 @@ def main(): > > ], > > "dependencies": [ > > cyclonedx_dependency("buildroot", > list(filtered_show_info_dict)), > > - *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref, > show_info_dict, args.virtual)) > > + *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref, > filtered_show_info_dict)) > > for ref in filtered_show_info_dict], > > ], > > - "vulnerabilities": cyclonedx_vulnerabilities(show_info_dict), > > + "vulnerabilities": > cyclonedx_vulnerabilities(filtered_show_info_dict), > > } > > > > args.out_file.write(json.dumps(cyclonedx_dict, indent=2)) > > -- > > 2.43.0 > > > > _______________________________________________ > > buildroot mailing list > > buildroot@buildroot.org > > https://lists.buildroot.org/mailman/listinfo/buildroot >
Hi Fabien, In reply of: > --000000000000a745f706479f6b60 > Content-Type: text/plain; charset="UTF-8" > Content-Transfer-Encoding: quoted-printable > > Le ven. 2 janv. 2026 =C3=A0 16:48, Thomas Perale <thomas.perale@mind.be> a > =C3=A9crit : > > > In reply of: > > > - Filter packages to include only those with external download sources > > (i.e., not bundled within Buildroot). > > > - Remove virtual package handling from dependency analysis. > > > > > > Signed-off-by: Fabien Lehoussel <fabien.lehoussel@smile.fr> > > > > Hi Fabien, > > > > Thanks for your submissions. > > > > Regarding this change could you be more specific in the commit message on > > why > > this is needed. What are the packages that aren't in the final SBOM when > > applying this change and why we don't want them in the final SBOM ? > > > > I think you want to remove the 'skeleton' packages from the final SBOM > > right ? > > > > > Hi Thomas, > > I added this filter to remove non "real packages", packages without real > sources and therefore not tracked with CPE or PURL. The SBOMs are useful for tracking vulnerabilities but they should still give a proper representation of the system so traceability should not necessary the only criteria I think. > For example, using the log function added in the next commit, this new > filter removes the following packages from my custom configuration: > > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D > Difference between original and filtered dictionary > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D > =F0=9F=93=8A Statistics: > - Total original: 121 packages > - Kept: 101 packages > - Removed: 20 packages > > =E2=9D=8C Removed packages (20): > - host-gettext > - host-makedevs > - host-mkpasswd > - host-openssl > - host-rustc > - host-skeleton > - host-zlib > - ifupdown-scripts > - initscripts > - openssl > - rootfs-common > - rootfs-cpio > - rootfs-ext2 > - skeleton > - skeleton-init-common > - skeleton-init-sysv > - toolchain > - toolchain-external > - udev > - zlib > > Test with entire Buildroot packages : > > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D > Difference between original and filtered dictionary > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D > Statistics: > - Total original: 3688 packages > - Kept: 3626 packages > - Removed: 62 packages > I won't go over all the entries but take some examples. > Removed packages (62): > - at91bootstrap3 > - b43-firmware > - cgroupfs-v2-mount > - flutter-engine > - getent > - gettext > - host-environment-setup > - host-fakedate > - host-gettext > - host-go > - host-luainterpreter > - host-makedevs > - host-mkpasswd > - host-mkpimage > - host-nodejs > - host-openssl > - host-rustc > - host-skeleton > - host-zlib > - hwclock-initscript > - ifupdown-scripts > - initscripts > - jpeg > - libegl > - libgbm > - libgl > - libgles > - libopencl > - libopenmax > - libopenvg > - linux-tools > - luainterpreter > - mcookie > - nodejs > - opensbi > - openssl > - optee-client > - optee-examples > - optee-test > - powervr > - rootfs-common > - rootfs-cpio > - rootfs-ext2 > - skeleton > - skeleton-custom > - skeleton-init-common > - skeleton-init-none > - skeleton-init-openrc > - skeleton-init-systemd > - skeleton-init-sysv > - tinyinit This 'tinyinit' package is a good example I think. This package doesn't have any download location it just copy a simple file from the Buildroot tree as a init script. There are no download location for it but I still think it needs to be tracked in the SBOM the same way other init system are tracked. Also this package if used will be a dependency of the busybox package. Packages that are just copying stuff from the directory directly into the location might be rare in the core repo of Buildroot but I think it's a more common practice in user's external. > - toolchain > - toolchain-bare-metal-buildroot > - toolchain-buildroot > - toolchain-external > - toolchain-external-bootlin > - tz > - udev > - urandom-scripts > - xilinx-embeddedsw > - xilinx-prebuilt > - zlib > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D A lot of the packages that get removed here are the virtual packages. Your submission does highlight to me that right now the dependencies are broken when not including the virtual packages. For instance the `lua-argon2` package will generate the following dependencies. ``` { "ref": "lua-argon2", "dependsOn": [ "host-skeleton", "libargon2", "skeleton-init-common", "skeleton-init-sysv", "toolchain-external-bootlin" ] }, ``` While it should have a `luainterpreter` dependency (provided by the lua package). Can you re-generate the overall diff of the SBOM with `generate-cyclonedx --virtual` to keep the virtual package and only highlight the one that get filtered out by your changes ? Thanks, PERALE Thomas > > Fabien > > You can provide a analysis over the entire Buildroot packages by changing a > > single line in `Makefile`: > > > > ``` > > diff --git a/Makefile b/Makefile > > index d2257ed3a3..0735f39d68 100644 > > --- a/Makefile > > +++ b/Makefile > > @@ -920,7 +920,7 @@ show-info: > > @: > > $(info $(call clean-json, \ > > { $(foreach p, \ > > - $(sort $(foreach i,$(PACKAGES) > > $(TARGETS_ROOTFS), \ > > + $(sort $(foreach i,$(PACKAGES_ALL) > > $(TARGETS_ROOTFS), \ > > $(i) \ > > $($(call > > UPPERCASE,$(i))_FINAL_RECURSIVE_DEPENDENCIES) \ > > ) \ > > ``` > > > > Regards, > > PERALE Thomas > > > > > --- > > > utils/generate-cyclonedx | 23 +++++++++-------------- > > > 1 file changed, 9 insertions(+), 14 deletions(-) > > > > > > diff --git a/utils/generate-cyclonedx b/utils/generate-cyclonedx > > > index a103b7b707..29c73f3616 100755 > > > --- a/utils/generate-cyclonedx > > > +++ b/utils/generate-cyclonedx > > > @@ -280,7 +280,7 @@ def cyclonedx_component(name, comp): > > > **({ > > > "version": comp["version"], > > > **(cyclonedx_licenses(comp["licenses"]) if "licenses" in > > comp else {}), > > > - } if not comp["virtual"] else {}), > > > + } if "version" in comp else {}), > > > **({ > > > "cpe": comp["cpe-id"], > > > } if "cpe-id" in comp else {}), > > > @@ -337,12 +337,10 @@ def cyclonedx_vulnerabilities(show_info_dict): > > > } for cve, components in cves.items()] > > > > > > > > > -def br2_parse_deps_recursively(ref, show_info_dict, virtual=3DFalse, > > deps=3D[]): > > > +def br2_parse_deps_recursively(ref, show_info_dict, deps=3D[]): > > > """Parse dependencies from the show-info output. This function wil= > l > > > recursively collect all dependencies, and return a list where each > > dependency > > > is stated at most once. > > > - The dependency on virtual package will collect the final dependenc= > y > > without > > > - including the virtual one. > > > > > > Args: > > > ref (str): The identifier of the package for which the > > dependencies have > > > @@ -359,9 +357,8 @@ def br2_parse_deps_recursively(ref, show_info_dict, > > virtual=3DFalse, deps=3D[]): > > > """ > > > for dep in show_info_dict.get(ref, {}).get("dependencies", []): > > > if dep not in deps: > > > - if virtual or show_info_dict.get(dep, {}).get("virtual") i= > s > > False: > > > - deps.append(dep) > > > - br2_parse_deps_recursively(dep, show_info_dict, virtual, > > deps) > > > + deps.append(dep) > > > + br2_parse_deps_recursively(dep, show_info_dict, deps) > > > > > > return deps > > > > > > @@ -376,8 +373,6 @@ def main(): > > > default=3D(None if sys.stdin.isatty() else > > sys.stdin)) > > > parser.add_argument("-o", "--out-file", nargs=3D"?", > > type=3Dargparse.FileType("w"), > > > default=3Dsys.stdout) > > > - parser.add_argument("--virtual", default=3DFalse, action=3D'store_= > true', > > > - help=3D"This option includes virtual packages = > to > > the CycloneDX output") > > > parser.add_argument("--project-name", type=3Dstr, default=3D"build= > root", > > > help=3D"Specify the project name to use in the > > SBOM metadata (default:'buildroot')") > > > parser.add_argument("--project-version", type=3Dstr, > > default=3Df"{BR2_VERSION_FULL}", > > > @@ -391,10 +386,10 @@ def main(): > > > > > > show_info_dict =3D json.load(args.in_file) > > > > > > - # Remove rootfs and virtual packages if not explicitly included > > > - # from the cli arguments > > > + # Keep only packages with at least one download source defined > > > + # (packages with _SITE variable in Buildroot makefile). > > > filtered_show_info_dict =3D {k: v for k, v in show_info_dict.items= > () > > > - if ("rootfs" not in v["type"]) and > > (args.virtual or v["virtual"] is False)} > > > + if (len(v.get('downloads', [])) > 0)} > > > > > > cyclonedx_dict =3D { > > > "bomFormat": "CycloneDX", > > > @@ -429,10 +424,10 @@ def main(): > > > ], > > > "dependencies": [ > > > cyclonedx_dependency("buildroot", > > list(filtered_show_info_dict)), > > > - *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref= > , > > show_info_dict, args.virtual)) > > > + *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref= > , > > filtered_show_info_dict)) > > > for ref in filtered_show_info_dict], > > > ], > > > - "vulnerabilities": cyclonedx_vulnerabilities(show_info_dict), > > > + "vulnerabilities": > > cyclonedx_vulnerabilities(filtered_show_info_dict), > > > } > > > > > > args.out_file.write(json.dumps(cyclonedx_dict, indent=3D2)) > > > -- > > > 2.43.0 > > > > > > _______________________________________________ > > > buildroot mailing list > > > buildroot@buildroot.org > > > https://lists.buildroot.org/mailman/listinfo/buildroot > > > > <div dir="ltr"><div dir="ltr"><br></div><br><div class="gmail_quote gmail_quote_container"><div dir="ltr" class="gmail_attr">Le ven. 2 janv. 2026 à 16:48, Thomas Perale <<a href="mailto:thomas.perale@mind.be">thomas.perale@mind.be</a>> a écrit :<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">In reply of:<br> > > - Filter packages to include only those with external download sources (i.e., not bundled within Buildroot).<br> > > - Remove virtual package handling from dependency analysis.<br> > > <br> > > Signed-off-by: Fabien Lehoussel <<a href="mailto:fabien.lehoussel@smile.fr" target="_blank">fabien.lehoussel@smile.fr</a>><br> > <br> > Hi Fabien,<br> > <br> > Thanks for your submissions.<br> > <br> > Regarding this change could you be more specific in the commit message on why<br> > this is needed. What are the packages that aren't in the final SBOM when<br> > applying this change and why we don't want them in the final SBOM ?<br> > <br> > I think you want to remove the 'skeleton' packages from the final SBOM right ?<br> > <br></blockquote><div><br></div><div>Hi Thomas, </div><div><br></div><div>I added this filter to remove non "real packages", packages without real sources and therefore not tracked with CPE or PURL.</div><div><br></div><div>For example, using the log function added in the next commit, this new filter removes the following packages from my custom configuration:</div><div><br></div><div>============================================================<br>Difference between original and filtered dictionary<br>============================================================<br>📊 Statistics:<br> - Total original: 121 packages<br> - Kept: 101 packages<br> - Removed: 20 packages<br><br></div><div>❌ Removed packages (20):<br> - host-gettext<br> - host-makedevs<br> - host-mkpasswd<br> - host-openssl<br> - host-rustc<br> - host-skeleton<br> - host-zlib<br> - ifupdown-scripts<br> - initscripts<br> - openssl<br> - rootfs-common<br> - rootfs-cpio<br> - rootfs-ext2<br> - skeleton<br> - skeleton-init-common<br> - skeleton-init-sysv<br> - toolchain<br> - toolchain-external<br> - udev<br> - zlib<br></div><div> </div><div>Test with entire Buildroot packages : </div><div><br></div><div>============================================================<br>Difference between original and filtered dictionary<br>============================================================<br>Statistics:<br> - Total original: 3688 packages<br> - Kept: 3626 packages<br> - Removed: 62 packages<br><br>Removed packages (62):<br> - at91bootstrap3<br> - b43-firmware<br> - cgroupfs-v2-mount<br> - flutter-engine<br> - getent<br> - gettext<br> - host-environment-setup<br> - host-fakedate<br> - host-gettext<br> - host-go<br> - host-luainterpreter<br> - host-makedevs<br> - host-mkpasswd<br> - host-mkpimage<br> - host-nodejs<br> - host-openssl<br> - host-rustc<br> - host-skeleton<br> - host-zlib<br> - hwclock-initscript<br> - ifupdown-scripts<br> - initscripts<br> - jpeg<br> - libegl<br> - libgbm<br> - libgl<br> - libgles<br> - libopencl<br> - libopenmax<br> - libopenvg<br> - linux-tools<br> - luainterpreter<br> - mcookie<br> - nodejs<br> - opensbi<br> - openssl<br> - optee-client<br> - optee-examples<br> - optee-test<br> - powervr<br> - rootfs-common<br> - rootfs-cpio<br> - rootfs-ext2<br> - skeleton<br> - skeleton-custom<br> - skeleton-init-common<br> - skeleton-init-none<br> - skeleton-init-openrc<br> - skeleton-init-systemd<br> - skeleton-init-sysv<br> - tinyinit<br> - toolchain<br> - toolchain-bare-metal-buildroot<br> - toolchain-buildroot<br> - toolchain-external<br> - toolchain-external-bootlin<br> - tz<br> - udev<br> - urandom-scripts<br> - xilinx-embeddedsw<br> - xilinx-prebuilt<br> - zlib<br>============================================================<br></div><div><br></div><div>Fabien</div><div><br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"> > You can provide a analysis over the entire Buildroot packages by changing a<br> > single line in `Makefile`:<br> > <br> > ```<br> > diff --git a/Makefile b/Makefile<br> > index d2257ed3a3..0735f39d68 100644<br> > --- a/Makefile<br> > +++ b/Makefile<br> > @@ -920,7 +920,7 @@ show-info:<br> > @:<br> > $(info $(call clean-json, \<br> > { $(foreach p, \<br> > - $(sort $(foreach i,$(PACKAGES) $(TARGETS_ROOTFS), \<br> > + $(sort $(foreach i,$(PACKAGES_ALL) $(TARGETS_ROOTFS), \<br> > $(i) \<br> > $($(call UPPERCASE,$(i))_FINAL_RECURSIVE_DEPENDENCIES) \<br> > ) \<br> > ```<br> > <br> > Regards,<br> > PERALE Thomas<br> > <br> > > ---<br> > > utils/generate-cyclonedx | 23 +++++++++--------------<br> > > 1 file changed, 9 insertions(+), 14 deletions(-)<br> > > <br> > > diff --git a/utils/generate-cyclonedx b/utils/generate-cyclonedx<br> > > index a103b7b707..29c73f3616 100755<br> > > --- a/utils/generate-cyclonedx<br> > > +++ b/utils/generate-cyclonedx<br> > > @@ -280,7 +280,7 @@ def cyclonedx_component(name, comp):<br> > > **({<br> > > "version": comp["version"],<br> > > **(cyclonedx_licenses(comp["licenses"]) if "licenses" in comp else {}),<br> > > - } if not comp["virtual"] else {}),<br> > > + } if "version" in comp else {}),<br> > > **({<br> > > "cpe": comp["cpe-id"],<br> > > } if "cpe-id" in comp else {}),<br> > > @@ -337,12 +337,10 @@ def cyclonedx_vulnerabilities(show_info_dict):<br> > > } for cve, components in cves.items()]<br> > > <br> > > <br> > > -def br2_parse_deps_recursively(ref, show_info_dict, virtual=False, deps=[]):<br> > > +def br2_parse_deps_recursively(ref, show_info_dict, deps=[]):<br> > > """Parse dependencies from the show-info output. This function will<br> > > recursively collect all dependencies, and return a list where each dependency<br> > > is stated at most once.<br> > > - The dependency on virtual package will collect the final dependency without<br> > > - including the virtual one.<br> > > <br> > > Args:<br> > > ref (str): The identifier of the package for which the dependencies have<br> > > @@ -359,9 +357,8 @@ def br2_parse_deps_recursively(ref, show_info_dict, virtual=False, deps=[]):<br> > > """<br> > > for dep in show_info_dict.get(ref, {}).get("dependencies", []):<br> > > if dep not in deps:<br> > > - if virtual or show_info_dict.get(dep, {}).get("virtual") is False:<br> > > - deps.append(dep)<br> > > - br2_parse_deps_recursively(dep, show_info_dict, virtual, deps)<br> > > + deps.append(dep)<br> > > + br2_parse_deps_recursively(dep, show_info_dict, deps)<br> > > <br> > > return deps<br> > > <br> > > @@ -376,8 +373,6 @@ def main():<br> > > default=(None if sys.stdin.isatty() else sys.stdin))<br> > > parser.add_argument("-o", "--out-file", nargs="?", type=argparse.FileType("w"),<br> > > default=sys.stdout)<br> > > - parser.add_argument("--virtual", default=False, action='store_true',<br> > > - help="This option includes virtual packages to the CycloneDX output")<br> > > parser.add_argument("--project-name", type=str, default="buildroot",<br> > > help="Specify the project name to use in the SBOM metadata (default:'buildroot')")<br> > > parser.add_argument("--project-version", type=str, default=f"{BR2_VERSION_FULL}",<br> > > @@ -391,10 +386,10 @@ def main():<br> > > <br> > > show_info_dict = json.load(args.in_file)<br> > > <br> > > - # Remove rootfs and virtual packages if not explicitly included<br> > > - # from the cli arguments<br> > > + # Keep only packages with at least one download source defined<br> > > + # (packages with _SITE variable in Buildroot makefile).<br> > > filtered_show_info_dict = {k: v for k, v in show_info_dict.items()<br> > > - if ("rootfs" not in v["type"]) and (args.virtual or v["virtual"] is False)}<br> > > + if (len(v.get('downloads', [])) > 0)}<br> > > <br> > > cyclonedx_dict = {<br> > > "bomFormat": "CycloneDX",<br> > > @@ -429,10 +424,10 @@ def main():<br> > > ],<br> > > "dependencies": [<br> > > cyclonedx_dependency("buildroot", list(filtered_show_info_dict)),<br> > > - *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref, show_info_dict, args.virtual))<br> > > + *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref, filtered_show_info_dict))<br> > > for ref in filtered_show_info_dict],<br> > > ],<br> > > - "vulnerabilities": cyclonedx_vulnerabilities(show_info_dict),<br> > > + "vulnerabilities": cyclonedx_vulnerabilities(filtered_show_info_dict),<br> > > }<br> > > <br> > > args.out_file.write(json.dumps(cyclonedx_dict, indent=2))<br> > > -- <br> > > 2.43.0<br> > > <br> > > _______________________________________________<br> > > buildroot mailing list<br> > > <a href="mailto:buildroot@buildroot.org" target="_blank">buildroot@buildroot.org</a><br> > > <a href="https://lists.buildroot.org/mailman/listinfo/buildroot" rel="noreferrer" target="_blank">https://lists.buildroot.org/mailman/listinfo/buildroot</a><br> > </blockquote></div></div> > > > _______________________________________________ > buildroot mailing list > buildroot@buildroot.org > https://lists.buildroot.org/mailman/listinfo/buildroot > >
diff --git a/utils/generate-cyclonedx b/utils/generate-cyclonedx index a103b7b707..29c73f3616 100755 --- a/utils/generate-cyclonedx +++ b/utils/generate-cyclonedx @@ -280,7 +280,7 @@ def cyclonedx_component(name, comp): **({ "version": comp["version"], **(cyclonedx_licenses(comp["licenses"]) if "licenses" in comp else {}), - } if not comp["virtual"] else {}), + } if "version" in comp else {}), **({ "cpe": comp["cpe-id"], } if "cpe-id" in comp else {}), @@ -337,12 +337,10 @@ def cyclonedx_vulnerabilities(show_info_dict): } for cve, components in cves.items()] -def br2_parse_deps_recursively(ref, show_info_dict, virtual=False, deps=[]): +def br2_parse_deps_recursively(ref, show_info_dict, deps=[]): """Parse dependencies from the show-info output. This function will recursively collect all dependencies, and return a list where each dependency is stated at most once. - The dependency on virtual package will collect the final dependency without - including the virtual one. Args: ref (str): The identifier of the package for which the dependencies have @@ -359,9 +357,8 @@ def br2_parse_deps_recursively(ref, show_info_dict, virtual=False, deps=[]): """ for dep in show_info_dict.get(ref, {}).get("dependencies", []): if dep not in deps: - if virtual or show_info_dict.get(dep, {}).get("virtual") is False: - deps.append(dep) - br2_parse_deps_recursively(dep, show_info_dict, virtual, deps) + deps.append(dep) + br2_parse_deps_recursively(dep, show_info_dict, deps) return deps @@ -376,8 +373,6 @@ def main(): default=(None if sys.stdin.isatty() else sys.stdin)) parser.add_argument("-o", "--out-file", nargs="?", type=argparse.FileType("w"), default=sys.stdout) - parser.add_argument("--virtual", default=False, action='store_true', - help="This option includes virtual packages to the CycloneDX output") parser.add_argument("--project-name", type=str, default="buildroot", help="Specify the project name to use in the SBOM metadata (default:'buildroot')") parser.add_argument("--project-version", type=str, default=f"{BR2_VERSION_FULL}", @@ -391,10 +386,10 @@ def main(): show_info_dict = json.load(args.in_file) - # Remove rootfs and virtual packages if not explicitly included - # from the cli arguments + # Keep only packages with at least one download source defined + # (packages with _SITE variable in Buildroot makefile). filtered_show_info_dict = {k: v for k, v in show_info_dict.items() - if ("rootfs" not in v["type"]) and (args.virtual or v["virtual"] is False)} + if (len(v.get('downloads', [])) > 0)} cyclonedx_dict = { "bomFormat": "CycloneDX", @@ -429,10 +424,10 @@ def main(): ], "dependencies": [ cyclonedx_dependency("buildroot", list(filtered_show_info_dict)), - *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref, show_info_dict, args.virtual)) + *[cyclonedx_dependency(ref, br2_parse_deps_recursively(ref, filtered_show_info_dict)) for ref in filtered_show_info_dict], ], - "vulnerabilities": cyclonedx_vulnerabilities(show_info_dict), + "vulnerabilities": cyclonedx_vulnerabilities(filtered_show_info_dict), } args.out_file.write(json.dumps(cyclonedx_dict, indent=2))
- Filter packages to include only those with external download sources (i.e., not bundled within Buildroot). - Remove virtual package handling from dependency analysis. Signed-off-by: Fabien Lehoussel <fabien.lehoussel@smile.fr> --- utils/generate-cyclonedx | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-)