From patchwork Wed Feb 26 19:43:43 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Thomas Petazzoni X-Patchwork-Id: 1245313 Return-Path: X-Original-To: incoming-buildroot@patchwork.ozlabs.org Delivered-To: patchwork-incoming-buildroot@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=busybox.net (client-ip=140.211.166.136; helo=silver.osuosl.org; envelope-from=buildroot-bounces@busybox.net; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 48SR7j0Wj0z9sP7 for ; Thu, 27 Feb 2020 06:44:00 +1100 (AEDT) Received: from localhost (localhost [127.0.0.1]) by silver.osuosl.org (Postfix) with ESMTP id EA618221F5; Wed, 26 Feb 2020 19:43:57 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from silver.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 5bfnRjGeTIC2; Wed, 26 Feb 2020 19:43:56 +0000 (UTC) Received: from ash.osuosl.org (ash.osuosl.org [140.211.166.34]) by silver.osuosl.org (Postfix) with ESMTP id 9258421577; Wed, 26 Feb 2020 19:43:56 +0000 (UTC) X-Original-To: buildroot@lists.busybox.net Delivered-To: buildroot@osuosl.org Received: from fraxinus.osuosl.org (smtp4.osuosl.org [140.211.166.137]) by ash.osuosl.org (Postfix) with ESMTP id 430CE1BF41D for ; Wed, 26 Feb 2020 19:43:54 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by fraxinus.osuosl.org (Postfix) with ESMTP id 3D42186C95 for ; Wed, 26 Feb 2020 19:43:54 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from fraxinus.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id wm87MXVKfbHw for ; Wed, 26 Feb 2020 19:43:53 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by fraxinus.osuosl.org (Postfix) with ESMTPS id 40E8886C93 for ; Wed, 26 Feb 2020 19:43:53 +0000 (UTC) X-Originating-IP: 86.210.146.109 Received: from localhost (lfbn-tou-1-915-109.w86-210.abo.wanadoo.fr [86.210.146.109]) (Authenticated sender: thomas.petazzoni@bootlin.com) by relay7-d.mail.gandi.net (Postfix) with ESMTPSA id 4FC6920004; Wed, 26 Feb 2020 19:43:49 +0000 (UTC) From: Thomas Petazzoni To: Buildroot List , Thomas De Schampheleire , "Yann E. MORIN" Date: Wed, 26 Feb 2020 20:43:43 +0100 Message-Id: <20200226194345.1087529-2-thomas.petazzoni@bootlin.com> X-Mailer: git-send-email 2.24.1 In-Reply-To: <20200226194345.1087529-1-thomas.petazzoni@bootlin.com> References: <20200226194345.1087529-1-thomas.petazzoni@bootlin.com> MIME-Version: 1.0 Subject: [Buildroot] [PATCH 1/2] package/pkg-generic.mk: simplify step_pkg_size X-BeenThere: buildroot@busybox.net X-Mailman-Version: 2.1.29 Precedence: list List-Id: Discussion and development of buildroot List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Thomas Petazzoni Errors-To: buildroot-bounces@busybox.net Sender: "buildroot" Use the same trick in step_pkg_size as the one used in check_bin_arch: factorize the two $(filter ...) calls into one, checking in one step the step and whether it's the beginning or end of the step. Signed-off-by: Thomas Petazzoni --- package/pkg-generic.mk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package/pkg-generic.mk b/package/pkg-generic.mk index f4f525f41a..6687ac9198 100644 --- a/package/pkg-generic.mk +++ b/package/pkg-generic.mk @@ -95,12 +95,12 @@ define step_pkg_size_inner endef define step_pkg_size - $(if $(filter install-target,$(2)),\ - $(if $(filter end,$(1)),$(call step_pkg_size_inner,$(3),$(TARGET_DIR)))) - $(if $(filter install-staging,$(2)),\ - $(if $(filter end,$(1)),$(call step_pkg_size_inner,$(3),$(STAGING_DIR),-staging))) - $(if $(filter install-host,$(2)),\ - $(if $(filter end,$(1)),$(call step_pkg_size_inner,$(3),$(HOST_DIR),-host))) + $(if $(filter end-install-target,$(1)-$(2)),\ + $(call step_pkg_size_inner,$(3),$(TARGET_DIR))) + $(if $(filter end-install-staging,$(1)-$(2)),\ + $(call step_pkg_size_inner,$(3),$(STAGING_DIR),-staging)) + $(if $(filter end-install-host,$(1)-$(2)),\ + $(call step_pkg_size_inner,$(3),$(HOST_DIR),-host)) endef GLOBAL_INSTRUMENTATION_HOOKS += step_pkg_size From patchwork Wed Feb 26 19:43:44 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Thomas Petazzoni X-Patchwork-Id: 1245314 Return-Path: X-Original-To: incoming-buildroot@patchwork.ozlabs.org Delivered-To: patchwork-incoming-buildroot@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=busybox.net (client-ip=140.211.166.136; helo=silver.osuosl.org; envelope-from=buildroot-bounces@busybox.net; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=bootlin.com Received: from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 48SR7q1zXdz9sP7 for ; Thu, 27 Feb 2020 06:44:07 +1100 (AEDT) Received: from localhost (localhost [127.0.0.1]) by silver.osuosl.org (Postfix) with ESMTP id 4A3D922875; Wed, 26 Feb 2020 19:44:05 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from silver.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id o6pppLcz8C8M; Wed, 26 Feb 2020 19:43:58 +0000 (UTC) Received: from ash.osuosl.org (ash.osuosl.org [140.211.166.34]) by silver.osuosl.org (Postfix) with ESMTP id A9105221BB; Wed, 26 Feb 2020 19:43:57 +0000 (UTC) X-Original-To: buildroot@lists.busybox.net Delivered-To: buildroot@osuosl.org Received: from silver.osuosl.org (smtp3.osuosl.org [140.211.166.136]) by ash.osuosl.org (Postfix) with ESMTP id 3C4EF1BF41D for ; Wed, 26 Feb 2020 19:43:56 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by silver.osuosl.org (Postfix) with ESMTP id 2B4CD21577 for ; Wed, 26 Feb 2020 19:43:56 +0000 (UTC) X-Virus-Scanned: amavisd-new at osuosl.org Received: from silver.osuosl.org ([127.0.0.1]) by localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 1pLgT9JQALr6 for ; Wed, 26 Feb 2020 19:43:54 +0000 (UTC) X-Greylist: domain auto-whitelisted by SQLgrey-1.7.6 Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by silver.osuosl.org (Postfix) with ESMTPS id 9D2E720448 for ; Wed, 26 Feb 2020 19:43:53 +0000 (UTC) X-Originating-IP: 86.210.146.109 Received: from localhost (lfbn-tou-1-915-109.w86-210.abo.wanadoo.fr [86.210.146.109]) (Authenticated sender: thomas.petazzoni@bootlin.com) by relay6-d.mail.gandi.net (Postfix) with ESMTPSA id 8ED74C0003; Wed, 26 Feb 2020 19:43:50 +0000 (UTC) From: Thomas Petazzoni To: Buildroot List , Thomas De Schampheleire , "Yann E. MORIN" Date: Wed, 26 Feb 2020 20:43:44 +0100 Message-Id: <20200226194345.1087529-3-thomas.petazzoni@bootlin.com> X-Mailer: git-send-email 2.24.1 In-Reply-To: <20200226194345.1087529-1-thomas.petazzoni@bootlin.com> References: <20200226194345.1087529-1-thomas.petazzoni@bootlin.com> MIME-Version: 1.0 Subject: [Buildroot] [PATCH 2/2] package/pkg-generic: make file list logic parallel build compatible X-BeenThere: buildroot@busybox.net X-Mailman-Version: 2.1.29 Precedence: list List-Id: Discussion and development of buildroot List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Thomas Petazzoni Errors-To: buildroot-bounces@busybox.net Sender: "buildroot" The current solution use to collect the list of files installed by packages does not work for top-level parallel build. Indeed, we rely on a file created after the installation of the previous package to build the list of files installed by the current package. This works well when packages are built sequentially, but badly fails when using top-level parallel build. More specifically, top-level parallel build can fail with: comm: /home/thomas/buildroot/output/build/.files-list-host.new: No such file or directory Because that file has been removed concurrently by the build process of another package. This commit reworks the logic in a very straight-forward way. Before the installation of each package, we store the list of files that are already installed and store it in the package build directory. After the installation of each package, we store again that list of files, calculate the difference with the before file, and store that as the list of files installed by that package, still in the package build directory. At the end of the build, in target-finalize we collect all the collected information into the global package file lists, that continue to be installed in the same location as before, with the same name. There are however some differences: (1) The files are no longer ordered in build order, but by alphabetic ordering of packages. Indeed, "build order" no longer makes any sense in the context of top-level parallel build. (2) Some files which were incorrectly tracked are no longer tracked. For example, the toolchain package is a target package, but it installs files in $(HOST_DIR). In the previous logic, the files installed by the toolchain package in $(HOST_DIR) were incorrectly affected to the next host package that was installed after the toolchain package. With our new logic, those files are no longer tracked at all. To fix this, we would have to change the logic to scan HOST_DIR/TARGET_DIR/STAGING_DIR for all installation steps, not just for the install-host, install-target and install-staging steps respecitively. But the result was already incorrect anyway, and therefore this should be fixed separately. Note that the check_bin_arch hook needs to be adjusted: it was using the global package-file-list.txt file, but this file is now created only at the very end of the build. So instead, we use the current package .file-list.txt file to know which packages have been installed by the current package in $(TARGET_DIR). Fixes: http://autobuild.buildroot.net/results/4e60fa31b1cd08bc7fdf9c5dd3a3f4941e029ba3/ Signed-off-by: Thomas Petazzoni --- Makefile | 15 +++++------ package/pkg-generic.mk | 59 +++++++++++++++++++----------------------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 40e71ffbf4..63e0dec392 100644 --- a/Makefile +++ b/Makefile @@ -804,15 +804,12 @@ endif # merged /usr touch $(TARGET_DIR)/usr -# AFTER ALL FILE-CHANGING ACTIONS: -# Update timestamps in internal file list to fix attribution of files -# to packages on subsequent builds - @$(call step_pkg_size_file_list,$(TARGET_DIR)) - @$(call step_pkg_size_finalize) - @$(call step_pkg_size_file_list,$(STAGING_DIR),-staging) - @$(call step_pkg_size_finalize,-staging) - @$(call step_pkg_size_file_list,$(HOST_DIR),-host) - @$(call step_pkg_size_finalize,-host) + cat $(sort $(wildcard $(BUILD_DIR)/*/.files-list.txt)) > \ + $(BUILD_DIR)/packages-file-list.txt + cat $(sort $(wildcard $(BUILD_DIR)/*/.files-list-host.txt)) > \ + $(BUILD_DIR)/packages-file-list-host.txt + cat $(sort $(wildcard $(BUILD_DIR)/*/.files-list-staging.txt)) > \ + $(BUILD_DIR)/packages-file-list-staging.txt .PHONY: target-post-image target-post-image: $(TARGETS_ROOTFS) target-finalize staging-finalize diff --git a/package/pkg-generic.mk b/package/pkg-generic.mk index 6687ac9198..7b240ca012 100644 --- a/package/pkg-generic.mk +++ b/package/pkg-generic.mk @@ -57,50 +57,45 @@ GLOBAL_INSTRUMENTATION_HOOKS += step_time # Hooks to collect statistics about installed files -# Helper function to create the file list -- also used from target-finalize -# $(1): base directory to search in -# $(2): suffix of file (optional) -define step_pkg_size_file_list - cd $(1); \ +# $(1): package name +# $(2): base directory to search in +# $(3): suffix of file (optional) +define step_pkg_size_before + cd $(2); \ LC_ALL=C find . \( -type f -o -type l \) -printf '%T@:%i:%#m:%y:%s,%p\n' \ - | LC_ALL=C sort > $(BUILD_DIR)/.files-list$(2).new -endef - -# Helper function to mark the latest file list as the reference for next -# iteration -- also used from target-finalize -# $(1): suffix of file (optional) -define step_pkg_size_finalize - mv $(BUILD_DIR)/.files-list$(1).new \ - $(BUILD_DIR)/.files-list$(1).stat + | LC_ALL=C sort > $($(PKG)_BUILDDIR)/.files-list$(3).before endef -# The suffix is typically empty for the target variant, for legacy backward -# compatibility. # $(1): package name # $(2): base directory to search in -# $(3): suffix of file (optional) -define step_pkg_size_inner - @touch $(BUILD_DIR)/.files-list$(3).stat - @touch $(BUILD_DIR)/packages-file-list$(3).txt - $(SED) '/^$(1),/d' $(BUILD_DIR)/packages-file-list$(3).txt - $(call step_pkg_size_file_list,$(2),$(3)) +# $(3): suffix of file (optional) +define step_pkg_size_after + cd $(2); \ + LC_ALL=C find . \( -type f -o -type l \) -printf '%T@:%i:%#m:%y:%s,%p\n' \ + | LC_ALL=C sort > $($(PKG)_BUILDDIR)/.files-list$(3).after LC_ALL=C comm -13 \ - $(BUILD_DIR)/.files-list$(3).stat \ - $(BUILD_DIR)/.files-list$(3).new \ + $($(PKG)_BUILDDIR)/.files-list$(3).before \ + $($(PKG)_BUILDDIR)/.files-list$(3).after \ + | sed -r -e 's/^[^,]+/$(1)/' \ > $($(PKG)_BUILDDIR)/.files-list$(3).txt - sed -r -e 's/^[^,]+/$(1)/' \ - $($(PKG)_BUILDDIR)/.files-list$(3).txt \ - >> $(BUILD_DIR)/packages-file-list$(3).txt - $(call step_pkg_size_finalize,$(3)) + rm -f $($(PKG)_BUILDDIR)/.files-list$(3).before + rm -f $($(PKG)_BUILDDIR)/.files-list$(3).after endef define step_pkg_size + $(if $(filter start-install-target,$(1)-$(2)),\ + $(call step_pkg_size_before,$(3),$(TARGET_DIR))) + $(if $(filter start-install-staging,$(1)-$(2)),\ + $(call step_pkg_size_before,$(3),$(STAGING_DIR),-staging)) + $(if $(filter start-install-host,$(1)-$(2)),\ + $(call step_pkg_size_before,$(3),$(HOST_DIR),-host)) + $(if $(filter end-install-target,$(1)-$(2)),\ - $(call step_pkg_size_inner,$(3),$(TARGET_DIR))) + $(call step_pkg_size_after,$(3),$(TARGET_DIR))) $(if $(filter end-install-staging,$(1)-$(2)),\ - $(call step_pkg_size_inner,$(3),$(STAGING_DIR),-staging)) + $(call step_pkg_size_after,$(3),$(STAGING_DIR),-staging)) $(if $(filter end-install-host,$(1)-$(2)),\ - $(call step_pkg_size_inner,$(3),$(HOST_DIR),-host)) + $(call step_pkg_size_after,$(3),$(HOST_DIR),-host)) endef GLOBAL_INSTRUMENTATION_HOOKS += step_pkg_size @@ -108,7 +103,7 @@ GLOBAL_INSTRUMENTATION_HOOKS += step_pkg_size define check_bin_arch $(if $(filter end-install-target,$(1)-$(2)),\ support/scripts/check-bin-arch -p $(3) \ - -l $(BUILD_DIR)/packages-file-list.txt \ + -l $($(PKG)_BUILDDIR)/.files-list.txt \ $(foreach i,$($(PKG)_BIN_ARCH_EXCLUDE),-i "$(i)") \ -r $(TARGET_READELF) \ -a $(BR2_READELF_ARCH_NAME))