diff mbox series

scripts: Add Buildbot dumpinfo.pl script

Message ID 20200713031629.1716736-1-mail@aparcar.org
State Changes Requested
Delegated to: Petr Štetiar
Headers show
Series scripts: Add Buildbot dumpinfo.pl script | expand

Commit Message

Paul Spooren July 13, 2020, 3:16 a.m. UTC 
The dumpinfo.pl script comes from buildbot.git[0] and is used to print
available targets and architectures, which are then build.

As the buildbot clones openwrt.git anyway, the script might as well live
here as it can be used for other cases as well, e.g. determining what
architectures are available when building Docker containers or show
developers an overview which architectures are used by which target.

It's called with either the parameter `architectures` or `targets`,
showing architectures followed by supported targets or targets, followed
by the supported architectures:

$ ./scripts/dumpinfo.pl architectures
aarch64_cortex-a53 bcm27xx/bcm2710 mediatek/mt7622 mvebu/cortexa53 sunxi/cortexa53
aarch64_cortex-a72 bcm27xx/bcm2711 mvebu/cortexa72
...

$ ./scripts/dumpinfo.pl targets
apm821xx/nand powerpc_464fp
apm821xx/sata powerpc_464fp
...

In the future the `dumpinfo.pl` script could be removed from the
buildbot repository and maintained only here.

[0]: https://git.openwrt.org/?p=buildbot.git;a=blob;f=scripts/dumpinfo.pl;h=aa97f8d60379076a41b968402e9337cea824ece5;hb=HEAD

Signed-off-by: Paul Spooren <mail@aparcar.org>
---
 scripts/dumpinfo.pl | 91 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)
 create mode 100755 scripts/dumpinfo.pl

Comments

Paul Spooren July 13, 2020, 8:56 a.m. UTC | #1 
On 12.07.20 17:16, Paul Spooren wrote:
> The dumpinfo.pl script comes from buildbot.git[0] and is used to print
> available targets and architectures, which are then build.
>
> As the buildbot clones openwrt.git anyway, the script might as well live
> here as it can be used for other cases as well, e.g. determining what
> architectures are available when building Docker containers or show
> developers an overview which architectures are used by which target.

With the dumpinfo script we could clean up the current docker 
implementation which uses a single targets.yml file with some special 
cases to handle target changes between snapshot and releases.[1]

[1]: https://github.com/openwrt/docker/blob/master/.gitlab/ci/targets.yml

GitLab supports now dynamic child piplines[2], so a parsed dumpinfo 
could automatically create CI jobs to create Docker IB/SDK containers.

[2]: https://gitlab.com/gitlab-org/gitlab/-/issues/35632

> It's called with either the parameter `architectures` or `targets`,
> showing architectures followed by supported targets or targets, followed
> by the supported architectures:
>
> $ ./scripts/dumpinfo.pl architectures
> aarch64_cortex-a53 bcm27xx/bcm2710 mediatek/mt7622 mvebu/cortexa53 sunxi/cortexa53
> aarch64_cortex-a72 bcm27xx/bcm2711 mvebu/cortexa72
> ...
>
> $ ./scripts/dumpinfo.pl targets
> apm821xx/nand powerpc_464fp
> apm821xx/sata powerpc_464fp
> ...
>
> In the future the `dumpinfo.pl` script could be removed from the
> buildbot repository and maintained only here.
>
> [0]: https://git.openwrt.org/?p=buildbot.git;a=blob;f=scripts/dumpinfo.pl;h=aa97f8d60379076a41b968402e9337cea824ece5;hb=HEAD
>
> Signed-off-by: Paul Spooren <mail@aparcar.org>
> ---
>   scripts/dumpinfo.pl | 91 +++++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 91 insertions(+)
>   create mode 100755 scripts/dumpinfo.pl
>
> diff --git a/scripts/dumpinfo.pl b/scripts/dumpinfo.pl
> new file mode 100755
> index 0000000000..aa97f8d603
> --- /dev/null
> +++ b/scripts/dumpinfo.pl
> @@ -0,0 +1,91 @@
> +#!/usr/bin/env perl
> +
> +use strict;
> +use warnings;
> +use Cwd;
> +
> +my (%targets, %architectures);
> +
> +$ENV{'TOPDIR'} = Cwd::getcwd();
> +
> +
> +sub parse_targetinfo {
> +	my ($target_dir, $subtarget) = @_;
> +
> +	if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 SUBTARGET='$subtarget' |") {
> +		my ($target_name, $target_arch, @target_features);
> +		while (defined(my $line = readline M)) {
> +			chomp $line;
> +
> +			if ($line =~ /^Target: (.+)$/) {
> +				$target_name = $1;
> +			}
> +			elsif ($line =~ /^Target-Arch-Packages: (.+)$/) {
> +				$target_arch = $1;
> +			}
> +			elsif ($line =~ /^Target-Features: (.+)$/) {
> +				@target_features = split /\s+/, $1;
> +			}
> +			elsif ($line =~ /^@\@$/) {
> +				if ($target_name && $target_arch &&
> +				    !grep { $_ eq 'broken' or $_ eq 'source-only' } @target_features) {
> +					$targets{$target_name} = $target_arch;
> +					$architectures{$target_arch} ||= [];
> +					push @{$architectures{$target_arch}}, $target_name;
> +				}
> +
> +				undef $target_name;
> +				undef $target_arch;
> +				@target_features = ();
> +			}
> +		}
> +		close M;
> +	}
> +}
> +
> +sub get_targetinfo {
> +	foreach my $target_makefile (glob "target/linux/*/Makefile") {
> +		my ($target_dir) = $target_makefile =~ m!^(.+)/Makefile$!;
> +		my @subtargets;
> +
> +		if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 val.FEATURES V=s 2>/dev/null |") {
> +			if (defined(my $line = readline M)) {
> +				chomp $line;
> +				if (grep { $_ eq 'broken' or $_ eq 'source-only' } split /\s+/, $line) {
> +					next;
> +				}
> +			}
> +		}
> +
> +		if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 val.SUBTARGETS V=s 2>/dev/null |") {
> +			if (defined(my $line = readline M)) {
> +				chomp $line;
> +				@subtargets = split /\s+/, $line;
> +			}
> +			close M;
> +		}
> +
> +		push @subtargets, 'generic' if @subtargets == 0;
> +
> +		foreach my $subtarget (@subtargets) {
> +			parse_targetinfo($target_dir, $subtarget);
> +		}
> +	}
> +}
> +
> +if (@ARGV == 1 && $ARGV[0] eq 'targets') {
> +	get_targetinfo();
> +	foreach my $target_name (sort keys %targets) {
> +		printf "%s %s\n", $target_name, $targets{$target_name};
> +	}
> +}
> +elsif (@ARGV == 1 && $ARGV[0] eq 'architectures') {
> +	get_targetinfo();
> +	foreach my $target_arch (sort keys %architectures) {
> +		printf "%s %s\n", $target_arch, join ' ', @{$architectures{$target_arch}};
> +	}
> +}
> +else {
> +	print "Usage: $0 targets\n";
> +	print "Usage: $0 architectures\n";
> +}
Petr Štetiar Aug. 3, 2020, 7:46 a.m. UTC | #2 
Paul Spooren <mail@aparcar.org> [2020-07-12 22:56:33]:

Hi,

> >   scripts/dumpinfo.pl | 91 +++++++++++++++++++++++++++++++++++++++++++++

that scripts directory is mess already, please rename it to something
meaningful, like `dump-target-info.pl` as that `dumpinfo.pl` is too generic.

-- ynezz
diff mbox series

Patch

diff --git a/scripts/dumpinfo.pl b/scripts/dumpinfo.pl
new file mode 100755
index 0000000000..aa97f8d603
--- /dev/null
+++ b/scripts/dumpinfo.pl
@@ -0,0 +1,91 @@ 
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use Cwd;
+
+my (%targets, %architectures);
+
+$ENV{'TOPDIR'} = Cwd::getcwd();
+
+
+sub parse_targetinfo {
+	my ($target_dir, $subtarget) = @_;
+
+	if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 SUBTARGET='$subtarget' |") {
+		my ($target_name, $target_arch, @target_features);
+		while (defined(my $line = readline M)) {
+			chomp $line;
+
+			if ($line =~ /^Target: (.+)$/) {
+				$target_name = $1;
+			}
+			elsif ($line =~ /^Target-Arch-Packages: (.+)$/) {
+				$target_arch = $1;
+			}
+			elsif ($line =~ /^Target-Features: (.+)$/) {
+				@target_features = split /\s+/, $1;
+			}
+			elsif ($line =~ /^@\@$/) {
+				if ($target_name && $target_arch &&
+				    !grep { $_ eq 'broken' or $_ eq 'source-only' } @target_features) {
+					$targets{$target_name} = $target_arch;
+					$architectures{$target_arch} ||= [];
+					push @{$architectures{$target_arch}}, $target_name;
+				}
+
+				undef $target_name;
+				undef $target_arch;
+				@target_features = ();
+			}
+		}
+		close M;
+	}
+}
+
+sub get_targetinfo {
+	foreach my $target_makefile (glob "target/linux/*/Makefile") {
+		my ($target_dir) = $target_makefile =~ m!^(.+)/Makefile$!;
+		my @subtargets;
+
+		if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 val.FEATURES V=s 2>/dev/null |") {
+			if (defined(my $line = readline M)) {
+				chomp $line;
+				if (grep { $_ eq 'broken' or $_ eq 'source-only' } split /\s+/, $line) {
+					next;
+				}
+			}
+		}
+
+		if (open M, "make -C '$target_dir' --no-print-directory DUMP=1 TARGET_BUILD=1 val.SUBTARGETS V=s 2>/dev/null |") {
+			if (defined(my $line = readline M)) {
+				chomp $line;
+				@subtargets = split /\s+/, $line;
+			}
+			close M;
+		}
+
+		push @subtargets, 'generic' if @subtargets == 0;
+
+		foreach my $subtarget (@subtargets) {
+			parse_targetinfo($target_dir, $subtarget);
+		}
+	}
+}
+
+if (@ARGV == 1 && $ARGV[0] eq 'targets') {
+	get_targetinfo();
+	foreach my $target_name (sort keys %targets) {
+		printf "%s %s\n", $target_name, $targets{$target_name};
+	}
+}
+elsif (@ARGV == 1 && $ARGV[0] eq 'architectures') {
+	get_targetinfo();
+	foreach my $target_arch (sort keys %architectures) {
+		printf "%s %s\n", $target_arch, join ' ', @{$architectures{$target_arch}};
+	}
+}
+else {
+	print "Usage: $0 targets\n";
+	print "Usage: $0 architectures\n";
+}