diff mbox

PATCH: Add capability to contrib/compare_tests to handle directories

Message ID CAEhygDrtBbbQKQj4EVBrN5qZMru_+3Mbk+GinZ+yNdd3wc3Hyg@mail.gmail.com
State New
Headers show

Commit Message

Quentin Neill Nov. 5, 2011, 3:23 a.m. UTC 
On Wed, Nov 2, 2011 at 12:32 PM, Quentin Neill
<quentin.neill.gnu@gmail.com> wrote:
> On Tue, Oct 4, 2011 at 4:57 PM, Mike Stump <mikestump@comcast.net> wrote:
>> On Oct 4, 2011, at 2:37 PM, Quentin Neill wrote:
>>> Ping?
>>
>> The problem with this patch is it reorders the listing so that lower priority things are after higher priority things.  The entire point of the routine is to list the high priority things first, so that a casual user can read the first line or two and not immediately if there is something they care about.
>>
>> Now, this property can be preserved simply by concatenating all the .sum files found in a directory into an all.sum file, and then running the script on those two files.  If one does that, then the priority order is preserved.
>
> Hi Mike,
> Thanks for the feedback.  From the comments, I assumed the usage was
> to compare two .log files not .sum files.
> Maybe it is a new scenario I'm imagining - comparing two builds to see
> if ANY test results changed, regardless of order.
> In any case, I will work on your changes when I get time (after stage1 probably)
> --
> Quentin

I got to this anyway.
My scenario about "ANY test results changed" is what I added with -strict.
This patch concatenates the common .sum files before comparing.

Okay to commit?

Comments

Mike Stump Nov. 5, 2011, 9:26 p.m. UTC | #1 
On Nov 4, 2011, at 8:23 PM, Quentin Neill wrote:
> This patch concatenates the common .sum files before comparing.
> 
> Okay to commit?

Ok, thanks for the contribution.
Mike Stump Feb. 11, 2012, 2:13 p.m. UTC | #2 
On Nov 4, 2011, at 8:23 PM, Quentin Neill wrote:
> My scenario about "ANY test results changed" is what I added with -strict.
> This patch concatenates the common .sum files before comparing.

So, how exactly does this work for you:

+	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
+	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
+	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
+	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2

sum1 and sum2 appear to be variables that aren't set.
diff mbox

Patch

diff --git a/contrib/ChangeLog b/contrib/ChangeLog
index eddf6ec..b0afb27 100644
--- a/contrib/ChangeLog
+++ b/contrib/ChangeLog
@@ -1,3 +1,8 @@ 
+2011-11-04   Quentin Neill  <quentin.neill@amd.com>
+
+	* compare_tests: Add ability to compare all .sum
+	files from two build directories.
+
 2011-09-13   Diego Novillo  <dnovillo@google.com>
 
 	* testsuite-management: New.
diff --git a/contrib/compare_tests b/contrib/compare_tests
index bed9742..a23fcf6 100755
--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -2,13 +2,35 @@ 
 # This script automatically test the given tool with the tool's test cases,
 # reporting anything of interest.
 
-# exits with 0 if there is nothing of interest
-# exits with 1 if there is something interesting
-# exits with 2 if an error occurred
-
-# Give two .sum files to compare them
+usage()
+{
+	if [ -n "$1" ] ; then
+		echo "$0: Error: $1" >&2
+		echo >&2
+	fi
+	cat >&2 <<EOUSAGE
+Usage: $0 [-strict] PREVIOUS CURRENT
+
+Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
+
+	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
+
+	Unless -strict is given, these discrepancies are not counted as errors:
+		missing/extra .sum files when comparing directories
+		tests that failed in PREVIOUS but pass in CURRENT
+		tests that were not in PREVIOUS but appear in CURRENT
+		tests in PREVIOUS that are missing in CURRENT
+
+	Exit with the following values:
+		0 if there is nothing of interest
+		1 if there are errors when comparing single test case files
+		N for the number of errors found when comparing directories
+EOUSAGE
+	exit 2
+}
 
 # Written by Mike Stump <mrs@cygnus.com>
+# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
 
 tool=gxx
 
@@ -16,10 +38,69 @@  tmp1=/tmp/$tool-testing.$$a
 tmp2=/tmp/$tool-testing.$$b
 now_s=/tmp/$tool-testing.$$d
 before_s=/tmp/$tool-testing.$$e
+lst1=/tmp/$tool-lst1.$$
+lst2=/tmp/$tool-lst2.$$
+lst3=/tmp/$tool-lst3.$$
+lst4=/tmp/$tool-lst4.$$
+lst5=/tmp/$tool-lst5.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
+
+[ "$1" = "-strict" ] && strict=$1 && shift
+[ "$1" = "-?" ] && usage
+[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
+
+trap "rm -f $tmps" 0 1 2 3 5 9 13 15
+exit_status=0
 
-if [ "$2" = "" ]; then
-	echo "Usage: $0 previous current" >&2
-	exit 2
+if [ -d "$1" -a -d "$2" ] ; then
+	find "$1" -name '*.sum' >$lst1
+	find "$2" -name '*.sum' >$lst2
+	echo "# Comparing directories"
+	echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
+	echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
+	echo
+	# remove leading directory components to compare
+	sed -e "s|^$1/||" $lst1 | sort >$lst3
+	sed -e "s|^$2/||" $lst2 | sort >$lst4
+	comm -23 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra sum files in Dir1=$1"
+		sed -e "s|^|< $1/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -13 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra sum files in Dir2=$2"
+		sed -e "s|^|> $2/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -12 $lst3 $lst4 | sort -u >$lst5
+	if [ ! -s $lst5 ] ; then
+		echo "# No common sum files"
+		exit_status=`expr $exit_status + 1`
+		exit $exit_status
+	fi
+	cmnsums=`cat $lst5 | wc -l`
+	echo "# Comparing $cmnsums common sum files"
+	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
+	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
+	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
+	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		exit_status=`expr $exit_status + 1`
+		echo "## Differences found: $fname"
+	fi
+	if [ $exit_status -ne 0 ]; then
+		echo "# $exit_status differences in $cmnsums common sum files found"
+	else
+		echo "# No differences found in $cmnsums common sum files"
+	fi
+	exit $exit_status
+elif [ -d "$1" -o -d "$2" ] ; then
+	usage "Must specify either two directories or two files"
 fi
 
 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp1
@@ -28,8 +109,6 @@  sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target =
 before=$tmp1
 now=$tmp2
 
-exit_status=0
-trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
 
 if sort -k 2 </dev/null >/dev/null 2>&1; then
   skip1='-k 2'
@@ -60,6 +139,7 @@  if [ $? = 0 ]; then
 	echo "Tests that now work, but didn't before:"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -83,6 +163,7 @@  if [ $? = 0 ]; then
 	echo "New tests that PASS:"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -94,6 +175,7 @@  if [ $? = 0 ]; then
 	echo "Old tests that passed, that have disappeared: (Eeek!)"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi
 
@@ -105,6 +187,7 @@  if [ $? = 0 ]; then
 	echo "Old tests that failed, that have disappeared: (Eeek!)"
 	echo
 	cat $tmp2
+	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
 	echo
 fi