contrib/

	Backport from trunk r240288.
	2016-09-20  Christophe Lyon  <christophe.lyon@linaro.org>

	* compare_tests: Take ERROR messages into account when
          comparing.

	contrib/
	Backport from trunk r240289.
	2016-09-20  Christophe Lyon  <christophe.lyon@linaro.org>

	* dg-extract-results.py: Report DejaGnu error in the final
	summary.
	* dg-extract-results.sh: Likewise.

Change-Id: Ib15f783dedc6b017a94e01f09b1e350b0459a890

3 files changed, 19 insertions, 8 deletions

diff --git a/contrib/compare_tests b/contrib/compare_tests
index 8ecf8245b8d..d16e7e9737e 100755
--- a/contrib/compare_tests
+++ b/contrib/compare_tests
@@ -107,8 +107,8 @@ elif [ -d "$1" -o -d "$2" ] ; then
 	usage "Must specify either two directories or two files"
 fi
 
-sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
-sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
+sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
+sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
 
 before=$tmp1
 now=$tmp2
diff --git a/contrib/dg-extract-results.py b/contrib/dg-extract-results.py
index 7db5e64cfe3..4b02a5bea92 100644
--- a/contrib/dg-extract-results.py
+++ b/contrib/dg-extract-results.py
@@ -134,6 +134,7 @@ class Prog:
         self.end_line = None
         # Known summary types.
         self.count_names = [
+            '# of DejaGnu errors\t\t',
             '# of expected passes\t\t',
             '# of unexpected failures\t',
             '# of unexpected successes\t',
@@ -245,6 +246,10 @@ class Prog:
             segment = Segment (filename, file.tell())
             variation.header = segment
 
+        # Parse the rest of the summary (the '# of ' lines).
+        if len (variation.counts) == 0:
+            variation.counts = self.zero_counts()
+
         # Parse up until the first line of the summary.
         if num_variations == 1:
             end = '\t\t=== ' + tool.name + ' Summary ===\n'
@@ -291,6 +296,11 @@ class Prog:
                 harness.results.append ((key, line))
                 if not first_key and sort_logs:
                     first_key = key
+                if line.startswith ('ERROR: (DejaGnu)'):
+                    for i in range (len (self.count_names)):
+                        if 'DejaGnu errors' in self.count_names[i]:
+                            variation.counts[i] += 1
+                            break
 
             # 'Using ...' lines are only interesting in a header.  Splitting
             # the test up into parallel runs leads to more 'Using ...' lines
@@ -309,9 +319,6 @@ class Prog:
             segment.lines -= final_using
             harness.add_segment (first_key, segment)
 
-        # Parse the rest of the summary (the '# of ' lines).
-        if len (variation.counts) == 0:
-            variation.counts = self.zero_counts()
         while True:
             before = file.tell()
             line = file.readline()
diff --git a/contrib/dg-extract-results.sh b/contrib/dg-extract-results.sh
index 5a8e67ea2fa..519d49c00c3 100755
--- a/contrib/dg-extract-results.sh
+++ b/contrib/dg-extract-results.sh
@@ -369,10 +369,11 @@ EOF
 BEGIN {
   variant="$VAR"
   tool="$TOOL"
-  passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kpasscnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0;
+  passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kpasscnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0; dgerrorcnt=0;
   curvar=""; insummary=0
 }
 /^Running target /		{ curvar = \$3; next }
+/^ERROR: \(DejaGnu\)/		{ if (variant == curvar) dgerrorcnt += 1 }
 /^# of /			{ if (variant == curvar) insummary = 1 }
 /^# of expected passes/		{ if (insummary == 1) passcnt += \$5; next; }
 /^# of unexpected successes/	{ if (insummary == 1) xpasscnt += \$5; next; }
@@ -390,6 +391,7 @@ BEGIN {
 { next }
 END {
   printf ("\t\t=== %s Summary for %s ===\n\n", tool, variant)
+  if (dgerrorcnt != 0) printf ("# of DejaGnu errors\t\t%d\n", dgerrorcnt)
   if (passcnt != 0) printf ("# of expected passes\t\t%d\n", passcnt)
   if (failcnt != 0) printf ("# of unexpected failures\t%d\n", failcnt)
   if (xpasscnt != 0) printf ("# of unexpected successes\t%d\n", xpasscnt)
@@ -419,8 +421,9 @@ TOTAL_AWK=${TMP}/total.awk
 cat << EOF > $TOTAL_AWK
 BEGIN {
   tool="$TOOL"
-  passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0
+  passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0; dgerrorcnt=0
 }
+/^# of DejaGnu errors/		{ dgerrorcnt += \$5 }
 /^# of expected passes/		{ passcnt += \$5 }
 /^# of unexpected failures/	{ failcnt += \$5 }
 /^# of unexpected successes/	{ xpasscnt += \$5 }
@@ -431,7 +434,8 @@ BEGIN {
 /^# of unresolved testcases/	{ unrescnt += \$5 }
 /^# of unsupported tests/	{ unsupcnt += \$5 }
 END {
-  printf ("\n\t\t=== %s Summary ===\n\n", tool)
+  printf ("\n\t\t=== %s MySummary ===\n\n", tool)
+  if (dgerrorcnt != 0) printf ("# of DejaGnu errors\t\t%d\n", dgerrorcnt)
   if (passcnt != 0) printf ("# of expected passes\t\t%d\n", passcnt)
   if (failcnt != 0) printf ("# of unexpected failures\t%d\n", failcnt)
   if (xpasscnt != 0) printf ("# of unexpected successes\t%d\n", xpasscnt)