Make use of unstable-tests file.

Update list of unstable tests, and improve syntax:
target:testname
so that we can filter unstable tests by target.
The current list has been built after several executions of the Backport job,
comparing the latest release, and the "Bump after release" commit, which should
have no difference.

Count such tests as unstable, and report them none-the-less,
in the minor-to-be-checked category.

Change-Id: I01bfcfc0f957318077cb58300e8ac2e7f66b0f07

4 files changed, 143 insertions, 63 deletions

diff --git a/scripts/compare_dg_tests.pl b/scripts/compare_dg_tests.pl
index 2b55464e..51266dc6 100755
--- a/scripts/compare_dg_tests.pl
+++ b/scripts/compare_dg_tests.pl
@@ -30,7 +30,7 @@ my $XFAIL_NOW_PASSES      = "Expected fail passes      [XFAIL=>XPASS]";
 my $FAIL_NOW_PASSES       = "Fail now passes           [FAIL => PASS]";
 my $NEW_PASSES		  = "New pass                  [     => PASS]";
 my $UNHANDLED_CASES	  = "Unhandled cases           [   ..??..   ]";
-my $UNSTABLE_CASES	  = "Unstable cases            [~RANDOM     ]";
+my $UNSTABLE_CASES	  = "Unstable cases, ignored   [~RANDOM     ]";
 
 # ERRORS
 my $PASSED_NOW_FAILS      = "Passed now fails          [PASS => FAIL]";
@@ -80,6 +80,7 @@ my ($testroot, $basename);
 my ($ref_file_name, $res_file_name);
 my $nounstable=0;
 my $unstablefile=0;
+my $target="";
 
 GetOptions ("l"           => \$long,
             "s"           => \$short,
@@ -89,7 +90,8 @@ GetOptions ("l"           => \$long,
             "testroot=s"  => \$testroot,
             "basename=s"  => \$basename,
             "no-unstable"  => \$nounstable,
-            "unstable-tests=s" => \$unstablefile);
+            "unstable-tests=s" => \$unstablefile,
+            "target=s"    => \$target);
 
 $ref_file_name = $ARGV[0] if ($#ARGV == 1);
 $res_file_name = $ARGV[1] if ($#ARGV == 1);
@@ -195,14 +197,28 @@ sub read_unstable($)
    open UNSTABLEFILE, $unstable_file or die $!;
    while (<UNSTABLEFILE>)
    {
-       # Skip lines starting with '#'
-       if (/^#/)
+       # Skip lines starting with '#', or with spaces only
+       if ((/^#/) || (/^[ 	]*$/))
        {
        }
        else
        {
 	   chomp;
-	   push @unstable_tests, $_;
+
+	   my $test = $_;
+
+	   # Check if line is of type: target:testname
+	   if (/^(.*):/)
+	   {
+	       if ($target eq $1) {
+		   # If target matches the one supplied as script
+		   # argument, add the testname to the list
+		   $test =~ s/.*://;
+		   push @unstable_tests, $test;
+	       }
+	   } else {
+	       push @unstable_tests, $test;
+	   }
        }
    }
    close UNSTABLEFILE;
@@ -262,31 +278,31 @@ sub compare_results($$)
    #### ACTIONS FOR EACH CASES
    foreach my $key (sort (keys %{$ref->{testcases}}))
    {
-       # If testcase is listed as 'unstable' mark it as such and skip
-       # other processing.
-       if (grep { (index $key,$_)!=-1} @unstablelist)
-       {
-	   print "[unstable] $key\n" if ($debug);
-	   push @{$res->{$UNSTABLE_CASES}}, $key if ($nounstable == 0);
-       }
-       else
+       foreach my $diag_diag (@handler_list)
        {
-	   foreach my $diag_diag (@handler_list)
+	   if ($ref->{testcases}->{$key}->{$diag_diag->{was}} != $res->{testcases}->{$key}->{$diag_diag->{was}}
+	       and $res->{testcases}->{$key}->{$diag_diag->{is}})
 	   {
-		  if ($ref->{testcases}->{$key}->{$diag_diag->{was}} != $res->{testcases}->{$key}->{$diag_diag->{was}}
-		  and $res->{testcases}->{$key}->{$diag_diag->{is}})
-		  {
-
-			print "[$diag_diag->{was} => $diag_diag->{is}] $key\n" if ($debug);
-			if ($diag_diag->{handler})
-			{
-			  $diag_diag->{handler} ($ref, $res, $diag_diag, $key);
-			}
-			else
-			{
-			  push @{$res->{$diag_diag->{cat}}}, $key;
-			}
-		  }
+	       # If testcase is listed as 'unstable' mark it as
+	       # such and skip other processing.
+	       {
+		   if (grep { (index $key,$_)!=-1} @unstablelist)
+		   {
+		       print "[unstable] $key\n" if ($debug);
+		       push @{$res->{$UNSTABLE_CASES}}, $key if ($nounstable == 0);
+		   }
+		   else {
+		       print "[$diag_diag->{was} => $diag_diag->{is}] $key\n" if ($debug);
+		       if ($diag_diag->{handler})
+		       {
+			   $diag_diag->{handler} ($ref, $res, $diag_diag, $key);
+		       }
+		       else
+		       {
+			   push @{$res->{$diag_diag->{cat}}}, $key;
+		       }
+		   }
+	       }
 	   }
        }
    }
diff --git a/scripts/compare_tests b/scripts/compare_tests
index 7c78b398..79a56adb 100755
--- a/scripts/compare_tests
+++ b/scripts/compare_tests
@@ -13,7 +13,7 @@ usage()
 		echo >&2
 	fi
 	cat >&2 <<EOUSAGE
-Usage: $0 [-strict] PREVIOUS CURRENT
+Usage: $0 [-strict] [-target target-triplet] PREVIOUS CURRENT
 
 Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
 
@@ -25,6 +25,9 @@ Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of int
 		tests that were not in PREVIOUS but appear in CURRENT
 		tests in PREVIOUS that are missing in CURRENT
 
+        -target enables to provide the target name to use when parsing
+         the file containing the list of unstable tests.
+
 	Exit with the following values:
 		0 if there is nothing of interest
 		1 if there are errors when comparing single test case files
@@ -54,6 +57,7 @@ sum2=/tmp/$tool-sum2.$$
 tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
 
 [ "$1" = "-strict" ] && strict=$1 && shift
+[ "$1" = "-target" ] && target=$2 && shift 2
 [ "$1" = "-?" ] && usage
 [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
 
@@ -95,7 +99,10 @@ if [ -d "$1" -a -d "$2" ] ; then
 	( for fname in `cat $lst5`; do xzcat $1/$fname | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/'; done ) >$sum1
 	( for fname in `cat $lst5`; do xzcat $2/$fname | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/'; done ) >$sum2
 	# At some point, we'll add --no-unstable --unstable-test=${my_path}/unstable-tests.txt
-	${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl -l $sum1 $sum2
+	if [ "x${target}" != "x" ] ; then
+	    target="--target ${target}"
+	fi
+	${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl -l --unstable-test=${my_path}/unstable-tests.txt ${target} $sum1 $sum2
 	ret=$?
 	if [ $ret -eq 2 ]; then
 		exit_status=`expr $exit_status + 1`
diff --git a/scripts/unstable-tests.txt b/scripts/unstable-tests.txt
index 6790bb84..3990f333 100644
--- a/scripts/unstable-tests.txt
+++ b/scripts/unstable-tests.txt
@@ -1,36 +1,93 @@
+# List of unstable tests.
+# Syntax:
+# target:test name
+#
 # gcc
-gcc.dg/atomic/c11-atomic-exec-4.c
+aarch64-linux-gnu:gcc.c-torture/execute/20000801-2.c execution, -O2 -flto -fno-use-linker-plugin -flto-partition=none
+aarch64-linux-gnu:gcc.c-torture/execute/20000819-1.c execution,  -Og -g
+aarch64-linux-gnu:gcc.c-torture/execute/20010711-1.c execution,  -O3 -fomit-frame-pointer
+aarch64-linux-gnu:gcc.c-torture/execute/20051215-1.c execution,  -Og -g 
+aarch64-linux-gnu:gcc.c-torture/execute/920604-1.c execution,  -O0
+aarch64-linux-gnu:gcc.c-torture/execute/920625-1.c execution,  -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects
+aarch64-linux-gnu:gcc.c-torture/execute/921124-1.c execution,  -O2 -flto -fno-use-linker-plugin -flto-partition=none
+aarch64-linux-gnu:gcc.c-torture/execute/930111-1.c execution,  -O0
+aarch64-linux-gnu:gcc.c-torture/execute/941101-1.c execution,  -O3 -g
+aarch64-linux-gnu:gcc.c-torture/execute/990525-1.c execution,  -O2
+aarch64-linux-gnu:gcc.c-torture/execute/anon-1.c execution,  -O2
+aarch64-linux-gnu:gcc.c-torture/execute/anon-1.c execution,  -O3 -fomit-frame-pointer
+aarch64-linux-gnu:gcc.c-torture/execute/builtins/pr22237.c execution,  -O1
+aarch64-linux-gnu:gcc.c-torture/execute/memset-1.c execution,  -Os
+aarch64-linux-gnu:gcc.c-torture/execute/pr28865.c execution,  -O1
+aarch64-linux-gnu:gcc.c-torture/execute/pr33779-1.c execution,  -Os
+aarch64-linux-gnu:gcc.c-torture/execute/pr40668.c execution,  -O2
+aarch64-linux-gnu:gcc.c-torture/execute/pr40668.c execution,  -O3 -fomit-frame-pointer
+aarch64-linux-gnu:gcc.dg/atomic/stdatomic-vm.c  -O3 -fomit-frame-pointer  execution test
 gcc.dg/atomic/c11-atomic-exec-5.c
-gcc.dg/di-sync-multithread
-gcc.dg/tree-prof/time-profiler-2.c
+aarch64-linux-gnu:gcc.dg/torture/vec-cvt-1.c  -O3 -fomit-frame-pointer -funroll-loops  execution test
+aarch64-linux-gnu:gcc.target/aarch64/aapcs64/test_19.c execution,  -O2
+
 # g++
-g++.dg/tls/thread_local4.C
-g++.dg/tls/thread_local4g.C
+aarch64-linux-gnu:g++.dg/opt/temp2.C -std=c++1y execution test
+aarch64-linux-gnu:g++.old-deja/g++.eh/new2.C -std=gnu++11 execution test
+aarch64-linux-gnu:g++.old-deja/g++.jason/temporary.C -std=c++1y execution test
+aarch64-linux-gnu:g++.old-deja/g++.law/refs4.C -std=c++11 execution test
+aarch64-linux-gnu:g++.old-deja/g++.mike/p3708a.C -std=c++1y execution test
+
+# gfortran
+aarch64-linux-gnu:gfortran.dg/backslash_1.f90  -O3 -fomit-frame-pointer -funroll-all-loops -finline-functions  execution test
+aarch64-linux-gnu:gfortran.dg/convert_implied_open.f90  -O2  execution test
+aarch64-linux-gnu:gfortran.dg/finalize_15.f90  -O3 -fomit-frame-pointer  execution test
+aarch64-linux-gnu:gfortran.dg/minlocval_3.f90
+arm-linux-gnueabi:gfortran.dg/minlocval_3.f90  -O3 -fomit-frame-pointer -funroll-loops  execution test
+armeb-linux-gnueabihf:gfortran.dg/minlocval_3.f90  -O3 -fomit-frame-pointer -funroll-loops  execution test
+arm-linux-gnueabihf:gfortran.dg/streamio_1.f90  -O3 -g  execution test
+aarch64-linux-gnu:gfortran.dg/tiny_2.f90  -Os  execution test
+aarch64-linux-gnu:gfortran.dg/unf_io_convert_3.f90  -O3 -g  execution test
+
+#libgomp
+aarch64-linux-gnu:libgomp.fortran/allocatable10.f90  -O3 -fomit-frame-pointer -funroll-loops  execution test
+aarch64-linux-gnu:libgomp.fortran/allocatable7.f90  -O2  execution test
+aarch64-linux-gnu:libgomp.fortran/omp_atomic1.f90  -O3 -fomit-frame-pointer -funroll-all-loops -finline-functions  execution test
+aarch64-linux-gnu:libgomp.fortran/reference1.f90  -O2  execution test
+aarch64-linux-gnu:libgomp.fortran/simd7.f90  -O3 -fomit-frame-pointer -funroll-loops  execution test
+aarch64-linux-gnu:libgomp.fortran/task3.f90  -O2  execution test
+
 # libstdc++
-21_strings/basic_string/pthread4.cc execution test
-26_numerics/random/negative_binomial_distribution/operators/values.cc
-26_numerics/random/binomial_distribution/operators/values.cc
-27_io/basic_filebuf/close/char/4879.cc
-27_io/basic_filebuf/close/char/9964.cc
-27_io/basic_filebuf/seekoff/char/26777.cc
-27_io/basic_filebuf/underflow/char/10097.cc
-27_io/basic_ofstream/pthread2.cc
-27_io/basic_ostringstream/pthread3.cc
-27_io/objects/char/7.cc
-27_io/objects/wchar_t/7.cc
-30_threads/call_once/39909.cc
-30_threads/condition_variable_any/50862.cc execution test
-30_threads/lock/2.cc
-30_threads/mutex/try_lock/2.cc execution test
-30_threads/thread/members/2.cc
-30_threads/thread/members/3.cc execution test
-30_threads/thread/native_handle/cancel.cc
-30_threads/shared_timed_mutex/try_lock/2.cc execution test
-30_threads/this_thread/4.cc execution test
-# Sanitizers
-c-c++-common/asan/
-c-c++-common/ubsan/
-gcc.dg/asan/
-gcc.dg/ubsan/
-g++.dg/asan/
-g++.dg/ubsan/
+aarch64-linux-gnu:21_strings/basic_string/operations/find/wchar_t/3.cc execution test
+aarch64-linux-gnu:22_locale/num_get/get/wchar_t/2.cc execution test
+aarch64-linux-gnu:22_locale/numpunct/members/pod/1.cc execution test
+aarch64-linux-gnu:26_numerics/random/lognormal_distribution/operators/serialize.cc execution test
+arm-linux-gnueabihf:26_numerics/complex/inserters_extractors/char/1.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/seekoff/char/1-io.cc execution test
+aarch64-linux-gnu:27_io/basic_filebuf/sgetc/char/1-out.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sbumpc/char/1-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sbumpc/char/1-io.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sbumpc/char/2-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sbumpc/char/2-io.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/seekpos/char/2-io.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sgetc/char/1-io.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sgetc/char/2-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sgetc/char/2-io.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/snextc/char/1-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/snextc/char/1-io.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/snextc/char/2-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/snextc/char/2-io.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sputbackc/char/1-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sputbackc/char/2-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sungetc/char/1-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_filebuf/sungetc/char/2-in.cc execution test
+armeb-linux-gnueabihf:27_io/basic_ofstream/pthread2.cc execution test
+arm-linux-gnueabi:27_io/basic_ostream/inserters_other/wchar_t/1.cc execution test
+armeb-linux-gnueabihf:30_threads/condition_variable/54185.cc execution test
+aarch64-linux-gnu:experimental/string_view/operations/find/wchar_t/2.cc execution test
+aarch64-linux-gnu:ext/slist/check_construct_destroy.cc execution test
+aarch64-linux-gnu:ext/random/simd_fast_mersenne_twister_engine/cons/default.cc execution test
+aarch64-linux-gnu:tmpdir-g++.dg-struct-layout-1/t011 cp_compat_x_tst.o-cp_compat_y_tst.o execute 
+aarch64-linux-gnu:tr1/4_metaprogramming/is_enum/24808.cc execution test
+aarch64-linux-gnu:tr1/5_numerical_facilities/special_functions/07_conf_hyperg/check_value.cc execution test
+aarch64-linux-gnu:tr1/5_numerical_facilities/special_functions/14_ellint_3/check_value.cc execution test
+aarch64-linux-gnu:tr1/5_numerical_facilities/special_functions/17_hyperg/check_nan.cc execution test
+
+# asan
+arm-linux-gnueabi:c-c++-common/asan/swapcontext-test-1.c
+arm-linux-gnueabihf:c-c++-common/asan/swapcontext-test-1.c
diff --git a/test-backport.sh b/test-backport.sh
index 93089649..fcf50212 100755
--- a/test-backport.sh
+++ b/test-backport.sh
@@ -195,7 +195,7 @@ if test x"${fileserver}" != x; then
 #	fi
 #	ssh ${fileserver} cat ${toplevel}/diff-${i}.txt
 #    done
-    ssh ${fileserver} "${tmp}/compare_tests ${dir2} ${dir1} > ${toplevel}/diff.txt"
+    ssh ${fileserver} "${tmp}/compare_tests -target ${target} ${dir2} ${dir1} > ${toplevel}/diff.txt"
     if test $? -ne 0; then
 	ret=1
     fi