From 758c750b307b3016c4054e41e38c925071794711 Mon Sep 17 00:00:00 2001 From: Christophe Lyon Date: Wed, 22 Mar 2017 16:07:44 +0000 Subject: Add reference output with -l option. Change-Id: I3518a9d866770af2996c338309c5c4a9abc5e3a4 --- testsuite/Makefile | 1 + testsuite/expected-1-2-long.txt | 337 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 338 insertions(+) create mode 100644 testsuite/expected-1-2-long.txt diff --git a/testsuite/Makefile b/testsuite/Makefile index cabb3b6..8925ba8 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -1,5 +1,6 @@ check: LC_ALL=C ../compare_dg_tests.pl sum-1.txt sum-2.txt > cmp-1-2.txt ; diff cmp-1-2.txt expected-1-2.txt + LC_ALL=C ../compare_dg_tests.pl -l sum-1.txt sum-2.txt > cmp-1-2-long.txt ; diff cmp-1-2-long.txt expected-1-2-long.txt clean: rm -f cmp-1-2.txt diff --git a/testsuite/expected-1-2-long.txt b/testsuite/expected-1-2-long.txt new file mode 100644 index 0000000..83ecb5c --- /dev/null +++ b/testsuite/expected-1-2-long.txt @@ -0,0 +1,337 @@ +Comparing: +REFERENCE:sum-1.txt +CURRENT: sum-2.txt + + ` +---------+---------+ +o RUN STATUS: | REF | RES | + +------------------------------------------+---------+---------+ + | Passes [PASS] | 9 | 11 | + | Unexpected fails [FAIL] | 10 | 10 | + | Errors [ERROR] | 1 | 1 | + | Unexpected passes [XPASS] | 8 | 9 | + | Expected fails [XFAIL] | 8 | 9 | + | Unresolved [UNRESOLVED] | 7 | 7 | + | Unsupported [UNSUPPORTED] | 7 | 7 | + | Untested [UNTESTED] | 7 | 7 | + +------------------------------------------+---------+---------+ + + REF PASS ratio: 0.303571 + RES PASS ratio: 0.333333 + ***** ERROR: REF PASS ratio is abnormally low ***** + ***** ERROR: RES PASS ratio is abnormally low ***** + ***** ERROR: No REF execution test PASSed. Check execution engine configuration. ***** + ***** WARNING: No RES execution test FAILed. Check execution engine configuration. ***** + +o REGRESSIONS: + +------------------------------------------+---------+ + | PASS now XPASS [PASS =>XPASS] | 1 | + | PASS now FAIL [PASS => FAIL] | 1 | + | PASS now UNRESOLVED [PASS =>UNRES] | 1 | + | ERROR appears [ =>ERROR] | 1 | + | XPASS now FAIL [XPASS=> FAIL] | 1 | + | XPASS appears [ =>XPASS] | 1 | + | XPASS now UNRESOLVED [XPASS=>UNRES] | 1 | + | FAIL appears [ => FAIL] | 1 | + | FAIL now UNRESOLVED [FAIL =>UNRES] | 1 | + | XFAIL now FAIL [XFAIL=> FAIL] | 1 | + | XFAIL disappears [XFAIL=> ] | 1 | + | XFAIL now UNRESOLVED [XFAIL=>UNRES] | 1 | + | UNSUPPORTED now FAIL [UNSUP=> FAIL] | 1 | + | UNSUPPORTED now XPASS [UNSUP=>XPASS] | 1 | + | UNSUPP now UNRESOLVED [UNSUP=>UNRES] | 1 | + | UNTESTED now FAIL [UNTES=> FAIL] | 1 | + | UNTESTED now XPASS [UNTES=>XPASS] | 1 | + | UNTESTED now UNRESOLVED [UNTES=>UNRES] | 1 | + | UNRESOLVED now FAIL [UNRES=> FAIL] | 1 | + | UNRESOLVED now XPASS [UNRES=>XPASS] | 1 | + | UNRESOLVED appears [ =>UNRES] | 1 | + +------------------------------------------+---------+ + | TOTAL_REGRESSIONS | 21 | + +------------------------------------------+---------+ + + - PASS now XPASS [PASS =>XPASS]: + + test2-pass-xpass + + - PASS now FAIL [PASS => FAIL]: + + test3-pass-fail + + - PASS now UNRESOLVED [PASS =>UNRES]: + + test8-pass-unresolved + + - ERROR appears [ =>ERROR]: + + (DejaGnu) proc "scan-dump-tree-not fail_test optimized 3" does not exist. + + - XPASS now FAIL [XPASS=> FAIL]: + + test3-xpass-fail + + - XPASS appears [ =>XPASS]: + + test2-xpass-appears + + - XPASS now UNRESOLVED [XPASS=>UNRES]: + + test8-xpass-unresolved + + - FAIL appears [ => FAIL]: + + test3-fail-appears + + - FAIL now UNRESOLVED [FAIL =>UNRES]: + + test8-fail-unresolved + + - XFAIL now FAIL [XFAIL=> FAIL]: + + test3-xfail-fail + + - XFAIL disappears [XFAIL=> ]: + + test5-xfail-disappears + + - XFAIL now UNRESOLVED [XFAIL=>UNRES]: + + test8-xfail-unresolved + + - UNSUPPORTED now FAIL [UNSUP=> FAIL]: + + test3-unsupported-fail + + - UNSUPPORTED now XPASS [UNSUP=>XPASS]: + + test2-unsupported-xpass + + - UNSUPP now UNRESOLVED [UNSUP=>UNRES]: + + test7-unsupported-unresolved + + - UNTESTED now FAIL [UNTES=> FAIL]: + + test3-untested-fail + + - UNTESTED now XPASS [UNTES=>XPASS]: + + test2-untested-xpass + + - UNTESTED now UNRESOLVED [UNTES=>UNRES]: + + test7-untested-unresolved + + - UNRESOLVED now FAIL [UNRES=> FAIL]: + + test3-unresolved-fail + + - UNRESOLVED now XPASS [UNRES=>XPASS]: + + test2-unresolved-xpass + + - UNRESOLVED appears [ =>UNRES]: + + test4-unresolved-appears + + +o MINOR TO BE CHECKED: + +------------------------------------------+---------+ + | PASS disappears [PASS => ] | 1 | + | PASS now XFAIL [PASS =>XFAIL] | 1 | + | New PASS [ => PASS] | 1 | + | PASS now UNSUPPORTED [PASS =>UNSUP] | 1 | + | PASS now UNTESTED [PASS =>UNTES] | 1 | + | XPASS now PASS [XPASS=> PASS] | 1 | + | XPASS now XFAIL [XPASS=>XFAIL] | 1 | + | XPASS now UNSUPPORTED [XPASS=>UNSUP] | 1 | + | XPASS now UNTESTED [XPASS=>UNTES] | 1 | + | XPASS disappears [XPASS=> ] | 1 | + | FAIL now PASS [FAIL => PASS] | 2 | + | FAIL now XPASS [FAIL =>XPASS] | 1 | + | FAIL now XFAIL [FAIL =>XFAIL] | 1 | + | FAIL now UNSUPPORTED [FAIL =>UNSUP] | 1 | + | FAIL now UNTESTED [FAIL =>UNTES] | 1 | + | FAIL disappears [FAIL => ] | 1 | + | XFAIL now PASS [XFAIL=> PASS] | 1 | + | XFAIL now XPASS [XFAIL=>XPASS] | 1 | + | XFAIL now UNSUPPORTED [XFAIL=>UNSUP] | 1 | + | XFAIL now UNTESTED [XFAIL=>UNTES] | 1 | + | XFAIL appears [ =>XFAIL] | 1 | + | UNSUPPORTED now PASS [UNSUP=> PASS] | 1 | + | UNSUPPORTED now XFAIL [UNSUP=>XFAIL] | 1 | + | UNSUPP now UNTESTED [UNSUP=>UNTES] | 1 | + | UNSUPPORTED disappears [UNSUP=> ] | 1 | + | UNSUPPORTED appears [ =>UNSUP] | 1 | + | UNTESTED now PASS [UNTES=> PASS] | 1 | + | UNTESTED now XFAIL [UNTES=>XFAIL] | 1 | + | UNTESTED now UNSUPP [UNTES=>UNSUP] | 1 | + | UNTESTED disappears [UNTES=> ] | 1 | + | UNTESTED appears [ =>UNTES] | 1 | + | UNRESOLVED now PASS [UNRES=> PASS] | 1 | + | UNRESOLVED now XFAIL [UNRES=>XFAIL] | 1 | + | UNRESOLVED now UNSUPP [UNRES=>UNSUP] | 1 | + | UNRESOLVED now UNTESTED [UNRES=>UNTES] | 1 | + | UNRESOLVED disappears [UNRES=> ] | 1 | + | ERROR disappears [ERROR=> ] | 1 | + +------------------------------------------+---------+ + | TOTAL_MINOR_TO_BE_CHECKED | 38 | + +------------------------------------------+---------+ + + - PASS disappears [PASS => ]: + + test5-pass-disappears + + - PASS now XFAIL [PASS =>XFAIL]: + + test4-pass-xfail + + - New PASS [ => PASS]: + + test1-pass-appears + + - PASS now UNSUPPORTED [PASS =>UNSUP]: + + test6-pass-unsupported + + - PASS now UNTESTED [PASS =>UNTES]: + + test7-pass-untested + + - XPASS now PASS [XPASS=> PASS]: + + test1-xpass-pass + + - XPASS now XFAIL [XPASS=>XFAIL]: + + test4-xpass-xfail + + - XPASS now UNSUPPORTED [XPASS=>UNSUP]: + + test6-xpass-unsupported + + - XPASS now UNTESTED [XPASS=>UNTES]: + + test7-xpass-untested + + - XPASS disappears [XPASS=> ]: + + test5-xpass-disappears + + - XFAIL now PASS [XFAIL=> PASS]: + + test1-xfail-pass + + - XFAIL now XPASS [XFAIL=>XPASS]: + + test2-xfail-xpass + + - FAIL now PASS [FAIL => PASS]: + + test1-fail-pass + test1-fail-pass execution + + - FAIL now XPASS [FAIL =>XPASS]: + + test2-fail-xpass + + - FAIL now XFAIL [FAIL =>XFAIL]: + + test4-fail-xfail + + - FAIL now UNSUPPORTED [FAIL =>UNSUP]: + + test6-fail-unsupported + + - FAIL now UNTESTED [FAIL =>UNTES]: + + test7-fail-untested + + - FAIL disappears [FAIL => ]: + + test5-fail-disappears + + - XFAIL now PASS [XFAIL=> PASS]: + + test1-xfail-pass + + - XFAIL now XPASS [XFAIL=>XPASS]: + + test2-xfail-xpass + + - XFAIL now UNSUPPORTED [XFAIL=>UNSUP]: + + test6-xfail-unsupported + + - XFAIL now UNTESTED [XFAIL=>UNTES]: + + test7-xfail-untested + + - XFAIL appears [ =>XFAIL]: + + test4-xfail-appears + + - UNSUPPORTED now PASS [UNSUP=> PASS]: + + test1-unsupported-pass + + - UNSUPPORTED now XFAIL [UNSUP=>XFAIL]: + + test4-unsupported-xfail + + - UNSUPP now UNTESTED [UNSUP=>UNTES]: + + test6-unsupported-untested + + - UNSUPPORTED disappears [UNSUP=> ]: + + test5-unsupported-disappears + + - UNSUPPORTED appears [ =>UNSUP]: + + test4-unsupported-appears + + - UNTESTED now PASS [UNTES=> PASS]: + + test1-untested-pass + + - UNTESTED now XFAIL [UNTES=>XFAIL]: + + test4-untested-xfail + + - UNTESTED now UNSUPP [UNTES=>UNSUP]: + + test6-untested-unsupported + + - UNTESTED disappears [UNTES=> ]: + + test5-untested-disappears + + - UNTESTED appears [ =>UNTES]: + + test4-untested-appears + + - UNRESOLVED now PASS [UNRES=> PASS]: + + test1-unresolved-pass + + - UNRESOLVED now XFAIL [UNRES=>XFAIL]: + + test4-unresolved-xfail + + - UNRESOLVED now UNSUPP [UNRES=>UNSUP]: + + test6-unresolved-unsupported + + - UNRESOLVED now UNTESTED [UNRES=>UNTES]: + + test7-unresolved-untested + + - UNRESOLVED disappears [UNRES=> ]: + + test5-unresolved-disappears + + - ERROR disappears [ERROR=> ]: + + (DejaGnu) proc "scan-dump-tree-not fail_test optimized 1" does not exist. + + -- cgit v1.2.3