diff options
author | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2015-11-20 12:43:32 +0000 |
---|---|---|
committer | Linaro Code Review <review@review.linaro.org> | 2015-11-20 12:43:32 +0000 |
commit | 0476a094113c0a8d3f2d6ab7d0108688926882bb (patch) | |
tree | 7d13e5e9eec94526ee3c7275a9047b109c494255 | |
parent | 66765aa7abf337c45db78e8d4e236dce52992e6a (diff) | |
parent | f47a7b289605161c4b0e34b42dbcc0d94104ec3f (diff) |
Merge "Remove scripts/compare* scripts/CompareJobs.job and test-backport.sh."
-rwxr-xr-x | scripts/CompareJobs.job | 73 | ||||
-rwxr-xr-x | scripts/compare_dg_tests.pl | 435 | ||||
-rwxr-xr-x | scripts/compare_jobs.sh | 211 | ||||
-rwxr-xr-x | scripts/compare_tests | 137 | ||||
-rwxr-xr-x | test-backport.sh | 316 |
5 files changed, 0 insertions, 1172 deletions
diff --git a/scripts/CompareJobs.job b/scripts/CompareJobs.job deleted file mode 100755 index 5d8648ce..00000000 --- a/scripts/CompareJobs.job +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -e - -# Improve debug logs -PRGNAME=`basename $0` -PS4='+ $PRGNAME: ${FUNCNAME+"$FUNCNAME : "}$LINENO: ' - -cat << EOF > ${WORKSPACE}/BUILD-INFO.txt -Format-Version: 0.5 - -Files-Pattern: * -License-Type: open -EOF - -abe_dir="$(dirname $0)/.." -fileserver="148.251.136.42" -status=0 - -dest=/tmp/CompareResults.$$ - -trap "ssh ${fileserver} rm -rf ${dest}" 0 1 2 3 5 9 13 15 - -BUILD_URL=${BUILD_URL:-} - -WORKSPACE=${WORKSPACE:-$PWD/workspace} -LOGSDIR=${WORKSPACE}/artifacts/logs - -rm -rf ${LOGSDIR} -mkdir -p ${LOGSDIR} - -# Expected input parameters (from Jenkins): -# reference_job_name -# reference_number -# this_job_name -# this_number - -if test x"${reference_job_name}" = x ; then - echo "ERROR: No reference_job_name provided." - exit 1 -fi - -if test x"${reference_number}" = x ; then - echo "ERROR: No reference_number provided." - exit 1 -fi - -if test x"${this_job_name}" = x ; then - echo "ERROR: No this_job_name provided." - exit 1 -fi - -if test x"${this_number}" = x ; then - echo "ERROR: No this_number provided." - exit 1 -fi - -ssh ${fileserver} mkdir -p ${dest} -scp ${abe_dir}/scripts/compare_jobs.sh \ - ${abe_dir}/scripts/compare_tests \ - ${abe_dir}/scripts/compare_dg_tests.pl \ - ${abe_dir}/scripts/unstable-tests.txt ${fileserver}:${dest} || status=1 -ssh ${fileserver} bash ${dest}/compare_jobs.sh \ - ${reference_job_name} ${reference_number} \ - ${this_job_name} ${this_number} || status=1 - -# Copy the generated reports -scp ${fileserver}:${dest}/diff-*.txt ${LOGSDIR} -scp ${fileserver}:${dest}/*.xml ${LOGSDIR} -sed -i ${LOGSDIR}/report0.xml -e "s|BUILD_URL|${BUILD_URL}|" - -ssh ${fileserver} rm -rf ${dest} - -exit $status diff --git a/scripts/compare_dg_tests.pl b/scripts/compare_dg_tests.pl deleted file mode 100755 index 6eb4fd12..00000000 --- a/scripts/compare_dg_tests.pl +++ /dev/null @@ -1,435 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; - -use File::Glob; -use Getopt::Long; -use Term::ANSIColor qw(:constants); -use File::Basename; -use Cwd; - -my $app = $0; - -sub read_sum($); -sub read_unstable($); -sub dump_result($); -sub compare_results($$); -sub usage(); -sub print_compare_results_summary($$); -sub nothing($$$$); - - # OK -my $STILL_PASSES = "Still passes [PASS => PASS]"; -my $STILL_FAILS = "Still fails [FAIL => FAIL]"; - -# TO BE CHECKED -my $XFAIL_APPEARS = "Xfail appears [PASS =>XFAIL]"; -my $PASSED_NOW_TIMEOUTS = "Timeout [PASS =>T.OUT]"; -my $FAIL_DISAPPEARS = "Fail disappears [FAIL => ]"; -my $XFAIL_NOW_PASSES = "Expected fail passes [XFAIL=>XPASS]"; -my $FAIL_NOW_PASSES = "Fail now passes [FAIL => PASS]"; -my $NEW_PASSES = "New pass [ => PASS]"; -my $UNHANDLED_CASES = "Unhandled cases [ ..??.. ]"; -my $UNSTABLE_CASES = "Unstable cases, ignored [~RANDOM ]"; - -# ERRORS -my $PASSED_NOW_FAILS = "Passed now fails [PASS => FAIL]"; -my $PASS_DISAPPEARS = "Pass disappears [PASS => ]"; -my $FAIL_APPEARS = "Fail appears [ => FAIL]"; - -my @handler_list = ( - {was=>"PASS", is=>"PASS", cat=>$STILL_PASSES}, - {was=>"PASS", is=>"XPASS", cat=>$STILL_PASSES}, - {was=>"XPASS", is=>"PASS", cat=>$STILL_PASSES}, - {was=>"XPASS", is=>"XPASS", cat=>$STILL_PASSES}, - {was=>"FAIL", is=>"FAIL", cat=>$STILL_FAILS}, - {was=>"FAIL", is=>"XFAIL", cat=>$STILL_FAILS}, - {was=>"XFAIL", is=>"FAIL", cat=>$STILL_FAILS}, - {was=>"XFAIL", is=>"XFAIL", cat=>$STILL_FAILS}, - - {was=>"XPASS", is=>"XFAIL", cat=>$XFAIL_APPEARS}, - {was=>"PASS", is=>"XFAIL", cat=>$XFAIL_APPEARS}, - {was=>"FAIL", is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS}, - {was=>"XFAIL", is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS}, - {was=>"XFAIL", is=>"PASS", cat=>$XFAIL_NOW_PASSES}, - {was=>"XFAIL", is=>"XPASS", cat=>$XFAIL_NOW_PASSES}, - {was=>"FAIL", is=>"PASS", cat=>$FAIL_NOW_PASSES}, - {was=>"FAIL", is=>"XPASS", cat=>$FAIL_NOW_PASSES}, - {was=>"NO_EXIST", is=>"PASS", cat=>$NEW_PASSES}, - {was=>"NO_EXIST", is=>"XPASS", cat=>$NEW_PASSES}, - - {was=>"PASS", is=>"FAIL", handler=>\&handle_pass_fail}, - {was=>"XPASS", is=>"FAIL", handler=>\&handle_pass_fail}, - {was=>"PASS", is=>"NO_EXIST", cat=>$PASS_DISAPPEARS}, - {was=>"XPASS", is=>"NO_EXIST", cat=>$PASS_DISAPPEARS}, - {was=>"NO_EXIST", is=>"FAIL", cat=>$FAIL_APPEARS}, - {was=>"NO_EXIST", is=>"XFAIL", cat=>$XFAIL_APPEARS}, - -# {was=>"NO_EXIST", is=>"NO_EXIST", handler=>\&handle_not_yet_supported} -); - -###################################################### -# TREAT ARGUMENTS - -my $verbose=0; -my $quiet=0; -my $long=0; -my $short=0; -my $debug=0; -my ($testroot, $basename); -my ($ref_file_name, $res_file_name); -my $nounstable=0; -my $unstablefile=0; -my @unstable_markers=(); - -GetOptions ("l" => \$long, - "s" => \$short, - "q" => \$quiet, - "v" => \$verbose, - "dbg" => \$debug, - "testroot=s" => \$testroot, - "basename=s" => \$basename, - "no-unstable" => \$nounstable, - "unstable-tests=s" => \$unstablefile, - "unstable-marker=s" => \@unstable_markers); - -$ref_file_name = $ARGV[0] if ($#ARGV == 1); -$res_file_name = $ARGV[1] if ($#ARGV == 1); - -$ref_file_name = $testroot."/expected_results/".$basename if ($testroot and $basename); -$res_file_name = $testroot."/testing/run/".$basename if ($testroot and $basename); -&usage if (not $ref_file_name or not $res_file_name); - -my ($col_boldred, $col_red, $col_boldgreen, $col_green, $col_boldpink, $col_pink, $col_reset) - = ("\033[31;1m","\033[31;3m","\033[32;1m","\033[32;3m","\033[35;1m","\033[35;2m","\033[0m"); -($col_boldred, $col_red, $col_boldgreen, $col_green, $col_boldpink, $col_pink, $col_reset) - = ("","","","","","","") if (not I_am_interactive()); - -###################################################### -# MAIN PROGRAM -# print "comparing $ref_file_name $res_file_name\n"; - -# If none of the 2 .sum exists, nothing to compare: exit early. -exit 0 if ( (! -e $ref_file_name) && (! -e $res_file_name )); - -my $ref = read_sum($ref_file_name) ; -my $res = read_sum($res_file_name) ; -my @unstablelist = (); - -@unstablelist = read_unstable($unstablefile) if ($unstablefile ne 0); - -compare_results($ref, $res); - -my $final_result = print_compare_results_summary($ref, $res); - -exit $final_result; - -###################################################### -# UTILITIES - -sub empty_result() -{ - my %empty_result;# = {PASS=>0, FAIL=>0, XPASS=>0, XFAIL=>0, UNSUPPORTED=>0, UNTESTED=>0, UNRESOLVED=>0}; - $empty_result{PASS}=$empty_result{FAIL}=$empty_result{XPASS}=$empty_result{XFAIL}=0; - $empty_result{UNSUPPORTED}=$empty_result{UNTESTED}=$empty_result{UNRESOLVED}=$empty_result{NO_EXIST}=0; - return \%empty_result; -} -sub I_am_interactive { - return -t STDIN && -t STDOUT; -} -sub usage() -{ - print "Usage : $app <ref_file.sum> <result_file.sum>\n"; - exit 1; -} - - -###################################################### -# PARSING -sub read_sum($) -{ - my ($sum_file) = @_; - my $res = empty_result(); - my %testcases; - my %unsupported; - $res->{testcases} = \%testcases; - my $pending_timeout=0; - - open SUMFILE, $sum_file or die $!; - while (<SUMFILE>) - { - if (m/^(PASS|XPASS|FAIL|XFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED): (.*)/) - { - my ($diag,$tc) = ($1,$2); - my %tcresult; - $tc =~ s/==[0-9]+== Shadow memory range interleaves with an existing memory mapping. ASan cannot proceed correctly./==<pid>== Shadow memory range interleaves with an existing memory mapping. ASan cannot proceed correctly./; - $testcases{$tc} = empty_result() if (not exists $testcases{$tc}); - $testcases{$tc}->{$diag}++; - $testcases{$tc}->{HAS_TIMED_OUT} = $pending_timeout; - $pending_timeout = 0; - $res->{$diag}++; - } - elsif (m/WARNING: program timed out/) - { - $pending_timeout = 1; - } - elsif (m/^(# of expected passes|# of unexpected failures|# of expected failures|# of known failures|# of unsupported tests|# of untested testcases)\s+(.*)/) - { - $res->{"summary - "."$1"} = $2; - } - elsif (m/^\/.*\/([^\/]+)\s+version\s+(.*)/) - { - $res->{tool} = $1; - $res->{version} = $2; - $res->{version} =~ s/ [-(].*//; - } - } - close SUMFILE; - return $res; -} - -# Parse list on unstable tests -sub read_unstable($) -{ - my ($unstable_file) = @_; - my @unstable_tests = (); - - open UNSTABLEFILE, $unstable_file or die $!; - while (<UNSTABLEFILE>) - { - # Skip lines starting with '#', or with spaces only - if ((/^#/) || (/^[ ]*$/)) - { - } - else - { - chomp; - - my $test = $_; - - # Check if line is of type: target:testname - if (/^(.*):/) - { - foreach my $unstable_marker (@unstable_markers) - { - if ($unstable_marker eq $1) { - # If target matches the one supplied as script - # argument, add the testname to the list - $test =~ s/.*://; - push @unstable_tests, $test; - } - } - } else { - push @unstable_tests, $test; - } - } - } - close UNSTABLEFILE; - return @unstable_tests; -} - -###################################################### -# DIFFING -sub handle_pass_fail($$$$) -{ - my ($ref, $res, $diag_diag, $tc) = @_; - if ($res->{testcases}->{$tc}->{HAS_TIMED_OUT}) - { - push @{$res->{$PASSED_NOW_TIMEOUTS}}, $tc; - } - else - { - push @{$res->{$PASSED_NOW_FAILS}}, $tc; - } -} - -sub compare_results($$) -{ - my ($ref, $res) = @_; - - @{$res->{$STILL_PASSES}} = (); - @{$res->{$STILL_FAILS}} = (); - @{$res->{$PASSED_NOW_FAILS}} = (); - @{$res->{$PASS_DISAPPEARS}} = (); - @{$res->{$FAIL_APPEARS}} = (); - @{$res->{$NEW_PASSES}} = (); - @{$res->{$FAIL_DISAPPEARS}} = (); - @{$res->{$XFAIL_APPEARS}} = (); - @{$res->{$XFAIL_NOW_PASSES}} = (); - @{$res->{$FAIL_NOW_PASSES}} = (); - @{$res->{$PASSED_NOW_TIMEOUTS}} = (); - @{$res->{$UNHANDLED_CASES}} = (); - @{$res->{$UNSTABLE_CASES}} = (); - - #### MERGE REF AND RES - foreach my $key (sort (keys %{$res->{testcases}})) - { - if (not exists $ref->{testcases}->{$key}) { - $ref->{testcases}->{$key} = empty_result(); - $ref->{testcases}->{$key}->{NO_EXIST} = 1; - } - } - foreach my $key (keys %{$ref->{testcases}}) - { - if (not exists $res->{testcases}->{$key}) - { - $res->{testcases}->{$key} = empty_result(); - $res->{testcases}->{$key}->{NO_EXIST} = 1; - } - } - - #### ACTIONS FOR EACH CASES - my %unstable_found; - - foreach my $key (sort (keys %{$ref->{testcases}})) - { - foreach my $diag_diag (@handler_list) - { - if ($ref->{testcases}->{$key}->{$diag_diag->{was}} != $res->{testcases}->{$key}->{$diag_diag->{was}} - and $res->{testcases}->{$key}->{$diag_diag->{is}}) - { - # If testcase is listed as 'unstable' mark it as - # such and skip other processing. - { - if (grep { (index $key,$_)!=-1} @unstablelist) - { - print "[unstable] $key\n" if ($debug); - $unstable_found{$key}=1; - } - else { - print "[$diag_diag->{was} => $diag_diag->{is}] $key\n" if ($debug); - if ($diag_diag->{handler}) - { - $diag_diag->{handler} ($ref, $res, $diag_diag, $key); - } - else - { - push @{$res->{$diag_diag->{cat}}}, $key; - } - } - } - } - } - } - push @{$res->{$UNSTABLE_CASES}}, (sort (keys (%unstable_found))) if ($nounstable == 0); - -} - -###################################################### -# PRINTING -sub print_tclist($@) -{ - my ($cat, @tclist) = @_; - print " - ".$cat.":\n\n ". join("\n ",@tclist) . "\n\n" if (scalar(@tclist)); -} - -sub print_compare_results_summary($$) -{ - my ($ref, $res) = @_; - my $return_value=0; - my $total = 0; - my $rtotal = 0; - my $quiet_reg = $quiet; - - if (not $quiet) - { - printf "Comparing:\n"; - printf "REFERENCE:$ref_file_name\n"; - printf "CURRENT: $res_file_name\n\n"; - } - - #### TESTS STATUS - if (not $quiet and not $short) - { - printf " ` +---------+---------+\n"; - printf "o RUN STATUS : | REF | RES |\n"; - printf " +------------------------------------------+---------+---------+\n"; - printf " | %-40s | %7d | %7d |\n", "Passes [PASS+XPASS]", $ref->{PASS} + $ref->{XPASS}, $res->{PASS} + $res->{XPASS}; - printf " | %-40s | %7d | %7d |\n", "Unexpected fails [FAIL]", $ref->{FAIL}, $res->{FAIL}; - printf " | %-40s | %7d | %7d |\n", "Expected fails [XFAIL]", $ref->{XFAIL}, $res->{XFAIL}; - printf " | %-40s | %7d | %7d |\n", "Unresolved [UNRESOLVED]", $ref->{UNRESOLVED}, $res->{UNRESOLVED}; - printf " | %-40s | %7d | %7d |\n", "Unsupported [UNTESTED+UNSUPPORTED]", $ref->{UNTESTED}+$ref->{UNSUPPORTED}, $res->{UNTESTED}+$res->{UNSUPPORTED}; - printf " +------------------------------------------+---------+---------+\n"; - printf "\n"; - } - - #### REGRESSIONS ? - $quiet_reg=1 if ($short and not scalar(@{$res->{$PASSED_NOW_FAILS}})+scalar(@{$res->{$PASS_DISAPPEARS}})+scalar(@{$res->{$FAIL_APPEARS}})+scalar(@{$res->{$PASSED_NOW_TIMEOUTS}})); - - if (not $quiet_reg) - { - $rtotal = scalar(@{$res->{$PASSED_NOW_FAILS}}) - +scalar(@{$res->{$PASS_DISAPPEARS}}) - +scalar(@{$res->{$FAIL_APPEARS}}) - +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}); - - printf "\n$col_red"."o REGRESSIONS : \n"; - printf " +------------------------------------------+---------+\n"; - printf " | %-40s | %7d |\n", $PASSED_NOW_FAILS, scalar(@{$res->{$PASSED_NOW_FAILS}}) if (scalar(@{$res->{$PASSED_NOW_FAILS}})); - printf " | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}})); - printf " | %-40s | %7d |\n", $PASS_DISAPPEARS, scalar(@{$res->{$PASS_DISAPPEARS}}) if (scalar(@{$res->{$PASS_DISAPPEARS}})); - printf " | %-40s | %7d |\n", $FAIL_APPEARS, scalar(@{$res->{$FAIL_APPEARS}}) if (scalar(@{$res->{$FAIL_APPEARS}})); - printf " +------------------------------------------+---------+\n"; - printf " | %-40s | %7d |\n", "TOTAL_REGRESSIONS", $rtotal; - printf " +------------------------------------------+---------+\n"; - printf "\n"; - - if ($long) - { - print_tclist($PASSED_NOW_FAILS, @{$res->{$PASSED_NOW_FAILS}}); - print_tclist($PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}}); - print_tclist($PASS_DISAPPEARS, @{$res->{$PASS_DISAPPEARS}}); - print_tclist($FAIL_APPEARS, @{$res->{$FAIL_APPEARS}}); - } - printf "$col_reset\n"; - } - - #### MINOR TO BE CHECKED ? - if (not $quiet and not $short) - { - $total = scalar(@{$res->{$XFAIL_NOW_PASSES}})+ - scalar(@{$res->{$FAIL_NOW_PASSES}})+ - scalar(@{$res->{$NEW_PASSES}})+ - scalar(@{$res->{$FAIL_DISAPPEARS}})+ - scalar(@{$res->{$XFAIL_APPEARS}})+ - scalar(@{$res->{$UNHANDLED_CASES}})+ - scalar(@{$res->{$UNSTABLE_CASES}}); - - printf "$col_pink"."o MINOR TO BE CHECKED : \n"; - printf " +------------------------------------------+---------+\n"; - printf " | %-40s | %7d |\n", $XFAIL_APPEARS, scalar(@{$res->{$XFAIL_APPEARS}}) if (scalar(@{$res->{$XFAIL_APPEARS}})); - - printf " | %-40s | %7d |\n", $FAIL_DISAPPEARS, scalar(@{$res->{$FAIL_DISAPPEARS}}) if (scalar(@{$res->{$FAIL_DISAPPEARS}})); - - printf " | %-40s | %7d |\n", $XFAIL_NOW_PASSES, scalar(@{$res->{$XFAIL_NOW_PASSES}}) if (scalar(@{$res->{$XFAIL_NOW_PASSES}})); - printf " | %-40s | %7d |\n", $FAIL_NOW_PASSES, scalar(@{$res->{$FAIL_NOW_PASSES}}) if (scalar(@{$res->{$FAIL_NOW_PASSES}})); - printf " | %-40s | %7d |\n", $NEW_PASSES, scalar(@{$res->{$NEW_PASSES}}) if (scalar(@{$res->{$NEW_PASSES}})); - printf " | %-40s | %7d |\n", $UNHANDLED_CASES, scalar(@{$res->{$UNHANDLED_CASES}}) if (scalar(@{$res->{$UNHANDLED_CASES}})); - printf " | %-40s | %7d |\n", $UNSTABLE_CASES, scalar(@{$res->{$UNSTABLE_CASES}}) if (scalar(@{$res->{$UNSTABLE_CASES}})); - printf " +------------------------------------------+---------+\n"; - printf " | %-40s | %7d |\n", "TOTAL_MINOR_TO_BE_CHECKED", $total; - printf " +------------------------------------------+---------+\n"; - printf "\n"; - - if ($long) - { - print_tclist($XFAIL_NOW_PASSES, @{$res->{$XFAIL_NOW_PASSES}}); - print_tclist($FAIL_NOW_PASSES, @{$res->{$FAIL_NOW_PASSES}}); - print_tclist($FAIL_DISAPPEARS, @{$res->{$FAIL_DISAPPEARS}}); - print_tclist($XFAIL_APPEARS, @{$res->{$XFAIL_APPEARS}}); - print_tclist($UNHANDLED_CASES, @{$res->{$UNHANDLED_CASES}}); - print_tclist($UNSTABLE_CASES, @{$res->{$UNSTABLE_CASES}}); - print_tclist($NEW_PASSES, @{$res->{$NEW_PASSES}}); - } - printf "$col_reset\n"; - } - - $return_value = 1 if ($total); - - $return_value = 2 if ($rtotal); - - # Error if there was no PASS (eg when sth went wrong and no .sum was generated - $return_value = 2 if (($res->{PASS} + $res->{XPASS}) == 0); - - return $return_value; -} diff --git a/scripts/compare_jobs.sh b/scripts/compare_jobs.sh deleted file mode 100755 index 6e8b3836..00000000 --- a/scripts/compare_jobs.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/bin/bash - -mydir="`dirname $0`" -status=0 - -if [ $# != 2 ] -then - echo "Usage: $0 ref_logs new_logs" - exit 1 -fi - -ref_logs=$1 -new_logs=$2 - -tmptargets=/tmp/targets.$$ -trap "rm -f ${tmptargets}" 0 1 2 3 5 9 13 15 - -rm -f ${tmptargets} - -function xml_report_print_row -{ - local target=${1?} - local failed=${2?} - local log_url=BUILD_URL/artifact/artifacts/logs/diff-${target}.txt - local color='#00FF00' - $failed && color='#FF0000' - local message=PASSED - $failed && message=FAILED - cat <<EOF -<tr> - <td>${target}</td> - <td fontattribute="bold" bgcolor="${color}">${message}</td> - <td><![CDATA[<a href="$log_url">log for ${target}</a>]]></td> -</tr> -EOF -} - -function html_report_print_row -{ - local target=${1?} - local failed=${2?} - local log_url=diff-${target}.txt - local color='#00FF00' - $failed && color='#FF0000' - local message=PASSED - $failed && message=FAILED - cat <<EOF - <tr> - <td>${target}</td> - <td style="background-color:$color"><a href="$log_url"><b>${message}</b></a></td> - </tr> -EOF -} - -function xml_report_print_header -{ - cat <<EOF -<section name="Results comparison ${ref_logs} vs ${new_logs}"><table> - <tr> - <td fontattribute="bold" width="120" align="center">Target</td> - <td fontattribute="bold" width="120" align="center">Status</td> - <td fontattribute="bold" width="120" align="center">Log</td> -</tr> -EOF -} - -function html_report_print_header -{ - cat <<EOF -<h1>Results comparison ${ref_logs} vs ${new_logs}</h1> -<table border="1"> - <tr> - <td><b>Target</b></td> - <td><b>Status</b></td> - </tr> -EOF -} - -function xml_report_print_footer -{ - cat <<EOF -</table></section> -EOF -} - -function html_report_print_footer -{ - cat <<EOF -</table> -EOF -} - -function xml_log_print_field -{ - local target=${1?} - local log=${2?} - cat <<EOF - <field name="${target}"> - <![CDATA[ -EOF -cat $log -cat <<EOF - ]]></field> -EOF -} - -function html_log_print_field -{ - local target=${1?} - local log=${2?} - cat <<EOF - <p>${target}</p> -EOF -cat $log -cat <<EOF -EOF -} - -function xml_log_print_header -{ - cat <<EOF -<section name="Logs"> -EOF -} - -function html_log_print_header -{ - cat <<EOF -<h1>Logs</h1> -EOF -} - -function xml_log_print_footer -{ - cat <<EOF -</section> -EOF -} - -function html_log_print_footer -{ - cat <<EOF -EOF -} - -# For the time being, we expect different jobs to store their results -# in similar directories. - -# Build list of all build-targets validated for ${ref_logs} -for dir in `find ${ref_logs}/ -mindepth 1 -maxdepth 1 -type d` -do - basename ${dir} >> ${tmptargets} -done - -# Build list of all build-targets validated for ${new_logs} -for dir in `find ${new_logs}/ -mindepth 1 -maxdepth 1 -type d` -do - basename ${dir} >> ${tmptargets} -done - -if [ -s ${tmptargets} ]; then - buildtargets=`sort -u ${tmptargets}` -fi -rm -f ${tmptargets} - -XML_REPORT=${mydir}/report0.xml -HTML_REPORT=${mydir}/report0.html -rm -f ${XML_REPORT} ${XML_REPORT}.part -rm -f ${HTML_REPORT} ${HTML_REPORT}.part -XML_LOG=${mydir}/report1.xml -HTML_LOG=${mydir}/report1.html -rm -f ${XML_LOG} ${XML_LOG}.part -rm -f ${HTML_LOG} ${HTML_LOG}.part - -xml_report_print_header > ${XML_REPORT}.part -html_report_print_header > ${HTML_REPORT}.part -xml_log_print_header > ${XML_LOG}.part -html_log_print_header > ${HTML_LOG}.part - -for buildtarget in ${buildtargets} -do - ref="${ref_logs}/${buildtarget}" - build="${new_logs}/${buildtarget}" - echo "REF = "${ref} - echo "BUILD = "${build} - failed=false - mylog=${mydir}/diff-${buildtarget}.txt - target=`echo ${buildtarget} | cut -d. -f2` - printf "\t# ============================================================== #\n" > ${mylog} - printf "\t#\t\t*** ${buildtarget} ***\n" >> ${mylog} - printf "\t# ============================================================== #\n\n" >> ${mylog} - [ -d "${build}" -a -d "${ref}" ] && ${mydir}/compare_tests -target ${target} \ - ${ref} ${build} >> ${mylog} || failed=true - - ${failed} && status=1 - xml_report_print_row "${buildtarget}" "${failed}" >> $XML_REPORT.part - html_report_print_row "${buildtarget}" "${failed}" >> $HTML_REPORT.part - xml_log_print_field "${buildtarget}" ${mylog} >> $XML_LOG.part - html_log_print_field "${buildtarget}" ${mylog} >> $HTML_LOG.part -done - -xml_report_print_footer >> ${XML_REPORT}.part -html_report_print_footer >> ${HTML_REPORT}.part -xml_log_print_footer >> ${XML_LOG}.part -html_log_print_footer >> ${HTML_LOG}.part -mv ${XML_REPORT}.part ${XML_REPORT} -mv ${HTML_REPORT}.part ${HTML_REPORT} -mv ${XML_LOG}.part ${XML_LOG} -mv ${HTML_LOG}.part ${HTML_LOG} - -exit ${status} diff --git a/scripts/compare_tests b/scripts/compare_tests deleted file mode 100755 index c2bcb2a7..00000000 --- a/scripts/compare_tests +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/sh -# This script automatically test the given tool with the tool's test cases, -# reporting anything of interest. - -# Written by Mike Stump <mrs@cygnus.com> -# Subdir comparison added by Quentin Neill <quentin.neill@amd.com> -# Modified by Yvan Roux <yvan.roux@linaro.org> and -# Christophe Lyon <christophe.lyon@linaro.org> -usage() -{ - if [ -n "$1" ] ; then - echo "$0: Error: $1" >&2 - echo >&2 - fi - cat >&2 <<EOUSAGE -Usage: $0 [-strict] [-target target-triplet] PREVIOUS CURRENT - -Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest. - - If PREVIOUS and CURRENT are directories, find and compare any *.sum files. - - Unless -strict is given, these discrepancies are not counted as errors: - missing/extra .sum files when comparing directories - tests that failed in PREVIOUS but pass in CURRENT - tests that were not in PREVIOUS but appear in CURRENT - tests in PREVIOUS that are missing in CURRENT - - -target enables to provide the target name to use when parsing - the file containing the list of unstable tests. - - Exit with the following values: - 0 if there is nothing of interest - 1 if there are errors when comparing single test case files - N for the number of errors found when comparing directories -EOUSAGE - exit 2 -} - -export LC_ALL=C - -me="`which $0`" -my_path="`dirname ${me}`" - -tool=gxx - -tmp1=/tmp/$tool-testing.$$a -tmp2=/tmp/$tool-testing.$$b -now_s=/tmp/$tool-testing.$$d -before_s=/tmp/$tool-testing.$$e -lst1=/tmp/$tool-lst1.$$ -lst2=/tmp/$tool-lst2.$$ -lst3=/tmp/$tool-lst3.$$ -lst4=/tmp/$tool-lst4.$$ -lst5=/tmp/$tool-lst5.$$ -sum1=/tmp/$tool-sum1.$$ -sum2=/tmp/$tool-sum2.$$ -tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2" - -[ "$1" = "-strict" ] && strict=$1 && shift -[ "$1" = "-target" ] && target=$2 && shift 2 -[ "$1" = "-?" ] && usage -[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT" - -trap "rm -f $tmps" 0 1 2 3 5 9 13 15 -exit_status=0 - -if [ -d "$1" -a -d "$2" ] ; then - find "$1/" \( -name '*.sum.xz' ! -name '*go.sum.xz' ! -name 'libgo-all.sum.xz' \)>$lst1 - find "$2/" \( -name '*.sum.xz' ! -name '*go.sum.xz' ! -name 'libgo-all.sum.xz' \)>$lst2 - echo "# Comparing directories" - echo "# REFERENCE: $1" - echo "# CURRENT: $2" - echo - # remove leading directory components to compare - sed -e "s|^$1[/]*||" $lst1 | sort >$lst3 - sed -e "s|^$2[/]*||" $lst2 | sort >$lst4 - comm -23 $lst3 $lst4 >$lst5 - if [ -s $lst5 ] ; then - echo "# Extra sum files in Dir1=$1" - sed -e "s|^|< $1/|" $lst5 - echo - [ -n "$strict" ] && exit_status=`expr $exit_status + 1` - fi - comm -13 $lst3 $lst4 >$lst5 - if [ -s $lst5 ] ; then - echo "# Extra sum files in Dir2=$2" - sed -e "s|^|> $2/|" $lst5 - echo - [ -n "$strict" ] && exit_status=`expr $exit_status + 1` - fi - comm -12 $lst3 $lst4 | sort -u >$lst5 - if [ ! -s $lst5 ] ; then - echo "# No common sum files" - exit_status=`expr $exit_status + 1` - exit $exit_status - fi - cmnsums=`cat $lst5 | wc -l` - echo "# Comparing $cmnsums common sum files:" - cat $lst5 - ( for fname in `cat $lst5`; do - bname=`basename $fname .sum.xz` - xzcat $1/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' - done ) >$sum1 - ( for fname in `cat $lst5`; do - bname=`basename $fname .sum.xz` - xzcat $2/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' - done ) >$sum2 - if [ "x${target}" != "x" ] ; then - unstable_target="--unstable-marker ${target}" - fi - for sum in $sum1 $sum2; do - board="$(grep "Running target " $sum | head -n 1 | sed -e "s/Running target //")" - if [ x"$board" != x"" ]; then - unstable_target="$unstable_target --unstable-marker $board" - fi - done - rm -rf ${my_path}/gcc-unstable-tests - git clone http://git.linaro.org/toolchain/gcc-unstable-tests.git ${my_path}/gcc-unstable-tests || exit 1 - ${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl -l --unstable-test=${my_path}/gcc-unstable-tests/unstable-tests.txt ${unstable_target} $sum1 $sum2 - ret=$? - if [ $ret -eq 2 ]; then - exit_status=`expr $exit_status + 1` - echo "# Regressions found" - else - if [ $ret -eq 1 ]; then - echo "# Improvements found" - fi - fi - if [ $exit_status -ne 0 ]; then - echo "# Regressions in $cmnsums common sum files found" - else - echo "# No regression found in $cmnsums common sum files" - fi - exit $exit_status -elif [ -d "$1" -o -d "$2" ] ; then - usage "Must specify either two directories or two files" -fi diff --git a/test-backport.sh b/test-backport.sh deleted file mode 100755 index 2580923d..00000000 --- a/test-backport.sh +++ /dev/null @@ -1,316 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014,2015 Linaro, Inc -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -# - -# To run, this script takes arguments in the same format as abe.sh. The two -# arguments it needs are the target architecture to build, and the gcc backport -# branch name. Example: -# $PATH/test-backport.sh --target arm-linux-gnueabihf gcc.git~4.9-backport-209419 -usage() -{ - # Format this section with 75 columns. - cat << EOF - test-backport.sh [--help] - [-f| --fileserver [remote file server] - [-t] --target triplet [config triplet] - [-s] --snapshots [path to snapshots directory] - [-r] --revisions [revision1,revision2] - [-g] --gitref [git reference directory] - [-b] --branch [GCC git branch] - [-w] --workspace [alternate workspace] - -For example: - -test-backport.sh --fileserver fileserver.linaro.org --target arm-none-eabi --gitref /linaro/shared/snapshots --snapshots ~/workspace/snapshots --branch gcc.git~linaro-4.9-branch - -EOF - exit 0 -} - -# Improve debug logs -PRGNAME=`basename $0` -PS4='+ $PRGNAME: ${FUNCNAME+"$FUNCNAME : "}$LINENO: ' - -if test $# -lt 2; then - echo "ERROR: No branch to build!" - usage - exit 1 -fi - -# load commonly used functions -which_dir="`which $0`" -topdir="`dirname ${which_dir}`" -. "${topdir}/lib/common.sh" || exit 1 - -# since host.conf isn't loaded, get the build architecture -build="`${topdir}/config.guess`" - -# Configure Abe itself. Force the use of bash instead of the Ubuntu -# default of dash as some configure scripts go into an infinite loop with -# dash. Not good... -# In addition, we may have overridden CONFIG_SHELL to use a schroot -# before, and need to reset it now that we are in the schroot. -export CONFIG_SHELL="/bin/bash" -if test x"${debug}" = x"true"; then - export CONFIG_SHELL="/bin/bash -x" -fi - -if test x"${abe_dir}" = x; then - abe_dir=${topdir} -fi - -# Non matrix builds use node_selector, but matrix builds use NODE_NAME -if test x"${node_selector}" != x; then - node="`echo ${node_selector} | tr '-' '_'`" - job=${JOB_NAME} -else - node="`echo ${NODE_NAME} | tr '-' '_'`" - job="`echo ${JOB_NAME} | cut -d '/' -f 1`" -fi - -node=${node:-`uname -n`} -job=${job:-TestBackport} -basedir="/work/logs" -repo="gcc.git" -fileserver="${fileserver:-148.251.136.42}" -branch="linaro-4.9-branch" -user_workspace=${WORKSPACE:-${HOME}/workspace/${job}} -user_snapshots="${user_workspace}/snapshots" -snapshots_ref="/linaro/shared/snapshots" -revision_str="" -user_options="" - -# These are needed by the functions in the ABE library. -local_snapshots=${user_snapshots} -sources_conf=${topdir}/config/sources.conf -NEWWORKDIR=/usr/local/bin/git-new-workdir - -# Whether to exclude some component from 'make check' -excludecheck= - -OPTS="`getopt -o s:r:f:w:o:t:b:g:c:h -l target:,fileserver:,help,snapshots:,branch:,gitref:,repo:,workspace:,revisions:,options,check,excludecheck: -- "$@"`" - -while test $# -gt 0; do - case $1 in - -s|--snapshots) user_snapshots=$2; shift ;; - -f|--fileserver) fileserver=$2; shift ;; - -r|--revisions) revision_str=$2; shift ;; - -g|--gitref) git_reference_dir=$2; shift ;; - -b|--branch) branch=$2; shift ;; - -w|--workspace) user_workspace=$2; shift ;; - -o|--options) user_options=$2; shift ;; - -t|--target) target=$2; shift ;; - -c|--check) check=$2; shift ;; - -h|--help) usage ;; - --excludecheck) excludecheck=$2; shift ;; - *) branch=$1;; - --) break ;; - esac - shift -done - -# If triggered by Gerrit, use the REST API. This assumes the lava-bot account -# is supported by Gerrit, and the public SSH key is available. -if test x"${GERRIT_CHANGE_ID}" != x; then - eval `gerrit_info $HOME` - gerrit_trigger=yes - #gerrit_query_status gcc - - eval "`gerrit_query_patchset ${GERRIT_CHANGE_ID}`" - - # Check out the revision made before this patch gets merged in - checkout "`get_URL gcc.git@${records['parents']}`" - - gerrit_cherry_pick ${gerrit['REFSPEC']} -else - gerrit_trigger=no -fi - -# The two revisions are specified on the command line -if test x"${revision_str}" != x; then - GIT_COMMIT="`echo ${revision_str} | cut -d ',' -f 1`" - GIT_PREVIOUS_COMMIT="`echo ${revision_str} | cut -d ',' -f 2`" -fi - -if test x"${target}" != x"native" -a x"${target}" != x; then - platform="--target ${target}" - targetname=${target} -else - # For native builds, we need to know the effective target name to - # be able to find the results - targetname=${build} -fi - -check="--check ${check:-all}" -check="${check}${excludecheck:+ --excludecheck ${excludecheck}}" - -if test "`echo ${branch} | grep -c gcc.git`" -gt 0; then - branch="`echo ${branch} | sed -e 's:gcc.git~::'`" -fi - -if test x"${git_reference_dir}" != x; then - srcdir="${git_reference_dir}/gcc.git~${branch}" - snapshots_ref="${git_reference_dir}" -else - git_reference_dir="${user_snapshots}" - snapshots_ref="${user_snapshots}" - srcdir="${user_snapshots}/gcc.git~${branch}" -fi - -# Create a build directory -topbuild="${user_workspace}/_build" -if test -d ${topbuild}; then - rm -fr ${topbuild} -fi -mkdir -p ${topbuild} - -local_builds="${topbuild}/builds/${build}/${targetname}" - -# Use the newly created build directory -pushd ${topbuild} - -$CONFIG_SHELL ${abe_dir}/configure --enable-schroot-test --with-local-snapshots=${user_snapshots} --with-git-reference-dir=${snapshots_ref} --with-fileserver=${fileserver} --with-remote-snapshots=/snapshots-ref - -# If Gerrit is specifing the two git revisions, don't try to extract them. -if test x"${gerrit_trigger}" != xyes; then - checkout "`get_URL gcc.git`" - srcdir="`get_srcdir gcc.git`" - - # Due to update cycles, sometimes the branch isn't in the repository yet. - exists="`cd ${srcdir} && git branch -a | grep -c "${branch}"`" - if test "${exists}" -eq 0; then - warning "Branch isn't in GCC repository yet!" - pushd ${srcdir} && git pull - popd - fi - - checkout "`get_URL gcc.git~${branch}`" - srcdir="`get_srcdir gcc.git~${branch}`" - - # Get the last two revisions - declare -a revisions=(`cd ${srcdir} && git log -n 2 | grep ^commit | cut -d ' ' -f 2`) - notice "Validating: ${revisions[0]} and ${revisions[1]}" -else - declare -a revisions=(${records['parents']} ${records['revision']}) -fi - -# Don't update any sources, we should be in sync already from the above. -update="--disable update" - -# Force GCC to not build the docs -export BUILD_INFO="" - -# Don't try to add comments to Gerrit if run manually -if test x"${gerrit_trigger}" != x; then - gerritopt="--enable gerrit" -else - gerritopt="" -fi - -resultsdir="${local_builds}/${node}/abe$$/${target}@" - -i=0 -while test $i -lt ${#revisions[@]}; do - # Don't build if a previous build of this revision exists - dir="${basedir}/gcc-linaro/${branch}/${job}${BUILD_NUMBER}/${build}.${target}/gcc.git@${revisions[$i]}" - exists="`ssh ${fileserver} "if test -d ${dir}; then echo YES; else echo NO; fi"`" - if test x"${exists}" = x"YES"; then - echo "${dir} already exists" - i="`expr $i + 1`" - continue - fi - - bash -x ${topdir}/abe.sh ${gerrit_opt} ${update} ${platform} gcc=gcc.git@${revisions[$i]} --build all --disable make_docs ${check} ${user_options} - if test $? -gt 0; then - echo "ERROR: Abe failed!" - exit 1 - fi - - # Don't update any sources for the other revision. - if test x"${update}" = x; then - update="--disable update" - fi - - # Compress .sum and .log files - sums="`find ${local_builds} -name \*.sum`" - logs="`find ${local_builds} -name \*.log | egrep -v 'config.log|check-.*.log|install.log'`" - xz ${sums} ${logs} - - # FIXME: The way this is currently implemented only handles GCC backports. If binutils - # backports are desired, this will have to be implented here. - sums="`find ${local_builds}/binutils-* -name \*.sum.xz`" - sums="${sums} `find ${local_builds}/gcc.git@${revisions[$i]}-stage2 -name \*.sum.xz`" - # Copy only the log files we want - logs="`find ${local_builds}/binutils-* -name \*.log.xz | egrep -v 'config.log|check-.*.log|install.log'`" - logs="${logs} `find ${local_builds}/gcc.git@${revisions[$i]}-stage2 -name \*.log.xz | egrep -v 'config.log|check-.*.log|install.log'`" - - manifest="`find ${local_builds} -name gcc.git@${revisions[$i]}\*manifest.txt`" - - # xz ${resultsdir}${revisions[$i]}/*.sum ${resultsdir}${revisions[$i]}/*.log - echo "Copying test results files to ${fileserver}:${dir}/ which will take some time..." - ssh ${fileserver} mkdir -p ${dir} - scp -C ${manifest} ${sums} ${logs} ${fileserver}:${dir}/ - # rm -fr ${resultsdir}${revisions[$i]} - - i="`expr $i + 1`" -done - -ret=0 - -# Test results and logs have been copied to this fileserver, so the validation is -# done remotely. -if test x"${fileserver}" != x; then - # Diff the two directories - tmp="/tmp/${node}/abe$$" - ssh ${fileserver} mkdir -p ${tmp} - # report.sh does not support .sum.xz for the moment - #scp -r ${topdir}/scripts/report.sh ${fileserver}:${tmp} - # For comparison with the perl script: - scp ${topdir}/scripts/compare_tests ${fileserver}:${tmp} - scp ${topdir}/scripts/compare_dg_tests.pl ${fileserver}:${tmp} - scp ${topdir}/scripts/unstable-tests.txt ${fileserver}:${tmp} - toplevel="`dirname ${dir}`" - dir1="${toplevel}/${revisions[0]}" - dir2="${toplevel}/${revisions[1]}" -# for i in gcc g++ gfortran libstdc++ ld gas binutils libgomp libitm; do -# ssh ${fileserver} "${tmp}/report.sh ${toplevel} ${i}.sum > ${toplevel}/diff-${i}.txt" -# if test $? -gt 0; then -# ret=1 -# fi -# ssh ${fileserver} cat ${toplevel}/diff-${i}.txt -# done - ssh ${fileserver} "${tmp}/compare_tests -target ${target} ${dir2} ${dir1} > ${toplevel}/diff.txt" - if test $? -ne 0; then - ret=1 - fi - ssh ${fileserver} cat ${toplevel}/diff.txt - - ssh ${fileserver} rm -fr ${tmp} - - echo "### Compared REFERENCE:" - man="`find ${local_builds} -name gcc.git@${revisions[1]}\*manifest.txt`" - cat ${man} - - echo "### with NEW COMMIT:" - man="`find ${local_builds} -name gcc.git@${revisions[0]}\*manifest.txt`" - cat ${man} - - wwwpath="`echo ${toplevel} | sed -e 's:/work::' -e 's:/space::'`" - echo "Full build logs: http://${fileserver}${wwwpath}/" -fi - -exit ${ret} |