From 40f145fa90f6b1f6a1998a528f47cfce1a77203d Mon Sep 17 00:00:00 2001 From: Andrew McDermott Date: Tue, 26 Nov 2013 19:11:52 +0000 Subject: jtreg: add failed results as discrete attachments For each test that fails add the corresponding .jtr (jtreg result) for easier observation from the LAVA dashboard. Also attach a zip of the JTwork and JTreport directory sans the class files. This zip is now attached directly to the overall LAVA test using lava-test-run-attach. The overall execution is now run using jtreg and the results are then parsed and analysed for success/fail. Signed-off-by: Andrew McDermott Change-Id: Id5e918e196892e6935b3361f822f6f129f59772c --- openembedded/jtreg-hotspot-compiler.yaml | 6 +-- openembedded/jtreg-hotspot-gc.yaml | 6 +-- openembedded/jtreg-hotspot-runtime.yaml | 6 +-- openembedded/jtreg-hotspot-sanity.yaml | 6 +-- openembedded/jtreg-hotspot-serviceability.yaml | 6 +-- openembedded/jtreg-hotspot-testlibrary.yaml | 6 +-- openembedded/jtreg-jdk_beans.yaml | 6 +-- openembedded/jtreg-jdk_io.yaml | 6 +-- openembedded/jtreg-jdk_lang.yaml | 6 +-- openembedded/jtreg-jdk_math.yaml | 6 +-- openembedded/jtreg-jdk_net.yaml | 6 +-- openembedded/jtreg-jdk_other.yaml | 6 +-- openembedded/jtreg-jdk_security1.yaml | 6 +-- openembedded/jtreg-jdk_text.yaml | 6 +-- openembedded/jtreg-jdk_time.yaml | 6 +-- openembedded/jtreg-jdk_util.yaml | 6 +-- openembedded/scripts/jtreg-test | 59 ++++++++++++++++++-------- 17 files changed, 90 insertions(+), 65 deletions(-) diff --git a/openembedded/jtreg-hotspot-compiler.yaml b/openembedded/jtreg-hotspot-compiler.yaml index ce821d1..d29b3c7 100644 --- a/openembedded/jtreg-hotspot-compiler.yaml +++ b/openembedded/jtreg-hotspot-compiler.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t hotspot-compiler -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/compiler' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-hotspot-gc.yaml b/openembedded/jtreg-hotspot-gc.yaml index 42a8043..3ba1584 100644 --- a/openembedded/jtreg-hotspot-gc.yaml +++ b/openembedded/jtreg-hotspot-gc.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t hotspot-gc -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/gc' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-hotspot-runtime.yaml b/openembedded/jtreg-hotspot-runtime.yaml index ab51dde..4459ab2 100644 --- a/openembedded/jtreg-hotspot-runtime.yaml +++ b/openembedded/jtreg-hotspot-runtime.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t hotspot-runtime -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/runtime' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-hotspot-sanity.yaml b/openembedded/jtreg-hotspot-sanity.yaml index 9514d02..a6b861a 100644 --- a/openembedded/jtreg-hotspot-sanity.yaml +++ b/openembedded/jtreg-hotspot-sanity.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t hotspot-sanity -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/sanity' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-hotspot-serviceability.yaml b/openembedded/jtreg-hotspot-serviceability.yaml index ed6a221..f96b6ca 100644 --- a/openembedded/jtreg-hotspot-serviceability.yaml +++ b/openembedded/jtreg-hotspot-serviceability.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t hotspot-serviceability -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/serviceability' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-hotspot-testlibrary.yaml b/openembedded/jtreg-hotspot-testlibrary.yaml index 03f5d38..c5b7673 100644 --- a/openembedded/jtreg-hotspot-testlibrary.yaml +++ b/openembedded/jtreg-hotspot-testlibrary.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t hotspot-testlibrary -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/hotspot/test/testlibrary' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_beans.yaml b/openembedded/jtreg-jdk_beans.yaml index 8027e5a..a26a224 100644 --- a/openembedded/jtreg-jdk_beans.yaml +++ b/openembedded/jtreg-jdk_beans.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_beans -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/beans' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_io.yaml b/openembedded/jtreg-jdk_io.yaml index a56145c..3ea069d 100644 --- a/openembedded/jtreg-jdk_io.yaml +++ b/openembedded/jtreg-jdk_io.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_io -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/io' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_lang.yaml b/openembedded/jtreg-jdk_lang.yaml index 9441eb1..9bf99c7 100644 --- a/openembedded/jtreg-jdk_lang.yaml +++ b/openembedded/jtreg-jdk_lang.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_lang -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/lang' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_math.yaml b/openembedded/jtreg-jdk_math.yaml index 61b2760..f5247e0 100644 --- a/openembedded/jtreg-jdk_math.yaml +++ b/openembedded/jtreg-jdk_math.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_math -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/math' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_net.yaml b/openembedded/jtreg-jdk_net.yaml index ad554c2..0e1e136 100644 --- a/openembedded/jtreg-jdk_net.yaml +++ b/openembedded/jtreg-jdk_net.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_net -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/net' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_other.yaml b/openembedded/jtreg-jdk_other.yaml index bb830c4..27ebcbf 100644 --- a/openembedded/jtreg-jdk_other.yaml +++ b/openembedded/jtreg-jdk_other.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_other -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/other' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_security1.yaml b/openembedded/jtreg-jdk_security1.yaml index 839ac83..bca59e7 100644 --- a/openembedded/jtreg-jdk_security1.yaml +++ b/openembedded/jtreg-jdk_security1.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_security1 -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/security1' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_text.yaml b/openembedded/jtreg-jdk_text.yaml index 3ff85d0..5db963a 100644 --- a/openembedded/jtreg-jdk_text.yaml +++ b/openembedded/jtreg-jdk_text.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_text -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/text' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_time.yaml b/openembedded/jtreg-jdk_time.yaml index 7e828f4..664e4f7 100644 --- a/openembedded/jtreg-jdk_time.yaml +++ b/openembedded/jtreg-jdk_time.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_time -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/time' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/jtreg-jdk_util.yaml b/openembedded/jtreg-jdk_util.yaml index 428bff1..e7b33f8 100644 --- a/openembedded/jtreg-jdk_util.yaml +++ b/openembedded/jtreg-jdk_util.yaml @@ -42,7 +42,7 @@ run: - 'cd /lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts' - './jtreg-test -t jdk_util -j $JAVA_VM -p $PRODUCT_HOME -- $JTREG_CONC $JTREG_TEST_MODE $JTREG_TIMEOUT -exclude:$JTREG_EXCLUDE_FILE $JTREG_VERBOSE $PRODUCT_HOME/jtreg/jdk/test/java/util' parse: - pattern: "(?P^(Passed|FAILED)):\\s+(?P(.*))" + pattern: "(?P^(JTREG-SUCCESS|JTREG-FAIL))\\s+(?P(.*))" fixupdict: - Passed: pass - FAILED: fail + JTREG-SUCCESS: pass + JTREG-FAIL: fail diff --git a/openembedded/scripts/jtreg-test b/openembedded/scripts/jtreg-test index 7f3b82b..261e1fa 100755 --- a/openembedded/scripts/jtreg-test +++ b/openembedded/scripts/jtreg-test @@ -58,14 +58,13 @@ shift $((OPTIND-1)) output_dir=/tmp/jtreg/$test_case.$$ rm -rf $output_dir -lava-test-case $test_case --shell \ - jtreg \ - -J${java_vm} \ - -vmoption:${java_vm} \ - -jdk:"${product_home}" \ - -w:$output_dir/JTwork \ - -r:$output_dir/JTreport \ - "$@" +jtreg \ + -J${java_vm} \ + -vmoption:${java_vm} \ + -jdk:"${product_home}" \ + -w:$output_dir/JTwork \ + -r:$output_dir/JTreport \ + "$@" exit_code=$? @@ -75,26 +74,52 @@ shopt -s nullglob # By pushd'ing we make the attached filenames in the LAVA dashboard # much smaller. +mkdir -p $output_dir/JTwork/failed + if [ -d $output_dir/JTreport/text ]; then pushd $output_dir/JTreport/text for i in *.txt; do + lava-test-run-attach $i if [ $i = "summary.txt" ]; then grep 'Failed\.' $i > failed.txt - lava-test-case-attach $test_case failed.txt grep 'Passed\.' $i > success.txt - lava-test-case-attach $test_case success.txt - sed -i '/Not run\./d' $i + lava-test-run-attach failed.txt + lava-test-run-attach success.txt + while read -r line; do + tc=$(echo $line | awk '{print $1}') + y=$(dirname $tc) + z=$(basename $tc .java) + lava-test-case $z --result pass + echo "JTREG-SUCCESS $z" + done < success.txt + while read -r line; do + tc=$(echo $line | awk '{print $1}') + y=$(dirname $tc) + z=$(basename $tc .java) + test_result=$output_dir/JTwork/$y/${z}.jtr + if [ -e $test_result ]; then + pushd $output_dir/JTwork + mkdir -p failed-results/$y + cp $test_result failed-results/$y/${z}.jtr + pushd failed-results + lava-test-case $z --result fail + lava-test-case-attach $z $y/${z}.jtr text/plain + popd + echo "JTREG-FAIL $z" + popd + fi + done < failed.txt fi - lava-test-case-attach $test_case $i done popd fi -if [ -d $output_dir/JTwork/scratch ]; then - pushd $output_dir/JTwork/scratch - for i in hs*.log; do - lava-test-case-attach $test_case $i text/plain - done +if [ -d $output_dir ]; then + pushd $output_dir + # Remove .class files as they are quite large. + find . -name \*.class -print | xargs rm + zip --quiet -r RESULTS-BUNDLE.zip * + lava-test-run-attach RESULTS-BUNDLE.zip popd fi -- cgit v1.2.3