sync tradefed with upstream

Change-Id: I3235541be788e8230cd245057f5b090bf1bfb897
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>

6 files changed, 166 insertions, 62 deletions

diff --git a/automated/android/tradefed/monitor_fastboot.sh b/automated/android/tradefed/monitor_fastboot.sh
new file mode 100755
index 0000000..2345cab
--- /dev/null
+++ b/automated/android/tradefed/monitor_fastboot.sh
@@ -0,0 +1,4 @@
+#!/bin/sh -x
+while true; do
+fastboot boot /lava-lxc/boot*.img
+done
diff --git a/automated/android/tradefed/setup.sh b/automated/android/tradefed/setup.sh
index 82dd0f0..86d8e69 100755
--- a/automated/android/tradefed/setup.sh
+++ b/automated/android/tradefed/setup.sh
@@ -5,13 +5,18 @@
 . ../../lib/sh-test-lib
 . ../../lib/android-test-lib
 
-JDK="openjdk-8-jdk-headless"
-PKG_DEPS="usbutils curl wget zip xz-utils python-lxml python-setuptools python-pexpect aapt lib32z1-dev libc6-dev-i386 lib32gcc1 libc6:i386 libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386 python-dev python-protobuf protobuf-compiler python-virtualenv python-pip python-pexpect"
+if echo "$ANDROID_VERSION" | grep aosp-master ; then
+    JDK="openjdk-9-jdk-headless"
+else
+    JDK="openjdk-8-jdk-headless"
+fi
+PKG_DEPS="usbutils curl wget zip xz-utils python-lxml python-setuptools python-pexpect aapt lib32z1-dev libc6-dev-i386 lib32gcc1 libc6:i386 libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386 python-dev python-protobuf protobuf-compiler python-virtualenv python-pip python-pexpect psmisc"
 
 dist_name
 case "${dist}" in
     ubuntu)
         dpkg --add-architecture i386
+        apt-get update -q
         install_deps "${PKG_DEPS} ${JDK}"
         ;;
     *)
diff --git a/automated/android/tradefed/tradefed-runner.py b/automated/android/tradefed/tradefed-runner.py
index 3143d5b..36172fc 100755
--- a/automated/android/tradefed/tradefed-runner.py
+++ b/automated/android/tradefed/tradefed-runner.py
@@ -60,6 +60,7 @@ def result_parser(xml_file, result_format):
         sys.exit(1)
     logger.info('Test modules in %s: %s'
                 % (xml_file, str(len(root.findall('Module')))))
+    failures_count = 0
     for elem in root.findall('Module'):
         # Naming: Module Name + Test Case Name + Test Name
         if 'abi' in elem.attrib.keys():
@@ -93,19 +94,33 @@ def result_parser(xml_file, result_format):
                 result = '%s_done pass' % module_name
             py_test_lib.add_result(RESULT_FILE, result)
 
-            # print failed test cases for debug
-            test_cases = elem.findall('.//TestCase')
-            for test_case in test_cases:
-                failed_tests = test_case.findall('.//Test[@result="fail"]')
-                for failed_test in failed_tests:
-                    test_name = '%s/%s.%s' % (module_name, test_case.get("name"), failed_test.get("name"))
-                    failures = failed_test.findall('.//Failure')
-                    failure_msg = ''
-                    for failure in failures:
-                        failure_msg = '%s \n %s' % (failure_msg, failure.get('message'))
-
-                    logger.info('%s %s' % (test_name, failure_msg.strip()))
-
+            if args.FAILURES_PRINTED > 0 and failures_count < args.FAILURES_PRINTED:
+                # print failed test cases for debug
+                test_cases = elem.findall('.//TestCase')
+                for test_case in test_cases:
+                    failed_tests = test_case.findall('.//Test[@result="fail"]')
+                    for failed_test in failed_tests:
+                        test_name = '%s/%s.%s' % (module_name,
+                                                  test_case.get("name"),
+                                                  failed_test.get("name"))
+                        failures = failed_test.findall('.//Failure')
+                        failure_msg = ''
+                        for failure in failures:
+                            failure_msg = '%s \n %s' % (failure_msg,
+                                                        failure.get('message'))
+
+                        logger.info('%s %s' % (test_name, failure_msg.strip()))
+                        failures_count = failures_count + 1
+                        if failures_count > args.FAILURES_PRINTED:
+                            logger.info('There are more than %d test cases '
+                                        'failed, the output for the rest '
+                                        'failed test cases will be '
+                                        'skipped.' % (args.FAILURES_PRINTED))
+                            #break the for loop of failed_tests
+                            break
+                    if failures_count > args.FAILURES_PRINTED:
+                        #break the for loop of test_cases
+                        break
 
         if result_format == ATOMIC:
             test_cases = elem.findall('.//TestCase')
@@ -131,6 +146,15 @@ parser.add_argument('-r', dest='RESULTS_FORMAT', required=False,
                     help="The format of the saved results. 'aggregated' means number of \
                     passed and failed tests are recorded for each module. 'atomic' means \
                     each test result is recorded separately")
+
+## The total number of failed test cases to be printed for this job
+## Print too much failures would cause the lava job timed out
+## Default to not print any failures
+parser.add_argument('-f', dest='FAILURES_PRINTED', type=int,
+                    required=False, default=0,
+                    help="Speciy the number of failed test cases to be\
+                    printed, 0 means not print any failures.")
+
 args = parser.parse_args()
 # TEST_PARAMS = args.TEST_PARAMS
 
@@ -180,7 +204,7 @@ if command == 'android-vts/tools/vts-tradefed' and \
     monitor_cmd = 'android-vts/testcases/vts/script/monitor-runner-output.py -m'
     monitor_vts_output = subprocess.Popen(shlex.split(monitor_cmd), stderr=subprocess.STDOUT, stdout=vts_run_details)
 
-child = pexpect.spawn(command, logfile=tradefed_stdout)
+child = pexpect.spawn(command, logfile=tradefed_stdout, searchwindowsize=1024)
 try:
     child.expect(prompt, timeout=60)
     child.sendline(args.TEST_PARAMS)
@@ -188,6 +212,7 @@ except pexpect.TIMEOUT:
     result = 'lunch-tf-shell fail'
     py_test_lib.add_result(RESULT_FILE, result)
 
+fail_to_complete = False
 while child.isalive():
     subprocess.call('echo')
     subprocess.call(['echo', '--- line break ---'])
@@ -195,49 +220,36 @@ while child.isalive():
     adb_command = "adb shell echo OK"
     adb_check = subprocess.Popen(shlex.split(adb_command))
     if adb_check.wait() != 0:
-        logger.debug('adb connection lost! Trying to dump logs of all invocations...')
-        child.sendline('d l')
-        time.sleep(30)
-        subprocess.call(['sh', '-c', '. ../../lib/sh-test-lib && . ../../lib/android-test-lib && adb_debug_info'])
-        logger.debug('"adb devices" output')
-        subprocess.call(['adb', 'devices'])
-        logger.error('adb connection lost!! Will wait for 5 minutes and terminating tradefed shell test as adb connection is lost!')
+        logger.debug('adb connection lost! maybe device is rebooting. Lets check again in 5 minute')
         time.sleep(300)
-        child.terminate(force=True)
-        result = 'check-adb-connectivity fail'
-        py_test_lib.add_result(RESULT_FILE, result)
-        break
+        adb_check = subprocess.Popen(shlex.split(adb_command))
+        if adb_check.wait() != 0:
+            logger.debug('adb connection lost! Trying to dump logs of all invocations...')
+            child.sendline('d l')
+            time.sleep(30)
+            subprocess.call(['sh', '-c', '. ../../lib/sh-test-lib && . ../../lib/android-test-lib && adb_debug_info'])
+            logger.debug('"adb devices" output')
+            subprocess.call(['adb', 'devices'])
+            logger.error('adb connection lost!! Will wait for 5 minutes and terminating tradefed shell test as adb connection is lost!')
+            time.sleep(300)
+            child.terminate(force=True)
+            result = 'check-adb-connectivity fail'
+            py_test_lib.add_result(RESULT_FILE, result)
+            break
     else:
         logger.info('adb device is alive')
+        time.sleep(300)
 
     # Check if all tests finished every minute.
     m = child.expect(['ResultReporter: Full Result:',
                       'ConsoleReporter:.*Test run failed to complete.',
                       pexpect.TIMEOUT],
+                     searchwindowsize=1024,
                      timeout=60)
-    # CTS tests finished correctly.
-    if m == 0:
-        logger.info('Output for debug purposea: m=%d: ResultReporter: Full Result: matched' % m)
-        py_test_lib.add_result(RESULT_FILE, 'tradefed-test-run pass')
-    # CTS tests ended with failure.
-    elif m == 1:
-        logger.info('Output for debug purposea: m=%d: ConsoleReporter:.*Test run failed to complete. matched' % m)
-        py_test_lib.add_result(RESULT_FILE, 'tradefed-test-run fail')
-    # CTS not finished yet, continue to wait.
-    elif m == 2:
-        # Flush pexpect input buffer.
-        child.expect(['.+', pexpect.TIMEOUT, pexpect.EOF], timeout=1)
-        logger.info('Printing tradefed recent output...')
-        subprocess.call(['tail', TRADEFED_STDOUT])
-    else:
-        logger.info('Output for debug purpose: m=%d' % m)
-
     # Once all tests finshed, exit from tf shell to throw EOF, which sets child.isalive() to false.
-    #if m == 0 or m == 1:
     if m == 0:
         try:
-            logger.debug('Try to stop the execution...m=%d' % m)
-            child.expect(prompt, timeout=60)
+            child.expect(prompt, searchwindowsize=1024, timeout=60)
             logger.debug('Sending "exit" command to TF shell...')
             child.sendline('exit')
             child.expect(pexpect.EOF, timeout=60)
@@ -247,9 +259,20 @@ while child.isalive():
             logger.debug('Unsuccessful clean exit, force killing child process...')
             child.terminate(force=True)
             break
-    else:
-        logger.debug('Test execution continued...')
+    # Mark test run as fail when a module or the whole run failed to complete.
+    elif m == 1:
+        fail_to_complete = True
+    # CTS not finished yet, continue to wait.
+    elif m == 2:
+        # Flush pexpect input buffer.
+        child.expect(['.+', pexpect.TIMEOUT, pexpect.EOF], timeout=1)
+        logger.info('Printing tradefed recent output...')
+        subprocess.call(['tail', TRADEFED_STDOUT])
 
+if fail_to_complete:
+    py_test_lib.add_result(RESULT_FILE, 'tradefed-test-run fail')
+else:
+    py_test_lib.add_result(RESULT_FILE, 'tradefed-test-run pass')
 
 logger.info('Tradefed test finished')
 tradefed_logcat.kill()
diff --git a/automated/android/tradefed/tradefed.sh b/automated/android/tradefed/tradefed.sh
index f020542..a03fd5a 100755
--- a/automated/android/tradefed/tradefed.sh
+++ b/automated/android/tradefed/tradefed.sh
@@ -13,13 +13,19 @@ TEST_PATH="android-cts"
 RESULT_FORMAT="aggregated"
 RESULT_FILE="$(pwd)/output/result.txt"
 export RESULT_FILE
+# the default number of failed test cases to be printed
+FAILURES_PRINTED="0"
+# WIFI AP SSID
+AP_SSID=""
+# WIFI AP KEY
+AP_KEY=""
 
 usage() {
-    echo "Usage: $0 [-o timeout] [-n serialno] [-c cts_url] [-t test_params] [-p test_path] [-r <aggregated|atomic>]" 1>&2
+    echo "Usage: $0 [-o timeout] [-n serialno] [-c cts_url] [-t test_params] [-p test_path] [-r <aggregated|atomic>] [-f failures_printed] [-a <ap_ssid>] [-k <ap_key>]" 1>&2
     exit 1
 }
 
-while getopts ':o:n:c:t:p:r:' opt; do
+while getopts ':o:n:c:t:p:r:f:a:k:' opt; do
     case "${opt}" in
         o) TIMEOUT="${OPTARG}" ;;
         n) export ANDROID_SERIAL="${OPTARG}" ;;
@@ -27,6 +33,9 @@ while getopts ':o:n:c:t:p:r:' opt; do
         t) TEST_PARAMS="${OPTARG}" ;;
         p) TEST_PATH="${OPTARG}" ;;
         r) RESULT_FORMAT="${OPTARG}" ;;
+        f) FAILURES_PRINTED="${OPTARG}" ;;
+        a) AP_SSID="${OPTARG}" ;;
+        k) AP_KEY="${OPTARG}" ;;
         *) usage ;;
     esac
 done
@@ -37,6 +46,7 @@ fi
 
 disable_suspend
 wait_boot_completed "${TIMEOUT}"
+disable_suspend
 # wait_homescreen() searches logcat output for
 # 'Displayed com.android.launcher', but the log might be washed away when
 # a lot of logs generated after it. When the function not executed in
@@ -54,23 +64,14 @@ java -version
 
 # Download CTS/VTS test package or copy it from local disk.
 if echo "${TEST_URL}" | grep "^http" ; then
-    df -h
-    ls -l ./
-    ls -l ../
     wget -S --progress=dot:giga "${TEST_URL}"
 else
     cp "${TEST_URL}" ./
 fi
-echo "before run basename"
 file_name=$(basename "${TEST_URL}")
-
-echo "before run unzip"
-pwd
-ls -l ./*
 unzip -q "${file_name}"
 rm -f "${file_name}"
 
-echo "after run unzip"
 if [ -d "${TEST_PATH}/results" ]; then
     mv "${TEST_PATH}/results" "${TEST_PATH}/results_$(date +%Y%m%d%H%M%S)"
 fi
@@ -80,6 +81,9 @@ if [ -e "${TEST_PATH}/testcases/vts/testcases/kernel/linux_kselftest/kselftest_c
     sed -i "/suspend/d" "${TEST_PATH}"/testcases/vts/testcases/kernel/linux_kselftest/kselftest_config.py
 fi
 
+# try to connect wifi if AP information specified
+adb_join_wifi "${AP_SSID}" "${AP_KEY}"
+
 # Run tradefed test.
 info_msg "About to run tradefed shell on device ${ANDROID_SERIAL}"
-./tradefed-runner.py -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULT_FORMAT}"
+./tradefed-runner.py -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULT_FORMAT}" -f "${FAILURES_PRINTED}"
diff --git a/automated/android/tradefed/tradefed.yaml b/automated/android/tradefed/tradefed.yaml
index f9d4f9c..44af753 100644
--- a/automated/android/tradefed/tradefed.yaml
+++ b/automated/android/tradefed/tradefed.yaml
@@ -29,6 +29,11 @@ params:
     # Specify url and token for file uploading.
     URL: "https://archive.validation.linaro.org/artifacts/team/qa/"
     TOKEN: ""
+    AP_SSID: ""
+    AP_KEY: ""
+    # Specify the failures number to be printed
+    FAILURES_PRINTED: "0"
+    TEST_REBOOT_EXPECTED: "false"
 
 run:
     steps:
@@ -41,7 +46,8 @@ run:
         # create test use to run the cts/vts tests
         - useradd -m testuser && echo "testuser created successfully"
         - chown testuser:testuser .
-        - sudo -u testuser ./tradefed.sh  -o "${TIMEOUT}" -c "${TEST_URL}" -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULTS_FORMAT}" -n "${ANDROID_SERIAL}"
+        - if [[ ${TEST_REBOOT_EXPECTED} == "true" ]]; then ./monitor_fastboot.sh & fi
+        - sudo -u testuser ./tradefed.sh  -o "${TIMEOUT}" -c "${TEST_URL}" -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULTS_FORMAT}" -n "${ANDROID_SERIAL}" -f "${FAILURES_PRINTED}" -a "${AP_SSID}" -k "${AP_KEY}"
         # Upload test log and result files to artifactorial.
         - cp -r ./${TEST_PATH}/results ./output/ || true
         - cp -r ./${TEST_PATH}/logs ./output/ || true
@@ -55,3 +61,4 @@ run:
         - userdel testuser -f -r || true
         # When adb device lost, end test job to mark it as 'incomplete'.
         - if ! adb shell echo ok; then error_fatal "adb device $ANDROID_SERIAL lost!"; fi
+        - if [[ ${TEST_REBOOT_EXPECTED} == "true" ]]; then killall monitor_fastboot.sh; fi
diff --git a/automated/android/tradefed/tradefed.yaml.mobile b/automated/android/tradefed/tradefed.yaml.mobile
new file mode 100644
index 0000000..c63196b
--- /dev/null
+++ b/automated/android/tradefed/tradefed.yaml.mobile
@@ -0,0 +1,61 @@
+metadata:
+    name: cts
+    format: "Lava-Test-Shell Test Definition 1.0"
+    description: "Run tradefed based tests in LAVA."
+    maintainer:
+        - milosz.wasilewski@linaro.org
+        - chase.qi@linaro.org
+    os:
+        - debian
+        - ubuntu
+    devices:
+        - lxc
+    scope:
+        - functional
+
+params:
+    SKIP_INSTALL: "false"
+    # Specify timeout in seconds for wait_boot_completed and wait_homescreen.
+    TIMEOUT: "300"
+    # Download CTS package or copy it from local disk.
+    # CTS_URL: "/root/android-cts/linaro/7.1_r1/android-cts-7.1_r1.zip"
+    TEST_URL: "http://testdata.validation.linaro.org/cts/android-cts-7.1_r1.zip"
+    TEST_PARAMS: "run cts -m CtsBionicTestCases --abi arm64-v8a --disable-reboot --skip-preconditions --skip-device-info"
+    # set to the name of the top directory in TEST_URL archive
+    # This should be 'android-cts' for CTS and android-vts for VTS
+    TEST_PATH: "android-cts"
+    # Specify result format: aggregated or atomic
+    RESULTS_FORMAT: "aggregated"
+    # Specify url and token for file uploading.
+    URL: "https://archive.validation.linaro.org/artifacts/team/mobile/"
+    TOKEN: ""
+    AP_SSID: ""
+    AP_KEY: ""
+    # Specify the failures number to be printed
+    FAILURES_PRINTED: "0"
+
+run:
+    steps:
+        - cd ./automated/android/tradefed
+        # Run setup.sh in the original shell to reserve env variables.
+        - . ./setup.sh
+        - echo "after ./setup.sh"
+        # delete the test user to clean environment
+        - userdel testuser -r -f || true
+        # create test use to run the cts/vts tests
+        - useradd -m testuser && echo "testuser created successfully"
+        - chown testuser:testuser .
+        - sudo -u testuser ./tradefed.sh  -o "${TIMEOUT}" -c "${TEST_URL}" -t "${TEST_PARAMS}" -p "${TEST_PATH}" -r "${RESULTS_FORMAT}" -n "${ANDROID_SERIAL}" -f "${FAILURES_PRINTED}"
+        # Upload test log and result files to artifactorial.
+        - cp -r ./${TEST_PATH}/results ./output/ || true
+        - cp -r ./${TEST_PATH}/logs ./output/ || true
+        # Include logs dumped from TF shell 'd l' command.
+        - if ls /tmp/tradefed*; then cp -r /tmp/tradefed* ./output || true; fi
+        - tar caf tradefed-output-$(date +%Y%m%d%H%M%S).tar.xz ./output
+        - ATTACHMENT=$(ls tradefed-output-*.tar.xz)
+        - ../../utils/upload-to-artifactorial.sh -a "${ATTACHMENT}" -u "${URL}" -t "${TOKEN}"
+        # Send test result to LAVA.
+        - ../../utils/send-to-lava.sh ./output/result.txt
+        - userdel testuser -f -r || true
+        # When adb device lost, end test job to mark it as 'incomplete'.
+        - if ! adb shell echo ok; then error_fatal "adb device $ANDROID_SERIAL lost!"; fi