diff options
-rw-r--r-- | android/cts-host.yaml | 23 | ||||
-rw-r--r-- | android/cts-target.yaml | 11 | ||||
-rwxr-xr-x | android/scripts/cts.py | 165 |
3 files changed, 155 insertions, 44 deletions
diff --git a/android/cts-host.yaml b/android/cts-host.yaml index e4632b9..85252a5 100644 --- a/android/cts-host.yaml +++ b/android/cts-host.yaml @@ -5,7 +5,7 @@ metadata: Get user defined CTS test command from JSON and run; The value put in params section in this file is default, user can overwrite them by the values in JSON file." maintainer: - - botao.sun@linaro.org + - milosz.wasilewski@linaro.org os: - ubuntu devices: @@ -18,6 +18,12 @@ install: - xz-utils - python-lxml - python-setuptools + - android-tools-adb + - android-tools-fastboot + - zip + - libc6:i386 + - libncurses5:i386 + - libstdc++6:i386 params: JDK: "default-jdk" @@ -28,12 +34,15 @@ params: run: steps: - - apt-add-repository -y http://ppa.launchpad.net/nilarimogard/webupd8/ubuntu - - apt-get update -y - - apt-get install -y android-tools-adb android-tools-fastboot zip $JDK $JRE libc6:i386 libncurses5:i386 libstdc++6:i386 + - apt-get install -y $JDK $JRE - lava-wait $TEST_NAME-send-ip - IPADDR=`awk -F '=' '{print $2}' /tmp/lava_multi_node_cache.txt` - - adb connect $IPADDR - - adb wait-for-device - - "./android/scripts/cts.py $CTS_URL $TEST_PARAMS" + - export REGEX="([1-9]?[0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" + - echo $IPADDR | grep -P $REGEX + - if [ $? -eq 0 ]; then adb connect $IPADDR; IPADDR=$IPADDR:5555; fi + - adb -s $IPADDR wait-for-device + - "./android/scripts/cts.py $CTS_URL $IPADDR $TEST_PARAMS" + - echo "processes running" + - ps - lava-sync $TEST_NAME-finished + diff --git a/android/cts-target.yaml b/android/cts-target.yaml index e2200be..a742a7a 100644 --- a/android/cts-target.yaml +++ b/android/cts-target.yaml @@ -4,7 +4,7 @@ metadata: description: "Run CTS on Linaro android. Target side. The value put in params section in this file is default, user can overwrite them by the values in JSON file." maintainer: - - botao.sun@linaro.org + - milosz.wasilewski@linaro.org os: - android devices: @@ -19,13 +19,18 @@ metadata: params: TEST_NAME: "android-cts-5.0-armv8" + run: steps: - lava-test-case step1-cat-build-info --shell cat /system/build.prop + - SERIALNO=$(getprop ro.serialno) - IPADDR=$(getprop dhcp.eth0.ipaddress) - if [ -z $IPADDR ]; then netcfg eth0 up; netcfg eth0 dhcp; IPADDR=$(getprop dhcp.eth0.ipaddress); fi - - lava-test-case step2-get-adb --shell getprop service.adb.tcp.port - - lava-test-case step3-set-adb --shell setprop service.adb.tcp.port 5555 + # if serial number exists it's preferred over IP based adb connection + # This means we're on 'usb only' device with WiFi connectivity + - if [ ! -z $SERIALNO ]; then IPADDR=$SERIALNO; fi + - if [ -z $SERIALNO ]; then lava-test-case step2-get-adb --shell getprop service.adb.tcp.port; fi + - if [ -z $SERIALNO ]; then lava-test-case step3-set-adb --shell setprop service.adb.tcp.port 5555; fi - lava-test-case step3-set-adb --shell setprop service.adb.root 1 - lava-test-case step4-stop-adbd --shell stop adbd - lava-test-case step5-start-adbd --shell start adbd diff --git a/android/scripts/cts.py b/android/scripts/cts.py index af08881..c5e9633 100755 --- a/android/scripts/cts.py +++ b/android/scripts/cts.py @@ -19,26 +19,92 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # Author: Botao Sun <botao.sun@linaro.org> +# Author: Milosz Wasilewski <milosz.wasilewski@linaro.org> +import datetime +import gzip import os import sys import shlex -import urllib -import zipfile +import shutil import subprocess +import threading +import urllib import xml.etree.ElementTree as ET +import zipfile + +CTS_STDOUT = "cts_stdout.txt" +CTS_LOGCAT = "cts_logcat.txt" + +class Command(object): + def __init__(self, cmd): + self.cmd = cmd + self.process = None + + def run(self, timeout): + def target(): + print '%s' % datetime.datetime.now() + self.process = subprocess.Popen(self.cmd, shell=True) + self.process.communicate() + + thread = threading.Thread(target=target) + thread.start() + + thread.join(timeout) + if thread.is_alive(): + print 'Terminating process' + self.process.terminate() + thread.join() + return self.process.returncode + + +class Heartbeat(threading.Thread): + def __init__(self, serial, process_list): + threading.Thread.__init__(self) + self.serial = serial + self.process_list = process_list + self.adb_ping = Command("adb -s %s shell echo \"OK\"" % serial) + self._finished = threading.Event() + self._interval = 30.0 + + def setInterval(self, interval): + self._interval = interval + + def shutdown(self): + for process in self.process_list: + print "terminating process: %s" % process.pid + if process.poll() is None: + process.kill() + self._finished.set() + + def run(self): + while 1: + if self._finished.isSet(): return + return_code = self.adb_ping.run(timeout=1) + if return_code != 0: + # terminate the test as adb connection is lost + print "terminating CTS for %s" % self.serial + for process in self.process_list: + print "terminating process: %s" % process.pid + if process.poll() is None: + process.kill() + self._finished.set() + else: + print "%s is alive" % self.serial + self._finished.wait(self._interval) + # Switch to home path of current user to avoid any permission issue home_path = os.environ['HOME'] -os.chdir(home_path) +#os.chdir(home_path) print os.getcwd() debug_switcher = False -def collect_result(testcase_id, result): - if debug_switcher is False: - subprocess.call(['lava-test-case', testcase_id, '--result', result]) - else: - print ['lava-test-case', testcase_id, '--result', result] +#def collect_result(testcase_id, result): +# if debug_switcher is False: +# subprocess.call(['lava-test-case', testcase_id, '--result', result]) +# else: +# print ['lava-test-case', testcase_id, '--result', result] def result_parser(xml_file): tree = ET.parse(xml_file) @@ -47,56 +113,87 @@ def result_parser(xml_file): ET.dump(tree) root = tree.getroot() print 'There are ' + str(len(root.findall('TestPackage'))) + ' Test Packages in this test result file: ' + xml_file - testcase_counter = 0 + #testcase_counter = 0 for elem in root.findall('TestPackage'): # Naming: Package Name + Test Case Name + Test Name if 'abi' in elem.attrib.keys(): package_name = '.'.join([elem.attrib['abi'], elem.attrib['appPackageName']]) else: package_name = elem.attrib['appPackageName'] - for testcase in elem.iter('TestCase'): - testcase_name = testcase.attrib['name'] - for test in testcase.iter('Test'): - testcase_counter = testcase_counter + 1 - test_name = test.attrib['name'] - testcase_id = '.'.join([package_name, testcase_name, test_name]) - result = test.attrib['result'] - collect_result(testcase_id, result) - print 'There are ' + str(testcase_counter) + ' test cases in this test result file: ' + xml_file + tests_executed = len(elem.findall('.//Test')) + tests_passed = len(elem.findall('.//Test[@result="pass"]')) + tests_failed = len(elem.findall('.//Test[@result="fail"]')) + subprocess.call(['lava-test-case', package_name + '_executed', '--result', 'pass', '--measurement', str(tests_executed)]) + subprocess.call(['lava-test-case', package_name + '_passed', '--result', 'pass', '--measurement', str(tests_passed)]) + failed_result = 'pass' + if tests_failed > 0: + failed_result = 'fail' + subprocess.call(['lava-test-case', package_name + '_failed', '--result', failed_result, '--measurement', str(tests_failed)]) + # leave the below code for now as commented + # might be used in future (unlikely) + # for testcase in elem.iter('TestCase'): + # testcase_name = testcase.attrib['name'] + # for test in testcase.iter('Test'): + # testcase_counter = testcase_counter + 1 + # test_name = test.attrib['name'] + # testcase_id = '.'.join([package_name, testcase_name, test_name]) + # result = test.attrib['result'] + # collect_result(testcase_id, result) + #print 'There are ' + str(testcase_counter) + ' test cases in this test result file: ' + xml_file # download and extract the CTS zip package ctsurl = sys.argv[1] +# ToDo this might fail and exit ungracefully ctsfile = urllib.urlretrieve(ctsurl, ctsurl.split('/')[-1]) +print "downloaded %s" % sys.argv[1] +print "unzipping %s" % ctsurl.split('/')[-1] +# ToDo this might fail and exit ungracefully with zipfile.ZipFile(ctsurl.split('/')[-1]) as z: z.extractall() z.close() +print "unzipped CTS package" os.chmod('android-cts/tools/cts-tradefed', 0755) +target_device = sys.argv[2] # receive user input from JSON file and run -cts_stdout = open('cts_stdout.txt', 'w') -command = 'android-cts/tools/cts-tradefed ' + ' '.join([str(para) for para in sys.argv[2:]]) +cts_stdout = open(CTS_STDOUT, 'w') +command = 'android-cts/tools/cts-tradefed ' + ' '.join([str(para) for para in sys.argv[3:]]) print command -return_check = subprocess.call(shlex.split(command), stdout=cts_stdout) -print 'The return value of CTS command run is ' + str(return_check) -if return_check != 0: +return_check = subprocess.Popen(shlex.split(command), stdout=cts_stdout) +cts_logcat_out = open(CTS_LOGCAT, 'w') +cts_logcat_command = "adb logcat" +cts_logcat = subprocess.Popen(shlex.split(cts_logcat_command), stdout=cts_logcat_out) +# start heartbeat process +heartbeat = Heartbeat(target_device, [return_check, cts_logcat]) +heartbeat.daemon=True +heartbeat.start() +if return_check.wait() != 0: # even though the whole command may not run successfully, continue to submit the existing result anyway # add test case CTS-Command-Check to indicate this incident print 'CTS command: ' + command + ' run failed!' - collect_result(testcase_id='CTS-Command-Check', result='fail') + #collect_result(testcase_id='CTS-Command-Check', result='fail') + subprocess.call(['lava-test-case', 'CTS-Command-Check', '--result', 'fail']) +heartbeat.shutdown() cts_stdout.close() +cts_logcat_out.close() # compress then attach the CTS stdout file to LAVA bundle -compress = 'xz -9 cts_stdout.txt' -subprocess.call(shlex.split(compress)) -subprocess.call(['lava-test-run-attach', 'cts_stdout.txt.xz']) +with open(CTS_STDOUT, 'rb') as f_in, gzip.open(CTS_STDOUT + '.gz', 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) +with open(CTS_LOGCAT, 'rb') as f_in, gzip.open(CTS_LOGCAT + '.gz', 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) +subprocess.call(['lava-test-run-attach', CTS_STDOUT + '.gz']) +subprocess.call(['lava-test-run-attach', CTS_LOGCAT + '.gz']) # locate and parse the test result result_dir = 'android-cts/repository/results' test_result = 'testResult.xml' -dir_list = [os.path.join(result_dir, item) for item in os.listdir(result_dir) if os.path.isdir(os.path.join(result_dir, item))==True] -print dir_list -for item in dir_list: - if test_result in os.listdir(item): - result_parser(xml_file=os.path.join(item, test_result)) - else: - print 'Could not find the test result file in ' + item + ', Skip!' +if os.path.exists(result_dir) and os.path.isdir(result_dir): + for root, dirs, files in os.walk(result_dir): + for name in files: + if name.endswith(".zip"): + subprocess.call(['lava-test-run-attach', os.path.join(root, name)]) + if name == test_result: + result_parser(xml_file=os.path.join(root, name)) +# set exit code so LAVA can trust the results +sys.exit(0) |