diff options
Diffstat (limited to 'wlauto/workloads')
185 files changed, 11056 insertions, 0 deletions
diff --git a/wlauto/workloads/__init__.py b/wlauto/workloads/__init__.py new file mode 100644 index 00000000..cd5d64d6 --- /dev/null +++ b/wlauto/workloads/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + diff --git a/wlauto/workloads/andebench/__init__.py b/wlauto/workloads/andebench/__init__.py new file mode 100644 index 00000000..9c62623e --- /dev/null +++ b/wlauto/workloads/andebench/__init__.py @@ -0,0 +1,88 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re + +from wlauto import AndroidUiAutoBenchmark, Parameter, Alias +from wlauto.exceptions import ConfigError + + +class Andebench(AndroidUiAutoBenchmark): + + name = 'andebench' + description = """ + AndEBench is an industry standard Android benchmark provided by The + Embedded Microprocessor Benchmark Consortium (EEMBC). + + http://www.eembc.org/andebench/about.php + + From the website: + + - Initial focus on CPU and Dalvik interpreter performance + - Internal algorithms concentrate on integer operations + - Compares the difference between native and Java performance + - Implements flexible multicore performance analysis + - Results displayed in Iterations per second + - Detailed log file for comprehensive engineering analysis + + """ + package = 'com.eembc.coremark' + activity = 'com.eembc.coremark.splash' + summary_metrics = ['AndEMark Java', 'AndEMark Native'] + + parameters = [ + Parameter('number_of_threads', kind=int, + description='Number of threads that will be spawned by AndEBench.'), + Parameter('single_threaded', kind=bool, + description=""" + If ``true``, AndEBench will run with a single thread. Note: this must + not be specified if ``number_of_threads`` has been specified. + """), + ] + + aliases = [ + Alias('andebenchst', number_of_threads=1), + ] + + regex = re.compile('\s*(?P<key>(AndEMark Native|AndEMark Java))\s*:' + '\s*(?P<value>\d+)') + + def validate(self): + if (self.number_of_threads is not None) and (self.single_threaded is not None): # pylint: disable=E1101 + raise ConfigError('Can\'t specify both number_of_threads and single_threaded parameters.') + + def setup(self, context): + if self.number_of_threads is None: # pylint: disable=access-member-before-definition + if self.single_threaded: # pylint: disable=E1101 + self.number_of_threads = 1 # pylint: disable=attribute-defined-outside-init + else: + self.number_of_threads = self.device.number_of_cores # pylint: disable=W0201 + self.logger.debug('Using {} threads'.format(self.number_of_threads)) + self.uiauto_params['number_of_threads'] = self.number_of_threads + # Called after this setup as modifying uiauto_params + super(Andebench, self).setup(context) + + def update_result(self, context): + super(Andebench, self).update_result(context) + results = {} + with open(self.logcat_log) as fh: + for line in fh: + match = self.regex.search(line) + if match: + data = match.groupdict() + results[data['key']] = data['value'] + for key, value in results.iteritems(): + context.result.add_metric(key, value) + diff --git a/wlauto/workloads/andebench/com.arm.wlauto.uiauto.andebench.jar b/wlauto/workloads/andebench/com.arm.wlauto.uiauto.andebench.jar Binary files differnew file mode 100644 index 00000000..cc1bb880 --- /dev/null +++ b/wlauto/workloads/andebench/com.arm.wlauto.uiauto.andebench.jar diff --git a/wlauto/workloads/andebench/uiauto/build.sh b/wlauto/workloads/andebench/uiauto/build.sh new file mode 100755 index 00000000..d36878cf --- /dev/null +++ b/wlauto/workloads/andebench/uiauto/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +${ANDROID_HOME}/tools/android update project -p . +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.andebench.jar ]]; then + cp bin/com.arm.wlauto.uiauto.andebench.jar .. +fi diff --git a/wlauto/workloads/andebench/uiauto/build.xml b/wlauto/workloads/andebench/uiauto/build.xml new file mode 100644 index 00000000..8d0957f1 --- /dev/null +++ b/wlauto/workloads/andebench/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.andebench" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/andebench/uiauto/project.properties b/wlauto/workloads/andebench/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/andebench/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/andebench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/andebench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..41e36800 --- /dev/null +++ b/wlauto/workloads/andebench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,108 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.andebench; + +import java.util.concurrent.TimeUnit; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "andebench"; + + private static int initialTimeoutSeconds = 20; + private static int shortDelaySeconds = 3; + + public void runUiAutomation() throws Exception{ + Bundle status = new Bundle(); + Bundle params = getParams(); + String numThreads = params.getString("number_of_threads"); + status.putString("product", getUiDevice().getProductName()); + + waitForStartButton(); + setNumberOfThreads(numThreads); + hitStart(); + waitForAndExtractResuts(); + + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + + public void waitForStartButton() throws Exception { + UiSelector selector = new UiSelector(); + UiObject startButton = new UiObject(selector.className("android.widget.ImageButton") + .packageName("com.eembc.coremark")); + if (!startButton.waitForExists(TimeUnit.SECONDS.toMillis(initialTimeoutSeconds))) { + throw new UiObjectNotFoundException("Did not see start button."); + } + } + + public void setNumberOfThreads(String numThreads) throws Exception { + UiSelector selector = new UiSelector(); + getUiDevice().pressMenu(); + + UiObject settingsButton = new UiObject(selector.clickable(true)); + settingsButton.click(); + UiObject threadNumberField = new UiObject(selector.className("android.widget.EditText")); + threadNumberField.clearTextField(); + threadNumberField.setText(numThreads); + + getUiDevice().pressBack(); + sleep(shortDelaySeconds); + // If the device does not have a physical keyboard, a virtual one might have + // poped up when setting the number of threads. If that happend, then the above + // backpress would dismiss the vkb and another one will be necessary to return + // from the settings screen. + if(threadNumberField.exists()) + { + getUiDevice().pressBack(); + sleep(shortDelaySeconds); + } + } + + public void hitStart() throws Exception { + UiSelector selector = new UiSelector(); + UiObject startButton = new UiObject(selector.className("android.widget.ImageButton") + .packageName("com.eembc.coremark")); + startButton.click(); + sleep(shortDelaySeconds); + } + + public void waitForAndExtractResuts() throws Exception { + UiSelector selector = new UiSelector(); + UiObject runningText = new UiObject(selector.textContains("Running...") + .className("android.widget.TextView") + .packageName("com.eembc.coremark")); + runningText.waitUntilGone(TimeUnit.SECONDS.toMillis(600)); + + UiObject resultText = new UiObject(selector.textContains("Results in Iterations/sec:") + .className("android.widget.TextView") + .packageName("com.eembc.coremark")); + resultText.waitForExists(TimeUnit.SECONDS.toMillis(shortDelaySeconds)); + Log.v(TAG, resultText.getText()); + sleep(shortDelaySeconds); + } +} diff --git a/wlauto/workloads/angrybirds/__init__.py b/wlauto/workloads/angrybirds/__init__.py new file mode 100644 index 00000000..92ef6828 --- /dev/null +++ b/wlauto/workloads/angrybirds/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from wlauto import GameWorkload + + +class AngryBirds(GameWorkload): + + name = 'angrybirds' + description = """ + Angry Birds game. + + A very popular Android 2D game. + """ + package = 'com.rovio.angrybirds' + activity = 'com.rovio.ka3d.App' + diff --git a/wlauto/workloads/angrybirds/angrybirds_classic.revent b/wlauto/workloads/angrybirds/angrybirds_classic.revent Binary files differnew file mode 100644 index 00000000..74a46c70 --- /dev/null +++ b/wlauto/workloads/angrybirds/angrybirds_classic.revent diff --git a/wlauto/workloads/angrybirds/revent_files/.empty b/wlauto/workloads/angrybirds/revent_files/.empty new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wlauto/workloads/angrybirds/revent_files/.empty diff --git a/wlauto/workloads/angrybirds_rio/__init__.py b/wlauto/workloads/angrybirds_rio/__init__.py new file mode 100644 index 00000000..c413fd97 --- /dev/null +++ b/wlauto/workloads/angrybirds_rio/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from wlauto import GameWorkload + + +class AngryBirdsRio(GameWorkload): + + name = 'angrybirds_rio' + description = """ + Angry Birds Rio game. + + The sequel to the very popular Android 2D game. + """ + package = 'com.rovio.angrybirdsrio' + activity = 'com.rovio.ka3d.App' + diff --git a/wlauto/workloads/angrybirds_rio/revent_files/.empty b/wlauto/workloads/angrybirds_rio/revent_files/.empty new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wlauto/workloads/angrybirds_rio/revent_files/.empty diff --git a/wlauto/workloads/angrybirds_rio/revent_files/Nexus10.run.revent b/wlauto/workloads/angrybirds_rio/revent_files/Nexus10.run.revent Binary files differnew file mode 100644 index 00000000..bb0e7018 --- /dev/null +++ b/wlauto/workloads/angrybirds_rio/revent_files/Nexus10.run.revent diff --git a/wlauto/workloads/angrybirds_rio/revent_files/Nexus10.setup.revent b/wlauto/workloads/angrybirds_rio/revent_files/Nexus10.setup.revent Binary files differnew file mode 100644 index 00000000..5f2ae879 --- /dev/null +++ b/wlauto/workloads/angrybirds_rio/revent_files/Nexus10.setup.revent diff --git a/wlauto/workloads/anomaly2/__init__.py b/wlauto/workloads/anomaly2/__init__.py new file mode 100644 index 00000000..8060c34c --- /dev/null +++ b/wlauto/workloads/anomaly2/__init__.py @@ -0,0 +1,63 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import json + +from wlauto.common.android.workload import GameWorkload +from wlauto.exceptions import WorkloadError, DeviceError + + +class Anomaly2(GameWorkload): + + name = 'anomaly2' + description = """ + Anomaly 2 game demo and benchmark. + + Plays three scenes from the game, benchmarking each one. Scores reported are intended to + represent overall perceived quality of the game, based not only on raw FPS but also factors + like smoothness. + + """ + package = 'com.elevenbitstudios.anomaly2Benchmark' + activity = 'com.android.Game11Bits.MainActivity' + loading_time = 30 + asset_file = 'obb:com.elevenbitstudios.anomaly2Benchmark.tar.gz' + + def reset(self, context): + pass + + def update_result(self, context): + super(Anomaly2, self).update_result(context) + sent_blobs = {'data': []} + with open(self.logcat_log) as fh: + for line in fh: + if 'sendHttpRequest: json = ' in line: + data = json.loads(line.split('json = ')[1]) + sent_blobs['data'].append(data) + if 'scene' not in data['intValues']: + continue + scene = data['intValues']['scene'] + score = data['intValues']['score'] + fps = data['floatValues']['fps'] + context.result.add_metric('scene_{}_score'.format(scene), score) + context.result.add_metric('scene_{}_fps'.format(scene), fps) + outfile = os.path.join(context.output_directory, 'anomaly2.json') + with open(outfile, 'wb') as wfh: + json.dump(sent_blobs, wfh, indent=4) + + def teardown(self, context): + self.device.execute('am force-stop {}'.format(self.package)) + diff --git a/wlauto/workloads/antutu/__init__.py b/wlauto/workloads/antutu/__init__.py new file mode 100644 index 00000000..305baa7b --- /dev/null +++ b/wlauto/workloads/antutu/__init__.py @@ -0,0 +1,136 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from collections import defaultdict, OrderedDict + +from wlauto import AndroidUiAutoBenchmark, Parameter + + +class Antutu(AndroidUiAutoBenchmark): + + name = 'antutu' + description = """ + AnTuTu Benchmark is an benchmarking tool for Android Mobile Phone/Pad. It + can run a full test of a key project, through the "Memory Performance","CPU + Integer Performance","CPU Floating point Performance","2D 3D Graphics + Performance","SD card reading/writing speed","Database IO" performance + testing, and gives accurate analysis for Andriod smart phones. + + http://www.antutulabs.com/AnTuTu-Benchmark + + From the website: + + AnTuTu Benchmark can support the latest quad-core cpu. In reaching the + overall and individual scores of the hardware, AnTuTu Benchmark could judge + your phone by the scores of the performance of the hardware. By uploading + the scores, Benchmark can view your device in the world rankings, allowing + points to let you know the level of hardware performance equipment. + + """ + #pylint: disable=E1101 + + package = "com.antutu.ABenchMark" + activity = ".ABenchMarkStart" + summary_metrics = ['score', 'Overall_Score'] + + valid_versions = ['3.3.2', '4.0.3', '5.3.0'] + + device_prefs_directory = '/data/data/com.antutu.ABenchMark/shared_prefs' + device_prefs_file = '/'.join([device_prefs_directory, 'com.antutu.ABenchMark_preferences.xml']) + local_prefs_directory = os.path.join(os.path.dirname(__file__), 'shared_prefs') + + parameters = [ + Parameter('version', allowed_values=valid_versions, default=sorted(valid_versions, reverse=True)[0], + description=('Specify the version of AnTuTu to be run. If not specified, the latest available ' + 'version will be used.')), + Parameter('times', kind=int, default=1, + description=('The number of times the benchmark will be executed in a row (i.e. ' + 'without going through the full setup/teardown process). Note: this does ' + 'not work with versions prior to 4.0.3.')), + Parameter('enable_sd_tests', kind=bool, default=False, + description=('If ``True`` enables SD card tests in pre version 4 AnTuTu. These tests ' + 'were know to cause problems on platforms without an SD card. This parameter ' + 'will be ignored on AnTuTu version 4 and higher.')), + ] + + def __init__(self, device, **kwargs): # pylint: disable=W0613 + super(Antutu, self).__init__(device, **kwargs) + self.run_timeout = 6 * 60 * self.times + self.uiauto_params['version'] = self.version + self.uiauto_params['times'] = self.times + self.uiauto_params['enable_sd_tests'] = self.enable_sd_tests + + def update_result(self, context): + super(Antutu, self).update_result(context) + with open(self.logcat_log) as fh: + if self.version == '4.0.3': + metrics = extract_version4_metrics(fh) + else: + metrics = extract_older_version_metrics(fh) + for key, value in metrics.iteritems(): + key = key.replace(' ', '_') + context.result.add_metric(key, value) + + +# Utility functions + +def extract_version4_metrics(fh): + metrics = OrderedDict() + metric_counts = defaultdict(int) + for line in fh: + if 'ANTUTU RESULT:' in line: + parts = line.split(':') + metric = parts[2].strip() + # If times prameter > 1 the same metric will appear + # multiple times in logcat -- we want to collet all of + # them as they're from different iterations. + metric_counts[metric] += 1 + if metric_counts[metric] > 1: + metric += '_' + str(metric_counts[metric]) + + value_string = parts[3].strip() + # Grahics results report resolution in square brackets + # as part of value string. + if ']' in value_string: + value = int(value_string.split(']')[1].strip()) + else: + value = int(value_string) + + metrics[metric] = value + return metrics + + +def extract_older_version_metrics(fh): + metrics = {} + metric_counts = defaultdict(int) + for line in fh: + if 'i/antutu' in line.lower(): + parts = line.split(':') + if not len(parts) == 3: + continue + metric = parts[1].strip() + value = int(parts[2].strip()) + + # If times prameter > 1 the same metric will appear + # multiple times in logcat -- we want to collet all of + # them as they're from different iterations. + metric_counts[metric] += 1 + if metric_counts[metric] > 1: + metric += ' ' + str(metric_counts[metric]) + + metrics[metric] = value + return metrics + diff --git a/wlauto/workloads/antutu/com.arm.wlauto.uiauto.antutu.jar b/wlauto/workloads/antutu/com.arm.wlauto.uiauto.antutu.jar Binary files differnew file mode 100644 index 00000000..35618ede --- /dev/null +++ b/wlauto/workloads/antutu/com.arm.wlauto.uiauto.antutu.jar diff --git a/wlauto/workloads/antutu/uiauto/build.sh b/wlauto/workloads/antutu/uiauto/build.sh new file mode 100755 index 00000000..7cd2e7f0 --- /dev/null +++ b/wlauto/workloads/antutu/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.antutu.jar ]]; then + cp bin/com.arm.wlauto.uiauto.antutu.jar .. +fi diff --git a/wlauto/workloads/antutu/uiauto/build.xml b/wlauto/workloads/antutu/uiauto/build.xml new file mode 100644 index 00000000..a649f2fd --- /dev/null +++ b/wlauto/workloads/antutu/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.antutu" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/antutu/uiauto/project.properties b/wlauto/workloads/antutu/uiauto/project.properties new file mode 100644 index 00000000..4ab12569 --- /dev/null +++ b/wlauto/workloads/antutu/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-19 diff --git a/wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..339f7d72 --- /dev/null +++ b/wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,295 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.antutu; + +import java.util.Set; +import java.util.HashSet; +import java.util.concurrent.TimeUnit; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.core.UiCollection; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "antutu"; + + private static int initialTimeoutSeconds = 20; + + public void runUiAutomation() throws Exception{ + Bundle parameters = getParams(); + + String version = parameters.getString("version"); + boolean enableSdTests = Boolean.parseBoolean(parameters.getString("enable_sd_tests")); + + int times = Integer.parseInt(parameters.getString("times")); + if (times < 1) { + times = 1; + } + + if (version.equals("4.0.3") || version.equals("5.3.0")){ + int iteration = 0; + dismissNewVersionNotificationIfNecessary(); + hitTestButton(); + while (true) { + if (version.equals("5.3.0")) + hitTestButtonVersion5(); + else + hitTestButton(); + + waitForVersion4Results(); + viewDetails(); + extractResults(); + iteration++; + if (iteration >= times) { + break; + } + returnToTestScreen(); + } + } else { // version earlier than 4.0.3 + dismissReleaseNotesDialogIfNecessary(); + if(!enableSdTests){ + disableSdCardTests(); + } + hitStart(); + waitForAndViewResults(); + } + + Bundle status = new Bundle(); + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + + public boolean dismissNewVersionNotificationIfNecessary() throws Exception { + UiSelector selector = new UiSelector(); + UiObject closeButton = new UiObject(selector.text("Cancel")); + if (closeButton.waitForExists(TimeUnit.SECONDS.toMillis(initialTimeoutSeconds))) { + closeButton.click(); + sleep(1); // diaglog dismissal + return true; + } else { + return false; + } + } + + public boolean dismissReleaseNotesDialogIfNecessary() throws Exception { + UiSelector selector = new UiSelector(); + UiObject closeButton = new UiObject(selector.text("Close")); + if (closeButton.waitForExists(TimeUnit.SECONDS.toMillis(initialTimeoutSeconds))) { + closeButton.click(); + sleep(1); // diaglog dismissal + return true; + } else { + return false; + } + } + + public void hitTestButton() throws Exception { + UiSelector selector = new UiSelector(); + UiObject test = new UiObject(selector.text("Test") + .className("android.widget.Button")); + test.waitForExists(initialTimeoutSeconds); + test.click(); + sleep(1); // possible tab transtion + } + + /* In version 5 of antutu, the test has been changed from a button widget to a textview */ + + public void hitTestButtonVersion5() throws Exception { + UiSelector selector = new UiSelector(); + UiObject test = new UiObject(selector.resourceId("com.antutu.ABenchMark:id/start_test_region") + .className("android.widget.TextView")); + test.waitForExists(initialTimeoutSeconds); + test.click(); + sleep(1); // possible tab transtion + } + + + + public void hitTest() throws Exception { + UiSelector selector = new UiSelector(); + UiObject test = new UiObject(selector.text("Test")); + test.click(); + sleep(1); // possible tab transtion + } + + public void disableSdCardTests() throws Exception { + UiSelector selector = new UiSelector(); + UiObject custom = new UiObject(selector.textContains("Custom")); + custom.click(); + sleep(1); // tab transition + + UiObject sdCardButton = new UiObject(selector.text("SD card IO")); + sdCardButton.click(); + } + + public void hitStart() throws Exception { + UiSelector selector = new UiSelector(); + Log.v(TAG, "Start the test"); + UiObject startButton = new UiObject(selector.text("Start Test") + .className("android.widget.Button")); + startButton.click(); + } + + public void waitForVersion4Results() throws Exception { + // The observed behaviour seems to vary between devices. On some platforms, + // the benchmark terminates in the barchart screen; on others, it terminates in + // details screen. So we have to wait for either and then act appropriatesl (on the barchart + // screen a back button press is required to get to the details screen. + UiSelector selector = new UiSelector(); + UiObject barChart = new UiObject(new UiSelector().className("android.widget.TextView") + .text("Bar Chart")); + UiObject detailsButton = new UiObject(new UiSelector().className("android.widget.Button") + .text("Details")); + for (int i = 0; i < 60; i++) { + if (detailsButton.exists() || barChart.exists()) { + break; + } + sleep(5); + } + + if (barChart.exists()) { + getUiDevice().pressBack(); + } + } + + public void viewDetails() throws Exception { + UiSelector selector = new UiSelector(); + UiObject detailsButton = new UiObject(new UiSelector().className("android.widget.Button") + .text("Details")); + detailsButton.clickAndWaitForNewWindow(); + } + + public void extractResults() throws Exception { + extractOverallResult(); + extractSectionResults(); + } + + public void extractOverallResult() throws Exception { + UiSelector selector = new UiSelector(); + UiSelector resultTextSelector = selector.className("android.widget.TextView").index(0); + UiSelector relativeLayoutSelector = selector.className("android.widget.RelativeLayout").index(1); + UiObject result = new UiObject(selector.className("android.widget.LinearLayout") + .childSelector(relativeLayoutSelector) + .childSelector(resultTextSelector)); + if (result.exists()) { + Log.v(TAG, String.format("ANTUTU RESULT: Overall Score: %s", result.getText())); + } + } + + public void extractSectionResults() throws Exception { + UiSelector selector = new UiSelector(); + Set<String> processedMetrics = new HashSet<String>(); + + actuallyExtractSectionResults(processedMetrics); + UiScrollable resultsList = new UiScrollable(selector.className("android.widget.ScrollView")); + // Note: there is an assumption here that the entire results list fits on at most + // two screens on the device. Given then number of entries in the current + // antutu verion and the devices we're dealing with, this is a reasonable + // assumption. But if this changes, this will need to be adapted to scroll more + // slowly. + resultsList.scrollToEnd(10); + actuallyExtractSectionResults(processedMetrics); + } + + public void actuallyExtractSectionResults(Set<String> processedMetrics) throws Exception { + UiSelector selector = new UiSelector(); + + for (int i = 1; i < 8; i += 2) { + UiObject table = new UiObject(selector.className("android.widget.TableLayout").index(i)); + for (int j = 0; j < 3; j += 2) { + UiObject row = table.getChild(selector.className("android.widget.TableRow").index(j)); + UiObject metric = row.getChild(selector.className("android.widget.TextView").index(0)); + UiObject value = row.getChild(selector.className("android.widget.TextView").index(1)); + + if (metric.exists() && value.exists()) { + String metricText = metric.getText(); + if (!processedMetrics.contains(metricText)) { + Log.v(TAG, String.format("ANTUTU RESULT: %s %s", metric.getText(), value.getText())); + processedMetrics.add(metricText); + } + } + } + } + } + + public void returnToTestScreen() throws Exception { + getUiDevice().pressBack(); + UiSelector selector = new UiSelector(); + UiObject retestButton = new UiObject(selector.text("Test Again") + .className("android.widget.Button")); + retestButton.clickAndWaitForNewWindow(); + } + + public void waitForAndViewResults() throws Exception { + UiSelector selector = new UiSelector(); + UiObject submitTextView = new UiObject(selector.text("Submit Scores") + .className("android.widget.TextView")); + UiObject detailTextView = new UiObject(selector.text("Detailed Scores") + .className("android.widget.TextView")); + UiObject commentTextView = new UiObject(selector.text("User comment") + .className("android.widget.TextView")); + boolean foundResults = false; + for (int i = 0; i < 60; i++) { + if (detailTextView.exists() || submitTextView.exists() || commentTextView.exists()) { + foundResults = true; + break; + } + sleep(5); + } + + if (!foundResults) { + throw new UiObjectNotFoundException("Did not see AnTuTu results screen."); + } + + if (commentTextView.exists()) { + getUiDevice().pressBack(); + } + // Yes, sometimes, it needs to be done twice... + if (commentTextView.exists()) { + getUiDevice().pressBack(); + } + + if (detailTextView.exists()) { + detailTextView.click(); + sleep(1); // tab transition + + UiObject testTextView = new UiObject(selector.text("Test") + .className("android.widget.TextView")); + if (testTextView.exists()) { + testTextView.click(); + sleep(1); // tab transition + } + + UiObject scoresTextView = new UiObject(selector.text("Scores") + .className("android.widget.TextView")); + if (scoresTextView.exists()) { + scoresTextView.click(); + sleep(1); // tab transition + } + } + } +} diff --git a/wlauto/workloads/applaunch/__init__.py b/wlauto/workloads/applaunch/__init__.py new file mode 100644 index 00000000..4e2cac14 --- /dev/null +++ b/wlauto/workloads/applaunch/__init__.py @@ -0,0 +1,169 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101 + +from __future__ import division +import os + +try: + import jinja2 +except ImportError: + jinja2 = None + +from wlauto import Workload, settings, Parameter +from wlauto.exceptions import WorkloadError +from wlauto.utils.hwmon import discover_sensors +from wlauto.utils.misc import get_meansd +from wlauto.utils.types import boolean, identifier, list_of_strs + + +THIS_DIR = os.path.dirname(__file__) +TEMPLATE_NAME = 'device_script.template' +SCRIPT_TEMPLATE = os.path.join(THIS_DIR, TEMPLATE_NAME) + +APP_CONFIG = { + 'browser': { + 'package': 'com.android.browser', + 'activity': '.BrowserActivity', + 'options': '-d about:blank', + }, + 'calculator': { + 'package': 'com.android.calculator2', + 'activity': '.Calculator', + 'options': '', + }, + 'calendar': { + 'package': 'com.android.calendar', + 'activity': '.LaunchActivity', + 'options': '', + }, +} + + +class ApplaunchWorkload(Workload): + + name = 'applaunch' + description = """ + Measures the time and energy used in launching an application. + + """ + + parameters = [ + Parameter('app', default='browser', allowed_values=['calculator', 'browser', 'calendar'], + description='The name of the application to measure.'), + Parameter('set_launcher_affinity', kind=bool, default=True, + description=('If ``True``, this will explicitly set the affinity of the launcher ' + 'process to the A15 cluster.')), + Parameter('times', kind=int, default=8, + description='Number of app launches to do on the device.'), + Parameter('measure_energy', kind=boolean, default=False, + description=""" + Specfies wether energy measurments should be taken during the run. + + .. note:: This depends on appropriate sensors to be exposed through HWMON. + + """), + Parameter('cleanup', kind=boolean, default=True, + description='Specifies whether to clean up temporary files on the device.'), + ] + + def __init__(self, device, **kwargs): + super(ApplaunchWorkload, self).__init__(device, **kwargs) + if not jinja2: + raise WorkloadError('Please install jinja2 Python package: "sudo pip install jinja2"') + filename = '{}-{}.sh'.format(self.name, self.app) + self.host_script_file = os.path.join(settings.meta_directory, filename) + self.device_script_file = os.path.join(self.device.working_directory, filename) + self._launcher_pid = None + self._old_launcher_affinity = None + self.sensors = [] + + def on_run_init(self, context): # pylint: disable=W0613 + if self.measure_energy: + self.sensors = discover_sensors(self.device, ['energy']) + for sensor in self.sensors: + sensor.label = identifier(sensor.label).upper() + + def setup(self, context): + self.logger.debug('Creating script {}'.format(self.host_script_file)) + with open(self.host_script_file, 'w') as wfh: + env = jinja2.Environment(loader=jinja2.FileSystemLoader(THIS_DIR)) + template = env.get_template(TEMPLATE_NAME) + wfh.write(template.render(device=self.device, # pylint: disable=maybe-no-member + sensors=self.sensors, + iterations=self.times, + package=APP_CONFIG[self.app]['package'], + activity=APP_CONFIG[self.app]['activity'], + options=APP_CONFIG[self.app]['options'], + )) + self.device_script_file = self.device.install(self.host_script_file) + if self.set_launcher_affinity: + self._set_launcher_affinity() + self.device.clear_logcat() + + def run(self, context): + self.device.execute('sh {}'.format(self.device_script_file), timeout=300) + + def update_result(self, context): + result_files = ['time.result'] + result_files += ['{}.result'.format(sensor.label) for sensor in self.sensors] + for filename in result_files: + host_result_file = os.path.join(context.output_directory, filename) + device_result_file = self.device.path.join(self.device.working_directory, filename) + self.device.pull_file(device_result_file, host_result_file) + + with open(host_result_file) as fh: + if filename == 'time.result': + values = [v / 1000 for v in map(int, fh.read().split())] + _add_metric(context, 'time', values, 'Seconds') + else: + metric = filename.replace('.result', '').lower() + numbers = iter(map(int, fh.read().split())) + deltas = [(after - before) / 1000000 for before, after in zip(numbers, numbers)] + _add_metric(context, metric, deltas, 'Joules') + + def teardown(self, context): + if self.set_launcher_affinity: + self._reset_launcher_affinity() + if self.cleanup: + self.device.delete_file(self.device_script_file) + + def _set_launcher_affinity(self): + try: + self._launcher_pid = self.device.get_pids_of('com.android.launcher')[0] + result = self.device.execute('taskset -p {}'.format(self._launcher_pid), busybox=True, as_root=True) + self._old_launcher_affinity = int(result.split(':')[1].strip(), 16) + + cpu_ids = [i for i, x in enumerate(self.device.core_names) if x == 'a15'] + if not cpu_ids or len(cpu_ids) == len(self.device.core_names): + self.logger.debug('Cannot set affinity.') + return + + new_mask = reduce(lambda x, y: x | y, cpu_ids, 0x0) + self.device.execute('taskset -p 0x{:X} {}'.format(new_mask, self._launcher_pid), busybox=True, as_root=True) + except IndexError: + raise WorkloadError('Could not set affinity of launcher: PID not found.') + + def _reset_launcher_affinity(self): + command = 'taskset -p 0x{:X} {}'.format(self._old_launcher_affinity, self._launcher_pid) + self.device.execute(command, busybox=True, as_root=True) + + +def _add_metric(context, metric, values, units): + mean, sd = get_meansd(values) + context.result.add_metric(metric, mean, units) + context.result.add_metric(metric + ' sd', sd, units, lower_is_better=True) + diff --git a/wlauto/workloads/applaunch/device_script.template b/wlauto/workloads/applaunch/device_script.template new file mode 100644 index 00000000..d1313db9 --- /dev/null +++ b/wlauto/workloads/applaunch/device_script.template @@ -0,0 +1,69 @@ +#!{{ device.binaries_directory.rstrip('/') }}/sh + + +{% for sensor in sensors %} +GET_{{ sensor.label }}="cat {{ sensor.filepath }}" +{% endfor %} + +LAUNCH_COMMAND="am start -W -n {{ package }}/{{ activity }} {{ options }}" +STOP_COMMAND="am force-stop {{ package }}" +TEMP_FILE=tmp.txt + +TIME_RESULT="" +{% for sensor in sensors %} +{{ sensor.label }}="" +{% endfor %} + +cd {{ device.working_directory }} + +# esc esc down down down ENTER (this should bring up the apps menu) +input keyevent 111 +sleep 1 +input keyevent 111 +sleep 1 +input keyevent 20 +sleep 1 +input keyevent 20 +sleep 1 +input keyevent 20 +sleep 1 +input keyevent 66 +sleep 1 + +# Warm up caches. +$LAUNCH_COMMAND +$STOP_COMMAND +$LAUNCH_COMMAND +$STOP_COMMAND +$LAUNCH_COMMAND +$STOP_COMMAND + +for i in $(busybox seq 1 {{ iterations }}) +do + {% for sensor in sensors %} + {{ sensor.label }}="${{ sensor.label }} `$GET_{{ sensor.label }}`" + {% endfor %} + + $LAUNCH_COMMAND > $TEMP_FILE + + {% for sensor in sensors %} + {{ sensor.label }}="${{ sensor.label }} `$GET_{{ sensor.label }}`" + {% endfor %} + + TIME=`busybox awk '{if($1~"TotalTime") print $2}' $TEMP_FILE` + TIME_RESULT="$TIME_RESULT $TIME" + {% if cleanup %} + rm $TEMP_FILE + {% endif %} + + $STOP_COMMAND + sleep 2 +done + +{% for sensor in sensors %} +echo ${{ sensor.label }} > {{ sensor.label }}.result +{% endfor %} +echo $TIME_RESULT > time.result +# esc esc down down down ENTER (this should bring up the apps menu) +input keyevent 111 +sleep 1 diff --git a/wlauto/workloads/audio/__init__.py b/wlauto/workloads/audio/__init__.py new file mode 100644 index 00000000..75244e45 --- /dev/null +++ b/wlauto/workloads/audio/__init__.py @@ -0,0 +1,102 @@ +# Copyright 2012-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=E1101,W0201 +import os +import time +import urllib + +from wlauto import settings, Workload, Parameter +from wlauto.exceptions import ConfigError +from wlauto.utils.types import boolean + + +DEFAULT_AUDIO_FILE_URL = "http://archive.org/download/PachelbelsCanoninD/Canon_in_D_Piano.mp3" + + +class Audio(Workload): + + name = 'audio' + description = """ + Audio workload plays an MP3 file using the built-in music player. By default, + it plays Canon_in_D_Pieano.mp3 for 30 seconds. + + """ + + parameters = [ + Parameter('duration', kind=int, default=30, + description='The duration the music will play for in seconds.'), + Parameter('audio_file', default=os.path.join(settings.dependencies_directory, 'Canon_in_D_Piano.mp3'), + description='''The (on-host) path to the audio file to be played. + + .. note:: If the default file is not present locally, it will be downloaded. + '''), + Parameter('perform_cleanup', kind=boolean, default=False, + description='If ``True``, workload files on the device will be deleted after execution.'), + Parameter('clear_file_cache', kind=boolean, default=True, + description='Clear the the file cache on the target device prior to running the workload.') + ] + + def init_resources(self, context): + if not os.path.isfile(self.audio_file): + self._download_audio_file() + + def setup(self, context): + self.on_device_file = os.path.join(self.device.working_directory, + os.path.basename(self.audio_file)) + + self.device.push_file(self.audio_file, self.on_device_file, timeout=120) + + # Open the browser with default page + self.device.execute('am start -n com.android.browser/.BrowserActivity about:blank') + time.sleep(5) + + # Stop the browser if already running and wait for it to stop + self.device.execute('am force-stop com.android.browser') + time.sleep(5) + + # Clear the logs + self.device.clear_logcat() + + # Clear browser cache + self.device.execute('pm clear com.android.browser') + + if self.clear_file_cache: + self.device.execute('sync') + self.device.set_sysfile_value('/proc/sys/vm/drop_caches', 3) + + # Start the background music + self.device.execute('am start -W -S -n com.android.music/.MediaPlaybackActivity -d {}'.format(self.on_device_file)) + + # Launch the browser to blank the screen + self.device.execute('am start -W -n com.android.browser/.BrowserActivity about:blank') + time.sleep(5) # Wait for browser to be properly launched + + def run(self, context): + time.sleep(self.duration) + + def update_result(self, context): + # Stop the browser + self.device.execute('am force-stop com.android.browser') + # Stop the audio + self.device.execute('am force-stop com.android.music') + + def teardown(self, context): + if self.perform_cleanup: + self.device.delete_file(self.on_device_file) + + def _download_audio_file(self): + self.logger.debug('Downloading audio file from {}'.format(DEFAULT_AUDIO_FILE_URL)) + urllib.urlretrieve(DEFAULT_AUDIO_FILE_URL, self.audio_file) + diff --git a/wlauto/workloads/bbench/__init__.py b/wlauto/workloads/bbench/__init__.py new file mode 100644 index 00000000..1843ad69 --- /dev/null +++ b/wlauto/workloads/bbench/__init__.py @@ -0,0 +1,231 @@ +# Copyright 2012-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=E1101,W0201 +import os +import time +import urllib +import tarfile +import shutil +import json +import re + +from collections import defaultdict + +from wlauto import settings, Workload, Parameter, Alias, Executable +from wlauto.exceptions import ConfigError +from wlauto.utils.types import boolean + +DEFAULT_BBENCH_FILE = "http://bbench.eecs.umich.edu/bbench/bbench_2.0.tgz" +DOWNLOADED_FILE_NAME = "bbench_2.0.tgz" +BBENCH_SERVER_NAME = 'bbench_server' +PATCH_FILES = os.path.join(os.path.dirname(__file__), "patches") +DEFAULT_AUDIO_FILE = "http://archive.org/download/PachelbelsCanoninD/Canon_in_D_Piano.mp3" +DEFAULT_AUDIO_FILE_NAME = 'Canon_in_D_Piano.mp3' + + +class BBench(Workload): + + name = 'bbench' + description = """ + BBench workload opens the built-in browser and navigates to, and + scrolls through, some preloaded web pages and ends the workload by trying to + connect to a local server it runs after it starts. It can also play the + workload while it plays an audio file in the background. + + """ + + summary_metrics = ['Mean Latency'] + + parameters = [ + Parameter('with_audio', kind=boolean, default=False, + description=('Specifies whether an MP3 should be played in the background during ' + 'workload execution.')), + Parameter('server_timeout', kind=int, default=300, + description='Specifies the timeout (in seconds) before the server is stopped.'), + Parameter('force_dependency_push', kind=boolean, default=False, + description=('Specifies whether to push dependency files to the device to the device ' + 'if they are already on it.')), + Parameter('audio_file', default=os.path.join(settings.dependencies_directory, 'Canon_in_D_Piano.mp3'), + description=('The (on-host) path to the audio file to be played. This is only used if ' + '``with_audio`` is ``True``.')), + Parameter('perform_cleanup', kind=boolean, default=False, + description='If ``True``, workload files on the device will be deleted after execution.'), + Parameter('clear_file_cache', kind=boolean, default=True, + description='Clear the the file cache on the target device prior to running the workload.'), + Parameter('browser_package', default='com.android.browser', + description='Specifies the package name of the device\'s browser app.'), + Parameter('browser_activity', default='.BrowserActivity', + description='Specifies the startup activity name of the device\'s browser app.'), + ] + + aliases = [ + Alias('bbench_with_audio', with_audio=True), + ] + + def setup(self, context): # NOQA + self.bbench_on_device = '/'.join([self.device.working_directory, 'bbench']) + self.bbench_server_on_device = os.path.join(self.device.working_directory, BBENCH_SERVER_NAME) + self.audio_on_device = os.path.join(self.device.working_directory, DEFAULT_AUDIO_FILE_NAME) + self.index_noinput = 'file:///{}'.format(self.bbench_on_device) + '/index_noinput.html' + self.luanch_server_command = '{} {}'.format(BBENCH_SERVER_NAME, self.server_timeout) + + if not os.path.isdir(os.path.join(self.dependencies_directory, "sites")): + self._download_bbench_file() + if self.with_audio and not os.path.isfile(self.audio_file): + self._download_audio_file() + + if not os.path.isdir(self.dependencies_directory): + raise ConfigError('Bbench directory does not exist: {}'.format(self.dependencies_directory)) + self._apply_patches() + + if self.with_audio: + if self.force_dependency_push or not self.device.file_exists(self.audio_on_device): + self.device.push_file(self.audio_file, self.audio_on_device, timeout=120) + + # Push the bbench site pages and http server to target device + if self.force_dependency_push or not self.device.file_exists(self.bbench_on_device): + self.logger.debug('Copying bbench sites to device.') + self.device.push_file(self.dependencies_directory, self.bbench_on_device, timeout=300) + + # Push the bbench server + host_binary = context.resolver.get(Executable(self, self.device.abi, 'bbench_server')) + self.device.install(host_binary) + + # Open the browser with default page + self.device.execute('am start -n {}/{} about:blank'.format(self.browser_package, self.browser_activity)) + time.sleep(5) + + # Stop the browser if already running and wait for it to stop + self.device.execute('am force-stop {}'.format(self.browser_package)) + time.sleep(5) + + # Clear the logs + self.device.clear_logcat() + + # clear browser cache + self.device.execute('pm clear {}'.format(self.browser_package)) + if self.clear_file_cache: + self.device.execute('sync') + self.device.set_sysfile_value('/proc/sys/vm/drop_caches', 3) + + # Launch the background music + if self.with_audio: + self.device.execute('am start -W -S -n com.android.music/.MediaPlaybackActivity -d {}'.format(self.audio_on_device)) + + def run(self, context): + # Launch the bbench + self.device.execute('am start -n {}/{} {}'.format(self.browser_package, self.browser_activity, self.index_noinput)) + time.sleep(5) # WA1 parity + # Launch the server waiting for Bbench to complete + self.device.execute(self.luanch_server_command, self.server_timeout) + + def update_result(self, context): + # Stop the browser + self.device.execute('am force-stop {}'.format(self.browser_package)) + + # Stop the music + if self.with_audio: + self.device.execute('am force-stop com.android.music') + + # Get index_no_input.html + indexfile = os.path.join(self.device.working_directory, 'bbench/index_noinput.html') + self.device.pull_file(indexfile, context.output_directory) + + # Get the logs + output_file = os.path.join(self.device.working_directory, 'browser_bbench_logcat.txt') + self.device.execute('logcat -v time -d > {}'.format(output_file)) + self.device.pull_file(output_file, context.output_directory) + + metrics = _parse_metrics(os.path.join(context.output_directory, 'browser_bbench_logcat.txt'), + os.path.join(context.output_directory, 'index_noinput.html'), + context.output_directory) + for key, values in metrics: + for i, value in enumerate(values): + metric = '{}_{}'.format(key, i) if i else key + context.result.add_metric(metric, value, units='ms', lower_is_better=True) + + def teardown(self, context): + if self.perform_cleanup: + self.device.execute('rm -r {}'.format(self.bbench_on_device)) + self.device.execute('rm {}'.format(self.audio_on_device)) + + def _download_audio_file(self): + self.logger.debug('Downloadling audio file.') + urllib.urlretrieve(DEFAULT_AUDIO_FILE, self.audio_file) + + def _download_bbench_file(self): + # downloading the file to bbench_dir + self.logger.debug('Downloading bbench dependencies.') + full_file_path = os.path.join(self.dependencies_directory, DOWNLOADED_FILE_NAME) + urllib.urlretrieve(DEFAULT_BBENCH_FILE, full_file_path) + + # Extracting Bbench to bbench_dir/ + self.logger.debug('Extracting bbench dependencies.') + tar = tarfile.open(full_file_path) + tar.extractall(os.path.dirname(self.dependencies_directory)) + + # Removing not needed files and the compressed file + os.remove(full_file_path) + youtube_dir = os.path.join(self.dependencies_directory, 'sites', 'youtube') + os.remove(os.path.join(youtube_dir, 'www.youtube.com', 'kp.flv')) + os.remove(os.path.join(youtube_dir, 'kp.flv')) + + def _apply_patches(self): + self.logger.debug('Applying patches.') + shutil.copy(os.path.join(PATCH_FILES, "bbench.js"), self.dependencies_directory) + shutil.copy(os.path.join(PATCH_FILES, "results.html"), self.dependencies_directory) + shutil.copy(os.path.join(PATCH_FILES, "index_noinput.html"), self.dependencies_directory) + + +def _parse_metrics(logfile, indexfile, output_directory): # pylint: disable=R0914 + regex_bbscore = re.compile(r'(?P<head>\w+)=(?P<val>\w+)') + regex_bbmean = re.compile(r'Mean = (?P<mean>[0-9\.]+)') + regex_pagescore_head = re.compile(r'metrics:(\w+),(\d+)') + regex_pagescore_tail = re.compile(r',(\d+.\d+)') + regex_indexfile = re.compile(r'<body onload="startTest\((.*)\)">') + settings_dict = defaultdict() + + with open(indexfile) as fh: + for line in fh: + match = regex_indexfile.search(line) + if match: + settings_dict['iterations'], settings_dict['scrollDelay'], settings_dict['scrollSize'] = match.group(1).split(',') + with open(logfile) as fh: + results_dict = defaultdict(list) + for line in fh: + if 'metrics:Mean' in line: + results_list = regex_bbscore.findall(line) + results_dict['Mean Latency'].append(regex_bbmean.search(line).group('mean')) + if results_list: + break + elif 'metrics:' in line: + page_results = [0] + match = regex_pagescore_head.search(line) + name, page_results[0] = match.groups() + page_results.extend(regex_pagescore_tail.findall(line[match.end():])) + for val in page_results[:-2]: + results_list.append((name, int(float(val)))) + + setting_names = ['siteIndex', 'CGTPreviousTime', 'scrollDelay', 'scrollSize', 'iterations'] + for k, v in results_list: + if k not in setting_names: + results_dict[k].append(v) + + sorted_results = sorted(results_dict.items()) + + with open(os.path.join(output_directory, 'settings.json'), 'w') as wfh: + json.dump(settings_dict, wfh) + + return sorted_results diff --git a/wlauto/workloads/bbench/bin/arm64/bbench_server b/wlauto/workloads/bbench/bin/arm64/bbench_server Binary files differnew file mode 100755 index 00000000..c33f5cfd --- /dev/null +++ b/wlauto/workloads/bbench/bin/arm64/bbench_server diff --git a/wlauto/workloads/bbench/bin/armeabi/bbench_server b/wlauto/workloads/bbench/bin/armeabi/bbench_server Binary files differnew file mode 100755 index 00000000..c33f5cfd --- /dev/null +++ b/wlauto/workloads/bbench/bin/armeabi/bbench_server diff --git a/wlauto/workloads/bbench/patches/bbench.js b/wlauto/workloads/bbench/patches/bbench.js new file mode 100644 index 00000000..05e2900f --- /dev/null +++ b/wlauto/workloads/bbench/patches/bbench.js @@ -0,0 +1,177 @@ +//Author: Anthony Gutierrez
+
+var bb_site = [];
+var bb_results = [];
+var globalSiteIndex = 0;
+var numWebsites = 9;
+var bb_path = document.location.pathname;
+var bb_home = "file:///" + bb_path.substr(1, bb_path.lastIndexOf("bbench") + 5);
+var num_iters = 0;
+var init = false;
+
+function generateSiteArray(numTimesToExecute) {
+ for (i = 0; i < numTimesToExecute * numWebsites; i += numWebsites) {
+ bb_site[i+0] = bb_home + "/sites/amazon/www.amazon.com/index.html";
+ bb_site[i+1] = bb_home + "/sites/bbc/www.bbc.co.uk/index.html";
+ bb_site[i+2] = bb_home + "/sites/cnn/www.cnn.com/index.html";
+ bb_site[i+3] = bb_home + "/sites/craigslist/newyork.craigslist.org/index.html";
+ bb_site[i+4] = bb_home + "/sites/ebay/www.ebay.com/index.html";
+ bb_site[i+5] = bb_home + "/sites/google/www.google.com/index.html";
+// bb_site[i+6] = bb_home + "/sites/youtube/www.youtube.com/index.html";
+ bb_site[i+6] = bb_home + "/sites/msn/www.msn.com/index.html";
+ bb_site[i+7] = bb_home + "/sites/slashdot/slashdot.org/index.html";
+ bb_site[i+8] = bb_home + "/sites/twitter/twitter.com/index.html";
+// bb_site[i+10] = bb_home + "/sites/espn/espn.go.com/index.html";
+ }
+
+ bb_site[i] = bb_home + "/results.html";
+}
+
+
+/* gets the URL parameters and removes from window href */
+function getAndRemoveURLParams(windowURL, param) {
+ var regex_string = "(.*)(\\?)" + param + "(=)([0-9]+)(&)(.*)";
+ var regex = new RegExp(regex_string);
+ var results = regex.exec(windowURL.value);
+
+ if (results == null)
+ return "";
+ else {
+ windowURL.value = results[1] + results[6];
+ return results[4];
+ }
+}
+
+/* gets the URL parameters */
+function getURLParams(param) {
+ var regex_string = "(.*)(\\?)" + param + "(=)([0-9]+)(&)(.*)";
+ var regex = new RegExp(regex_string);
+ var results = regex.exec(window.location.href);
+
+ if (results == null)
+ return "";
+ else
+ return results[4];
+}
+
+/* gets all the parameters */
+function getAllParams() {
+ var regex_string = "(\\?.*)(\\?siteIndex=)([0-9]+)(&)";
+ var regex = new RegExp(regex_string);
+ var results = regex.exec(window.location.href);
+ /*alert(" Result is 1: " + results[1] + " 2: " + results[2] + " 3: " + results[3]);*/
+
+ if (results == null)
+ return "";
+ else
+ return results[1];
+}
+
+/* sets a cookie */
+function setCookie(c_name, value) {
+ var c_value = escape(value) + ";";
+ document.cookie = c_name + "=" + c_value + " path=/";
+}
+
+/* gets a cookie */
+function getCookie(c_name) {
+ var cookies = document.cookie.split(";");
+ var i, x, y;
+
+ for (i = 0; i < cookies.length; ++i) {
+ x = cookies[i].substr(0, cookies[i].indexOf("="));
+ y = cookies[i].substr(cookies[i].indexOf("=") + 1);
+ x = x.replace(/^\s+|\s+$/g,"");
+
+ if (x == c_name)
+ return unescape(y);
+ }
+}
+
+/* start the test, simply go to site 1. */
+function startTest(n, del, y) {
+ //var start_time = (new Date()).getTime();
+ //setCookie("PreviousTime", start_time);
+
+ init = true;
+
+ generateSiteArray(n);
+ siteTest(bb_site[0], globalSiteIndex, new Date().getTime(), "scrollSize=" + y + "&?scrollDelay=" + del + "&?iterations=" + n + "&?" + "StartPage");
+ //siteTest(bb_site[0], globalSiteIndex, new Date().getTime(), "scrollDelay=" + del + "&?iterations=" + n + "&?" + "StartPage");
+ //goToSite(bb_site[0], new Date().getTime());
+}
+
+/* jump to the next site */
+function goToSite(site) {
+ curr_time = new Date().getTime();
+ setCookie("CGTPreviousTime", curr_time);
+ site+="?CGTPreviousTime="+curr_time+"&";
+ window.location.href = site;
+}
+
+/*
+ the test we want to run on the site.
+ for now, simply scroll to the bottom
+ and jump to the next site. in the
+ future we will want to do some more
+ realistic browsing tests.
+*/
+function siteTest(nextSite, siteIndex, startTime, siteName) {
+ if (!init) {
+ var iterations = getURLParams("iterations");
+ var params = getAllParams();
+ var delay = getURLParams("scrollDelay");
+ var verticalScroll = getURLParams("scrollSize");
+ generateSiteArray(iterations);
+ nextSite = bb_site[siteIndex] + params;
+ }
+ else {
+ var delay = 500;
+ var verticalScroll = 500;
+ }
+ var cgtPreviousTime = getURLParams("CGTPreviousTime");
+ var load_time = 0;
+ siteIndex++;
+ if (siteIndex > 1) {
+ cur_time = new Date().getTime();
+// alert("previous " + cgtPreviousTime + " foo " + getCookie("CGTPreviousTime"));
+ load_time = (cur_time - cgtPreviousTime);
+ setCookie("CGTLoadTime", load_time);
+// diff = cur_time-startTime;
+// alert("starttime "+startTime+" currtime "+ cur_time + " diff " + diff + "load_time " + load_time );
+ }
+ setTimeout(function() {
+ scrollToBottom(0, verticalScroll, delay,load_time,
+ function(load_time_param){
+ cur_time = new Date().getTime();
+ load_time = (cur_time - startTime);
+ //load_time = (cur_time - getCookie("PreviousTime"));
+ // alert("Done with this site! " + window.cur_time + " " + startTime + " " + window.load_time);
+ //alert("Done with this site! " + window.cur_time + " " + getCookie("PreviousTime") + " " + window.load_time);
+ //goToSite(nextSite + "?iterations=" + iterations + "&?" + siteName + "=" + load_time + "&" + "?siteIndex=" + siteIndex + "&" );
+// alert("loadtime in cookie="+ getCookie("CGTLoadTime")+" loadtime in var="+load_time_param);
+ goToSite(nextSite + "?" + siteName + "=" + load_time_param + "&" + "?siteIndex=" + siteIndex + "&" );
+ }
+ );},(siteIndex > 1) ? 1000 : 0);
+}
+
+/*
+ scroll to the bottom of the page in
+ num_y pixel increments. may want to
+ do some horizontal scrolling in the
+ future as well.
+*/
+function scrollToBottom(num_x, num_y, del, load_time, k) {
+ ++num_iters;
+ var diff = document.body.scrollHeight - num_y * num_iters;
+ //var num_scrolls = 0;
+
+ if (diff > num_y) {
+ //self.scrollBy(num_x, num_y);
+ //setTimeout(function(){self.scrollBy(num_x, num_y); /*diff -= 100;*/ scrollToBottom(num_x, num_y, k);}, 2);
+ setTimeout(function(){self.scrollBy(num_x, num_y); /*diff -= 100;*/ scrollToBottom(num_x, num_y, del, load_time,k);}, del);
+ }
+ else{
+ k(load_time);
+ }
+}
diff --git a/wlauto/workloads/bbench/patches/index_noinput.html b/wlauto/workloads/bbench/patches/index_noinput.html new file mode 100644 index 00000000..072c9ad8 --- /dev/null +++ b/wlauto/workloads/bbench/patches/index_noinput.html @@ -0,0 +1,56 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1.dtd">
+<!--
+ Author: Anthony Gutierrez
+-->
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+ <title>University of Michigan - BBench 2.0</title>
+ <script type="text/javascript" src="bbench.js"></script>
+ <script type="text/javascript" src="forms.js"></script>
+</head>
+
+<body onload="startTest(2,2000,500)">
+<!--
+<body>
+ <img src="mich_engin.png" width="35%"/>
+ <h2>University of Michigan BBench version 2.0</h2>
+
+ <form name="config_form">
+ <b>Number of iterations:</b> <input type="text" name="numIterations" value="5" size="4" onchange="setIters();">
+ <input type="button" value="-" name="iterPlusButton" onClick="document.config_form.numIterations.value=numItersDec(); return true;">
+ <input type="button" value="+" name="iterMinusButton" onClick="document.config_form.numIterations.value=numItersInc(); return true;">
+ (Number of times the page set is iterated through.)
+ <br/><br/>
+
+ <b>Scroll Delay (ms):</b> <input type="text" name="scrollDelay" value="0" size="8" onchange="setScrollDelay();">
+ <input type="button" value="-" name="scrollDelayPlusButton" onClick="document.config_form.scrollDelay.value=scrollDelayDec(); return true;">
+ <input type="button" value="+" name="scrollDelayMinusButton" onClick="document.config_form.scrollDelay.value=scrollDelayInc(); return true;">
+ (Number of milliseconds to pause before scrolling.)
+ <br/><br/>
+
+ <b>Scroll Size:</b> <input type="text" name="scrollSize" value="500" size="8" onchange="setScrollSize();">
+ <input type="button" value="-" name="scrollSizePlusButton" onClick="document.config_form.scrollSize.value=scrollSizeDec(); return true;">
+ <input type="button" value="+" name="scrollSizeMinusButton" onClick="document.config_form.scrollSize.value=scrollSizeInc(); return true;">
+ (Number of pixel to scroll.)
+ <br/><br/>
+ </form>
+
+ <p>
+ <b>Click on the start button to begin the benchmark.</b>
+ </p>
+ <button onclick="startTest(numIters, scrollDelay, scrollSize)">start</button>
+
+ <p>
+ If you use BBench in your work please cite our <a href="http://www.eecs.umich.edu/~atgutier/iiswc_2011.pdf">2011 IISWC paper</a>:<br/><br/>
+
+ A. Gutierrez, R.G. Dreslinksi, T.F. Wenisch, T. Mudge, A. Saidi, C. Emmons, and N. Paver. Full-System Analysis and Characterization
+ of Interactive Smartphone Applications. <i>IEEE International Symposium on Workload Characterization</i>, 2011.
+ </p>
+--!>
+</body>
+
+</html>
+
diff --git a/wlauto/workloads/bbench/patches/results.html b/wlauto/workloads/bbench/patches/results.html new file mode 100644 index 00000000..a7eb2e33 --- /dev/null +++ b/wlauto/workloads/bbench/patches/results.html @@ -0,0 +1,158 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1.dtd"> +<!-- + Author: Anthony Gutierrez +--> + +<html xmlns="http://www.w3.org/1999/xhtml"> + +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + <title>University of Michigan - BBench 2.0</title> + <script type="text/javascript" src="bbench.js"></script> + + <script type="text/javascript"> + var numTimesToExecute = getURLParams("iterations"); + + function closeWindow() { + window.open('','_self',''); + window.close(); + } + + function averageWarm(siteTimes) { + var sum = 0; + + if (numTimesToExecute == 1) + return siteTimes[0]; + + for (i = 0; i < numTimesToExecute - 1; ++i) + sum = eval(sum + siteTimes[i]); + + return (sum / (numTimesToExecute - 1)); + } + + function stdDevWarm(siteTimes) { + var avg = averageWarm(siteTimes) + var tmpArray = []; + + if (numTimesToExecute == 1) + return 0; + + for (i = 0; i < numTimesToExecute - 1; ++i) + tmpArray[i] = Math.pow((siteTimes[i] - avg), 2); + + avg = averageWarm(tmpArray); + + return Math.sqrt(avg); + } + + function geoMean(avgTimes) { + var prod = 1; + + for (i = 0; i < numWebsites; ++i) + prod = eval(prod * avgTimes[i]); + + return Math.pow(prod, (1/numWebsites)); + } + </script> +</head> + +<body> + <img src="mich_engin.png" width="35%"/> + <h2>University of Michigan BBench version 2.0</h2> + <h3>Results</h3> + + <script type="text/javascript"> + var bbSiteColdTimes = []; + var bbSiteTimes = []; + var bbSiteAvgRunTime = []; + var bbSiteStdDev = []; + var bbSiteCoeffVar = []; + var bbSiteNames = ["amazon", + "bbc", + "cnn", + "craigslist", + "ebay", +// "espn", + "google", + "msn", + "slashdot", + "twitter"]; +// "youtube"]; + + var windowURL = new Object(); + var windowURL2 = new Object(); + windowURL.value = window.location.href; + windowURL2.value = window.location.href; + + for (j = 0; j < numWebsites; ++j) { + + for (i = 0; i < numTimesToExecute; ++i) { + var site_time = getAndRemoveURLParams(windowURL, bbSiteNames[j]) - 0; + bbSiteTimes[i] = site_time; + } + + bbSiteColdTimes[j] = bbSiteTimes[i - 1]; + bbSiteAvgRunTime[j] = averageWarm(bbSiteTimes); + bbSiteStdDev[j] = stdDevWarm(bbSiteTimes); + bbSiteCoeffVar[j] = (bbSiteStdDev[j] / bbSiteAvgRunTime[j]) * 100; + } + + var bbSiteAvgGeoMean = geoMean(bbSiteAvgRunTime); + </script> + + <table border="1"> + <script type="text/javascript"> + document.write("<tr align=\"right\"><td>Site Name</td><td>Cold Start Time</td><td>Avg Warm Page Rendering Time (ms)</td><td>Std Dev of Warm Runs</td><td>%Coeff Var of Warm Runs</td>"); + for (i = 0; i < numWebsites; ++i) { + document.write("<tr align=\"right\">"); + document.write("<td>" + bbSiteNames[i] + "</td>"); + document.write("<td>" + bbSiteColdTimes[i] + "</td>"); + document.write("<td>" + bbSiteAvgRunTime[i].toFixed(2) + "</td>"); + document.write("<td>" + bbSiteStdDev[i].toFixed(2) + "</td>"); + document.write("<td>" + bbSiteCoeffVar[i].toFixed(2) + "</td>"); + document.write("</tr>"); + } + </script> + </table> + + <br /> + + <table border="1"> + <script type="text/javascript"> + document.write("<tr><td>Geometric Mean of Average Warm Runs</td><td>" + bbSiteAvgGeoMean.toFixed(2) + "</td></tr>"); + console.log("metrics:" + "Mean = " + bbSiteAvgGeoMean.toFixed(2) + ":") + </script> + </table> + + <h3>CSV version of the table:</h3> + + <script type="text/javascript"> + document.write("Site Name,Cold Start Time, Avg Warm Page Rendering Time (ms),Std Dev of Warm Runs,%Coeff Var of Warm Runs<br />"); + for (i = 0; i < numWebsites; ++i) { + document.write(bbSiteNames[i] + ","); + document.write(bbSiteColdTimes[i] + ","); + document.write(bbSiteAvgRunTime[i].toFixed(2) + ","); + document.write(bbSiteStdDev[i].toFixed(2) + ","); + document.write(bbSiteCoeffVar[i].toFixed(2) + "<br />"); + console.log("metrics:" + bbSiteNames[i] + "," + bbSiteColdTimes[i] + "," + bbSiteAvgRunTime[i].toFixed(2) + "," + bbSiteStdDev[i].toFixed(2) + "," + bbSiteCoeffVar[i].toFixed(2) + ":"); + } + + document.write("<h3>Individual Site Times:</h3>"); + for (j = 0; j < numWebsites; ++j) { + for (i = 0; i < numTimesToExecute; ++i) { + var site_time = getAndRemoveURLParams(windowURL2, bbSiteNames[j]) - 0; + bbSiteTimes[i] = site_time; + document.write(bbSiteNames[j] + " load time: " + site_time + "<br />"); + } + document.write("<br />"); + } + setTimeout("window.location.href='http://localhost:3030/'", 1); + </script> + + <p> + <b>Click the return button to go to the start page.</b> + </p> + <button onclick="window.location.href='index.html'">return</button> +</body> + +</html> diff --git a/wlauto/workloads/benchmarkpi/__init__.py b/wlauto/workloads/benchmarkpi/__init__.py new file mode 100644 index 00000000..c49f6d05 --- /dev/null +++ b/wlauto/workloads/benchmarkpi/__init__.py @@ -0,0 +1,63 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import re + +from wlauto import AndroidUiAutoBenchmark + + +class BenchmarkPi(AndroidUiAutoBenchmark): + + name = 'benchmarkpi' + description = """ + Measures the time the target device takes to run and complete the Pi + calculation algorithm. + + http://androidbenchmark.com/howitworks.php + + from the website: + + The whole idea behind this application is to use the same Pi calculation + algorithm on every Android Device and check how fast that proccess is. + Better calculation times, conclude to faster Android devices. This way you + can also check how lightweight your custom made Android build is. Or not. + + As Pi is an irrational number, Benchmark Pi does not calculate the actual Pi + number, but an approximation near the first digits of Pi over the same + calculation circles the algorithms needs. + + So, the number you are getting in miliseconds is the time your mobile device + takes to run and complete the Pi calculation algorithm resulting in a + approximation of the first Pi digits. + """ + package = 'gr.androiddev.BenchmarkPi' + activity = '.BenchmarkPi' + summary_metrics = ['pi calculation'] + + regex = re.compile('You calculated Pi in ([0-9]+)') + + def update_result(self, context): + super(BenchmarkPi, self).update_result(context) + result = None + with open(self.logcat_log) as fh: + for line in fh: + match = self.regex.search(line) + if match: + result = int(match.group(1)) + + if result is not None: + context.result.add_metric('pi calculation', result, + 'Milliseconds', lower_is_better=True) diff --git a/wlauto/workloads/benchmarkpi/com.arm.wlauto.uiauto.benchmarkpi.jar b/wlauto/workloads/benchmarkpi/com.arm.wlauto.uiauto.benchmarkpi.jar Binary files differnew file mode 100644 index 00000000..433334d2 --- /dev/null +++ b/wlauto/workloads/benchmarkpi/com.arm.wlauto.uiauto.benchmarkpi.jar diff --git a/wlauto/workloads/benchmarkpi/uiauto/build.sh b/wlauto/workloads/benchmarkpi/uiauto/build.sh new file mode 100755 index 00000000..be7ca104 --- /dev/null +++ b/wlauto/workloads/benchmarkpi/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.benchmarkpi.jar ]]; then + cp bin/com.arm.wlauto.uiauto.benchmarkpi.jar .. +fi diff --git a/wlauto/workloads/benchmarkpi/uiauto/build.xml b/wlauto/workloads/benchmarkpi/uiauto/build.xml new file mode 100644 index 00000000..67603ca8 --- /dev/null +++ b/wlauto/workloads/benchmarkpi/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.benchmarkpi" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/benchmarkpi/uiauto/project.properties b/wlauto/workloads/benchmarkpi/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/benchmarkpi/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/benchmarkpi/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/benchmarkpi/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..e4e8b7ad --- /dev/null +++ b/wlauto/workloads/benchmarkpi/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,62 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.benchmarkpi; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "benchmarkpi"; + + public void runUiAutomation() throws Exception{ + Bundle status = new Bundle(); + + startTest(); + waitForAndExtractResults(); + + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + + public void startTest() throws Exception{ + UiSelector selector = new UiSelector(); + UiObject benchButton = new UiObject(selector.text("Benchmark my Android!") + .className("android.widget.Button")); + benchButton.click(); + } + + public void waitForAndExtractResults() throws Exception{ + UiSelector selector = new UiSelector(); + UiObject submitButton = new UiObject(selector.text("Submit") + .className("android.widget.Button")); + submitButton.waitForExists(10 * 1000); + + UiObject resultsText = new UiObject(selector.textContains("You calculated Pi in") + .className("android.widget.TextView")); + Log.v(TAG, resultsText.getText()); + } +} diff --git a/wlauto/workloads/caffeinemark/__init__.py b/wlauto/workloads/caffeinemark/__init__.py new file mode 100644 index 00000000..6b2ae0f9 --- /dev/null +++ b/wlauto/workloads/caffeinemark/__init__.py @@ -0,0 +1,68 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re + +from wlauto import AndroidUiAutoBenchmark + + +class Caffeinemark(AndroidUiAutoBenchmark): + + name = 'caffeinemark' + description = """ + CaffeineMark is a series of tests that measure the speed of Java + programs running in various hardware and software configurations. + + http://www.benchmarkhq.ru/cm30/info.html + + From the website: + + CaffeineMark scores roughly correlate with the number of Java instructions + executed per second, and do not depend significantly on the the amount of + memory in the system or on the speed of a computers disk drives or internet + connection. + + The following is a brief description of what each test does: + + - Sieve: The classic sieve of eratosthenes finds prime numbers. + - Loop: The loop test uses sorting and sequence generation as to measure + compiler optimization of loops. + - Logic: Tests the speed with which the virtual machine executes + decision-making instructions. + - Method: The Method test executes recursive function calls to see how + well the VM handles method calls. + - Float: Simulates a 3D rotation of objects around a point. + - Graphics: Draws random rectangles and lines. + - Image: Draws a sequence of three graphics repeatedly. + - Dialog: Writes a set of values into labels and editboxes on a form. + + The overall CaffeineMark score is the geometric mean of the individual + scores, i.e., it is the 9th root of the product of all the scores. + """ + package = "com.flexycore.caffeinemark" + activity = ".Application" + summary_metrics = ['OverallScore'] + + regex = re.compile(r'CAFFEINEMARK RESULT: (?P<type>\w+) (?P<value>\S+)') + + def update_result(self, context): + super(Caffeinemark, self).update_result(context) + with open(self.logcat_log) as fh: + for line in fh: + match = self.regex.search(line) + if match: + metric = match.group('type') + value = float(match.group('value')) + context.result.add_metric(metric, value) diff --git a/wlauto/workloads/caffeinemark/com.arm.wlauto.uiauto.caffeinemark.jar b/wlauto/workloads/caffeinemark/com.arm.wlauto.uiauto.caffeinemark.jar Binary files differnew file mode 100644 index 00000000..2a75e9d2 --- /dev/null +++ b/wlauto/workloads/caffeinemark/com.arm.wlauto.uiauto.caffeinemark.jar diff --git a/wlauto/workloads/caffeinemark/uiauto/build.sh b/wlauto/workloads/caffeinemark/uiauto/build.sh new file mode 100755 index 00000000..148e101e --- /dev/null +++ b/wlauto/workloads/caffeinemark/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.caffeinemark.jar ]]; then + cp bin/com.arm.wlauto.uiauto.caffeinemark.jar .. +fi diff --git a/wlauto/workloads/caffeinemark/uiauto/build.xml b/wlauto/workloads/caffeinemark/uiauto/build.xml new file mode 100644 index 00000000..0b50bbf9 --- /dev/null +++ b/wlauto/workloads/caffeinemark/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.caffeinemark" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/caffeinemark/uiauto/project.properties b/wlauto/workloads/caffeinemark/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/caffeinemark/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/caffeinemark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/caffeinemark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..3979b675 --- /dev/null +++ b/wlauto/workloads/caffeinemark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,85 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.caffeinemark; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "caffeinemark"; + public String[] categories = {"Sieve", "Loop", "Logic", "String", "Float", "Method"}; + + public void runUiAutomation() throws Exception { + Bundle status = new Bundle(); + status.putString("product", getUiDevice().getProductName()); + + UiSelector selector = new UiSelector(); + UiObject runButton = new UiObject(selector.text("Run benchmark") + .className("android.widget.Button")); + runButton.click(); + + try { + waitText("CaffeineMark results"); + extractOverallScore(); + extractDetailedScores(); + + + } catch(UiObjectNotFoundException e) { + takeScreenshot("caffeine-error"); + } + + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + + public void extractOverallScore() throws Exception { + UiSelector selector = new UiSelector(); + UiObject linearLayoutOverallScore = new UiObject(selector.className("android.widget.LinearLayout") + .instance(1)); + UiObject overallScore = linearLayoutOverallScore.getChild(selector.className("android.widget.TextView") + .instance(2)); + Log.v(TAG, "CAFFEINEMARK RESULT: OverallScore " + overallScore.getText()); + } + + public void extractDetailedScores() throws Exception { + UiSelector selector = new UiSelector(); + UiObject detailsButton = new UiObject(selector.text("Details") + .className("android.widget.Button")); + detailsButton.click(); + sleep(2); + + UiObject linearObject; + UiObject detailedScore; + for (int i = 1; i <= 6; i++) { + linearObject = new UiObject(selector.className("android.widget.LinearLayout") + .instance(i)); + detailedScore = linearObject.getChild(selector.className("android.widget.TextView") + .instance(1)); + Log.v(TAG,"CAFFEINEMARK RESULT: " + categories[i-1] + " " + detailedScore.getText()); + } + } +} diff --git a/wlauto/workloads/cameracapture/__init__.py b/wlauto/workloads/cameracapture/__init__.py new file mode 100644 index 00000000..de72acea --- /dev/null +++ b/wlauto/workloads/cameracapture/__init__.py @@ -0,0 +1,51 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101 + +from wlauto import UiAutomatorWorkload, Parameter + + +class Cameracapture(UiAutomatorWorkload): + + name = 'cameracapture' + description = """ + Uses in-built Android camera app to take photos. + + """ + package = 'com.google.android.gallery3d' + activity = 'com.android.camera.CameraActivity' + + parameters = [ + Parameter('no_of_captures', kind=int, default=5, + description='Number of photos to be taken.'), + Parameter('time_between_captures', kind=int, default=5, + description='Time, in seconds, between two consecutive camera clicks.'), + ] + + def __init__(self, device, **kwargs): + super(Cameracapture, self).__init__(device, **kwargs) + self.uiauto_params['no_of_captures'] = self.no_of_captures + self.uiauto_params['time_between_captures'] = self.time_between_captures + + def setup(self, context): + super(Cameracapture, self).setup(context) + self.device.execute('am start -n {}/{}'.format(self.package, self.activity)) + + def update_result(self, context): + pass + + def teardown(self, context): + super(Cameracapture, self).teardown(context) diff --git a/wlauto/workloads/cameracapture/com.arm.wlauto.uiauto.cameracapture.jar b/wlauto/workloads/cameracapture/com.arm.wlauto.uiauto.cameracapture.jar Binary files differnew file mode 100644 index 00000000..0d37d0b2 --- /dev/null +++ b/wlauto/workloads/cameracapture/com.arm.wlauto.uiauto.cameracapture.jar diff --git a/wlauto/workloads/cameracapture/uiauto/build.sh b/wlauto/workloads/cameracapture/uiauto/build.sh new file mode 100755 index 00000000..6b54f4f6 --- /dev/null +++ b/wlauto/workloads/cameracapture/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.cameracapture.jar ]]; then + cp bin/com.arm.wlauto.uiauto.cameracapture.jar .. +fi diff --git a/wlauto/workloads/cameracapture/uiauto/build.xml b/wlauto/workloads/cameracapture/uiauto/build.xml new file mode 100644 index 00000000..bcd7ef9d --- /dev/null +++ b/wlauto/workloads/cameracapture/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.cameracapture" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/cameracapture/uiauto/project.properties b/wlauto/workloads/cameracapture/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/cameracapture/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/cameracapture/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/cameracapture/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..a5497468 --- /dev/null +++ b/wlauto/workloads/cameracapture/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,68 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.cameracapture; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.view.KeyEvent; + +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "cameracapture"; + + public void runUiAutomation() throws Exception { + int timeDurationBetweenEachCapture = 0; + int sleepTime = 2; + Bundle parameters = getParams(); + String noOfCaptures = ""; + int iterations = 0; + + if (parameters.size() > 0) { + iterations = Integer.parseInt(parameters + .getString("no_of_captures")); + timeDurationBetweenEachCapture = Integer.parseInt(parameters + .getString("time_between_captures")); + } + // switch to camera capture mode + UiObject clickModes = new UiObject(new UiSelector().descriptionMatches("Camera, video or panorama selector")); + clickModes.click(); + sleep(sleepTime); + + UiObject changeModeToCapture = new UiObject(new UiSelector().descriptionMatches("Switch to photo")); + + changeModeToCapture.click(); + sleep(sleepTime); + + // click to capture photos + UiObject clickCaptureButton = new UiObject(new UiSelector().descriptionMatches("Shutter button")); + + for (int i = 0; i < iterations; i++) { + clickCaptureButton.longClick(); + sleep(timeDurationBetweenEachCapture); + } + getUiDevice().pressBack(); + } +} diff --git a/wlauto/workloads/camerarecord/__init__.py b/wlauto/workloads/camerarecord/__init__.py new file mode 100644 index 00000000..7f237f2f --- /dev/null +++ b/wlauto/workloads/camerarecord/__init__.py @@ -0,0 +1,47 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from wlauto import UiAutomatorWorkload, Parameter + + +class Camerarecord(UiAutomatorWorkload): + + name = 'camerarecord' + description = """ + Uses in-built Android camera app to record the video for given interval + of time. + + """ + package = 'com.google.android.gallery3d' + activity = 'com.android.camera.CameraActivity' + run_timeout = 0 + + parameters = [ + Parameter('recording_time', kind=int, default=60, + description='The video recording time in seconds.'), + ] + + def __init__(self, device, **kwargs): + super(Camerarecord, self).__init__(device) + self.uiauto_params['recording_time'] = self.recording_time # pylint: disable=E1101 + self.run_timeout = 3 * self.uiauto_params['recording_time'] + + def setup(self, context): + super(Camerarecord, self).setup(context) + self.device.execute('am start -n {}/{}'.format(self.package, self.activity)) + + def teardown(self, context): + self.device.execute('am force-stop {}'.format(self.package)) + super(Camerarecord, self).teardown(context) diff --git a/wlauto/workloads/camerarecord/com.arm.wlauto.uiauto.camerarecord.jar b/wlauto/workloads/camerarecord/com.arm.wlauto.uiauto.camerarecord.jar Binary files differnew file mode 100644 index 00000000..7de7cfcf --- /dev/null +++ b/wlauto/workloads/camerarecord/com.arm.wlauto.uiauto.camerarecord.jar diff --git a/wlauto/workloads/camerarecord/uiauto/build.sh b/wlauto/workloads/camerarecord/uiauto/build.sh new file mode 100755 index 00000000..eff5293f --- /dev/null +++ b/wlauto/workloads/camerarecord/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.camerarecord.jar ]]; then + cp bin/com.arm.wlauto.uiauto.camerarecord.jar .. +fi diff --git a/wlauto/workloads/camerarecord/uiauto/build.xml b/wlauto/workloads/camerarecord/uiauto/build.xml new file mode 100644 index 00000000..31a4132e --- /dev/null +++ b/wlauto/workloads/camerarecord/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.camerarecord" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/camerarecord/uiauto/project.properties b/wlauto/workloads/camerarecord/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/camerarecord/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/camerarecord/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/camerarecord/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..c030a078 --- /dev/null +++ b/wlauto/workloads/camerarecord/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,65 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.camerarecord; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.view.KeyEvent; + +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "camerarecord"; + + public void runUiAutomation() throws Exception { + Bundle parameters = getParams(); + int timeToRecord = 0; + int timeout = 4; + int sleepTime = 2; + int recordingTime = 0; + if (parameters.size() > 0) { + recordingTime = Integer.parseInt(parameters + .getString("recording_time")); + } + + // switch to camera capture mode + UiObject clickModes = new UiObject(new UiSelector().descriptionMatches("Camera, video or panorama selector")); + clickModes.click(); + sleep(sleepTime); + + UiObject changeModeToCapture = new UiObject(new UiSelector().descriptionMatches("Switch to video")); + changeModeToCapture.click(); + sleep(sleepTime); + + UiObject clickRecordingButton = new UiObject(new UiSelector().descriptionMatches("Shutter button")); + clickRecordingButton.longClick(); + sleep(recordingTime); + + // Stop video recording + clickRecordingButton.longClick(); + getUiDevice().pressBack(); + } + +} diff --git a/wlauto/workloads/castlebuilder/__init__.py b/wlauto/workloads/castlebuilder/__init__.py new file mode 100644 index 00000000..5a527330 --- /dev/null +++ b/wlauto/workloads/castlebuilder/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from wlauto import GameWorkload + + +class Castlebuilder(GameWorkload): + + name = 'castlebuilder' + description = """ + Castle Builder game. + + """ + package = 'com.ettinentertainment.castlebuilder' + activity = 'com.unity3d.player.UnityPlayerProxyActivity' diff --git a/wlauto/workloads/castlebuilder/revent_files/.empty b/wlauto/workloads/castlebuilder/revent_files/.empty new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wlauto/workloads/castlebuilder/revent_files/.empty diff --git a/wlauto/workloads/castlebuilder/revent_files/Nexus10.run.revent b/wlauto/workloads/castlebuilder/revent_files/Nexus10.run.revent Binary files differnew file mode 100644 index 00000000..704231f5 --- /dev/null +++ b/wlauto/workloads/castlebuilder/revent_files/Nexus10.run.revent diff --git a/wlauto/workloads/castlebuilder/revent_files/Nexus10.setup.revent b/wlauto/workloads/castlebuilder/revent_files/Nexus10.setup.revent Binary files differnew file mode 100644 index 00000000..25370392 --- /dev/null +++ b/wlauto/workloads/castlebuilder/revent_files/Nexus10.setup.revent diff --git a/wlauto/workloads/castlemaster/__init__.py b/wlauto/workloads/castlemaster/__init__.py new file mode 100644 index 00000000..fa104b81 --- /dev/null +++ b/wlauto/workloads/castlemaster/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from wlauto import GameWorkload + + +class CastleMaster(GameWorkload): + + name = 'castlemaster' + description = """ + Castle Master v1.09 game. + + """ + package = 'com.alphacloud.castlemaster' + activity = 'com.unity3d.player.UnityPlayerActivity' + install_timeout = 500 + diff --git a/wlauto/workloads/castlemaster/revent_files/.empty b/wlauto/workloads/castlemaster/revent_files/.empty new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wlauto/workloads/castlemaster/revent_files/.empty diff --git a/wlauto/workloads/castlemaster/revent_files/Nexus10.run.revent b/wlauto/workloads/castlemaster/revent_files/Nexus10.run.revent Binary files differnew file mode 100644 index 00000000..404f7c71 --- /dev/null +++ b/wlauto/workloads/castlemaster/revent_files/Nexus10.run.revent diff --git a/wlauto/workloads/castlemaster/revent_files/Nexus10.setup.revent b/wlauto/workloads/castlemaster/revent_files/Nexus10.setup.revent Binary files differnew file mode 100644 index 00000000..680d2e34 --- /dev/null +++ b/wlauto/workloads/castlemaster/revent_files/Nexus10.setup.revent diff --git a/wlauto/workloads/cfbench/__init__.py b/wlauto/workloads/cfbench/__init__.py new file mode 100644 index 00000000..a9fab988 --- /dev/null +++ b/wlauto/workloads/cfbench/__init__.py @@ -0,0 +1,72 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import xml.etree.ElementTree as ET + +from wlauto import AndroidUiAutoBenchmark + + +class Cfbench(AndroidUiAutoBenchmark): + + name = 'cfbench' + description = """ + CF-Bench is (mainly) CPU and memory benchmark tool specifically designed to + be able to handle multi-core devices, produce a fairly stable score, and + test both native as well managed code performance. + + https://play.google.com/store/apps/details?id=eu.chainfire.cfbench&hl=en + + From the website: + + It tests specific device properties you do not regularly see tested by other + benchmarks, and runs in a set timeframe. + + It does produce some "final" scores, but as with every benchmark, you should + take those with a grain of salt. It is simply not theoretically possible to + produce a single number that accurately describes a device's performance. + + .. note:: This workload relies on the device being rooted + + """ + package = 'eu.chainfire.cfbench' + activity = '.MainActivity' + run_timeout = 5 * 60 # seconds + summary_metrics = ['overall_score'] + + cfbench_params = ['java_mdflops', 'native_memory_read', 'java_msflops', 'native_disk_read', 'native_score', 'java_efficiency_memory_read', + 'native_mips', 'native_mdflops', 'java_score', 'native_memory_write', 'java_memory_write', 'native_mallocs', 'native_msflops', + 'java_mips', 'java_efficiency_mdflops', 'overall_score', 'java_memory_read', 'java_efficiency_memory_write', 'java_efficiency_mips', + 'java_efficiency_msflops', 'native_disk_write'] + + def update_result(self, context): + super(Cfbench, self).update_result(context) + device_results_file = os.path.join(self.device.package_data_directory, + self.package, + 'shared_prefs', 'eu.chainfire.cfbench_preferences.xml ') + self.device.execute('cp {} {}'.format(device_results_file, self.device.working_directory), as_root=True) + self.device.pull_file(os.path.join(self.device.working_directory, 'eu.chainfire.cfbench_preferences.xml'), context.output_directory) + result_file = os.path.join(context.output_directory, 'eu.chainfire.cfbench_preferences.xml') + tree = ET.parse(result_file) + root = tree.getroot() + for child in root: + if child.attrib['name'] in self.cfbench_params: + if '%' in child.text: + value = float(child.text.split('%')[0]) / 100 + else: + value = int(child.text) + context.result.add_metric(child.attrib['name'], value) + + diff --git a/wlauto/workloads/cfbench/com.arm.wlauto.uiauto.cfbench.jar b/wlauto/workloads/cfbench/com.arm.wlauto.uiauto.cfbench.jar Binary files differnew file mode 100644 index 00000000..1b4ae753 --- /dev/null +++ b/wlauto/workloads/cfbench/com.arm.wlauto.uiauto.cfbench.jar diff --git a/wlauto/workloads/cfbench/uiauto/build.sh b/wlauto/workloads/cfbench/uiauto/build.sh new file mode 100755 index 00000000..d72e4d38 --- /dev/null +++ b/wlauto/workloads/cfbench/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.cfbench.jar ]]; then + cp bin/com.arm.wlauto.uiauto.cfbench.jar .. +fi diff --git a/wlauto/workloads/cfbench/uiauto/build.xml b/wlauto/workloads/cfbench/uiauto/build.xml new file mode 100644 index 00000000..994c34e7 --- /dev/null +++ b/wlauto/workloads/cfbench/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.cfbench" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/cfbench/uiauto/project.properties b/wlauto/workloads/cfbench/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/cfbench/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/cfbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/cfbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..0e61d92d --- /dev/null +++ b/wlauto/workloads/cfbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,63 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.cfbench; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "cfbench"; + + public void runUiAutomation() throws Exception{ + Bundle status = new Bundle(); + status.putString("product", getUiDevice().getProductName()); + UiSelector selector = new UiSelector(); + UiObject text_bench = new UiObject(selector.text("Full Benchmark") + .className("android.widget.TextView")); + + text_bench.click(); + sleep(2); + + try{ + UiObject stop_text = new UiObject(selector.textContains("Benchmarking ...") + .className("android.widget.TextView")); + waitUntilNoObject(stop_text, 600); + + sleep(2); + }finally{ + takeScreenshot("cf-bench"); + } + + UiScrollable res = new UiScrollable(new UiSelector());//.scrollable(true)); + res.flingToEnd(10); + sleep(2); + + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + +} diff --git a/wlauto/workloads/citadel/__init__.py b/wlauto/workloads/citadel/__init__.py new file mode 100644 index 00000000..71b433b9 --- /dev/null +++ b/wlauto/workloads/citadel/__init__.py @@ -0,0 +1,44 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101 +import time + +from wlauto import GameWorkload, Parameter + + +class EpicCitadel(GameWorkload): + + name = 'citadel' + description = """ + Epic Citadel demo showcasing Unreal Engine 3. + + The game has very rich graphics details. The workload only moves around its + environment for the specified time. + + """ + package = 'com.epicgames.EpicCitadel' + activity = '.UE3JavaApp' + install_timeout = 120 + + parameters = [ + Parameter('duration', kind=int, default=60, + description=('Duration, in seconds, of the run (may need to be adjusted for ' + 'different devices.')), + ] + + def run(self, context): + super(EpicCitadel, self).run(context) + time.sleep(self.duration) diff --git a/wlauto/workloads/citadel/revent_files/.empty b/wlauto/workloads/citadel/revent_files/.empty new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wlauto/workloads/citadel/revent_files/.empty diff --git a/wlauto/workloads/citadel/revent_files/Nexus10.run.revent b/wlauto/workloads/citadel/revent_files/Nexus10.run.revent Binary files differnew file mode 100644 index 00000000..ac580434 --- /dev/null +++ b/wlauto/workloads/citadel/revent_files/Nexus10.run.revent diff --git a/wlauto/workloads/citadel/revent_files/Nexus10.setup.revent b/wlauto/workloads/citadel/revent_files/Nexus10.setup.revent Binary files differnew file mode 100644 index 00000000..df45cf44 --- /dev/null +++ b/wlauto/workloads/citadel/revent_files/Nexus10.setup.revent diff --git a/wlauto/workloads/cyclictest/LICENSE b/wlauto/workloads/cyclictest/LICENSE new file mode 100644 index 00000000..58b35d03 --- /dev/null +++ b/wlauto/workloads/cyclictest/LICENSE @@ -0,0 +1,8 @@ +cyclictest binaries included here are part of the Linux kernel and are distributed +under GPL version 2; The full text of the license may be viewed here: + +http://www.gnu.org/licenses/gpl-2.0.html + +Source for these binaries can be obtained here: + +http://git.kernel.org/cgit/linux/kernel/git/clrkwllms/rt-tests.git diff --git a/wlauto/workloads/cyclictest/__init__.py b/wlauto/workloads/cyclictest/__init__.py new file mode 100644 index 00000000..700bd993 --- /dev/null +++ b/wlauto/workloads/cyclictest/__init__.py @@ -0,0 +1,141 @@ +# Copyright 2012-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=no-member +# pylint: disable=attribute-defined-outside-init + +import os +import time + +from wlauto import settings, Workload, Executable, Parameter +from wlauto.exceptions import ConfigError, WorkloadError +from wlauto.utils.types import boolean + +TXT_RESULT_NAME = 'cyclictest_result.txt' +RESULT_INTERPRETATION = { + 'T': 'Thread', + 'P': 'Priority', + 'C': 'Clock', +} + + +class Cyclictest(Workload): + + name = 'cyclictest' + description = """ + Measures the amount of time that passes between when a timer expires and + when the thread which set the timer actually runs. + + Cyclic test works by taking a time snapshot just prior to waiting for a specific + time interval (t1), then taking another time snapshot after the timer + finishes (t2), then comparing the theoretical wakeup time with the actual + wakeup time (t2 -(t1 + sleep_time)). This value is the latency for that + timers wakeup. + + """ + + parameters = [ + Parameter('clock', allowed_values=['monotonic', 'realtime'], default='realtime', + description=('specify the clock to be used during the test.')), + Parameter('duration', kind=int, default=30, + description=('Specify the length for the test to run in seconds.')), + Parameter('quiet', kind=boolean, default=True, + description=('Run the tests quiet and print only a summary on exit.')), + Parameter('thread', kind=int, default=8, + description=('Set the number of test threads')), + Parameter('latency', kind=int, default=1000000, + description=('Write the value to /dev/cpu_dma_latency')), + Parameter('extra_parameters', kind=str, default="", + description=('Any additional command line parameters to append to the ' + 'existing parameters above. A list can be found at ' + 'https://rt.wiki.kernel.org/index.php/Cyclictest or ' + 'in the help page ``cyclictest -h``')), + Parameter('clear_file_cache', kind=boolean, default=True, + description=('Clear file caches before starting test')), + Parameter('screen_off', kind=boolean, default=True, + description=('If true it will turn the screen off so that onscreen ' + 'graphics do not effect the score. This is predominantly ' + 'for devices without a GPU')), + + ] + + def setup(self, context): + self.cyclictest_on_device = 'cyclictest' + self.cyclictest_result = os.path.join(self.device.working_directory, TXT_RESULT_NAME) + self.cyclictest_command = '{} --clock={} --duration={}s --thread={} --latency={} {} {} > {}' + self.device_binary = None + + if not self.device.is_rooted: + raise WorkloadError("This workload requires a device with root premissions to run") + + if not self.device.is_installed('cyclictest'): + host_binary = context.resolver.get(Executable(self, self.device.abi, 'cyclictest')) + self.device_binary = self.device.install(host_binary) + else: + self.device_binary = 'cyclictest' + + self.cyclictest_command = self.cyclictest_command.format(self.device_binary, + 0 if self.clock == 'monotonic' else 1, + self.duration, + self.thread, + self.latency, + "--quiet" if self.quiet else "", + self.extra_parameters, + self.cyclictest_result) + + if self.clear_file_cache: + self.device.execute('sync') + self.device.set_sysfile_value('/proc/sys/vm/drop_caches', 3) + + if self.device.platform == 'android': + if self.screen_off and self.device.is_screen_on: + self.device.execute('input keyevent 26') + + def run(self, context): + self.device.execute(self.cyclictest_command, self.duration * 2, as_root=True) + + def update_result(self, context): + self.device.pull_file(self.cyclictest_result, context.output_directory) + + # Parsing the output + # Standard Cyclictest Output: + # T: 0 (31974) P:95 I:1000 C:4990 Min:9 Act:37 Avg:31 Max:59 + with open(os.path.join(context.output_directory, TXT_RESULT_NAME)) as f: + for line in f: + if line.find('C:') is not -1: + # Key = T: 0 (31974) P:95 I:1000 + # Remaing = 49990 Min:9 Act:37 Avg:31 Max:59 + # sperator = C: + (key, sperator, remaing) = line.partition('C:') + + index = key.find('T') + key = key.replace(key[index], RESULT_INTERPRETATION['T']) + index = key.find('P') + key = key.replace(key[index], RESULT_INTERPRETATION['P']) + + index = sperator.find('C') + sperator = sperator.replace(sperator[index], RESULT_INTERPRETATION['C']) + + metrics = (sperator + remaing).split() + # metrics is now in the from of ['Min:', '9', 'Act:', '37', 'Avg:', '31' , 'Max', '59'] + for i in range(0, len(metrics), 2): + full_key = key + ' ' + metrics[i][:-1] + value = int(metrics[i + 1]) + context.result.add_metric(full_key, value, 'microseconds') + + def teardown(self, context): + if self.device.platform == 'android': + if self.screen_off: + self.device.ensure_screen_is_on() + self.device.execute('rm -f {}'.format(self.cyclictest_result)) diff --git a/wlauto/workloads/cyclictest/bin/arm64/cyclictest b/wlauto/workloads/cyclictest/bin/arm64/cyclictest Binary files differnew file mode 100755 index 00000000..9d682da1 --- /dev/null +++ b/wlauto/workloads/cyclictest/bin/arm64/cyclictest diff --git a/wlauto/workloads/cyclictest/bin/armeabi/cyclictest b/wlauto/workloads/cyclictest/bin/armeabi/cyclictest Binary files differnew file mode 100755 index 00000000..e61f2076 --- /dev/null +++ b/wlauto/workloads/cyclictest/bin/armeabi/cyclictest diff --git a/wlauto/workloads/dex2oat/__init__.py b/wlauto/workloads/dex2oat/__init__.py new file mode 100644 index 00000000..440ed5b3 --- /dev/null +++ b/wlauto/workloads/dex2oat/__init__.py @@ -0,0 +1,121 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=no-member,attribute-defined-outside-init +import re +import os +import sys + +from wlauto import Workload, Parameter, ExtensionLoader +from wlauto.exceptions import WorkloadError +from wlauto.utils.android import ApkInfo +import wlauto.common.android.resources + + +class Dex2oatBenchmark(Workload): + + name = 'dex2oat' + description = """ + Benchmarks the execution time of dex2oat (a key part of APK installation process). + + ART is a new Android runtime in KitKat, which replaces Dalvik VM. ART uses Ahead-Of-Time + compilation. It pre-compiles ODEX files used by Dalvik using dex2oat tool as part of APK + installation process. + + This workload benchmarks the time it take to compile an APK using dex2oat, which has a + significant impact on the total APK installation time, and therefore user experience. + + """ + + command_template = 'dex2oat --dex-file={} --oat-file={} --instruction-set={} --dump-timing' + run_timeout = 5 * 60 + + parameters = [ + Parameter('instruction_set', default='arm64', + allowed_values=['arm', 'arm64', 'x86', 'x86_64', 'mips'], + description="""Specifies the instruction set to compile for. Only options supported by + the target device can be used."""), + ] + + def init_resources(self, context): + # TODO: find a better APK to use for this. + peacekeeper = ExtensionLoader().get_workload('peacekeeper', self.device) + self.apk_file = context.resolver.get(wlauto.common.android.resources.ApkFile(peacekeeper), version='chrome') + self.package = ApkInfo(self.apk_file).package + + def setup(self, context): + if self.device.getprop('persist.sys.dalvik.vm.lib.2') != 'libart.so': + raise WorkloadError('Android system must be using ART (rather than Dalvik) in order for dex2oat to work.') + supported = [eabi == 'armeabi' and 'arm' or eabi.split('-')[0] + for eabi in self.device.supported_eabi] + if self.instruction_set not in supported: + message = 'Instruction set "{}" is not supported by the device; (supported: {})' + raise WorkloadError(message.format(self.instruction_set, supported)) + + on_device_apk = self.device.path.join(self.device.working_directory, + os.path.basename(self.apk_file)) + self.on_device_oat = on_device_apk.replace('.apk', '-{}.oat'.format(self.instruction_set)) + self.command = self.command_template.format(on_device_apk, self.on_device_oat, self.instruction_set) + + if not self.device.file_exists(on_device_apk): + self.device.push_file(self.apk_file, on_device_apk) + + def run(self, context): + self.device.execute(self.command, self.run_timeout) + + def update_result(self, context): + """ + Retrieve the last dex2oat time from the logs. That will correspond with the run() method. + The compilation time does not. + + Pulls out the compilation time and dex2oat execution time: + I/dex2oat ( 2522): 1.8s Compile Dex File + I/dex2oat ( 2522): dex2oat took 2.366s (threads: 6) + + + """ + logcat_log = os.path.join(context.output_directory, 'logcat.log') + self.device.dump_logcat(logcat_log) + + regex_time = re.compile("^I\/dex2oat \( *[0-9]+\): dex2oat took (?P<time>[0-9]+\.?[0-9]*)(?P<unit>m?s)") + regex_comp_time = re.compile("^I\/dex2oat \( *[0-9]+\): +(?P<time>[0-9]*\.?[0-9]*)(?P<unit>m?s) Compile Dex File") + time_data, comp_time_data = None, None + with open(logcat_log) as fh: + for line in fh: + match = regex_time.search(line) + + if match: + time_data = match.groupdict() + + match = regex_comp_time.search(line) + + if match: + comp_time_data = match.groupdict() + # Last dex2oat time wins. + if time_data is not None: + time = time_data['time'] + if time_data['unit'] == "s": + time = float(time) * 1000.0 + context.result.add_metric('dex2oat_time', time, "ms", lower_is_better=True) + + if comp_time_data is not None: + time = comp_time_data['time'] + if comp_time_data['unit'] == "s": + time = float(time) * 1000.0 + context.result.add_metric('dex2oat_comp_time', time, "ms", lower_is_better=True) + + def teardown(self, context): + self.device.delete_file(self.on_device_oat) + diff --git a/wlauto/workloads/dhrystone/__init__.py b/wlauto/workloads/dhrystone/__init__.py new file mode 100644 index 00000000..b87ff99f --- /dev/null +++ b/wlauto/workloads/dhrystone/__init__.py @@ -0,0 +1,109 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#pylint: disable=E1101,W0201 + +import os +import re + +from wlauto import Workload, Parameter +from wlauto.exceptions import ConfigError + + +this_dir = os.path.dirname(__file__) + + +class Dhrystone(Workload): + + name = 'dhrystone' + description = """ + Runs the Dhrystone benchmark. + + Original source from:: + + http://classes.soe.ucsc.edu/cmpe202/benchmarks/standard/dhrystone.c + + This version has been modified to configure duration and the number of + threads used. + + """ + + bm_regex = re.compile(r'This machine benchmarks at (?P<score>\d+)') + dmips_regex = re.compile(r'(?P<score>\d+) DMIPS') + time_regex = re.compile(r'Total dhrystone run time: (?P<time>[0-9.]+)') + + default_mloops = 100 + + parameters = [ + Parameter('duration', kind=int, default=0, + description='The duration, in seconds, for which dhrystone will be executed. ' + 'Either this or ``mloops`` should be specified but not both.'), + Parameter('mloops', kind=int, default=0, + description='Millions of loops to run. Either this or ``duration`` should be ' + 'specified, but not both. If neither is specified, this will default ' + 'to ``{}``'.format(default_mloops)), + Parameter('threads', kind=int, default=4, + description='The number of separate dhrystone "threads" that will be forked.'), + Parameter('delay', kind=int, default=0, + description=('The delay, in seconds, between kicking off of dhrystone ' + 'threads (if ``threads`` > 1).')), + ] + + def setup(self, context): + host_exe = os.path.join(this_dir, 'dhrystone') + self.device_exe = self.device.install(host_exe) + execution_mode = '-l {}'.format(self.mloops) if self.mloops else '-r {}'.format(self.duration) + self.command = '{} {} -t {} -d {}'.format(self.device_exe, + execution_mode, + self.threads, self.delay) + self.timeout = self.duration and self.duration + self.delay * self.threads + 10 or 300 + + def run(self, context): + self.output = self.device.execute(self.command, timeout=self.timeout, check_exit_code=False) + + def update_result(self, context): + outfile = os.path.join(context.output_directory, 'dhrystone.output') + with open(outfile, 'w') as wfh: + wfh.write(self.output) + score_count = 0 + dmips_count = 0 + for line in self.output.split('\n'): + match = self.time_regex.search(line) + if match: + context.result.add_metric('time', float(match.group('time')), 'seconds', lower_is_better=True) + else: + match = self.bm_regex.search(line) + if match: + metric = 'thread {} score'.format(score_count) + value = int(match.group('score')) + context.result.add_metric(metric, value) + score_count += 1 + else: + match = self.dmips_regex.search(line) + if match: + metric = 'thread {} DMIPS'.format(dmips_count) + value = int(match.group('score')) + context.result.add_metric(metric, value) + dmips_count += 1 + + def teardown(self, context): + self.device.uninstall_executable('dhrystone') + + def validate(self): + if self.mloops and self.duration: # pylint: disable=E0203 + raise ConfigError('mloops and duration cannot be both specified at the same time for dhrystone.') + if not self.mloops and not self.duration: # pylint: disable=E0203 + self.mloops = self.default_mloops + diff --git a/wlauto/workloads/dhrystone/dhrystone b/wlauto/workloads/dhrystone/dhrystone Binary files differnew file mode 100755 index 00000000..68cd9b71 --- /dev/null +++ b/wlauto/workloads/dhrystone/dhrystone diff --git a/wlauto/workloads/dhrystone/src/build.sh b/wlauto/workloads/dhrystone/src/build.sh new file mode 100755 index 00000000..61fcce5d --- /dev/null +++ b/wlauto/workloads/dhrystone/src/build.sh @@ -0,0 +1,23 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +ndk-build +if [[ -f libs/armeabi/dhrystone ]]; then + echo "Dhrystone binary updated." + cp libs/armeabi/dhrystone .. + rm -rf libs + rm -rf obj +fi diff --git a/wlauto/workloads/dhrystone/src/jni/Android.mk b/wlauto/workloads/dhrystone/src/jni/Android.mk new file mode 100644 index 00000000..2f974319 --- /dev/null +++ b/wlauto/workloads/dhrystone/src/jni/Android.mk @@ -0,0 +1,11 @@ +LOCAL_PATH:= $(call my-dir) + +include $(CLEAR_VARS) +LOCAL_SRC_FILES:= dhrystone.c +LOCAL_MODULE := dhrystone +LOCAL_MODULE_TAGS := optional +LOCAL_STATIC_LIBRARIES := libc +LOCAL_SHARED_LIBRARIES := liblog +LOCAL_LDLIBS := -llog +LOCAL_CFLAGS := -O2 +include $(BUILD_EXECUTABLE) diff --git a/wlauto/workloads/dhrystone/src/jni/dhrystone.c b/wlauto/workloads/dhrystone/src/jni/dhrystone.c new file mode 100644 index 00000000..9f16003e --- /dev/null +++ b/wlauto/workloads/dhrystone/src/jni/dhrystone.c @@ -0,0 +1,959 @@ +/* ARM modifications to the original Dhrystone are */ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +/***** hpda:net.sources / homxb!gemini / 1:58 am Apr 1, 1986*/ +/* EVERBODY: Please read "APOLOGY" below. -rick 01/06/85 + * See introduction in net.arch, or net.micro + * + * "DHRYSTONE" Benchmark Program + * + * Version: C/1.1, 12/01/84 + * + * Date: PROGRAM updated 01/06/86, RESULTS updated 03/31/86 + * + * Author: Reinhold P. Weicker, CACM Vol 27, No 10, 10/84 pg. 1013 + * Translated from ADA by Rick Richardson + * Every method to preserve ADA-likeness has been used, + * at the expense of C-ness. + * + * Compile: cc -O dry.c -o drynr : No registers + * cc -O -DREG=register dry.c -o dryr : Registers + * + * Defines: Defines are provided for old C compiler's + * which don't have enums, and can't assign structures. + * The time(2) function is library dependant; Most + * return the time in seconds, but beware of some, like + * Aztec C, which return other units. + * The LOOPS define is initially set for 50000 loops. + * If you have a machine with large integers and is + * very fast, please change this number to 500000 to + * get better accuracy. Please select the way to + * measure the execution time using the TIME define. + * For single user machines, time(2) is adequate. For + * multi-user machines where you cannot get single-user + * access, use the times(2) function. If you have + * neither, use a stopwatch in the dead of night. + * Use a "printf" at the point marked "start timer" + * to begin your timings. DO NOT use the UNIX "time(1)" + * command, as this will measure the total time to + * run this program, which will (erroneously) include + * the time to malloc(3) storage and to compute the + * time it takes to do nothing. + * + * Run: drynr; dryr + * + * Results: If you get any new machine/OS results, please send to: + * + * ihnp4!castor!pcrat!rick + * + * and thanks to all that do. Space prevents listing + * the names of those who have provided some of these + * results. I'll be forwarding these results to + * Rheinhold Weicker. + * + * Note: I order the list in increasing performance of the + * "with registers" benchmark. If the compiler doesn't + * provide register variables, then the benchmark + * is the same for both REG and NOREG. + * + * PLEASE: Send complete information about the machine type, + * clock speed, OS and C manufacturer/version. If + * the machine is modified, tell me what was done. + * On UNIX, execute uname -a and cc -V to get this info. + * + * 80x8x NOTE: 80x8x benchers: please try to do all memory models + * for a particular compiler. + * + * APOLOGY (1/30/86): + * Well, I goofed things up! As pointed out by Haakon Bugge, + * the line of code marked "GOOF" below was missing from the + * Dhrystone distribution for the last several months. It + * *WAS* in a backup copy I made last winter, so no doubt it + * was victimized by sleepy fingers operating vi! + * + * The effect of the line missing is that the reported benchmarks + * are 15% too fast (at least on a 80286). Now, this creates + * a dilema - do I throw out ALL the data so far collected + * and use only results from this (corrected) version, or + * do I just keep collecting data for the old version? + * + * Since the data collected so far *is* valid as long as it + * is compared with like data, I have decided to keep + * TWO lists- one for the old benchmark, and one for the + * new. This also gives me an opportunity to correct one + * other error I made in the instructions for this benchmark. + * My experience with C compilers has been mostly with + * UNIX 'pcc' derived compilers, where the 'optimizer' simply + * fixes sloppy code generation (peephole optimization). + * But today, there exist C compiler optimizers that will actually + * perform optimization in the Computer Science sense of the word, + * by removing, for example, assignments to a variable whose + * value is never used. Dhrystone, unfortunately, provides + * lots of opportunities for this sort of optimization. + * + * I request that benchmarkers re-run this new, corrected + * version of Dhrystone, turning off or bypassing optimizers + * which perform more than peephole optimization. Please + * indicate the version of Dhrystone used when reporting the + * results to me. + * + * RESULTS BEGIN HERE + * + *----------------DHRYSTONE VERSION 1.1 RESULTS BEGIN-------------------------- + * + * MACHINE MICROPROCESSOR OPERATING COMPILER DHRYSTONES/SEC. + * TYPE SYSTEM NO REG REGS + * -------------------------- ------------ ----------- --------------- + * Apple IIe 65C02-1.02Mhz DOS 3.3 Aztec CII v1.05i 37 37 + * - Z80-2.5Mhz CPM-80 v2.2 Aztec CII v1.05g 91 91 + * - 8086-8Mhz RMX86 V6 Intel C-86 V2.0 197 203LM?? + * IBM PC/XT 8088-4.77Mhz COHERENT 2.3.43 Mark Wiiliams 259 275 + * - 8086-8Mhz RMX86 V6 Intel C-86 V2.0 287 304 ?? + * Fortune 32:16 68000-6Mhz V7+sys3+4.1BSD cc 360 346 + * PDP-11/34A w/FP-11C UNIX V7m cc 406 449 + * Macintosh512 68000-7.7Mhz Mac ROM O/S DeSmet(C ware) 625 625 + * VAX-11/750 w/FPA UNIX 4.2BSD cc 831 852 + * DataMedia 932 68000-10Mhz UNIX sysV cc 837 888 + * Plexus P35 68000-12.5Mhz UNIX sysIII cc 835 894 + * ATT PC7300 68010-10Mhz UNIX 5.0.3 cc 973 1034 + * Compaq II 80286-8Mhz MSDOS 3.1 MS C 3.0 1086 1140 LM + * IBM PC/AT 80286-7.5Mhz Venix/286 SVR2 cc 1159 1254 *15 + * Compaq II 80286-8Mhz MSDOS 3.1 MS C 3.0 1190 1282 MM + * MicroVAX II - Mach/4.3 cc 1361 1385 + * DEC uVAX II - Ultrix-32m v1.1 cc 1385 1399 + * Compaq II 80286-8Mhz MSDOS 3.1 MS C 3.0 1351 1428 + * VAX 11/780 - UNIX 4.2BSD cc 1417 1441 + * VAX-780/MA780 Mach/4.3 cc 1428 1470 + * VAX 11/780 - UNIX 5.0.1 cc 4.1.1.31 1650 1640 + * Ridge 32C V1 - ROS 3.3 Ridge C (older) 1628 1695 + * Gould PN6005 - UTX 1.1c+ (4.2) cc 1732 1884 + * Gould PN9080 custom ECL UTX-32 1.1C cc 4745 4992 + * VAX-784 - Mach/4.3 cc 5263 5555 &4 + * VAX 8600 - 4.3 BSD cc 6329 6423 + * Amdahl 5860 - UTS sysV cc 1.22 28735 28846 + * IBM3090/200 - ? ? 31250 31250 + * + * + *----------------DHRYSTONE VERSION 1.0 RESULTS BEGIN-------------------------- + * + * MACHINE MICROPROCESSOR OPERATING COMPILER DHRYSTONES/SEC. + * TYPE SYSTEM NO REG REGS + * -------------------------- ------------ ----------- --------------- + * Commodore 64 6510-1MHz C64 ROM C Power 2.8 36 36 + * HP-110 8086-5.33Mhz MSDOS 2.11 Lattice 2.14 284 284 + * IBM PC/XT 8088-4.77Mhz PC/IX cc 271 294 + * CCC 3205 - Xelos(SVR2) cc 558 592 + * Perq-II 2901 bitslice Accent S5c cc (CMU) 301 301 + * IBM PC/XT 8088-4.77Mhz COHERENT 2.3.43 MarkWilliams cc 296 317 + * Cosmos 68000-8Mhz UniSoft cc 305 322 + * IBM PC/XT 8088-4.77Mhz Venix/86 2.0 cc 297 324 + * DEC PRO 350 11/23 Venix/PRO SVR2 cc 299 325 + * IBM PC 8088-4.77Mhz MSDOS 2.0 b16cc 2.0 310 340 + * PDP11/23 11/23 Venix (V7) cc 320 358 + * Commodore Amiga ? Lattice 3.02 368 371 + * PC/XT 8088-4.77Mhz Venix/86 SYS V cc 339 377 + * IBM PC 8088-4.77Mhz MSDOS 2.0 CI-C86 2.20M 390 390 + * IBM PC/XT 8088-4.77Mhz PCDOS 2.1 Wizard 2.1 367 403 + * IBM PC/XT 8088-4.77Mhz PCDOS 3.1 Lattice 2.15 403 403 @ + * Colex DM-6 68010-8Mhz Unisoft SYSV cc 378 410 + * IBM PC 8088-4.77Mhz PCDOS 3.1 Datalight 1.10 416 416 + * IBM PC NEC V20-4.77Mhz MSDOS 3.1 MS 3.1 387 420 + * IBM PC/XT 8088-4.77Mhz PCDOS 2.1 Microsoft 3.0 390 427 + * IBM PC NEC V20-4.77Mhz MSDOS 3.1 MS 3.1 (186) 393 427 + * PDP-11/34 - UNIX V7M cc 387 438 + * IBM PC 8088, 4.77mhz PC-DOS 2.1 Aztec C v3.2d 423 454 + * Tandy 1000 V20, 4.77mhz MS-DOS 2.11 Aztec C v3.2d 423 458 + * Tandy TRS-16B 68000-6Mhz Xenix 1.3.5 cc 438 458 + * PDP-11/34 - RSTS/E decus c 438 495 + * Onyx C8002 Z8000-4Mhz IS/1 1.1 (V7) cc 476 511 + * Tandy TRS-16B 68000-6Mhz Xenix 1.3.5 Green Hills 609 617 + * DEC PRO 380 11/73 Venix/PRO SVR2 cc 577 628 + * FHL QT+ 68000-10Mhz Os9/68000 version 1.3 603 649 FH + * Apollo DN550 68010-?Mhz AegisSR9/IX cc 3.12 666 666 + * HP-110 8086-5.33Mhz MSDOS 2.11 Aztec-C 641 676 + * ATT PC6300 8086-8Mhz MSDOS 2.11 b16cc 2.0 632 684 + * IBM PC/AT 80286-6Mhz PCDOS 3.0 CI-C86 2.1 666 684 + * Tandy 6000 68000-8Mhz Xenix 3.0 cc 694 694 + * IBM PC/AT 80286-6Mhz Xenix 3.0 cc 684 704 MM + * Macintosh 68000-7.8Mhz 2M Mac Rom Mac C 32 bit int 694 704 + * Macintosh 68000-7.7Mhz - MegaMax C 2.0 661 709 + * Macintosh512 68000-7.7Mhz Mac ROM O/S DeSmet(C ware) 714 714 + * IBM PC/AT 80286-6Mhz Xenix 3.0 cc 704 714 LM + * Codata 3300 68000-8Mhz UniPlus+ (v7) cc 678 725 + * WICAT MB 68000-8Mhz System V WICAT C 4.1 585 731 ~ + * Cadmus 9000 68010-10Mhz UNIX cc 714 735 + * AT&T 6300 8086-8Mhz Venix/86 SVR2 cc 668 743 + * Cadmus 9790 68010-10Mhz 1MB SVR0,Cadmus3.7 cc 720 747 + * NEC PC9801F 8086-8Mhz PCDOS 2.11 Lattice 2.15 768 - @ + * ATT PC6300 8086-8Mhz MSDOS 2.11 CI-C86 2.20M 769 769 + * Burroughs XE550 68010-10Mhz Centix 2.10 cc 769 769 CT1 + * EAGLE/TURBO 8086-8Mhz Venix/86 SVR2 cc 696 779 + * ALTOS 586 8086-10Mhz Xenix 3.0b cc 724 793 + * DEC 11/73 J-11 micro Ultrix-11 V3.0 cc 735 793 + * ATT 3B2/300 WE32000-?Mhz UNIX 5.0.2 cc 735 806 + * Apollo DN320 68010-?Mhz AegisSR9/IX cc 3.12 806 806 + * IRIS-2400 68010-10Mhz UNIX System V cc 772 829 + * Atari 520ST 68000-8Mhz TOS DigResearch 839 846 + * IBM PC/AT 80286-6Mhz PCDOS 3.0 MS 3.0(large) 833 847 LM + * WICAT MB 68000-8Mhz System V WICAT C 4.1 675 853 S~ + * VAX 11/750 - Ultrix 1.1 4.2BSD cc 781 862 + * CCC 7350A 68000-8MHz UniSoft V.2 cc 821 875 + * VAX 11/750 - UNIX 4.2bsd cc 862 877 + * Fast Mac 68000-7.7Mhz - MegaMax C 2.0 839 904 + + * IBM PC/XT 8086-9.54Mhz PCDOS 3.1 Microsoft 3.0 833 909 C1 + * DEC 11/44 Ultrix-11 V3.0 cc 862 909 + * Macintosh 68000-7.8Mhz 2M Mac Rom Mac C 16 bit int 877 909 S + * CCC 3210 - Xelos R01(SVR2) cc 849 924 + * CCC 3220 - Ed. 7 v2.3 cc 892 925 + * IBM PC/AT 80286-6Mhz Xenix 3.0 cc -i 909 925 + * AT&T 6300 8086, 8mhz MS-DOS 2.11 Aztec C v3.2d 862 943 + * IBM PC/AT 80286-6Mhz Xenix 3.0 cc 892 961 + * VAX 11/750 w/FPA Eunice 3.2 cc 914 976 + * IBM PC/XT 8086-9.54Mhz PCDOS 3.1 Wizard 2.1 892 980 C1 + * IBM PC/XT 8086-9.54Mhz PCDOS 3.1 Lattice 2.15 980 980 C1 + * Plexus P35 68000-10Mhz UNIX System III cc 984 980 + * PDP-11/73 KDJ11-AA 15Mhz UNIX V7M 2.1 cc 862 981 + * VAX 11/750 w/FPA UNIX 4.3bsd cc 994 997 + * IRIS-1400 68010-10Mhz UNIX System V cc 909 1000 + * IBM PC/AT 80286-6Mhz Venix/86 2.1 cc 961 1000 + * IBM PC/AT 80286-6Mhz PCDOS 3.0 b16cc 2.0 943 1063 + * Zilog S8000/11 Z8001-5.5Mhz Zeus 3.2 cc 1011 1084 + * NSC ICM-3216 NSC 32016-10Mhz UNIX SVR2 cc 1041 1084 + * IBM PC/AT 80286-6Mhz PCDOS 3.0 MS 3.0(small) 1063 1086 + * VAX 11/750 w/FPA VMS VAX-11 C 2.0 958 1091 + * Stride 68000-10Mhz System-V/68 cc 1041 1111 + * Plexus P/60 MC68000-12.5Mhz UNIX SYSIII Plexus 1111 1111 + * ATT PC7300 68010-10Mhz UNIX 5.0.2 cc 1041 1111 + * CCC 3230 - Xelos R01(SVR2) cc 1040 1126 + * Stride 68000-12Mhz System-V/68 cc 1063 1136 + * IBM PC/AT 80286-6Mhz Venix/286 SVR2 cc 1056 1149 + * Plexus P/60 MC68000-12.5Mhz UNIX SYSIII Plexus 1111 1163 T + * IBM PC/AT 80286-6Mhz PCDOS 3.0 Datalight 1.10 1190 1190 + * ATT PC6300+ 80286-6Mhz MSDOS 3.1 b16cc 2.0 1111 1219 + * IBM PC/AT 80286-6Mhz PCDOS 3.1 Wizard 2.1 1136 1219 + * Sun2/120 68010-10Mhz Sun 4.2BSD cc 1136 1219 + * IBM PC/AT 80286-6Mhz PCDOS 3.0 CI-C86 2.20M 1219 1219 + * WICAT PB 68000-8Mhz System V WICAT C 4.1 998 1226 ~ + * MASSCOMP 500 68010-10MHz RTU V3.0 cc (V3.2) 1156 1238 + * Alliant FX/8 IP (68012-12Mhz) Concentrix cc -ip;exec -i 1170 1243 FX + * Cyb DataMate 68010-12.5Mhz Uniplus 5.0 Unisoft cc 1162 1250 + * PDP 11/70 - UNIX 5.2 cc 1162 1250 + * IBM PC/AT 80286-6Mhz PCDOS 3.1 Lattice 2.15 1250 1250 + * IBM PC/AT 80286-7.5Mhz Venix/86 2.1 cc 1190 1315 *15 + * Sun2/120 68010-10Mhz Standalone cc 1219 1315 + * Intel 380 80286-8Mhz Xenix R3.0up1 cc 1250 1315 *16 + * Sequent Balance 8000 NS32032-10MHz Dynix 2.0 cc 1250 1315 N12 + * IBM PC/DSI-32 32032-10Mhz MSDOS 3.1 GreenHills 2.14 1282 1315 C3 + * ATT 3B2/400 WE32100-?Mhz UNIX 5.2 cc 1315 1315 + * CCC 3250XP - Xelos R01(SVR2) cc 1215 1318 + * IBM PC/RT 032 RISC(801?)?Mhz BSD 4.2 cc 1248 1333 RT + * DG MV4000 - AOS/VS 5.00 cc 1333 1333 + * IBM PC/AT 80286-8Mhz Venix/86 2.1 cc 1275 1380 *16 + * IBM PC/AT 80286-6Mhz MSDOS 3.0 Microsoft 3.0 1250 1388 + * ATT PC6300+ 80286-6Mhz MSDOS 3.1 CI-C86 2.20M 1428 1428 + * COMPAQ/286 80286-8Mhz Venix/286 SVR2 cc 1326 1443 + * IBM PC/AT 80286-7.5Mhz Venix/286 SVR2 cc 1333 1449 *15 + * WICAT PB 68000-8Mhz System V WICAT C 4.1 1169 1464 S~ + * Tandy II/6000 68000-8Mhz Xenix 3.0 cc 1384 1477 + * MicroVAX II - Mach/4.3 cc 1513 1536 + * WICAT MB 68000-12.5Mhz System V WICAT C 4.1 1246 1537 ~ + * IBM PC/AT 80286-9Mhz SCO Xenix V cc 1540 1556 *18 + * Cyb DataMate 68010-12.5Mhz Uniplus 5.0 Unisoft cc 1470 1562 S + * VAX 11/780 - UNIX 5.2 cc 1515 1562 + * MicroVAX-II - - - 1562 1612 + * VAX-780/MA780 Mach/4.3 cc 1587 1612 + * VAX 11/780 - UNIX 4.3bsd cc 1646 1662 + * Apollo DN660 - AegisSR9/IX cc 3.12 1666 1666 + * ATT 3B20 - UNIX 5.2 cc 1515 1724 + * NEC PC-98XA 80286-8Mhz PCDOS 3.1 Lattice 2.15 1724 1724 @ + * HP9000-500 B series CPU HP-UX 4.02 cc 1724 - + * Ridge 32C V1 - ROS 3.3 Ridge C (older) 1776 - + * IBM PC/STD 80286-8Mhz MSDOS 3.0 Microsoft 3.0 1724 1785 C2 + * WICAT MB 68000-12.5Mhz System V WICAT C 4.1 1450 1814 S~ + * WICAT PB 68000-12.5Mhz System V WICAT C 4.1 1530 1898 ~ + * DEC-2065 KL10-Model B TOPS-20 6.1FT5 Port. C Comp. 1937 1946 + * Gould PN6005 - UTX 1.1(4.2BSD) cc 1675 1964 + * DEC2060 KL-10 TOPS-20 cc 2000 2000 NM + * Intel 310AP 80286-8Mhz Xenix 3.0 cc 1893 2009 + * VAX 11/785 - UNIX 5.2 cc 2083 2083 + * VAX 11/785 - VMS VAX-11 C 2.0 2083 2083 + * VAX 11/785 - UNIX SVR2 cc 2123 2083 + * VAX 11/785 - ULTRIX-32 1.1 cc 2083 2091 + * VAX 11/785 - UNIX 4.3bsd cc 2135 2136 + * WICAT PB 68000-12.5Mhz System V WICAT C 4.1 1780 2233 S~ + * Pyramid 90x - OSx 2.3 cc 2272 2272 + * Pyramid 90x FPA,cache,4Mb OSx 2.5 cc no -O 2777 2777 + * Pyramid 90x w/cache OSx 2.5 cc w/-O 3333 3333 + * IBM-4341-II - VM/SP3 Waterloo C 1.2 3333 3333 + * IRIS-2400T 68020-16.67Mhz UNIX System V cc 3105 3401 + * Celerity C-1200 ? UNIX 4.2BSD cc 3485 3468 + * SUN 3/75 68020-16.67Mhz SUN 4.2 V3 cc 3333 3571 + * IBM-4341 Model 12 UTS 5.0 ? 3685 3685 + * SUN-3/160 68020-16.67Mhz Sun 4.2 V3.0A cc 3381 3764 + * Sun 3/180 68020-16.67Mhz Sun 4.2 cc 3333 3846 + * IBM-4341 Model 12 UTS 5.0 ? 3910 3910 MN + * MC 5400 68020-16.67MHz RTU V3.0 cc (V4.0) 3952 4054 + * Intel 386/20 80386-12.5Mhz PMON debugger Intel C386v0.2 4149 4386 + * NCR Tower32 68020-16.67Mhz SYS 5.0 Rel 2.0 cc 3846 4545 + * MC 5600/5700 68020-16.67MHz RTU V3.0 cc (V4.0) 4504 4746 % + * Intel 386/20 80386-12.5Mhz PMON debugger Intel C386v0.2 4534 4794 i1 + * Intel 386/20 80386-16Mhz PMON debugger Intel C386v0.2 5304 5607 + * Gould PN9080 custom ECL UTX-32 1.1C cc 5369 5676 + * Gould 1460-342 ECL proc UTX/32 1.1/c cc 5342 5677 G1 + * VAX-784 - Mach/4.3 cc 5882 5882 &4 + * Intel 386/20 80386-16Mhz PMON debugger Intel C386v0.2 5801 6133 i1 + * VAX 8600 - UNIX 4.3bsd cc 7024 7088 + * VAX 8600 - VMS VAX-11 C 2.0 7142 7142 + * Alliant FX/8 CE Concentrix cc -ce;exec -c 6952 7655 FX + * CCI POWER 6/32 COS(SV+4.2) cc 7500 7800 + * CCI POWER 6/32 POWER 6 UNIX/V cc 8236 8498 + * CCI POWER 6/32 4.2 Rel. 1.2b cc 8963 9544 + * Sperry (CCI Power 6) 4.2BSD cc 9345 10000 + * CRAY-X-MP/12 105Mhz COS 1.14 Cray C 10204 10204 + * IBM-3083 - UTS 5.0 Rel 1 cc 16666 12500 + * CRAY-1A 80Mhz CTSS Cray C 2.0 12100 13888 + * IBM-3083 - VM/CMS HPO 3.4 Waterloo C 1.2 13889 13889 + * Amdahl 470 V/8 UTS/V 5.2 cc v1.23 15560 15560 + * CRAY-X-MP/48 105Mhz CTSS Cray C 2.0 15625 17857 + * Amdahl 580 - UTS 5.0 Rel 1.2 cc v1.5 23076 23076 + * Amdahl 5860 UTS/V 5.2 cc v1.23 28970 28970 + * + * NOTE + * * Crystal changed from 'stock' to listed value. + * + This Macintosh was upgraded from 128K to 512K in such a way that + * the new 384K of memory is not slowed down by video generator accesses. + * % Single processor; MC == MASSCOMP + * NM A version 7 C compiler written at New Mexico Tech. + * @ vanilla Lattice compiler used with MicroPro standard library + * S Shorts used instead of ints + * T with Chris Torek's patches (whatever they are). + * ~ For WICAT Systems: MB=MultiBus, PB=Proprietary Bus + * LM Large Memory Model. (Otherwise, all 80x8x results are small model) + * MM Medium Memory Model. (Otherwise, all 80x8x results are small model) + * C1 Univation PC TURBO Co-processor; 9.54Mhz 8086, 640K RAM + * C2 Seattle Telecom STD-286 board + * C3 Definicon DSI-32 coprocessor + * C? Unknown co-processor board? + * CT1 Convergent Technologies MegaFrame, 1 processor. + * MN Using Mike Newtons 'optimizer' (see net.sources). + * G1 This Gould machine has 2 processors and was able to run 2 dhrystone + * Benchmarks in parallel with no slowdown. + * FH FHC == Frank Hogg Labs (Hazelwood Uniquad 2 in an FHL box). + * FX The Alliant FX/8 is a system consisting of 1-8 CEs (computation + * engines) and 1-12 IPs (interactive processors). Note N8 applies. + * RT This is one of the RT's that CMU has been using for awhile. I'm + * not sure that this is identical to the machine that IBM is selling + * to the public. + * i1 Normally, the 386/20 starter kit has a 16k direct mapped cache + * which inserts 2 or 3 wait states on a write thru. These results + * were obtained by disabling the write-thru, or essentially turning + * the cache into 0 wait state memory. + * Nnn This machine has multiple processors, allowing "nn" copies of the + * benchmark to run in the same time as 1 copy. + * &nn This machine has "nn" processors, and the benchmark results were + * obtained by having all "nn" processors working on 1 copy of dhrystone. + * (Note, this is different than Nnn. Salesmen like this measure). + * ? I don't trust results marked with '?'. These were sent to me with + * either incomplete info, or with times that just don't make sense. + * ?? means I think the performance is too poor, ?! means too good. + * If anybody can confirm these figures, please respond. + * + * ABBREVIATIONS + * CCC Concurrent Computer Corp. (was Perkin-Elmer) + * MC Masscomp + * + *--------------------------------RESULTS END---------------------------------- + * + * The following program contains statements of a high-level programming + * language (C) in a distribution considered representative: + * + * assignments 53% + * control statements 32% + * procedure, function calls 15% + * + * 100 statements are dynamically executed. The program is balanced with + * respect to the three aspects: + * - statement type + * - operand type (for simple data types) + * - operand access + * operand global, local, parameter, or constant. + * + * The combination of these three aspects is balanced only approximately. + * + * The program does not compute anything meaningfull, but it is + * syntactically and semantically correct. + * + */ + +/* Accuracy of timings and human fatigue controlled by next two lines */ +/*#define LOOPS 5000 /* Use this for slow or 16 bit machines */ +/*#define LOOPS 50000 /* Use this for slow or 16 bit machines */ +#define LOOPS 500000 /* Use this for faster machines */ + +/* Compiler dependent options */ +#undef NOENUM /* Define if compiler has no enum's */ +#undef NOSTRUCTASSIGN /* Define if compiler can't assign structures */ + +/* define only one of the next three defines */ +#define GETRUSAGE /* Use getrusage(2) time function */ +/*#define TIMES /* Use times(2) time function */ +/*#define TIME /* Use time(2) time function */ + +/* define the granularity of your times(2) function (when used) */ +/*#define HZ 60 /* times(2) returns 1/60 second (most) */ +/*#define HZ 100 /* times(2) returns 1/100 second (WECo) */ + +/* for compatibility with goofed up version */ +/*#define GOOF /* Define if you want the goofed up version */ + +/* default number of threads that will be spawned */ +#define DEFAULT_THREADS 1 + +/* Dhrystones per second obtained on VAX11/780 -- a notional 1MIPS machine. */ +/* Used in DMIPS calculation. */ +#define ONE_MIPS 1757 + +#ifdef GOOF +char Version[] = "1.0"; +#else +char Version[] = "1.1"; +#endif + +#ifdef NOSTRUCTASSIGN +#define structassign(d, s) memcpy(&(d), &(s), sizeof(d)) +#else +#define structassign(d, s) d = s +#endif + +#ifdef NOENUM +#define Ident1 1 +#define Ident2 2 +#define Ident3 3 +#define Ident4 4 +#define Ident5 5 +typedef int Enumeration; +#else +typedef enum {Ident1, Ident2, Ident3, Ident4, Ident5} Enumeration; +#endif + +typedef int OneToThirty; +typedef int OneToFifty; +typedef char CapitalLetter; +typedef char String30[31]; +typedef int Array1Dim[51]; +typedef int Array2Dim[51][51]; + +struct Record +{ + struct Record *PtrComp; + Enumeration Discr; + Enumeration EnumComp; + OneToFifty IntComp; + String30 StringComp; +}; + +typedef struct Record RecordType; +typedef RecordType * RecordPtr; +typedef int boolean; + +//#define NULL 0 +#define TRUE 1 +#define FALSE 0 + +#ifndef REG +#define REG +#endif + +extern Enumeration Func1(); +extern boolean Func2(); + +#ifdef TIMES +#include <sys/param.h> +#include <sys/types.h> +#endif +#ifdef GETRUSAGE +#include <sys/resource.h> +#endif +#include <time.h> +#include <unistd.h> +#include <sys/wait.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <sys/time.h> + + +main(int argc, char** argv) +{ + int num_threads = DEFAULT_THREADS; + int runtime = 0; + int delay = 0; + long mloops = 0; + + int opt; + while ((opt = getopt(argc, argv, "ht:r:d:l:")) != -1) { + switch (opt) { + case 'h': + printhelp(); + exit(0); + break; + case 't': + num_threads = atoi(optarg); + break; + case 'r': + runtime = atoi(optarg); + break; + case 'd': + delay = atoi(optarg); + break; + case 'l': + mloops = atoll(optarg); + break; + } + } + + if (runtime && mloops) { + fprintf(stderr, "-r and -l options cannot be specified at the same time.\n"); + exit(1); + } else if (!runtime && !mloops) { + fprintf(stderr, "Must specify either -r or -l option; use -h to see help.\n"); + exit(1); + } + + long num_loops = mloops ? mloops * 1000000L : LOOPS * num_threads; + run_dhrystone(runtime, num_threads, num_loops, delay); +} + +run_dhrystone(int duration, int num_threads, long num_loops, int delay) { + printf("duration: %d seconds\n", duration); + printf("number of threads: %d\n", num_threads); + printf("number of loops: %ld\n", num_loops); + printf("delay between starting threads: %d seconds\n", delay); + printf("\n"); + + pid_t *children = malloc(num_threads* sizeof(pid_t)); + int loops_per_thread = num_loops / num_threads; + + clock_t run_start = clock(); + + long i; + int actual_duration; + for (i = 0; i < (num_threads - 1); i++) { + pid_t c = fork(); + if (c == 0) { + // child + actual_duration = duration - i * delay; + if (actual_duration < 0) + actual_duration = 0; + run_for_duration(actual_duration, loops_per_thread); + exit(0); + } + + children[i] = c; + sleep(delay); + } + + run_for_duration(duration - delay * (num_threads - 1), loops_per_thread); + + for (i = 0; i < num_threads; i++) { + int status, w; + do { + w= wait(&status); + } while (w != -1 && (!WIFEXITED(status) && !WIFSIGNALED(status))); + } + + clock_t run_end = clock(); + printf("\nTotal dhrystone run time: %f seconds.\n", (double)(run_end - run_start) / CLOCKS_PER_SEC); + + exit(0); +} + +run_for_duration(int duration, long num_loops) { + clock_t end = clock() + duration * CLOCKS_PER_SEC; + do { + Proc0(num_loops, duration == 0); + } while (clock() < end); +} + +printhelp() { + printf("Usage: dhrystone (-h | -l MLOOPS | -r DURATION) [-t THREADS [-d DELAY]]\n"); + printf("\n"); + printf("Runs dhrystone benchmark either for a specfied duration or for a specified\n"); + printf("number of iterations.\n"); + printf("\n"); + printf("Options:\n"); + printf(" -h Print this message and exit.\n"); + printf(" -l MLOOPS Run dhrystone for the specified number of millions\n"); + printf(" of iterations (i.e. the actual number of iterations is\n"); + printf(" MLOOPS * 1e6).\n"); + printf(" -r DURATION Run dhhrystone for the specified duration (in seconds). \n"); + printf(" dhrystone will be run 500000 iterations, looping until\n"); + printf(" the specified time period has passed.\n"); + printf("\n"); + printf(" Note: -r and -l options may not be specified at the same time.\n"); + printf("\n"); + printf(" -t THREADS Specified the number of concurrent threads (processes,\n"); + printf(" actually) that will be spawned. Defaults to 1.\n"); + printf(" -d DELAY if THREADS is > 1, this specifies the delay between\n"); + printf(" spawning the threads.\n"); + printf("\n"); +} + + +/* + * Package 1 + */ +int IntGlob; +boolean BoolGlob; +char Char1Glob; +char Char2Glob; +Array1Dim Array1Glob; +Array2Dim Array2Glob; +RecordPtr PtrGlb; +RecordPtr PtrGlbNext; + +Proc0(long numloops, boolean print_result) +{ + OneToFifty IntLoc1; + REG OneToFifty IntLoc2; + OneToFifty IntLoc3; + REG char CharLoc; + REG char CharIndex; + Enumeration EnumLoc; + String30 String1Loc; + String30 String2Loc; + // extern char *malloc(); + + register unsigned int i; +#ifdef TIME + long time(); + long starttime; + long benchtime; + long nulltime; + + starttime = time( (long *) 0); + for (i = 0; i < numloops; ++i); + nulltime = time( (long *) 0) - starttime; /* Computes o'head of loop */ +#endif +#ifdef TIMES + time_t starttime; + time_t benchtime; + time_t nulltime; + struct tms tms; + + times(&tms); starttime = tms.tms_utime; + for (i = 0; i < numloops; ++i); + times(&tms); + nulltime = tms.tms_utime - starttime; /* Computes overhead of looping */ +#endif +#ifdef GETRUSAGE + struct rusage starttime; + struct rusage endtime; + struct timeval nulltime; + + getrusage(RUSAGE_SELF, &starttime); + for (i = 0; i < numloops; ++i); + getrusage(RUSAGE_SELF, &endtime); + nulltime.tv_sec = endtime.ru_utime.tv_sec - starttime.ru_utime.tv_sec; + nulltime.tv_usec = endtime.ru_utime.tv_usec - starttime.ru_utime.tv_usec; +#endif + + PtrGlbNext = (RecordPtr) malloc(sizeof(RecordType)); + PtrGlb = (RecordPtr) malloc(sizeof(RecordType)); + PtrGlb->PtrComp = PtrGlbNext; + PtrGlb->Discr = Ident1; + PtrGlb->EnumComp = Ident3; + PtrGlb->IntComp = 40; + strcpy(PtrGlb->StringComp, "DHRYSTONE PROGRAM, SOME STRING"); +#ifndef GOOF + strcpy(String1Loc, "DHRYSTONE PROGRAM, 1'ST STRING"); /*GOOF*/ +#endif + Array2Glob[8][7] = 10; /* Was missing in published program */ + +/***************** +-- Start Timer -- +*****************/ +#ifdef TIME + starttime = time( (long *) 0); +#endif +#ifdef TIMES + times(&tms); starttime = tms.tms_utime; +#endif +#ifdef GETRUSAGE + getrusage (RUSAGE_SELF, &starttime); +#endif + for (i = 0; i < numloops; ++i) + { + + Proc5(); + Proc4(); + IntLoc1 = 2; + IntLoc2 = 3; + strcpy(String2Loc, "DHRYSTONE PROGRAM, 2'ND STRING"); + EnumLoc = Ident2; + BoolGlob = ! Func2(String1Loc, String2Loc); + while (IntLoc1 < IntLoc2) + { + IntLoc3 = 5 * IntLoc1 - IntLoc2; + Proc7(IntLoc1, IntLoc2, &IntLoc3); + ++IntLoc1; + } + Proc8(Array1Glob, Array2Glob, IntLoc1, IntLoc3); + Proc1(PtrGlb); + for (CharIndex = 'A'; CharIndex <= Char2Glob; ++CharIndex) + if (EnumLoc == Func1(CharIndex, 'C')) + Proc6(Ident1, &EnumLoc); + IntLoc3 = IntLoc2 * IntLoc1; + IntLoc2 = IntLoc3 / IntLoc1; + IntLoc2 = 7 * (IntLoc3 - IntLoc2) - IntLoc1; + Proc2(&IntLoc1); + } + +/***************** +-- Stop Timer -- +*****************/ + + if (print_result) { +#ifdef TIME + benchtime = time( (long *) 0) - starttime - nulltime; + printf("Dhrystone(%s) time for %ld passes = %ld\n", + Version, + (long) numloops, benchtime); + printf("This machine benchmarks at %ld dhrystones/second\n", + ((long) numloops) / benchtime); + printf(" %ld DMIPS\n", + ((long) numloops) / benchtime / ONE_MIPS); +#endif +#ifdef TIMES + times(&tms); + benchtime = tms.tms_utime - starttime - nulltime; + printf("Dhrystone(%s) time for %ld passes = %ld\n", + Version, + (long) numloops, benchtime/HZ); + printf("This machine benchmarks at %ld dhrystones/second\n", + ((long) numloops) * HZ / benchtime); + printf(" %ld DMIPS\n", + ((long) numloops) * HZ / benchtime / ONE_MIPS); +#endif +#ifdef GETRUSAGE + getrusage(RUSAGE_SELF, &endtime); + { + double t = (double)(endtime.ru_utime.tv_sec + - starttime.ru_utime.tv_sec + - nulltime.tv_sec) + + (double)(endtime.ru_utime.tv_usec + - starttime.ru_utime.tv_usec + - nulltime.tv_usec) * 1e-6; + printf("Dhrystone(%s) time for %ld passes = %.1f\n", + Version, + (long)numloops, + t); + printf("This machine benchmarks at %.0f dhrystones/second\n", + (double)numloops / t); + printf(" %.0f DMIPS\n", + (double)numloops / t / ONE_MIPS); + } +#endif + } + +} + +Proc1(PtrParIn) +REG RecordPtr PtrParIn; +{ +#define NextRecord (*(PtrParIn->PtrComp)) + + structassign(NextRecord, *PtrGlb); + PtrParIn->IntComp = 5; + NextRecord.IntComp = PtrParIn->IntComp; + NextRecord.PtrComp = PtrParIn->PtrComp; + Proc3(NextRecord.PtrComp); + if (NextRecord.Discr == Ident1) + { + NextRecord.IntComp = 6; + Proc6(PtrParIn->EnumComp, &NextRecord.EnumComp); + NextRecord.PtrComp = PtrGlb->PtrComp; + Proc7(NextRecord.IntComp, 10, &NextRecord.IntComp); + } + else + structassign(*PtrParIn, NextRecord); + +#undef NextRecord +} + +Proc2(IntParIO) +OneToFifty *IntParIO; +{ + REG OneToFifty IntLoc; + REG Enumeration EnumLoc; + + IntLoc = *IntParIO + 10; + for(;;) + { + if (Char1Glob == 'A') + { + --IntLoc; + *IntParIO = IntLoc - IntGlob; + EnumLoc = Ident1; + } + if (EnumLoc == Ident1) + break; + } +} + +Proc3(PtrParOut) +RecordPtr *PtrParOut; +{ + if (PtrGlb != NULL) + *PtrParOut = PtrGlb->PtrComp; + else + IntGlob = 100; + Proc7(10, IntGlob, &PtrGlb->IntComp); +} + +Proc4() +{ + REG boolean BoolLoc; + + BoolLoc = Char1Glob == 'A'; + BoolLoc |= BoolGlob; + Char2Glob = 'B'; +} + +Proc5() +{ + Char1Glob = 'A'; + BoolGlob = FALSE; +} + +extern boolean Func3(); + +Proc6(EnumParIn, EnumParOut) +REG Enumeration EnumParIn; +REG Enumeration *EnumParOut; +{ + *EnumParOut = EnumParIn; + if (! Func3(EnumParIn) ) + *EnumParOut = Ident4; + switch (EnumParIn) + { + case Ident1: *EnumParOut = Ident1; break; + case Ident2: if (IntGlob > 100) *EnumParOut = Ident1; + else *EnumParOut = Ident4; + break; + case Ident3: *EnumParOut = Ident2; break; + case Ident4: break; + case Ident5: *EnumParOut = Ident3; + } +} + +Proc7(IntParI1, IntParI2, IntParOut) +OneToFifty IntParI1; +OneToFifty IntParI2; +OneToFifty *IntParOut; +{ + REG OneToFifty IntLoc; + + IntLoc = IntParI1 + 2; + *IntParOut = IntParI2 + IntLoc; +} + +Proc8(Array1Par, Array2Par, IntParI1, IntParI2) +Array1Dim Array1Par; +Array2Dim Array2Par; +OneToFifty IntParI1; +OneToFifty IntParI2; +{ + REG OneToFifty IntLoc; + REG OneToFifty IntIndex; + + IntLoc = IntParI1 + 5; + Array1Par[IntLoc] = IntParI2; + Array1Par[IntLoc+1] = Array1Par[IntLoc]; + Array1Par[IntLoc+30] = IntLoc; + for (IntIndex = IntLoc; IntIndex <= (IntLoc+1); ++IntIndex) + Array2Par[IntLoc][IntIndex] = IntLoc; + ++Array2Par[IntLoc][IntLoc-1]; + Array2Par[IntLoc+20][IntLoc] = Array1Par[IntLoc]; + IntGlob = 5; +} + +Enumeration Func1(CharPar1, CharPar2) +CapitalLetter CharPar1; +CapitalLetter CharPar2; +{ + REG CapitalLetter CharLoc1; + REG CapitalLetter CharLoc2; + + CharLoc1 = CharPar1; + CharLoc2 = CharLoc1; + if (CharLoc2 != CharPar2) + return (Ident1); + else + return (Ident2); +} + +boolean Func2(StrParI1, StrParI2) +String30 StrParI1; +String30 StrParI2; +{ + REG OneToThirty IntLoc; + REG CapitalLetter CharLoc; + + IntLoc = 1; + while (IntLoc <= 1) + if (Func1(StrParI1[IntLoc], StrParI2[IntLoc+1]) == Ident1) + { + CharLoc = 'A'; + ++IntLoc; + } + if (CharLoc >= 'W' && CharLoc <= 'Z') + IntLoc = 7; + if (CharLoc == 'X') + return(TRUE); + else + { + if (strcmp(StrParI1, StrParI2) > 0) + { + IntLoc += 7; + return (TRUE); + } + else + return (FALSE); + } +} + +boolean Func3(EnumParIn) +REG Enumeration EnumParIn; +{ + REG Enumeration EnumLoc; + + EnumLoc = EnumParIn; + if (EnumLoc == Ident3) return (TRUE); + return (FALSE); +} + +#ifdef NOSTRUCTASSIGN +memcpy(d, s, l) +register char *d; +register char *s; +register int l; +{ + while (l--) *d++ = *s++; +} +#endif +/* ---------- */ diff --git a/wlauto/workloads/dungeondefenders/__init__.py b/wlauto/workloads/dungeondefenders/__init__.py new file mode 100644 index 00000000..da924202 --- /dev/null +++ b/wlauto/workloads/dungeondefenders/__init__.py @@ -0,0 +1,34 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=R0801 +import os +import time + +from wlauto import GameWorkload +from wlauto.exceptions import WorkloadError, DeviceError + + +class DungeonDefenders(GameWorkload): + + name = 'dungeondefenders' + description = """ + Dungeon Defenders game. + + """ + package = 'com.trendy.ddapp' + activity = 'com.trendy.ddapp.ddapp' + loading_time = 20 + asset_file = 'com.trendy.ddapp.tar.gz' diff --git a/wlauto/workloads/dungeondefenders/revent_files/Nexus10.run.revent b/wlauto/workloads/dungeondefenders/revent_files/Nexus10.run.revent Binary files differnew file mode 100644 index 00000000..42b13a84 --- /dev/null +++ b/wlauto/workloads/dungeondefenders/revent_files/Nexus10.run.revent diff --git a/wlauto/workloads/dungeondefenders/revent_files/Nexus10.setup.revent b/wlauto/workloads/dungeondefenders/revent_files/Nexus10.setup.revent Binary files differnew file mode 100644 index 00000000..d3575a75 --- /dev/null +++ b/wlauto/workloads/dungeondefenders/revent_files/Nexus10.setup.revent diff --git a/wlauto/workloads/facebook/__init__.py b/wlauto/workloads/facebook/__init__.py new file mode 100644 index 00000000..cbc9a7c8 --- /dev/null +++ b/wlauto/workloads/facebook/__init__.py @@ -0,0 +1,82 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os +import time +import sys + +from wlauto import AndroidUiAutoBenchmark +from wlauto import UiAutomatorWorkload +from wlauto import AndroidBenchmark + + +class Facebook(AndroidUiAutoBenchmark): + + name = 'facebook' + description = """ + Uses com.facebook.patana apk for facebook workload. + This workload does the following activities in facebook + + Login to facebook account. + Send a message. + Check latest notification. + Search particular user account and visit his/her facebook account. + Find friends. + Update the facebook status + + [NOTE: This workload starts disableUpdate workload as a part of setup to + disable online updates, which helps to tackle problem of uncertain + behavier during facebook workload run.] + + """ + package = 'com.facebook.katana' + activity = '.LoginActivity' + + #'du' specify 'disable update' + du_activity = 'com.android.vending/.AssetBrowserActivity' + du_method_string = 'com.arm.wlauto.uiauto.facebook.UiAutomation#disableUpdate' + du_jar_file = '/data/local/wa_usecases/com.arm.wlauto.uiauto.facebook.jar' + du_run_timeout = 4 * 60 + du_working_dir = '/data/local/wa_usecases' + du_apk_file = '/disableupdateapk/com.android.vending-4.3.10.apk' + DELAY = 5 + + def setup(self, context): + UiAutomatorWorkload.setup(self, context) + + #Start the play store activity + self.device.execute('am start {}'.format(self.du_activity)) + + #Creating command + command = 'uiautomator runtest {} -e workdir {} -c {}'.format(self.du_jar_file, + self.du_working_dir, + self.du_method_string) + + #Start the disable update workload + self.device.execute(command, self.du_run_timeout) + time.sleep(self.DELAY) + + #Stop the play store activity + self.device.execute('am force-stop com.android.vending') + + AndroidBenchmark.setup(self, context) + + def update_result(self, context): + super(Facebook, self).update_result(context) + + def teardown(self, context): + pass + diff --git a/wlauto/workloads/facebook/com.arm.wlauto.uiauto.facebook.jar b/wlauto/workloads/facebook/com.arm.wlauto.uiauto.facebook.jar Binary files differnew file mode 100644 index 00000000..098030b0 --- /dev/null +++ b/wlauto/workloads/facebook/com.arm.wlauto.uiauto.facebook.jar diff --git a/wlauto/workloads/facebook/uiauto/build.sh b/wlauto/workloads/facebook/uiauto/build.sh new file mode 100755 index 00000000..00535591 --- /dev/null +++ b/wlauto/workloads/facebook/uiauto/build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.facebook.jar ]]; then + cp bin/com.arm.wlauto.uiauto.facebook.jar .. +fi diff --git a/wlauto/workloads/facebook/uiauto/build.xml b/wlauto/workloads/facebook/uiauto/build.xml new file mode 100644 index 00000000..e39db0ff --- /dev/null +++ b/wlauto/workloads/facebook/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.facebook" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/facebook/uiauto/project.properties b/wlauto/workloads/facebook/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/facebook/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/facebook/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/facebook/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..3c9dbb2c --- /dev/null +++ b/wlauto/workloads/facebook/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,257 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.facebook; + +import android.app.Activity; +import android.os.Bundle; +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "facebook"; + + /* + * The 'runUiAutomation' method implements the following activities + * Login to facebook account. + * Send a message. + * Check latest notification. + * Search particular user account and visit his/her facebook account. + * Go to find friends. + * Update the facebook status + */ + public void runUiAutomation() throws Exception { + final int timeout = 5; + UiSelector selector = new UiSelector(); + + UiObject logInButton = new UiObject(selector + .className("android.widget.Button").index(3).text("Log In")); + + UiObject emailField = new UiObject(selector + .className("android.widget.EditText").index(1)); + emailField.clearTextField(); + emailField.setText("abkksathe@gmail.com"); + + UiObject passwordField = new UiObject(selector + .className("android.widget.EditText").index(2)); + passwordField.clearTextField(); + passwordField.setText("highelymotivated"); + + logInButton.clickAndWaitForNewWindow(timeout); + + sleep(timeout); + + //Click on message logo + UiObject messageLogo = new UiObject(new UiSelector() + .className("android.widget.RelativeLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(3) + .childSelector(new UiSelector() + .className("android.widget.RelativeLayout").index(1) + .childSelector(new UiSelector() + .className("android.widget.ImageButton").index(0))))); + messageLogo.clickAndWaitForNewWindow(timeout); + + //send message + UiObject clickMessage = new UiObject(new UiSelector() + .className("android.support.v4.view.ViewPager").index(0) + .childSelector(new UiSelector() + .className("android.widget.RelativeLayout").index(1))); + clickMessage.clickAndWaitForNewWindow(timeout); + + sleep(timeout); + + UiObject sendMessage = new UiObject(new UiSelector() + .className("android.widget.FrameLayout").index(4) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(2)) + .childSelector(new UiSelector() + .className("android.widget.EditText").index(0) + .text("Write a message"))); + sendMessage.click(); + + sleep(timeout); + + UiObject editMessage = new UiObject(new UiSelector() + .className("android.widget.EditText").text("Write a message")); + + editMessage.setText("Hi how are you?????"); + + UiObject sendButton = new UiObject(new UiSelector() + .className("android.widget.TextView").text("Send")); + sendButton.click(); + + getUiDevice().pressDPadDown(); + sleep(timeout); + getUiDevice().pressBack(); + sleep(timeout); + getUiDevice().pressBack(); + + //Check for notifications + UiObject clickNotificationsLogo = new UiObject(new UiSelector() + .className("android.widget.RelativeLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(3) + .childSelector(new UiSelector() + .className("android.widget.RelativeLayout").index(2) + .childSelector(new UiSelector() + .className("android.widget.ImageButton").index(0))))); + clickNotificationsLogo.clickAndWaitForNewWindow(timeout); + + //Click on latest notification + UiObject clickNotify = new UiObject(new UiSelector() + .className("android.support.v4.view.ViewPager").index(0) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(1))); + clickNotify.clickAndWaitForNewWindow(timeout); + + sleep(timeout); + getUiDevice().pressBack(); + sleep(timeout); + getUiDevice().pressBack(); + + //Search for the facebook account + UiObject clickBar = new UiObject(new UiSelector() + .className("android.view.View").index(0) + .childSelector(new UiSelector() + .className("android.widget.ImageButton").index(0) + .description("Main navigation menu"))); + clickBar.clickAndWaitForNewWindow(timeout); + + UiObject clickSearch = new UiObject(new UiSelector() + .className("android.widget.FrameLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.FrameLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.FrameLayout").index(1) + .childSelector(new UiSelector() + .className("android.widget.EditText").index(1) + .text("Search")))))); + clickSearch.clickAndWaitForNewWindow(timeout); + + UiObject editSearch = new UiObject(new UiSelector() + .className("android.widget.EditText").index(0).text("Search")); + + editSearch.clearTextField(); + editSearch.setText("amol kamble"); + sleep(timeout); + + UiObject clickOnSearchResult = new UiObject(new UiSelector() + .className("android.webkit.WebView").index(0)); + clickOnSearchResult.clickTopLeft(); + + sleep(2 * timeout); + + getUiDevice().pressBack(); + sleep(timeout); + getUiDevice().pressBack(); + + clickBar.click(); + + sleep(timeout); + + //Click on find friends + UiObject clickFriends = new UiObject(new UiSelector() + .className("android.widget.FrameLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.FrameLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.FrameLayout").index(1) + .childSelector(new UiSelector() + .className("android.widget.RelativeLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.ListView").index(2))))))); + + UiObject friends = clickFriends.getChild(new UiSelector() + .className("android.widget.RelativeLayout").index(3)); + friends.click(); + sleep(timeout); + getUiDevice().pressBack(); + + //Update the status + UiObject updateStatus = new UiObject(new UiSelector() + .className("android.widget.FrameLayout").index(1) + .childSelector(new UiSelector() + .className("android.widget.FrameLayout").index(1) + .childSelector(new UiSelector() + .className("android.widget.RelativeLayout").index(1) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(0))))))); + + updateStatus.clickAndWaitForNewWindow(timeout); + + UiObject editUpdateStatus = new UiObject(new UiSelector() + .className("android.widget.EditText") + .text("What's on your mind?")); + editUpdateStatus.clearTextField(); + editUpdateStatus.setText("hellllooooooo its done!!"); + + UiObject clickPost = new UiObject(new UiSelector() + .className("android.widget.RelativeLayout").index(0) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout").index(3))); + clickPost.clickAndWaitForNewWindow(timeout); + getUiDevice().pressHome(); + } + + //disable update using playstore + public void disableUpdate() throws UiObjectNotFoundException { + + UiObject accountSelect = new UiObject(new UiSelector() + .className("android.widget.Button").text("Accept")); + + if (accountSelect.exists()) + accountSelect.click(); + + UiObject moreOptions = new UiObject(new UiSelector() + .className("android.widget.ImageButton") + .description("More options")); + moreOptions.click(); + + UiObject settings = new UiObject(new UiSelector() + .className("android.widget.TextView").text("Settings")); + settings.clickAndWaitForNewWindow(); + + UiObject autoUpdate = new UiObject(new UiSelector() + .className("android.widget.TextView") + .text("Auto-update apps")); + + autoUpdate.clickAndWaitForNewWindow(); + + UiObject clickAutoUpdate = new UiObject(new UiSelector() + .className("android.widget.CheckedTextView") + .text("Do not auto-update apps")); + + clickAutoUpdate.clickAndWaitForNewWindow(); + + getUiDevice().pressBack(); + getUiDevice().pressHome(); + } +} diff --git a/wlauto/workloads/geekbench/__init__.py b/wlauto/workloads/geekbench/__init__.py new file mode 100644 index 00000000..84a048f6 --- /dev/null +++ b/wlauto/workloads/geekbench/__init__.py @@ -0,0 +1,351 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101 +import os +import re +import tempfile +import json +from collections import defaultdict + +from wlauto import AndroidUiAutoBenchmark, Parameter, Artifact +from wlauto.exceptions import ConfigError, WorkloadError +from wlauto.utils.misc import capitalize +import wlauto.common.android.resources + + +class Geekbench(AndroidUiAutoBenchmark): + + name = 'geekbench' + description = """ + Geekbench provides a comprehensive set of benchmarks engineered to quickly + and accurately measure processor and memory performance. + + http://www.primatelabs.com/geekbench/ + + From the website: + + Designed to make benchmarks easy to run and easy to understand, Geekbench + takes the guesswork out of producing robust and reliable benchmark results. + + Geekbench scores are calibrated against a baseline score of 1,000 (which is + the score of a single-processor Power Mac G5 @ 1.6GHz). Higher scores are + better, with double the score indicating double the performance. + + The benchmarks fall into one of four categories: + + - integer performance. + - floating point performance. + - memory performance. + - stream performance. + + Geekbench benchmarks: http://www.primatelabs.com/geekbench/doc/benchmarks.html + + Geekbench scoring methedology: + http://support.primatelabs.com/kb/geekbench/interpreting-geekbench-scores + + """ + summary_metrics = ['score', 'multicore_score'] + versions = { + '3': { + 'package': 'com.primatelabs.geekbench3', + 'activity': '.HomeActivity', + }, + '2': { + 'package': 'ca.primatelabs.geekbench2', + 'activity': '.HomeActivity', + }, + } + begin_regex = re.compile(r'^\s*D/WebViewClassic.loadDataWithBaseURL\(\s*\d+\s*\)' + r'\s*:\s*(?P<content>\<.*)\s*$') + replace_regex = re.compile(r'<[^>]*>') + + parameters = [ + Parameter('version', default=sorted(versions.keys())[-1], allowed_values=sorted(versions.keys()), + description='Specifies which version of the workload should be run.'), + Parameter('times', kind=int, default=1, + description=('Specfies the number of times the benchmark will be run in a "tight ' + 'loop", i.e. without performaing setup/teardown inbetween.')), + ] + + @property + def activity(self): + return self.versions[self.version]['activity'] + + @property + def package(self): + return self.versions[self.version]['package'] + + def __init__(self, device, **kwargs): + super(Geekbench, self).__init__(device, **kwargs) + self.uiauto_params['version'] = self.version + self.uiauto_params['times'] = self.times + self.run_timeout = 3 * 60 * self.times + + def init_resources(self, context): + self.apk_file = context.resolver.get(wlauto.common.android.resources.ApkFile(self), version=self.version) + self.uiauto_file = context.resolver.get(wlauto.common.android.resources.JarFile(self)) + self.device_uiauto_file = self.device.path.join(self.device.working_directory, + os.path.basename(self.uiauto_file)) + if not self.uiauto_package: + self.uiauto_package = os.path.splitext(os.path.basename(self.uiauto_file))[0] + + def update_result(self, context): + super(Geekbench, self).update_result(context) + update_method = getattr(self, 'update_result_{}'.format(self.version)) + update_method(context) + + def validate(self): + if (self.times > 1) and (self.version == '2'): + raise ConfigError('times parameter is not supported for version 2 of Geekbench.') + + def update_result_2(self, context): + score_calculator = GBScoreCalculator() + score_calculator.parse(self.logcat_log) + score_calculator.update_results(context) + + def update_result_3(self, context): + outfile_glob = self.device.path.join(self.device.package_data_directory, self.package, 'files', '*gb3') + on_device_output_files = [f.strip() for f in + self.device.execute('ls {}'.format(outfile_glob), as_root=True).split('\n')] + for i, on_device_output_file in enumerate(on_device_output_files): + host_temp_file = tempfile.mktemp() + self.device.pull_file(on_device_output_file, host_temp_file) + host_output_file = os.path.join(context.output_directory, os.path.basename(on_device_output_file)) + with open(host_temp_file) as fh: + data = json.load(fh) + os.remove(host_temp_file) + with open(host_output_file, 'w') as wfh: + json.dump(data, wfh, indent=4) + context.iteration_artifacts.append(Artifact('geekout', path=os.path.basename(on_device_output_file), + kind='data', + description='Geekbench 3 output from device.')) + context.result.add_metric(namemify('score', i), data['score']) + context.result.add_metric(namemify('multicore_score', i), data['multicore_score']) + for section in data['sections']: + context.result.add_metric(namemify(section['name'] + '_score', i), section['score']) + context.result.add_metric(namemify(section['name'] + '_multicore_score', i), + section['multicore_score']) + + +class GBWorkload(object): + """ + Geekbench workload (not to be confused with WA's workloads). This is a single test run by + geek bench, such as preforming compression or generating Madelbrot. + + """ + + # Index maps onto the hundreds digit of the ID. + categories = [None, 'integer', 'float', 'memory', 'stream'] + + # 2003 entry-level Power Mac G5 is considered to have a baseline score of + # 1000 for every category. + pmac_g5_base_score = 1000 + + units_conversion_map = { + 'K': 1, + 'M': 1000, + 'G': 1000000, + } + + def __init__(self, wlid, name, pmac_g5_st_score, pmac_g5_mt_score): + """ + :param wlid: A three-digit workload ID. Uniquely identifies a workload and also + determines the category a workload belongs to. + :param name: The name of the workload. + :param pmac_g5_st_score: Score achieved for this workload on 2003 entry-level + Power Mac G5 running in a single thread. + :param pmac_g5_mt_score: Score achieved for this workload on 2003 entry-level + Power Mac G5 running in multiple threads. + + """ + self.wlid = wlid + self.name = name + self.pmac_g5_st_score = pmac_g5_st_score + self.pmac_g5_mt_score = pmac_g5_mt_score + self.category = self.categories[int(wlid) // 100] + self.collected_results = [] + + def add_result(self, value, units): + self.collected_results.append(self.convert_to_kilo(value, units)) + + def convert_to_kilo(self, value, units): + return value * self.units_conversion_map[units[0]] + + def clear(self): + self.collected_results = [] + + def get_scores(self): + """ + Returns a tuple (single-thraded score, multi-threaded score) for this workload. + Some workloads only have a single-threaded score, in which case multi-threaded + score will be ``None``. + + Geekbench will perform four iterations of each workload in single-threaded and, + for some workloads, multi-threaded configurations. Thus there should always be + either four or eight scores collected for each workload. Single-threaded iterations + are always done before multi-threaded, so the ordering of the scores can be used + to determine which configuration they belong to. + + This method should not be called before score collection has finished. + + """ + no_of_results = len(self.collected_results) + if no_of_results == 4: + return (self._calculate(self.collected_results[:4], self.pmac_g5_st_score), None) + if no_of_results == 8: + return (self._calculate(self.collected_results[:4], self.pmac_g5_st_score), + self._calculate(self.collected_results[4:], self.pmac_g5_mt_score)) + else: + msg = 'Collected {} results for Geekbench {} workload;'.format(no_of_results, self.name) + msg += ' expecting either 4 or 8.' + raise WorkloadError(msg) + + def _calculate(self, values, scale_factor): + return max(values) * self.pmac_g5_base_score / scale_factor + + def __str__(self): + return self.name + + __repr__ = __str__ + + +class GBScoreCalculator(object): + """ + Parses logcat output to extract raw Geekbench workload values and converts them into + category and overall scores. + + """ + + result_regex = re.compile(r'workload (?P<id>\d+) (?P<value>[0-9.]+) ' + r'(?P<units>[a-zA-Z/]+) (?P<time>[0-9.]+)s') + + # Indicates contribution to the overall score. + category_weights = { + 'integer': 0.3357231, + 'float': 0.3594, + 'memory': 0.1926489, + 'stream': 0.1054738, + } + #pylint: disable=C0326 + workloads = [ + # ID Name Power Mac ST Power Mac MT + GBWorkload(101, 'Blowfish', 43971, 40979), + GBWorkload(102, 'Text Compress', 3202, 3280), + GBWorkload(103, 'Text Decompress', 4112, 3986), + GBWorkload(104, 'Image Compress', 8272, 8412), + GBWorkload(105, 'Image Decompress', 16800, 16330), + GBWorkload(107, 'Lua', 385, 385), + + GBWorkload(201, 'Mandelbrot', 665589, 653746), + GBWorkload(202, 'Dot Product', 481449, 455422), + GBWorkload(203, 'LU Decomposition', 889933, 877657), + GBWorkload(204, 'Primality Test', 149394, 185502), + GBWorkload(205, 'Sharpen Image', 2340, 2304), + GBWorkload(206, 'Blur Image', 791, 787), + + GBWorkload(302, 'Read Sequential', 1226708, None), + GBWorkload(304, 'Write Sequential', 683782, None), + GBWorkload(306, 'Stdlib Allocate', 3739, None), + GBWorkload(307, 'Stdlib Write', 2070681, None), + GBWorkload(308, 'Stdlib Copy', 1030360, None), + + GBWorkload(401, 'Stream Copy', 1367892, None), + GBWorkload(402, 'Stream Scale', 1296053, None), + GBWorkload(403, 'Stream Add', 1507115, None), + GBWorkload(404, 'Stream Triad', 1384526, None), + ] + + def __init__(self): + self.workload_map = {wl.wlid: wl for wl in self.workloads} + + def parse(self, filepath): + """ + Extract results from the specified file. The file should contain a logcat log of Geekbench execution. + Iteration results in the log appear as 'I/geekbench' category entries in the following format:: + + | worklod ID value units timing + | \------------- | ----/ ---/ + | | | | | + | I/geekbench(29026): [....] workload 101 132.9 MB/sec 0.0300939s + | | | + | | -----\ + | label random crap we don't care about + + """ + for wl in self.workloads: + wl.clear() + with open(filepath) as fh: + for line in fh: + match = self.result_regex.search(line) + if match: + wkload = self.workload_map[int(match.group('id'))] + wkload.add_result(float(match.group('value')), match.group('units')) + + def update_results(self, context): + """ + http://support.primatelabs.com/kb/geekbench/interpreting-geekbench-2-scores + + From the website: + + Each workload's performance is compared against a baseline to determine a score. These + scores are averaged together to determine an overall, or Geekbench, score for the system. + + Geekbench uses the 2003 entry-level Power Mac G5 as the baseline with a score of 1,000 + points. Higher scores are better, with double the score indicating double the performance. + + Geekbench provides three different kinds of scores: + + :Workload Scores: Each time a workload is executed Geekbench calculates a score based + on the computer's performance compared to the baseline + performance. There can be multiple workload scores for the + same workload as Geekbench can execute each workload multiple + times with different settings. For example, the "Dot Product" + workload is executed four times (single-threaded scalar code, + multi-threaded scalar code, single-threaded vector code, and + multi-threaded vector code) producing four "Dot Product" scores. + + :Section Scores: A section score is the average of all the workload scores for + workloads that are part of the section. These scores are useful + for determining the performance of the computer in a particular + area. See the section descriptions above for a summary on what + each section measures. + + :Geekbench Score: The Geekbench score is the weighted average of the four section + scores. The Geekbench score provides a way to quickly compare + performance across different computers and different platforms + without getting bogged down in details. + + """ + scores_by_category = defaultdict(list) + for wkload in self.workloads: + st_score, mt_score = wkload.get_scores() + scores_by_category[wkload.category].append(st_score) + context.result.add_metric(wkload.name + ' (single-threaded)', int(st_score)) + if mt_score is not None: + scores_by_category[wkload.category].append(mt_score) + context.result.add_metric(wkload.name + ' (multi-threaded)', int(mt_score)) + + overall_score = 0 + for category in scores_by_category: + scores = scores_by_category[category] + category_score = sum(scores) / len(scores) + overall_score += category_score * self.category_weights[category] + context.result.add_metric(capitalize(category) + ' Score', int(category_score)) + context.result.add_metric('Geekbench Score', int(overall_score)) + + +def namemify(basename, i): + return basename + (' {}'.format(i) if i else '') diff --git a/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar b/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar Binary files differnew file mode 100644 index 00000000..5359cc30 --- /dev/null +++ b/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar diff --git a/wlauto/workloads/geekbench/uiauto/build.sh b/wlauto/workloads/geekbench/uiauto/build.sh new file mode 100755 index 00000000..7da9f5fe --- /dev/null +++ b/wlauto/workloads/geekbench/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.geekbench.jar ]]; then + cp bin/com.arm.wlauto.uiauto.geekbench.jar .. +fi diff --git a/wlauto/workloads/geekbench/uiauto/build.xml b/wlauto/workloads/geekbench/uiauto/build.xml new file mode 100644 index 00000000..7fdf1685 --- /dev/null +++ b/wlauto/workloads/geekbench/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.geekbench" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/geekbench/uiauto/project.properties b/wlauto/workloads/geekbench/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/geekbench/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..968d2abc --- /dev/null +++ b/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,121 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.geekbench; + +import java.util.concurrent.TimeUnit; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.view.KeyEvent; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "geekbench"; + + public void runUiAutomation() throws Exception { + Bundle params = getParams(); + int version = Integer.parseInt(params.getString("version")); + int times = Integer.parseInt(params.getString("times")); + + for (int i = 0; i < times; i++) { + runBenchmarks(); + switch(version) { + case 2: + // In version 2, we scroll through the results WebView to make sure + // all results appear on the screen, which causes them to be dumped into + // logcat by the Linaro hacks. + waitForResultsv2(); + scrollThroughResults(); + break; + case 3: + // Attempting to share the results will generate the .gb3 file with + // results that can then be pulled from the device. This is not possible + // in verison 2 of Geekbench (Share option was added later). + waitForResultsv3(); + shareResults(); + break; + } + + if (i < (times - 1)) { + getUiDevice().pressBack(); + getUiDevice().pressBack(); // twice + } + } + + Bundle status = new Bundle(); + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + + public void runBenchmarks() throws Exception { + UiSelector selector = new UiSelector(); + UiObject runButton = new UiObject(selector.text("Run Benchmarks") + .className("android.widget.Button")); + if (!runButton.exists()) { + getUiDevice().pressBack(); + } + runButton.click(); + } + + public void waitForResultsv2() throws Exception { + UiSelector selector = new UiSelector(); + UiObject resultsWebview = new UiObject(selector.className("android.webkit.WebView")); + if (!resultsWebview.waitForExists(TimeUnit.SECONDS.toMillis(200))) { + throw new UiObjectNotFoundException("Did not see Geekbench results screen."); + } + } + + public void waitForResultsv3() throws Exception { + UiSelector selector = new UiSelector(); + UiObject runningTextView = new UiObject(selector.text("Running Benchmarks...") + .className("android.widget.TextView")); + runningTextView.waitForExists(TimeUnit.SECONDS.toMillis(2)); + if (!runningTextView.waitUntilGone(TimeUnit.SECONDS.toMillis(200))) { + throw new UiObjectNotFoundException("Did not get to Geekbench results screen."); + } + } + + public void scrollThroughResults() throws Exception { + UiSelector selector = new UiSelector(); + getUiDevice().pressKeyCode(KeyEvent.KEYCODE_PAGE_DOWN); + sleep(1); + getUiDevice().pressKeyCode(KeyEvent.KEYCODE_PAGE_DOWN); + sleep(1); + getUiDevice().pressKeyCode(KeyEvent.KEYCODE_PAGE_DOWN); + sleep(1); + getUiDevice().pressKeyCode(KeyEvent.KEYCODE_PAGE_DOWN); + } + + public void shareResults() throws Exception { + sleep(2); // transition + UiSelector selector = new UiSelector(); + getUiDevice().pressMenu(); + UiObject runButton = new UiObject(selector.text("Share") + .className("android.widget.TextView")); + runButton.waitForExists(500); + runButton.click(); + } +} diff --git a/wlauto/workloads/glbcorp/__init__.py b/wlauto/workloads/glbcorp/__init__.py new file mode 100644 index 00000000..2ffcdb0c --- /dev/null +++ b/wlauto/workloads/glbcorp/__init__.py @@ -0,0 +1,209 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,W0201,E0203 + +from __future__ import division +import os +import re +import time +import select +import json +import threading +import subprocess + +from wlauto import ApkWorkload, Parameter, Alias +from wlauto.exceptions import WorkloadError + + +DELAY = 2 + + +class GlbCorp(ApkWorkload): + + name = 'glb_corporate' + description = """ + GFXBench GL (a.k.a. GLBench) v3.0 Corporate version. + + This is a version of GLBench available through a corporate license (distinct + from the version available in Google Play store). + + """ + package = 'net.kishonti.gfxbench' + activity = 'net.kishonti.benchui.TestActivity' + + result_start_regex = re.compile(r'I/TfwActivity\s*\(\s*\d+\):\s+\S+\s+result: {') + preamble_regex = re.compile(r'I/TfwActivity\s*\(\s*\d+\):\s+') + + valid_test_ids = [ + 'gl_alu', + 'gl_alu_off', + 'gl_blending', + 'gl_blending_off', + 'gl_driver', + 'gl_driver_off', + 'gl_fill', + 'gl_fill_off', + 'gl_manhattan', + 'gl_manhattan_off', + 'gl_trex', + 'gl_trex_battery', + 'gl_trex_off', + 'gl_trex_qmatch', + 'gl_trex_qmatch_highp', + ] + + supported_resolutions = { + '720p': { + '-ei -w': 1280, + '-ei -h': 720, + }, + '1080p': { + '-ei -w': 1920, + '-ei -h': 1080, + } + } + + run_timeout = 3 * 60 + + parameters = [ + Parameter('times', kind=int, default=1, constraint=lambda x: x > 0, + description=('Specifies the number of times the benchmark will be run in a "tight ' + 'loop", i.e. without performaing setup/teardown inbetween.')), + Parameter('resolution', default=None, allowed_values=['720p', '1080p', '720', '1080'], + description=('Explicitly specifies the resultion under which the benchmark will ' + 'be run. If not specfied, device\'s native resoution will used.')), + Parameter('test_id', default='gl_manhattan_off', allowed_values=valid_test_ids, + description='ID of the GFXBench test to be run.') + ] + + aliases = [ + Alias('manhattan', test_id='gl_manhattan'), + Alias('manhattan_off', test_id='gl_manhattan_off'), + Alias('manhattan_offscreen', test_id='gl_manhattan_off'), + ] + + def setup(self, context): + super(GlbCorp, self).setup(context) + self.command = self._build_command() + self.monitor = GlbRunMonitor(self.device) + self.monitor.start() + + def start_activity(self): + # Unlike with most other APK workloads, we're invoking the use case + # directly by starting the activity with appropriate parameters on the + # command line during execution, so we dont' need to start activity + # during setup. + pass + + def run(self, context): + for _ in xrange(self.times): + result = self.device.execute(self.command, timeout=self.run_timeout) + if 'FAILURE' in result: + raise WorkloadError(result) + else: + self.logger.debug(result) + time.sleep(DELAY) + self.monitor.wait_for_run_end(self.run_timeout) + + def update_result(self, context): # NOQA + super(GlbCorp, self).update_result(context) + self.monitor.stop() + iteration = 0 + results = [] + with open(self.logcat_log) as fh: + try: + line = fh.next() + result_lines = [] + while True: + if self.result_start_regex.search(line): + result_lines.append('{') + line = fh.next() + while self.preamble_regex.search(line): + result_lines.append(self.preamble_regex.sub('', line)) + line = fh.next() + try: + result = json.loads(''.join(result_lines)) + results.append(result) + if iteration: + suffix = '_{}'.format(iteration) + else: + suffix = '' + for sub_result in result['results']: + frames = sub_result['score'] + elapsed_time = sub_result['elapsed_time'] / 1000 + fps = frames / elapsed_time + context.result.add_metric('score' + suffix, frames, 'frames') + context.result.add_metric('fps' + suffix, fps) + except ValueError: + self.logger.warning('Could not parse result for iteration {}'.format(iteration)) + result_lines = [] + iteration += 1 + line = fh.next() + except StopIteration: + pass # EOF + if results: + outfile = os.path.join(context.output_directory, 'glb-results.json') + with open(outfile, 'wb') as wfh: + json.dump(results, wfh, indent=4) + + def _build_command(self): + command_params = [] + command_params.append('-e test_ids "{}"'.format(self.test_id)) + if self.resolution: + if not self.resolution.endswith('p'): + self.resolution += 'p' + for k, v in self.supported_resolutions[self.resolution].iteritems(): + command_params.append('{} {}'.format(k, v)) + return 'am start -W -S -n {}/{} {}'.format(self.package, + self.activity, + ' '.join(command_params)) + + +class GlbRunMonitor(threading.Thread): + + regex = re.compile(r'I/Runner\s+\(\s*\d+\): finished:') + + def __init__(self, device): + super(GlbRunMonitor, self).__init__() + self.device = device + self.daemon = True + self.run_ended = threading.Event() + self.stop_event = threading.Event() + if self.device.adb_name: + self.command = ['adb', '-s', self.device.adb_name, 'logcat'] + else: + self.command = ['adb', 'logcat'] + + def run(self): + proc = subprocess.Popen(self.command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + while not self.stop_event.is_set(): + if self.run_ended.is_set(): + time.sleep(DELAY) + else: + ready, _, _ = select.select([proc.stdout, proc.stderr], [], [], 2) + if ready: + line = ready[0].readline() + if self.regex.search(line): + self.run_ended.set() + + def stop(self): + self.stop_event.set() + self.join() + + def wait_for_run_end(self, timeout): + self.run_ended.wait(timeout) + self.run_ended.clear() + diff --git a/wlauto/workloads/glbenchmark/__init__.py b/wlauto/workloads/glbenchmark/__init__.py new file mode 100644 index 00000000..9710c206 --- /dev/null +++ b/wlauto/workloads/glbenchmark/__init__.py @@ -0,0 +1,158 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,E0203 +import re +import os + +from wlauto import AndroidUiAutoBenchmark, Parameter, Alias +from wlauto.exceptions import ConfigError +import wlauto.common.android.resources + +# These maps provide use-friendly aliases for the most common options. +USE_CASE_MAP = { + 'egypt': 'GLBenchmark 2.5 Egypt HD', + 'egypt-classic': 'GLBenchmark 2.1 Egypt Classic', + 't-rex': 'GLBenchmark 2.7 T-Rex HD', +} + +VARIANT_MAP = { + 'onscreen': 'C24Z16 Onscreen Auto', + 'offscreen': 'C24Z16 Offscreen Auto', +} + + +class Glb(AndroidUiAutoBenchmark): + + name = 'glbenchmark' + description = """ + Measures the graphics performance of Android devices by testing + the underlying OpenGL (ES) implementation. + + http://gfxbench.com/about-gfxbench.jsp + + From the website: + + The benchmark includes console-quality high-level 3D animations + (T-Rex HD and Egypt HD) and low-level graphics measurements. + + With high vertex count and complex effects such as motion blur, parallax + mapping and particle systems, the engine of GFXBench stresses GPUs in order + provide users a realistic feedback on their device. + + """ + activity = 'com.glbenchmark.activities.GLBenchmarkDownloaderActivity' + view = 'com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBRender' + + packages = { + '2.7.0': 'com.glbenchmark.glbenchmark27', + '2.5.1': 'com.glbenchmark.glbenchmark25', + } + # If usecase is not specified the default usecase is the first supported usecase alias + # for the specified version. + supported_usecase_aliases = { + '2.7.0': ['t-rex', 'egypt'], + '2.5.1': ['egypt-classic', 'egypt'], + } + + default_iterations = 1 + install_timeout = 500 + + regex = re.compile(r'GLBenchmark (metric|FPS): (.*)') + + parameters = [ + Parameter('version', default='2.7.0', allowed_values=['2.7.0', '2.5.1'], + description=('Specifies which version of the benchmark to run (different versions ' + 'support different use cases).')), + Parameter('use_case', default=None, + description="""Specifies which usecase to run, as listed in the benchmark menu; e.g. + ``'GLBenchmark 2.5 Egypt HD'``. For convenience, two aliases are provided + for the most common use cases: ``'egypt'`` and ``'t-rex'``. These could + be use instead of the full use case title. For version ``'2.7.0'`` it defaults + to ``'t-rex'``, for version ``'2.5.1'`` it defaults to ``'egypt-classic'``. + """), + Parameter('variant', default='onscreen', + description="""Specifies which variant of the use case to run, as listed in the benchmarks + menu (small text underneath the use case name); e.g. ``'C24Z16 Onscreen Auto'``. + For convenience, two aliases are provided for the most common variants: + ``'onscreen'`` and ``'offscreen'``. These may be used instead of full variant + names. + """), + Parameter('times', kind=int, default=1, + description=('Specfies the number of times the benchmark will be run in a "tight ' + 'loop", i.e. without performaing setup/teardown inbetween.')), + Parameter('timeout', kind=int, default=200, + description="""Specifies how long, in seconds, UI automation will wait for results screen to + appear before assuming something went wrong. + """), + ] + + aliases = [ + Alias('glbench'), + Alias('egypt', use_case='egypt'), + Alias('t-rex', use_case='t-rex'), + Alias('egypt_onscreen', use_case='egypt', variant='onscreen'), + Alias('t-rex_onscreen', use_case='t-rex', variant='onscreen'), + Alias('egypt_offscreen', use_case='egypt', variant='offscreen'), + Alias('t-rex_offscreen', use_case='t-rex', variant='offscreen'), + ] + + def __init__(self, device, **kwargs): + super(Glb, self).__init__(device, **kwargs) + self.uiauto_params['version'] = self.version + + if self.use_case is None: + self.use_case = self.supported_usecase_aliases[self.version][0] + if self.use_case.lower() in USE_CASE_MAP: + if self.use_case not in self.supported_usecase_aliases[self.version]: + raise ConfigError('usecases {} is not supported in version {}'.format(self.use_case, self.version)) + self.use_case = USE_CASE_MAP[self.use_case.lower()] + self.uiauto_params['use_case'] = self.use_case.replace(' ', '_') + + if self.variant.lower() in VARIANT_MAP: + self.variant = VARIANT_MAP[self.variant.lower()] + self.uiauto_params['variant'] = self.variant.replace(' ', '_') + + self.uiauto_params['iterations'] = self.times + self.run_timeout = 4 * 60 * self.times + + self.uiauto_params['timeout'] = self.timeout + self.package = self.packages[self.version] + + def init_resources(self, context): + self.apk_file = context.resolver.get(wlauto.common.android.resources.ApkFile(self), version=self.version) + self.uiauto_file = context.resolver.get(wlauto.common.android.resources.JarFile(self)) + self.device_uiauto_file = self.device.path.join(self.device.working_directory, + os.path.basename(self.uiauto_file)) + if not self.uiauto_package: + self.uiauto_package = os.path.splitext(os.path.basename(self.uiauto_file))[0] + + def update_result(self, context): + super(Glb, self).update_result(context) + match_count = 0 + with open(self.logcat_log) as fh: + for line in fh: + match = self.regex.search(line) + if match: + metric = match.group(1) + value, units = match.group(2).split() + value = value.replace('*', '') + if metric == 'metric': + metric = 'Frames' + units = 'frames' + metric = metric + '_' + str(match_count // 2) + context.result.add_metric(metric, value, units) + match_count += 1 + diff --git a/wlauto/workloads/glbenchmark/com.arm.wlauto.uiauto.glb.jar b/wlauto/workloads/glbenchmark/com.arm.wlauto.uiauto.glb.jar Binary files differnew file mode 100644 index 00000000..57d0fb1e --- /dev/null +++ b/wlauto/workloads/glbenchmark/com.arm.wlauto.uiauto.glb.jar diff --git a/wlauto/workloads/glbenchmark/uiauto/build.sh b/wlauto/workloads/glbenchmark/uiauto/build.sh new file mode 100755 index 00000000..820eae37 --- /dev/null +++ b/wlauto/workloads/glbenchmark/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.glb.jar ]]; then + cp bin/com.arm.wlauto.uiauto.glb.jar .. +fi diff --git a/wlauto/workloads/glbenchmark/uiauto/build.xml b/wlauto/workloads/glbenchmark/uiauto/build.xml new file mode 100644 index 00000000..54ccc98b --- /dev/null +++ b/wlauto/workloads/glbenchmark/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.glb" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/glbenchmark/uiauto/project.properties b/wlauto/workloads/glbenchmark/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/glbenchmark/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..2c244d64 --- /dev/null +++ b/wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,164 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.glb; + +import java.lang.Runtime; +import java.lang.Process; +import java.util.concurrent.TimeUnit; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.view.KeyEvent; + +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "glb"; + public static int maxScrolls = 15; + + public void runUiAutomation() throws Exception { + Bundle parameters = getParams(); + String version = parameters.getString("version"); + String useCase = parameters.getString("use_case").replace('_', ' '); + String variant = parameters.getString("variant").replace('_', ' '); + int iterations = Integer.parseInt(parameters.getString("iterations")); + int testTimeoutSeconds = Integer.parseInt(parameters.getString("timeout")); + if (iterations < 1) + iterations = 1; + + goToPreformanceTestsMenu(); + selectUseCase(version, useCase, variant); + hitStart(); + waitForResults(version, useCase, testTimeoutSeconds); + extractResults(); + iterations -= 1; + + while (iterations > 0) { + getUiDevice().pressBack(); + goToPreformanceTestsMenu(); + hitStart(); + waitForResults(version, useCase, testTimeoutSeconds); + extractResults(); + iterations -= 1; + } + + Bundle status = new Bundle(); + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + + public void goToPreformanceTestsMenu() throws Exception { + UiSelector selector = new UiSelector(); + UiObject choosePerfTest = new UiObject(selector.text("Performance Tests") + .className("android.widget.TextView")); + choosePerfTest.clickAndWaitForNewWindow(); + } + + public void selectUseCase(String version, String useCase, String variant) throws Exception { + UiSelector selector = new UiSelector(); + UiScrollable testList = new UiScrollable(selector.className("android.widget.ListView")); + UiObject useCaseText = new UiObject(selector.className("android.widget.TextView") + .text(useCase) + ); + if (version.equals("2.7.0")){ + UiObject variantText = useCaseText.getFromParent(selector.className("android.widget.TextView") + .text(variant)); + int scrolls = 0; + while(!variantText.exists()) { + testList.scrollForward(); + scrolls += 1; + if (scrolls >= maxScrolls) { + break; + } + } + variantText.click(); + } + else if (version.equals("2.5.1")){ + int scrolls = 0; + while(!useCaseText.exists()) { + testList.scrollForward(); + scrolls += 1; + if (scrolls >= maxScrolls) { + break; + } + } + useCaseText.click(); + //UiSelector selector = new UiSelector(); + UiObject modeDisableModeButton = null; + if (variant.contains("Onscreen")) + modeDisableModeButton = new UiObject(selector.text("Offscreen")); + else + modeDisableModeButton = new UiObject(selector.text("Onscreen")); + modeDisableModeButton.click(); + } + } + + public void hitStart() throws Exception { + UiSelector selector = new UiSelector(); + UiObject startButton = new UiObject(selector.text("Start")); + startButton.clickAndWaitForNewWindow(); + } + + public void waitForResults(String version, String useCase, int timeout) throws Exception { + UiSelector selector = new UiSelector(); + UiObject results = null; + if (version.equals("2.7.0")) + results = new UiObject(selector.text("Results").className("android.widget.TextView")); + else + results = new UiObject(selector.text(useCase).className("android.widget.TextView")); + Log.v(TAG, "Waiting for results screen."); + // On some devices, the results screen sometimes gets "backgrounded" (or + // rather, doesn't seem to come to foreground to begin with). This code + // attemps to deal with that by explicitly bringing glbench to the + // foreground if results screen doesn't appear within testTimeoutSeconds seconds of + // starting GLB. + if (!results.waitForExists(TimeUnit.SECONDS.toMillis(timeout))) { + Log.v(TAG, "Results screen not found. Attempting to bring to foreground."); + String[] commandLine = {"am", "start", + "-a", "android.intent.action.MAIN", + "-c", "android.intent.category.LAUNCHER", + "-n", "com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBenchmarkDownloaderActivity"}; + Process proc = Runtime.getRuntime().exec(commandLine); + proc.waitFor(); + Log.v(TAG, String.format("am start exit value: %d", proc.exitValue())); + if (!results.exists()) { + throw new UiObjectNotFoundException("Could not find results screen."); + } + } + Log.v(TAG, "Results screen found."); + } + + public void extractResults() throws Exception { + Log.v(TAG, "Extracting results."); + sleep(2); // wait for the results screen to fully load. + UiSelector selector = new UiSelector(); + UiObject fpsText = new UiObject(selector.className("android.widget.TextView") + .textContains("fps") + ); + UiObject otherText = fpsText.getFromParent(selector.className("android.widget.TextView").index(0)); + + Log.v(TAG, String.format("GLBenchmark metric: %s", otherText.getText().replace('\n', ' '))); + Log.v(TAG, String.format("GLBenchmark FPS: %s", fpsText.getText().replace('\n', ' '))); + } +} diff --git a/wlauto/workloads/gunbros2/__init__.py b/wlauto/workloads/gunbros2/__init__.py new file mode 100644 index 00000000..be33dc54 --- /dev/null +++ b/wlauto/workloads/gunbros2/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=R0801 +import os +import time +import tarfile +import shutil + +from wlauto import settings +from wlauto.common.android.workload import GameWorkload +from wlauto.exceptions import WorkloadError, DeviceError +from wlauto.utils.misc import check_output +from wlauto.common.resources import ExtensionAsset + + +class GunBros(GameWorkload): + + name = 'gunbros2' + description = """ + Gun Bros. 2 game. + + """ + package = 'com.glu.gunbros2' + activity = 'com.google.android.vending.expansion.downloader_impl.DownloaderActivity' + asset_file = 'com.glu.gunbros2.tar.gz' + ondevice_asset_root = '/data' + loading_time = 20 + install_timeout = 500 + diff --git a/wlauto/workloads/homescreen/__init__.py b/wlauto/workloads/homescreen/__init__.py new file mode 100644 index 00000000..729054be --- /dev/null +++ b/wlauto/workloads/homescreen/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101 + +import time + +from wlauto import Workload, Parameter + + +class HomeScreen(Workload): + + name = 'homescreen' + description = """ + A workload that goes to the home screen and idles for the the + specified duration. + + """ + + parameters = [ + Parameter('duration', kind=int, default=20, + description='Specifies the duration, in seconds, of this workload.'), + ] + + def setup(self, context): + self.device.clear_logcat() + self.device.execute('input keyevent 3') # press the home key + + def run(self, context): + time.sleep(self.duration) diff --git a/wlauto/workloads/idle/__init__.py b/wlauto/workloads/idle/__init__.py new file mode 100644 index 00000000..03e72470 --- /dev/null +++ b/wlauto/workloads/idle/__init__.py @@ -0,0 +1,56 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101 + +import time + +from wlauto import Workload, Parameter +from wlauto.exceptions import WorkloadError + + +class IdleWorkload(Workload): + + name = 'idle' + description = """ + Stop Android and sleep for the specified duration before restarting it. + + .. note:: This workload requires the device to be rooted. + + """ + + parameters = [ + Parameter('duration', kind=int, default=20, + description='Specifies the duration, in seconds, of this workload.'), + ] + + def setup(self, context): + if not self.device.is_rooted: + raise WorkloadError('Idle workload requires the device to be rooted.') + + def run(self, context): + self.device.execute('stop && sleep {} && start'.format(self.duration), as_root=True) + + def update_result(self, context): + pass + + def teardown(self, context): + self.logger.debug('Waiting for Android restart to complete...') + # Wait for the boot animation to start and then to finish. + while self.device.execute('getprop init.svc.bootanim').strip() == 'stopped': + time.sleep(0.2) + while self.device.execute('getprop init.svc.bootanim').strip() == 'running': + time.sleep(1) + diff --git a/wlauto/workloads/ironman/__init__.py b/wlauto/workloads/ironman/__init__.py new file mode 100644 index 00000000..1bbef415 --- /dev/null +++ b/wlauto/workloads/ironman/__init__.py @@ -0,0 +1,35 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=R0801 +import os +import time + +from wlauto import GameWorkload +from wlauto.exceptions import WorkloadError, DeviceError +from wlauto.utils.misc import check_output + + +class IronMan(GameWorkload): + + name = 'ironman3' + description = """ + Iron Man 3 game. + + """ + package = 'com.gameloft.android.ANMP.GloftIMHM' + activity = '.GameActivity' + + asset_file = 'obb:com.gameloft.android.ANMP.GloftIMHM.tar.gz' diff --git a/wlauto/workloads/ironman/revent_files/Nexus10.run.revent b/wlauto/workloads/ironman/revent_files/Nexus10.run.revent Binary files differnew file mode 100644 index 00000000..96955bad --- /dev/null +++ b/wlauto/workloads/ironman/revent_files/Nexus10.run.revent diff --git a/wlauto/workloads/ironman/revent_files/Nexus10.setup.revent b/wlauto/workloads/ironman/revent_files/Nexus10.setup.revent Binary files differnew file mode 100644 index 00000000..8cc49d3a --- /dev/null +++ b/wlauto/workloads/ironman/revent_files/Nexus10.setup.revent diff --git a/wlauto/workloads/krazykart/__init__.py b/wlauto/workloads/krazykart/__init__.py new file mode 100644 index 00000000..055816a7 --- /dev/null +++ b/wlauto/workloads/krazykart/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from wlauto import GameWorkload + + +class KrazyKartRacing(GameWorkload): + + name = 'krazykart' + description = """ + Krazy Kart Racing game. + + """ + package = 'com.polarbit.sg2.krazyracers' + activity = '.krazyracers' diff --git a/wlauto/workloads/krazykart/revent_files/.empty b/wlauto/workloads/krazykart/revent_files/.empty new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wlauto/workloads/krazykart/revent_files/.empty diff --git a/wlauto/workloads/linpack/__init__.py b/wlauto/workloads/linpack/__init__.py new file mode 100644 index 00000000..3f728ab9 --- /dev/null +++ b/wlauto/workloads/linpack/__init__.py @@ -0,0 +1,64 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,E0203 + +import os +import re + +from wlauto import AndroidUiAutoBenchmark, Parameter + + +class Linpack(AndroidUiAutoBenchmark): + + name = 'linpack' + description = """ + The LINPACK Benchmarks are a measure of a system's floating point computing + power. + + http://en.wikipedia.org/wiki/LINPACK_benchmarks + + From the article: + + Introduced by Jack Dongarra, they measure how fast a computer solves + a dense n by n system of linear equations Ax = b, which is a common task in + engineering. + + """ + package = 'com.greenecomputing.linpackpro' + activity = '.Linpack' + summary_metrics = ['Linpack ST', 'Linpack MT'] + regex = re.compile(r'LINPACK RESULT: (?P<type>\w+) (?P<value>\S+)') + + parameters = [ + Parameter('output_file', default=None, + description='On-device output file path.'), + ] + + def __init__(self, device, **kwargs): + super(Linpack, self).__init__(device, **kwargs) + if self.output_file is None: + self.output_file = os.path.join(self.device.working_directory, 'linpack.txt') + self.uiauto_params['output_file'] = self.output_file + + def update_result(self, context): + super(Linpack, self).update_result(context) + with open(self.logcat_log) as fh: + for line in fh: + match = self.regex.search(line) + if match: + metric = 'Linpack ' + match.group('type') + value = float(match.group('value')) + context.result.add_metric(metric, value, 'MFLOPS') diff --git a/wlauto/workloads/linpack/com.arm.wlauto.uiauto.linpack.jar b/wlauto/workloads/linpack/com.arm.wlauto.uiauto.linpack.jar Binary files differnew file mode 100644 index 00000000..8835bdee --- /dev/null +++ b/wlauto/workloads/linpack/com.arm.wlauto.uiauto.linpack.jar diff --git a/wlauto/workloads/linpack/uiauto/build.sh b/wlauto/workloads/linpack/uiauto/build.sh new file mode 100755 index 00000000..5ff5da2e --- /dev/null +++ b/wlauto/workloads/linpack/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.linpack.jar ]]; then + cp bin/com.arm.wlauto.uiauto.linpack.jar .. +fi diff --git a/wlauto/workloads/linpack/uiauto/build.xml b/wlauto/workloads/linpack/uiauto/build.xml new file mode 100644 index 00000000..a532fd35 --- /dev/null +++ b/wlauto/workloads/linpack/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.linpack" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/linpack/uiauto/project.properties b/wlauto/workloads/linpack/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/linpack/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/linpack/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/linpack/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..de6c39ef --- /dev/null +++ b/wlauto/workloads/linpack/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,59 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.linpack; + +import java.util.concurrent.TimeUnit; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; +import com.arm.wlauto.uiauto.BaseUiAutomation; + + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "linpack"; + + public void runUiAutomation() throws Exception{ + UiSelector selector = new UiSelector(); + UiObject runSingleButton = new UiObject(selector.text("Run Single Thread")); + runSingleButton.click(); + runSingleButton.waitUntilGone(500); + runSingleButton.waitForExists(TimeUnit.SECONDS.toMillis(30)); + + UiObject mflops = new UiObject(new UiSelector().className("android.widget.TextView").instance(2)); + Log.v(TAG, String.format("LINPACK RESULT: ST %s", mflops.getText())); + + UiObject runMultiButton = new UiObject(selector.text("Run Multi-Thread")); + runMultiButton.click(); + runMultiButton.waitUntilGone(500); + runMultiButton.waitForExists(TimeUnit.SECONDS.toMillis(30)); + + Log.v(TAG, String.format("LINPACK RESULT: MT %s", mflops.getText())); + + Bundle status = new Bundle(); + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + +} diff --git a/wlauto/workloads/manual/__init__.py b/wlauto/workloads/manual/__init__.py new file mode 100644 index 00000000..344a71fb --- /dev/null +++ b/wlauto/workloads/manual/__init__.py @@ -0,0 +1,105 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=E1101,W0201,E0203 +import os +import time + +from wlauto import Workload, Parameter +from wlauto.exceptions import ConfigError +from wlauto.utils.misc import getch +from wlauto.utils.types import boolean + + +class ManualWorkloadConfig(object): + + default_duration = 30 + + def __init__(self, + duration=None, # Seconds + user_triggered=None, + view=None, + enable_logcat=True + ): + self.user_triggered = user_triggered if user_triggered is not None else (False if duration else True) + self.duration = duration or (None if self.user_triggered else self.default_duration) + self.view = view + self.enable_logcat = enable_logcat + + +class ManualWorkload(Workload): + + name = 'manual' + description = """ + Yields control to the user, either for a fixed period or based on user input, to perform + custom operations on the device, about which workload automation does not know of. + + """ + + parameters = [ + Parameter('duration', kind=int, default=None, + description=('Control of the devices is yielded for the duration (in seconds) specified. ' + 'If not specified, ``user_triggered`` is assumed.')), + Parameter('user_triggered', kind=boolean, default=None, + description="""If ``True``, WA will wait for user input after starting the workload; + otherwise fixed duration is expected. Defaults to ``True`` if ``duration`` + is not specified, and ``False`` otherwise. + """), + Parameter('view', default='SurfaceView', + description="""Specifies the View of the workload. This enables instruments that require a + View to be specified, such as the ``fps`` instrument."""), + Parameter('enable_logcat', kind=boolean, default=True, + description='If ``True``, ``manual`` workload will collect logcat as part of the results.'), + ] + + def setup(self, context): + self.logger.info('Any setup required by your workload should be done now.') + self.logger.info('As soon as you are done hit any key and wait for the message') + self.logger.info('"START NOW!" to begin your manual workload.') + self.logger.info('') + self.logger.info('hit any key to finalize your setup...') + getch() + + def run(self, context): + self.logger.info('START NOW!') + if self.duration: + time.sleep(self.duration) + elif self.user_triggered: + self.logger.info('') + self.logger.info('hit any key to end your workload execution...') + getch() + else: + raise ConfigError('Illegal parameters for manual workload') + self.logger.info('DONE! your results are now being collected!') + + def update_result(self, context): + if self.enable_logcat: + logcat_dir = os.path.join(context.output_directory, 'logcat') + self.device.dump_logcat(logcat_dir) + + def teardown(self, context): + pass + + def validate(self): + if self.duration is None: + if self.user_triggered is None: + self.user_triggered = True + elif self.user_triggered is False: + self.duration = self.default_duration + if self.user_triggered and self.duration: + message = 'Manual Workload can either specify duration or be user triggered, but not both' + raise ConfigError(message) + if not self.user_triggered and not self.duration: + raise ConfigError('Either user_triggered must be ``True`` or duration must be > 0.') + diff --git a/wlauto/workloads/memcpy/__init__.py b/wlauto/workloads/memcpy/__init__.py new file mode 100644 index 00000000..54508363 --- /dev/null +++ b/wlauto/workloads/memcpy/__init__.py @@ -0,0 +1,76 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,W0201 + +import os +import re + +from wlauto import Workload, Parameter + + +THIS_DIR = os.path.dirname(__file__) + + +RESULT_REGEX = re.compile('Total time: ([\d.]+) s.*Bandwidth: ([\d.]+) MB/s', re.S) + + +class MemcpyTest(Workload): + + name = 'memcpy' + description = """ + Runs memcpy in a loop. + + This will run memcpy in a loop for a specified number of times on a buffer + of a specified size. Additionally, the affinity of the test can be set to one + or more specific cores. + + This workload is single-threaded. It genrates no scores or metrics by itself. + + """ + + parameters = [ + Parameter('buffer_size', kind=int, default=1024 * 1024 * 5, + description='Specifies the size, in bytes, of the buffer to be copied.'), + Parameter('iterations', kind=int, default=1000, + description='Specfies the number of iterations that will be performed.'), + Parameter('cpus', kind=list, default=[], + description="""A list of integers specifying ordinals of cores to which the affinity + of the test process should be set. If not specified, all avaiable cores + will be used. + """), + ] + + def setup(self, context): + self.host_binary = os.path.join(THIS_DIR, 'memcpy') + if not self.device.is_installed('memcpy'): + self.device_binary = self.device.install(self.host_binary) + else: + self.device_binary = 'memcpy' + self.command = '{} -i {} -s {}'.format(self.device_binary, self.iterations, self.buffer_size) + if self.cpus: + for c in self.cpus: + self.command += ' -c {}'.format(c) + + def run(self, context): + self.result = self.device.execute(self.command, timeout=300) + + def update_result(self, context): + match = RESULT_REGEX.search(self.result) + context.result.add_metric('time', float(match.group(1)), 'seconds', lower_is_better=True) + context.result.add_metric('bandwidth', float(match.group(2)), 'MB/s') + + def teardown(self, context): + pass diff --git a/wlauto/workloads/memcpy/memcpy b/wlauto/workloads/memcpy/memcpy Binary files differnew file mode 100755 index 00000000..4af3239a --- /dev/null +++ b/wlauto/workloads/memcpy/memcpy diff --git a/wlauto/workloads/memcpy/src/build.sh b/wlauto/workloads/memcpy/src/build.sh new file mode 100755 index 00000000..3638949a --- /dev/null +++ b/wlauto/workloads/memcpy/src/build.sh @@ -0,0 +1,21 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +ndk-build + +if [[ $? -eq 0 ]]; then + cp libs/armeabi/memcpy .. +fi diff --git a/wlauto/workloads/memcpy/src/jni/Android.mk b/wlauto/workloads/memcpy/src/jni/Android.mk new file mode 100644 index 00000000..77d438e6 --- /dev/null +++ b/wlauto/workloads/memcpy/src/jni/Android.mk @@ -0,0 +1,11 @@ +LOCAL_PATH := $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := memcopy.c + +LOCAL_LD_LIBS := -lrt + +LOCAL_MODULE := memcpy + + +include $(BUILD_EXECUTABLE) diff --git a/wlauto/workloads/memcpy/src/jni/memcopy.c b/wlauto/workloads/memcpy/src/jni/memcopy.c new file mode 100644 index 00000000..19f569d3 --- /dev/null +++ b/wlauto/workloads/memcpy/src/jni/memcopy.c @@ -0,0 +1,114 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +#define _GNU_SOURCE +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <sched.h> +#include <unistd.h> +#include <sys/syscall.h> +#include <pthread.h> +#include <time.h> + +const int MAX_CPUS = 8; +const int DEFAULT_ITERATIONS = 1000; +const int DEFAULT_BUFFER_SIZE = 1024 * 1024 * 5; + +int set_affinity(size_t cpus_size, int* cpus) +{ + int i; + int mask = 0; + + for(i = 0; i < cpus_size; ++i) + { + mask |= 1 << cpus[i]; + } + + return syscall(__NR_sched_setaffinity, 0, sizeof(mask), &mask); +} + +int main(int argc, char** argv) +{ + int cpus[MAX_CPUS]; + int next_cpu = 0; + int iterations = DEFAULT_ITERATIONS; + int buffer_size = DEFAULT_BUFFER_SIZE; + + int c; + while ((c = getopt(argc, argv, "i:c:s:")) != -1) + switch (c) + { + case 'c': + cpus[next_cpu++] = atoi(optarg); + if (next_cpu == MAX_CPUS) + { + fprintf(stderr, "Max CPUs exceeded."); + abort(); + } + break; + case 'i': + iterations = atoi(optarg); + break; + case 's': + buffer_size = atoi(optarg); + break; + default: + abort(); + break; + } + + int ret; + if (next_cpu != 0) + if (ret = set_affinity(next_cpu, cpus)) + { + fprintf(stderr, "sched_setaffinity returnred %i.", ret); + abort(); + } + + char* source = malloc(buffer_size); + char* dest = malloc(buffer_size); + + struct timespec before, after; + if (clock_gettime(CLOCK_MONOTONIC, &before)) + { + fprintf(stderr, "Could not get start time."); + abort(); + } + + int i; + for (i = 0; i < iterations; ++i) + { + memcpy(dest, source, buffer_size); + } + + if (clock_gettime(CLOCK_MONOTONIC, &after)) + { + fprintf(stderr, "Could not get end time."); + abort(); + } + + free(dest); + free(source); + + long delta_sec = (long)(after.tv_sec - before.tv_sec); + long delta_nsec = after.tv_nsec - before.tv_nsec; + double delta = (double)delta_sec + delta_nsec / 1e9; + printf("Total time: %f s\n", delta); + printf("Bandwidth: %f MB/s\n", buffer_size / delta * iterations / 1e6); + + return 0; +} diff --git a/wlauto/workloads/nenamark/__init__.py b/wlauto/workloads/nenamark/__init__.py new file mode 100644 index 00000000..8a9f4a8d --- /dev/null +++ b/wlauto/workloads/nenamark/__init__.py @@ -0,0 +1,58 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os +import re +import time + +from wlauto import AndroidBenchmark + + +class Nenamark(AndroidBenchmark): + + name = 'nenamark' + description = """ + NenaMark is an OpenGL-ES 2.0 graphics performance benchmark for Android + devices. + + http://nena.se/nenamark_story + + From the website: + + The NenaMark2 benchmark scene averages about 45k triangles, with a span + between 26k and 68k triangles. It averages 96 batches per frame and contains + about 15 Mb of texture data (non-packed). + """ + package = 'se.nena.nenamark2' + activity = 'se.nena.nenamark2.NenaMark2' + + regex = re.compile('.*NenaMark2.*Score.*?([0-9\.]*)fps') + + def run(self, context): + time.sleep(5) # wait for nenamark menu to show up + self.device.execute('input keyevent 23') + time.sleep(120) # wait two minutes for nenamark to complete + + def update_result(self, context): + super(Nenamark, self).update_result(context) + with open(self.logcat_log) as fh: + for line in fh: + match = self.regex.search(line) + if match: + score = match.group(1) + context.result.add_metric('nenamark score', score) + break + diff --git a/wlauto/workloads/peacekeeper/__init__.py b/wlauto/workloads/peacekeeper/__init__.py new file mode 100644 index 00000000..61c65f7c --- /dev/null +++ b/wlauto/workloads/peacekeeper/__init__.py @@ -0,0 +1,129 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,W0201,E0203 +import os +import urllib2 +from HTMLParser import HTMLParser + +from wlauto import AndroidUiAutoBenchmark, Parameter +from wlauto.exceptions import WorkloadError + + +BROWSER_MAP = { + 'firefox': { + 'package': 'org.mozilla.firefox', + 'activity': '.App', + }, + 'chrome': { + 'package': 'com.android.chrome', + 'activity': 'com.google.android.apps.chrome.Main', + }, +} + + +class Peacekeeper(AndroidUiAutoBenchmark): + + name = 'peacekeeper' + description = """ + Peacekeeper is a free and fast browser test that measures a browser's speed. + + .. note:: + + This workload requires a network connection as well as support for + one of the two currently-supported browsers. Moreover, TC2 has + compatibility issue with chrome + + """ + run_timeout = 15 * 60 + + parameters = [ + Parameter('browser', default='firefox', allowed_values=['firefox', 'chrome'], + description='The browser to be benchmarked.'), + Parameter('output_file', default=None, + description="""The result URL of peacekeeper benchmark will be written + into this file on device after completion of peacekeeper benchmark. + Defaults to peacekeeper.txt in the device's ``working_directory``. + """), + Parameter('peacekeeper_url', default='http://peacekeeper.futuremark.com/run.action', + description='The URL to run the peacekeeper benchmark.'), + ] + + def __init__(self, device, **kwargs): + super(Peacekeeper, self).__init__(device, **kwargs) + self.version = self.browser + + def update_result(self, context): + super(Peacekeeper, self).update_result(context) + url = None + + # Pull the result page url, which contains the results, from the + # peacekeeper.txt file and process it + self.device.pull_file(self.output_file, context.output_directory) + result_file = os.path.join(context.output_directory, 'peacekeeper.txt') + with open(result_file) as fh: + for line in fh: + url = line + + # Fetch the html page containing the results + if not url: + raise WorkloadError('The url is empty, error while running peacekeeper benchmark') + + req = urllib2.Request(url) + response = urllib2.urlopen(req) + result_page = response.read() + + # Parse the HTML content using HTML parser + parser = PeacekeeperParser() + parser.feed(result_page) + + # Add peacekeeper_score into results file + context.result.add_metric('peacekeeper_score', parser.peacekeeper_score) + + def validate(self): + if self.output_file is None: + self.output_file = os.path.join(self.device.working_directory, 'peacekeeper.txt') + if self.browser == 'chrome' and self.device == 'TC2': + raise WorkloadError('Chrome not supported on TC2') + + self.uiauto_params['output_file'] = self.output_file + self.uiauto_params['browser'] = self.browser + self.uiauto_params['peacekeeper_url'] = self.peacekeeper_url + + self.package = BROWSER_MAP[self.browser]['package'] + self.activity = BROWSER_MAP[self.browser]['activity'] + + +class PeacekeeperParser(HTMLParser): + def __init__(self): + HTMLParser.__init__(self) + self.flag = False + self.peacekeeper_score = '' + + def handle_starttag(self, tag, attrs): + if tag == 'div': + for name, value in attrs: + if name == 'class' and value == 'resultBarContainer clearfix resultBarSelected': + self.flag = True + elif self.flag and name == 'class' and value == 'resultBarComment': + self.flag = False + self.peacekeeper_score = self.peacekeeper_score.split('details')[1] + + def handle_endtag(self, tag): + pass + + def handle_data(self, data): + if self.flag: + self.peacekeeper_score += data.strip() diff --git a/wlauto/workloads/peacekeeper/com.arm.wlauto.uiauto.peacekeeper.jar b/wlauto/workloads/peacekeeper/com.arm.wlauto.uiauto.peacekeeper.jar Binary files differnew file mode 100644 index 00000000..4dbd9465 --- /dev/null +++ b/wlauto/workloads/peacekeeper/com.arm.wlauto.uiauto.peacekeeper.jar diff --git a/wlauto/workloads/peacekeeper/uiauto/build.sh b/wlauto/workloads/peacekeeper/uiauto/build.sh new file mode 100755 index 00000000..96df2690 --- /dev/null +++ b/wlauto/workloads/peacekeeper/uiauto/build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.peacekeeper.jar ]]; then + cp bin/com.arm.wlauto.uiauto.peacekeeper.jar .. +fi diff --git a/wlauto/workloads/peacekeeper/uiauto/build.xml b/wlauto/workloads/peacekeeper/uiauto/build.xml new file mode 100644 index 00000000..7d60a557 --- /dev/null +++ b/wlauto/workloads/peacekeeper/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.peacekeeper" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/peacekeeper/uiauto/project.properties b/wlauto/workloads/peacekeeper/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/peacekeeper/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/peacekeeper/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/peacekeeper/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..2384b800 --- /dev/null +++ b/wlauto/workloads/peacekeeper/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,115 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.peacekeeper; + +import java.io.File; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; +import java.net.URLConnection; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.io.PrintWriter; +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.view.KeyEvent; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "peacekeeper"; + + public void runUiAutomation() throws Exception { + // maximum time for running peacekeeper benchmark 80 * 10 sec + final int TIMEOUT = 80; + + // reading the input parameter + Bundle parameters = getParams(); + String browser = parameters.getString("browser"); + String outputFile = parameters.getString("output_file"); + String peacekeeperUrl = parameters.getString("peacekeeper_url"); + + String urlAddress = ""; + + PrintWriter writer = new PrintWriter(outputFile, "UTF-8"); + + // firefox browser uiautomator code + if (browser.equals("firefox")) { + + UiObject addressBar = new UiObject(new UiSelector() + .className("android.widget.TextView") + .text("Enter Search or Address")); + addressBar.click(); + UiObject setUrl = new UiObject(new UiSelector() + .className("android.widget.EditText")); + setUrl.clearTextField(); + setUrl.setText(peacekeeperUrl); + getUiDevice().pressEnter(); + + UiObject currentUrl = new UiObject(new UiSelector() + .className("android.widget.TextView").index(1)); + for (int i = 0; i < TIMEOUT; i++) { + + if (currentUrl.getText() + .equals("Peacekeeper - free universal browser test for HTML5 from Futuremark")) { + + // write url address to peacekeeper.txt file + currentUrl.click(); + urlAddress = setUrl.getText(); + writer.println(urlAddress); + break; + } + sleep(10); + } + } else if (browser.equals("chrome")) { // Code for Chrome browser + UiObject adressBar = new UiObject(new UiSelector() + .className("android.widget.EditText") + .description("Search or type url")); + + adressBar.clearTextField(); + adressBar.setText(peacekeeperUrl); + getUiDevice().pressEnter(); + for (int i = 0; i < TIMEOUT; i++) { + + if (!adressBar.getText().contains("run.action")) { + + // write url address to peacekeeper.txt file + urlAddress = adressBar.getText(); + if (!urlAddress.contains("http")) + urlAddress = "http://" + urlAddress; + writer.println(urlAddress); + break; + } + sleep(10); + } + } + writer.close(); + getUiDevice().pressHome(); + } +} diff --git a/wlauto/workloads/quadrant/__init__.py b/wlauto/workloads/quadrant/__init__.py new file mode 100644 index 00000000..5670ceea --- /dev/null +++ b/wlauto/workloads/quadrant/__init__.py @@ -0,0 +1,112 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import re +from collections import defaultdict + +from wlauto import AndroidUiAutoBenchmark + + +TEST_TYPES = { + 'benchmark_cpu_branching_logic': 'time', + 'benchmark_cpu_matrix_int': 'time', + 'benchmark_cpu_matrix_long': 'time', + 'benchmark_cpu_matrix_short': 'time', + 'benchmark_cpu_matrix_byte': 'time', + 'benchmark_cpu_matrix_float': 'time', + 'benchmark_cpu_matrix_double': 'time', + 'benchmark_cpu_checksum': 'time', + 'benchmark_cpu': 'aggregate', + 'benchmark_memory_transfer': 'time', + 'benchmark_memory': 'aggregate', + 'benchmark_io_fs_write': 'time', + 'benchmark_io_fs_read': 'time', + 'benchmark_io_db_write': 'time', + 'benchmark_io_db_read': 'time', + 'benchmark_io': 'aggregate', + 'benchmark_g2d_fractal': 'rate', + 'benchmark_g2d': 'aggregate', + 'benchmark_g3d_corridor': 'rate', + 'benchmark_g3d_planet': 'rate', + 'benchmark_g3d_dna': 'rate', + 'benchmark_g3d': 'aggregate', + 'benchmark': 'aggregate', +} + +TYPE_TESTS = defaultdict(list) +for k, v in TEST_TYPES.iteritems(): + TYPE_TESTS[v].append(k) + +TYPE_UNITS = { + 'time': 'ms', + 'rate': 'Hz', +} + +REGEX_TEMPLATES = { + 'aggregate': r'(?P<metric>{}) aggregate score is (?P<score>\d+)', + 'time': r'(?P<metric>{}) executed in (?P<time>\d+) ms, ' + r'reference time: (?P<reference>\d+) ms, ' + r'score: (?P<score>\d+)', + 'rate': r'(?P<metric>{}) executed with a rate of (?P<rate>[0-9.]+)/sec, ' + r'reference rate: (?P<reference>[0-9.]+)/sec, ' + r'score: (?P<score>\d+)', +} + +TEST_REGEXES = {} +for test_, type_ in TEST_TYPES.items(): + TEST_REGEXES[test_] = re.compile(REGEX_TEMPLATES[type_].format(test_)) + + +class Quadrant(AndroidUiAutoBenchmark): + + name = 'quadrant' + description = """ + Quadrant is a benchmark for mobile devices, capable of measuring CPU, memory, + I/O and 3D graphics performance. + + http://www.aurorasoftworks.com/products/quadrant + + From the website: + Quadrant outputs a score for the following categories: 2D, 3D, Mem, I/O, CPU + , Total. + """ + package = 'com.aurorasoftworks.quadrant.ui.professional' + activity = '.QuadrantProfessionalLauncherActivity' + summary_metrics = ['benchmark_score'] + + run_timeout = 10 * 60 + + def __init__(self, device, **kwargs): + super(Quadrant, self).__init__(device, **kwargs) + self.uiauto_params['has_gpu'] = self.device.has_gpu + self.regex = {} + + def update_result(self, context): + super(Quadrant, self).update_result(context) + with open(self.logcat_log) as fh: + for line in fh: + for test, regex in TEST_REGEXES.items(): + match = regex.search(line) + if match: + test_type = TEST_TYPES[test] + data = match.groupdict() + if test_type != 'aggregate': + context.result.add_metric(data['metric'] + '_' + test_type, + data[test_type], + TYPE_UNITS[test_type]) + context.result.add_metric(data['metric'] + '_score', data['score']) + break + diff --git a/wlauto/workloads/quadrant/com.arm.wlauto.uiauto.quadrant.jar b/wlauto/workloads/quadrant/com.arm.wlauto.uiauto.quadrant.jar Binary files differnew file mode 100644 index 00000000..2c5aac3d --- /dev/null +++ b/wlauto/workloads/quadrant/com.arm.wlauto.uiauto.quadrant.jar diff --git a/wlauto/workloads/quadrant/uiauto/build.sh b/wlauto/workloads/quadrant/uiauto/build.sh new file mode 100755 index 00000000..eba2b1cc --- /dev/null +++ b/wlauto/workloads/quadrant/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.quadrant.jar ]]; then + cp bin/com.arm.wlauto.uiauto.quadrant.jar .. +fi diff --git a/wlauto/workloads/quadrant/uiauto/build.xml b/wlauto/workloads/quadrant/uiauto/build.xml new file mode 100644 index 00000000..113eccbe --- /dev/null +++ b/wlauto/workloads/quadrant/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.quadrant" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/quadrant/uiauto/project.properties b/wlauto/workloads/quadrant/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/quadrant/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/quadrant/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/quadrant/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..f8fe8749 --- /dev/null +++ b/wlauto/workloads/quadrant/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,120 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.quadrant; + +import java.util.concurrent.TimeUnit; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "quadrant"; + + public void runUiAutomation() throws Exception { + Bundle status = new Bundle(); + Bundle params = getParams(); + boolean hasGpu = Boolean.parseBoolean(params.getString("has_gpu").toLowerCase()); + + clearLogcat(); + handleFtuInfoDialogIfNecessary(); + goToRunCustomBenchmark(); + selectTestsToRun(hasGpu); + hitStart(); + handleWarningIfNecessary(); + waitForResults(); + + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + + public void handleFtuInfoDialogIfNecessary() throws Exception { + UiSelector selector = new UiSelector(); + UiObject infoText = new UiObject(selector.text("Information")); + if (infoText.waitForExists(TimeUnit.SECONDS.toMillis(10))) + { + UiObject okButton = new UiObject(selector.text("OK") + .className("android.widget.Button")); + okButton.click(); + } + else + { + // FTU dialog didn't come up. + } + } + + public void goToRunCustomBenchmark() throws Exception { + UiSelector selector = new UiSelector(); + UiObject runCustom = new UiObject(selector.text("Run custom benchmark") + .className("android.widget.TextView")); + runCustom.clickAndWaitForNewWindow(); + } + + // By default, all tests are selected. However, if our device does not have a GPU, then + // running graphics tests may cause a crash, so we disable those. + public void selectTestsToRun(boolean hasGpu) throws Exception { + if(!hasGpu) { + UiSelector selector = new UiSelector(); + UiObject gfx2d = new UiObject(selector.text("2D graphics") + .className("android.widget.CheckBox")); + gfx2d.click(); + + UiObject gfx3d = new UiObject(selector.text("3D graphics") + .className("android.widget.CheckBox")); + gfx3d.click(); + } + } + + public void hitStart() throws Exception { + UiSelector selector = new UiSelector(); + UiObject startButton = new UiObject(selector.text("Start") + .className("android.widget.Button") + .packageName("com.aurorasoftworks.quadrant.ui.professional")); + startButton.click(); + } + + // Even if graphics tests aren't selected, Quadrant will still show a warning about running + // with software rendering. + public void handleWarningIfNecessary() throws Exception { + UiSelector selector = new UiSelector(); + UiObject warning = new UiObject(selector.text("Warning")); + if (warning.waitForExists(TimeUnit.SECONDS.toMillis(2))) { + UiObject closeButton = new UiObject(selector.text("Close") + .className("android.widget.Button")); + if (closeButton.exists()) { + closeButton.click(); + } + } + else + { + // Warning dialog didn't come up. + } + } + + public void waitForResults() throws Exception { + waitForLogcatText("benchmark aggregate score is", TimeUnit.SECONDS.toMillis(200)); + } +} diff --git a/wlauto/workloads/real_linpack/__init__.py b/wlauto/workloads/real_linpack/__init__.py new file mode 100644 index 00000000..2f8121ab --- /dev/null +++ b/wlauto/workloads/real_linpack/__init__.py @@ -0,0 +1,66 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,W0201,E0203 + +import re + +from wlauto import AndroidUiAutoBenchmark, Parameter + + +class RealLinpack(AndroidUiAutoBenchmark): + + name = 'real-linpack' + description = """ + This version of `Linpack <http://en.wikipedia.org/wiki/LINPACK_benchmarks>` + was developed by Dave Butcher. RealLinpack tries to find the number of threads + that give you the maximum linpack score. + + RealLinpack runs 20 runs of linpack for each number of threads and + calculates the mean and confidence. It stops when the + score's confidence interval drops below the current best score + interval. That is, when (current_score + confidence) < (best_score - + best_score_confidence) + + """ + package = 'com.arm.RealLinpack' + activity = '.RealLinpackActivity' + + parameters = [ + Parameter('max_threads', kind=int, default=16, constraint=lambda x: x > 0, + description='The maximum number of threads that real linpack will try.'), + ] + + def __init__(self, device, **kwargs): + super(RealLinpack, self).__init__(device, **kwargs) + self.uiauto_params['max_threads'] = self.max_threads + self.run_timeout = 120 + 120 * self.max_threads # a base of 2 minutes plus 2 minutes for each thread + + def update_result(self, context): + super(RealLinpack, self).update_result(context) + score_regex = re.compile(r'Optimum.*threads:\s*([0-9])+.*score:\s*([0-9]+\.[0-9]+).*MFLOPS') + match_found = False + with open(self.logcat_log) as logcat_file: + for line in logcat_file: + match = re.search(score_regex, line) + if match: + number_of_threads = match.group(1) + score = match.group(2) + context.result.add_metric('optimal number of threads', number_of_threads, None) + context.result.add_metric('score', score, 'MFLOPS') + match_found = True + break + if not match_found: + self.logger.warning('Failed To collect results for real linpack') diff --git a/wlauto/workloads/real_linpack/com.arm.wlauto.uiauto.reallinpack.jar b/wlauto/workloads/real_linpack/com.arm.wlauto.uiauto.reallinpack.jar Binary files differnew file mode 100644 index 00000000..133435d0 --- /dev/null +++ b/wlauto/workloads/real_linpack/com.arm.wlauto.uiauto.reallinpack.jar diff --git a/wlauto/workloads/real_linpack/uiauto/build.sh b/wlauto/workloads/real_linpack/uiauto/build.sh new file mode 100755 index 00000000..645f225a --- /dev/null +++ b/wlauto/workloads/real_linpack/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.reallinpack.jar ]]; then + cp bin/com.arm.wlauto.uiauto.reallinpack.jar .. +fi diff --git a/wlauto/workloads/real_linpack/uiauto/build.xml b/wlauto/workloads/real_linpack/uiauto/build.xml new file mode 100644 index 00000000..7771a5d3 --- /dev/null +++ b/wlauto/workloads/real_linpack/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.reallinpack" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/real_linpack/uiauto/project.properties b/wlauto/workloads/real_linpack/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/real_linpack/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/real_linpack/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/real_linpack/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..a24d9783 --- /dev/null +++ b/wlauto/workloads/real_linpack/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,51 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.reallinpack; + +import android.app.Activity; +import android.os.Bundle; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public void runUiAutomation() throws Exception{ + Bundle status = new Bundle(); + status.putString("product", getUiDevice().getProductName()); + UiSelector selector = new UiSelector(); + // set the maximum number of threads + String maxThreads = getParams().getString("max_threads"); + UiObject maxThreadNumberField = new UiObject(selector.index(3)); + maxThreadNumberField.clearTextField(); + maxThreadNumberField.setText(maxThreads); + // start the benchamrk + UiObject btn_st = new UiObject(selector.text("Run")); + btn_st.click(); + btn_st.waitUntilGone(500); + // set timeout for the benchmark + btn_st.waitForExists(60 * 60 * 1000); + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + +} diff --git a/wlauto/workloads/realracing3/__init__.py b/wlauto/workloads/realracing3/__init__.py new file mode 100644 index 00000000..cfeaa416 --- /dev/null +++ b/wlauto/workloads/realracing3/__init__.py @@ -0,0 +1,35 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import time + +from wlauto.common.android.workload import GameWorkload +from wlauto.exceptions import WorkloadError, DeviceError + + +class RealRacing3(GameWorkload): + + name = 'realracing3' + description = """ + Real Racing 3 game. + """ + package = 'com.ea.games.r3_row' + activity = 'com.firemint.realracing3.MainActivity' + loading_time = 90 + asset_file = 'com.ea.games.r3_row.tar.gz' + saved_state_file = 'rr3-save.tar.gz' + + diff --git a/wlauto/workloads/shellscript/__init__.py b/wlauto/workloads/shellscript/__init__.py new file mode 100644 index 00000000..9662f288 --- /dev/null +++ b/wlauto/workloads/shellscript/__init__.py @@ -0,0 +1,65 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,W0201,E0203 + +import os + +from wlauto import Workload, Parameter +from wlauto.exceptions import ConfigError + + +class ShellScript(Workload): + + name = 'shellscript' + description = """ + Runs an arbitrary shellscript on the device. + + """ + + parameters = [ + Parameter('script_file', mandatory=True, + description=('The path (on the host) to the shell script file. This must be ' + 'an absolute path (though it may contain ~).')), + Parameter('argstring', default='', + description='A string that should contain arguments passed to the script.'), + Parameter('timeout', kind=int, default=60, + description='Timeout, in seconds, for the script run time.'), + ] + + def __init__(self, device, **kwargs): + super(ShellScript, self).__init__(device, **kwargs) + self.script_file = os.path.expanduser(self.script_file) + if not os.path.isfile(self.script_file): + raise ConfigError('Can\'t access file (is the path correct?): {}'.format(self.script_file)) + self.output = None + self.command = None + self.on_device_script_file = None + + def setup(self, context): + self.on_device_script_file = self.device.path.join(self.device.working_directory, + os.path.basename(self.script_file)) + self.device.push_file(self.script_file, self.on_device_script_file) + self.command = 'sh {} {}'.format(self.on_device_script_file, self.argstring) + + def run(self, context): + self.output = self.device.execute(self.command, timeout=self.timeout) + + def update_result(self, context): + with open(os.path.join(context.output_directory, 'output.txt'), 'w') as wfh: + wfh.write(self.output) + + def teardown(self, context): + self.device.delete_file(self.on_device_script_file) diff --git a/wlauto/workloads/skypevideo/__init__.py b/wlauto/workloads/skypevideo/__init__.py new file mode 100644 index 00000000..58959e1f --- /dev/null +++ b/wlauto/workloads/skypevideo/__init__.py @@ -0,0 +1,130 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,W0201,E0203 + +import time + +from wlauto import UiAutomatorWorkload, Parameter +from wlauto.utils.types import boolean + + +class SkypeVideo(UiAutomatorWorkload): + + name = 'skypevideo' + description = """ + Initiates Skype video call to a specified contact for a pre-determined duration. + (Note: requires Skype to be set up appropriately). + + This workload is intended for monitoring the behaviour of a device while a Skype + video call is in progress (a common use case). It does not produce any score or + metric and the intention is that some addition instrumentation is enabled while + running this workload. + + This workload, obviously, requires a network connection (ideally, wifi). + + This workload accepts the following parameters: + + + **Skype Setup** + + - You should install Skype client from Google Play Store on the device + (this was tested with client version 4.5.0.39600; other recent versions + should also work). + - You must have an account set up and logged into Skype on the device. + - The contact to be called must be added (and has accepted) to the + account. It's possible to have multiple contacts in the list, however + the contact to be called *must* be visible on initial navigation to the + list. + - The contact must be able to received the call. This means that there + must be a Skype client running (somewhere) with the contact logged in + and that client must have been configured to auto-accept calls from the + account on the device (how to set this varies between different versions + of Skype and between platforms -- please search online for specific + instructions). + https://support.skype.com/en/faq/FA3751/can-i-automatically-answer-all-my-calls-with-video-in-skype-for-windows-desktop + + """ + + package = 'com.skype.raider' + + parameters = [ + Parameter('duration', kind=int, default=300, + description='Duration of the video call in seconds.'), + Parameter('contact', mandatory=True, + description=""" + The name of the Skype contact to call. The contact must be already + added (see below). *If use_gui is set*, then this must be the skype + ID of the contact, *otherwise*, this must be the name of the + contact as it appears in Skype client's contacts list. In the latter case + it *must not* contain underscore characters (``_``); it may, however, contain + spaces. There is no default, you **must specify the name of the contact**. + + .. note:: You may alternatively specify the contact name as + ``skype_contact`` setting in your ``config.py``. If this is + specified, the ``contact`` parameter is optional, though + it may still be specified (in which case it will override + ``skype_contact`` setting). + """), + Parameter('use_gui', kind=boolean, default=False, + description=""" + Specifies whether the call should be placed directly through a + Skype URI, or by navigating the GUI. The URI is the recommended way + to place Skype calls on a device, but that does not seem to work + correctly on some devices (the URI seems to just start Skype, but not + place the call), so an alternative exists that will start the Skype app + and will then navigate the UI to place the call (incidentally, this method + does not seem to work on all devices either, as sometimes Skype starts + backgrounded...). Please note that the meaning of ``contact`` prameter + is different depending on whether this is set. Defaults to ``False``. + + .. note:: You may alternatively specify this as ``skype_use_gui`` setting + in your ``config.py``. + """), + + ] + + def __init__(self, device, **kwargs): + super(SkypeVideo, self).__init__(device, **kwargs) + if self.use_gui: + self.uiauto_params['name'] = self.contact.replace(' ', '_') + self.uiauto_params['duration'] = self.duration + self.run_timeout = self.duration + 30 + + def setup(self, context): + if self.use_gui: + super(SkypeVideo, self).setup(context) + self.device.execute('am force-stop {}'.format(self.package)) + self.device.execute('am start -W -a android.intent.action.VIEW -d skype:') + else: + self.device.execute('am force-stop {}'.format(self.package)) + + def run(self, context): + if self.use_gui: + super(SkypeVideo, self).run(context) + else: + command = "am start -W -a android.intent.action.VIEW -d \"skype:{}?call&video=true\"" + self.logger.debug(self.device.execute(command.format(self.contact))) + self.logger.debug('Call started; waiting for {} seconds...'.format(self.duration)) + time.sleep(self.duration) + self.device.execute('am force-stop com.skype.raider') + + def update_result(self, context): + pass + + def teardown(self, context): + if self.use_gui: + super(SkypeVideo, self).teardown(context) + self.device.execute('am force-stop {}'.format(self.package)) diff --git a/wlauto/workloads/skypevideo/com.arm.wlauto.uiauto.skypevideo.jar b/wlauto/workloads/skypevideo/com.arm.wlauto.uiauto.skypevideo.jar Binary files differnew file mode 100644 index 00000000..dff2302a --- /dev/null +++ b/wlauto/workloads/skypevideo/com.arm.wlauto.uiauto.skypevideo.jar diff --git a/wlauto/workloads/skypevideo/uiauto/build.sh b/wlauto/workloads/skypevideo/uiauto/build.sh new file mode 100755 index 00000000..db6f8ff4 --- /dev/null +++ b/wlauto/workloads/skypevideo/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.skypevideo.jar ]]; then + cp bin/com.arm.wlauto.uiauto.skypevideo.jar .. +fi diff --git a/wlauto/workloads/skypevideo/uiauto/build.xml b/wlauto/workloads/skypevideo/uiauto/build.xml new file mode 100644 index 00000000..c2fdeb90 --- /dev/null +++ b/wlauto/workloads/skypevideo/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.skypevideo" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/skypevideo/uiauto/project.properties b/wlauto/workloads/skypevideo/uiauto/project.properties new file mode 100644 index 00000000..ce39f2d0 --- /dev/null +++ b/wlauto/workloads/skypevideo/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-18 diff --git a/wlauto/workloads/skypevideo/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/skypevideo/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..0743372e --- /dev/null +++ b/wlauto/workloads/skypevideo/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,72 @@ +/* Copyright 2014-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.skypevideo; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.view.KeyEvent; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "skypevideo"; + public static String videoCallButtonResourceId = "com.skype.raider:id/chat_menu_item_call_video"; + public static String noContactMessage = "Could not find contact \"%s\" in the contacts list."; + + public void runUiAutomation() throws Exception { + Bundle parameters = getParams(); + String contactName = parameters.getString("name").replace('_', ' '); + int duration = Integer.parseInt(parameters.getString("duration")); + + selectContact(contactName); + initiateCall(duration); + } + + public void selectContact(String name) throws Exception { + UiSelector selector = new UiSelector(); + UiObject peopleTab = new UiObject(selector.text("People")); + peopleTab.click(); + sleep(1); // tab transition + + // Note: this assumes that the contact is in view and does not attempt to scroll to find it. + // The expectation is that this automation will be used with a dedicated account that was set + // up for the purpose and so would only have the intended target plus one or two other contacts + // at most in the list. If that is not the case, then this needs to be re-written to scroll to + // find the contact if necessary. + UiObject contactCard = new UiObject(selector.text(name)); + if (!contactCard.exists()) { + throw new UiObjectNotFoundException(String.format(noContactMessage, name)); + } + contactCard.clickAndWaitForNewWindow(); + } + + public void initiateCall(int duration) throws Exception { + UiSelector selector = new UiSelector(); + UiObject videoCallButton = new UiObject(selector.resourceId(videoCallButtonResourceId)); + videoCallButton.click(); + sleep(duration); + } +} diff --git a/wlauto/workloads/smartbench/__init__.py b/wlauto/workloads/smartbench/__init__.py new file mode 100644 index 00000000..4b7cbe3e --- /dev/null +++ b/wlauto/workloads/smartbench/__init__.py @@ -0,0 +1,59 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os +import re +import time + +from wlauto import AndroidUiAutoBenchmark + + +class Smartbench(AndroidUiAutoBenchmark): + + name = 'smartbench' + description = """ + Smartbench is a multi-core friendly benchmark application that measures the + overall performance of an android device. It reports both Productivity and + Gaming Index. + + https://play.google.com/store/apps/details?id=com.smartbench.twelve&hl=en + + From the website: + + It will be better prepared for the quad-core world. Unfortunately this also + means it will run slower on older devices. It will also run slower on + high-resolution tablet devices. All 3D tests are now rendered in full native + resolutions so naturally it will stress hardware harder on these devices. + This also applies to higher resolution hand-held devices. + """ + package = 'com.smartbench.twelve' + activity = '.Smartbench2012' + summary_metrics = ['Smartbench: valueGame', 'Smartbench: valueProd'] + run_timeout = 10 * 60 + + prod_regex = re.compile('valueProd=(\d+)') + game_regex = re.compile('valueGame=(\d+)') + + def update_result(self, context): + super(Smartbench, self).update_result(context) + with open(self.logcat_log) as fh: + text = fh.read() + match = self.prod_regex.search(text) + prod = int(match.group(1)) + match = self.game_regex.search(text) + game = int(match.group(1)) + context.result.add_metric('Smartbench: valueProd', prod) + context.result.add_metric('Smartbench: valueGame', game) diff --git a/wlauto/workloads/smartbench/com.arm.wlauto.uiauto.smartbench.jar b/wlauto/workloads/smartbench/com.arm.wlauto.uiauto.smartbench.jar Binary files differnew file mode 100644 index 00000000..f388cbe0 --- /dev/null +++ b/wlauto/workloads/smartbench/com.arm.wlauto.uiauto.smartbench.jar diff --git a/wlauto/workloads/smartbench/uiauto/build.sh b/wlauto/workloads/smartbench/uiauto/build.sh new file mode 100755 index 00000000..bf76a67e --- /dev/null +++ b/wlauto/workloads/smartbench/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.smartbench.jar ]]; then + cp bin/com.arm.wlauto.uiauto.smartbench.jar .. +fi diff --git a/wlauto/workloads/smartbench/uiauto/build.xml b/wlauto/workloads/smartbench/uiauto/build.xml new file mode 100644 index 00000000..ee913c41 --- /dev/null +++ b/wlauto/workloads/smartbench/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.smartbench" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/smartbench/uiauto/project.properties b/wlauto/workloads/smartbench/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wlauto/workloads/smartbench/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wlauto/workloads/smartbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/smartbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..e8c3aac1 --- /dev/null +++ b/wlauto/workloads/smartbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,62 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.smartbench; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "smartbench"; + + public void runUiAutomation() throws Exception { + Bundle status = new Bundle(); + status.putString("product", getUiDevice().getProductName()); + UiSelector selector = new UiSelector(); + sleep(3); + UiObject text_bench = new UiObject(selector.text("Run SmartBench") + .className("android.widget.TextView")); + text_bench.click(); + + try{ + UiObject complete_text = new UiObject(selector .textContains("Display Index Scores") + .className("android.widget.TextView")); + + waitObject(complete_text); + + sleep(2); + complete_text.click(); + } finally{ + //complete_text.click(); + } + + sleep(5); + takeScreenshot("SmartBench"); + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } + +} diff --git a/wlauto/workloads/spec2000/__init__.py b/wlauto/workloads/spec2000/__init__.py new file mode 100644 index 00000000..87529c63 --- /dev/null +++ b/wlauto/workloads/spec2000/__init__.py @@ -0,0 +1,356 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +#pylint: disable=E1101,W0201 +import os +import re +import string +import tarfile +from collections import defaultdict + +from wlauto import Workload, Parameter, Alias +from wlauto.exceptions import ConfigError, WorkloadError +from wlauto.common.resources import ExtensionAsset +from wlauto.utils.misc import get_cpu_mask +from wlauto.utils.types import boolean, list_or_string + + +class Spec2000(Workload): + + name = 'spec2000' + description = """ + SPEC2000 benchmarks measuring processor, memory and compiler. + + http://www.spec.org/cpu2000/ + + From the web site: + + SPEC CPU2000 is the next-generation industry-standardized CPU-intensive benchmark suite. SPEC + designed CPU2000 to provide a comparative measure of compute intensive performance across the + widest practical range of hardware. The implementation resulted in source code benchmarks + developed from real user applications. These benchmarks measure the performance of the + processor, memory and compiler on the tested system. + + .. note:: At the moment, this workload relies on pre-built SPEC binaries (included in an + asset bundle). These binaries *must* be built according to rules outlined here:: + + http://www.spec.org/cpu2000/docs/runrules.html#toc_2.0 + + in order for the results to be valid SPEC2000 results. + + .. note:: This workload does not attempt to generate results in an admissible SPEC format. No + metadata is provided (though some, but not all, of the required metdata is colleted + by WA elsewhere). It is upto the user to post-process results to generated + SPEC-admissible results file, if that is their intention. + + *base vs peak* + + SPEC2000 defines two build/test configuration: base and peak. Base is supposed to use basic + configuration (e.g. default compiler flags) with no tuning, and peak is specifically optimized for + a system. Since this workload uses externally-built binaries, there is no way for WA to be sure + what configuration is used -- the user is expected to keep track of that. Be aware that + base/peak also come with specfic requirements for the way workloads are run (e.g. how many instances + on multi-core systems):: + + http://www.spec.org/cpu2000/docs/runrules.html#toc_3 + + These are not enforced by WA, so it is again up to the user to ensure that correct workload + parameters are specfied inthe agenda, if they intend to collect "official" SPEC results. (Those + interested in collecting official SPEC results should also note that setting runtime parameters + would violate SPEC runs rules that state that no configuration must be done to the platform + after boot). + + *bundle structure* + + This workload expects the actual benchmark binaries to be provided in a tarball "bundle" that has + a very specific structure. At the top level of the tarball, there should be two directories: "fp" + and "int" -- for each of the SPEC2000 categories. Under those, there is a sub-directory per benchmark. + Each benchmark sub-directory contains three sub-sub-directorie: + + - "cpus" contains a subdirector for each supported cpu (e.g. a15) with a single executable binary + for that cpu, in addition to a "generic" subdirectory that has not been optimized for a specific + cpu and should run on any ARM system. + - "data" contains all additional files (input, configuration, etc) that the benchmark executable + relies on. + - "scripts" contains one or more one-liner shell scripts that invoke the benchmark binary with + appropriate command line parameters. The name of the script must be in the format + <benchmark name>[.<variant name>].sh, i.e. name of benchmark, optionally followed by variant + name, followed by ".sh" extension. If there is more than one script, then all of them must + have a variant; if there is only one script the it should not cotain a variant. + + A typical bundle may look like this:: + + |- fp + | |-- ammp + | | |-- cpus + | | | |-- generic + | | | | |-- ammp + | | | |-- a15 + | | | | |-- ammp + | | | |-- a7 + | | | | |-- ammp + | | |-- data + | | | |-- ammp.in + | | |-- scripts + | | | |-- ammp.sh + | |-- applu + . . . + . . . + . . . + |- int + . + + """ + + # TODO: This is a bit of a hack. Need to re-think summary metric indication + # (also more than just summary/non-summary classification?) + class _SPECSummaryMetrics(object): + def __contains__(self, item): + if item.endswith('_real'): + return True + else: + return False + + asset_file = 'spec2000-assets.tar.gz' + + aliases = [ + Alias('spec2k'), + ] + + summary_metrics = _SPECSummaryMetrics() + + parameters = [ + Parameter('benchmarks', kind=list_or_string, + description='Specfiles the SPEC benchmarks to run.'), + Parameter('mode', kind=str, allowed_values=['speed', 'rate'], default='speed', + description='SPEC benchmarks can report either speed to execute or throughput/rate. ' + 'In the latter case, several "threads" will be spawned.'), + Parameter('number_of_threads', kind=int, default=None, + description='Specify the number of "threads" to be used in \'rate\' mode. (Note: ' + 'on big.LITTLE systems this is the number of threads, for *each cluster*). '), + + Parameter('force_extract_assets', kind=boolean, default=False, + description='if set to ``True``, will extract assets from the bundle, even if they are ' + 'already extracted. Note: this option implies ``force_push_assets``.'), + Parameter('force_push_assets', kind=boolean, default=False, + description='If set to ``True``, assets will be pushed to device even if they\'re already ' + 'present.'), + Parameter('timeout', kind=int, default=20 * 60, + description='Timemout, in seconds, for the execution of single spec test.'), + ] + + speed_run_template = 'cd {datadir}; time ({launch_command})' + rate_run_template = 'cd {datadir}; time ({loop}; wait)' + loop_template = 'for i in $(busybox seq 1 {threads}); do {launch_command} 1>/dev/null 2>&1 & done' + launch_template = 'busybox taskset {cpumask} {command} 1>/dev/null 2>&1' + + timing_regex = re.compile(r'(?P<minutes>\d+)m(?P<seconds>[\d.]+)s\s+(?P<category>\w+)') + + def init_resources(self, context): + self._load_spec_benchmarks(context) + + def setup(self, context): + cpus = self.device.core_names + if not cpus: + raise WorkloadError('Device has not specifed CPU cores configruation.') + cpumap = defaultdict(list) + for i, cpu in enumerate(cpus): + cpumap[cpu.lower()].append(i) + for benchspec in self.benchmarks: + commandspecs = self._verify_and_deploy_benchmark(benchspec, cpumap) + self._build_command(benchspec, commandspecs) + + def run(self, context): + for name, command in self.commands: + self.timings[name] = self.device.execute(command, timeout=self.timeout) + + def update_result(self, context): + for benchmark, output in self.timings.iteritems(): + matches = self.timing_regex.finditer(output) + found = False + for match in matches: + category = match.group('category') + mins = float(match.group('minutes')) + secs = float(match.group('seconds')) + total = secs + 60 * mins + context.result.add_metric('_'.join([benchmark, category]), + total, 'seconds', + lower_is_better=True) + found = True + if not found: + self.logger.error('Could not get timings for {}'.format(benchmark)) + + def validate(self): + if self.force_extract_assets: + self.force_push_assets = True + if self.benchmarks is None: # pylint: disable=access-member-before-definition + self.benchmarks = 'all' + if isinstance(self.benchmarks, basestring): + if self.benchmarks == 'all': + self.benchmarks = self.loaded_benchmarks.keys() + else: + self.benchmarks = [self.benchmarks] + for benchname in self.benchmarks: + if benchname not in self.loaded_benchmarks: + raise ConfigError('Unknown SPEC benchmark: {}'.format(benchname)) + if self.mode == 'speed': + if self.number_of_threads is not None: + raise ConfigError('number_of_threads cannot be specified in speed mode.') + else: + raise ValueError('Unexpected SPEC2000 mode: {}'.format(self.mode)) # Should never get here + self.commands = [] + self.timings = {} + + def _load_spec_benchmarks(self, context): + self.loaded_benchmarks = {} + self.categories = set() + if self.force_extract_assets or len(os.listdir(self.dependencies_directory)) < 2: + bundle = context.resolver.get(ExtensionAsset(self, self.asset_file)) + with tarfile.open(bundle, 'r:gz') as tf: + tf.extractall(self.dependencies_directory) + for entry in os.listdir(self.dependencies_directory): + entrypath = os.path.join(self.dependencies_directory, entry) + if os.path.isdir(entrypath): + for bench in os.listdir(entrypath): + self.categories.add(entry) + benchpath = os.path.join(entrypath, bench) + self._load_benchmark(benchpath, entry) + + def _load_benchmark(self, path, category): + datafiles = [] + cpus = [] + for df in os.listdir(os.path.join(path, 'data')): + datafiles.append(os.path.join(path, 'data', df)) + for cpu in os.listdir(os.path.join(path, 'cpus')): + cpus.append(cpu) + commandsdir = os.path.join(path, 'commands') + for command in os.listdir(commandsdir): + bench = SpecBenchmark() + bench.name = os.path.splitext(command)[0] + bench.path = path + bench.category = category + bench.datafiles = datafiles + bench.cpus = cpus + with open(os.path.join(commandsdir, command)) as fh: + bench.command_template = string.Template(fh.read().strip()) + self.loaded_benchmarks[bench.name] = bench + + def _verify_and_deploy_benchmark(self, benchspec, cpumap): # pylint: disable=R0914 + """Verifies that the supplied benchmark spec is valid and deploys the required assets + to the device (if necessary). Returns a list of command specs (one for each CPU cluster) + that can then be used to construct the final command.""" + bench = self.loaded_benchmarks[benchspec] + basename = benchspec.split('.')[0] + datadir = self.device.path.join(self.device.working_directory, self.name, basename) + if self.force_push_assets or not self.device.file_exists(datadir): + self.device.execute('mkdir -p {}'.format(datadir)) + for datafile in bench.datafiles: + self.device.push_file(datafile, self.device.path.join(datadir, os.path.basename(datafile))) + + if self.mode == 'speed': + cpus = [self._get_fastest_cpu().lower()] + else: + cpus = cpumap.keys() + + cmdspecs = [] + for cpu in cpus: + try: + host_bin_file = bench.get_binary(cpu) + except ValueError, e: + try: + msg = e.message + msg += ' Attempting to use generic binary instead.' + self.logger.debug(msg) + host_bin_file = bench.get_binary('generic') + cpu = 'generic' + except ValueError, e: + raise ConfigError(e.message) # re-raising as user error + binname = os.path.basename(host_bin_file) + binary = self.device.install(host_bin_file, with_name='.'.join([binname, cpu])) + commandspec = CommandSpec() + commandspec.command = bench.command_template.substitute({'binary': binary}) + commandspec.datadir = datadir + commandspec.cpumask = get_cpu_mask(cpumap[cpu]) + cmdspecs.append(commandspec) + return cmdspecs + + def _build_command(self, name, commandspecs): + if self.mode == 'speed': + if len(commandspecs) != 1: + raise AssertionError('Must be exactly one command spec specifed in speed mode.') + spec = commandspecs[0] + launch_command = self.launch_template.format(command=spec.command, cpumask=spec.cpumask) + self.commands.append((name, + self.speed_run_template.format(datadir=spec.datadir, + launch_command=launch_command))) + elif self.mode == 'rate': + loops = [] + for spec in commandspecs: + launch_command = self.launch_template.format(command=spec.command, cpumask=spec.cpumask) + loops.append(self.loop_template.format(launch_command=launch_command, threads=spec.threads)) + self.commands.append((name, + self.rate_run_template.format(datadir=spec.datadir, + loop='; '.join(loops)))) + else: + raise ValueError('Unexpected SPEC2000 mode: {}'.format(self.mode)) # Should never get here + + def _get_fastest_cpu(self): + cpu_types = set(self.device.core_names) + if len(cpu_types) == 1: + return cpu_types.pop() + fastest_cpu = None + fastest_freq = 0 + for cpu_type in cpu_types: + try: + idx = self.device.get_core_cpu(cpu_type) + freq = self.device.get_cpu_max_frequency(idx) + if freq > fastest_freq: + fastest_freq = freq + fastest_cpu = cpu_type + except ValueError: + pass + if not fastest_cpu: + raise WorkloadError('No active CPUs found on device. Something is very wrong...') + return fastest_cpu + + +class SpecBenchmark(object): + + def __init__(self): + self.name = None + self.path = None + self.category = None + self.command_template = None + self.cpus = [] + self.datafiles = [] + + def get_binary(self, cpu): + if cpu not in self.cpus: + raise ValueError('CPU {} is not supported by {}.'.format(cpu, self.name)) + binpath = os.path.join(self.path, 'cpus', cpu, self.name.split('.')[0]) + if not os.path.isfile(binpath): + raise ValueError('CPU {} is not supported by {}.'.format(cpu, self.name)) + return binpath + + +class CommandSpec(object): + + def __init__(self): + self.cpumask = None + self.datadir = None + self.command = None + self.threads = None + diff --git a/wlauto/workloads/sqlite/__init__.py b/wlauto/workloads/sqlite/__init__.py new file mode 100644 index 00000000..60a01276 --- /dev/null +++ b/wlauto/workloads/sqlite/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import re + +from wlauto import AndroidUiAutoBenchmark + + +class Sqlite(AndroidUiAutoBenchmark): + + name = 'sqlitebm' + description = """ + Measures the performance of the sqlite database. It determines within + what time the target device processes a number of SQL queries. + + """ + package = 'com.redlicense.benchmark.sqlite' + activity = '.Main' + summary_metrics = ['Overall'] + + score_regex = re.compile(r'V/sqlite.*:\s+([\w ]+) = ([\d\.]+) sec') + + def update_result(self, context): + super(Sqlite, self).update_result(context) + with open(self.logcat_log) as fh: + text = fh.read() + for match in self.score_regex.finditer(text): + metric = match.group(1) + value = match.group(2) + try: + value = float(value) + except ValueError: + self.logger.warn("Reported results do not match expected format (seconds)") + context.result.add_metric(metric, value, 'Seconds', lower_is_better=True) + diff --git a/wlauto/workloads/sqlite/com.arm.wlauto.uiauto.sqlite.jar b/wlauto/workloads/sqlite/com.arm.wlauto.uiauto.sqlite.jar Binary files differnew file mode 100644 index 00000000..e8b77514 --- /dev/null +++ b/wlauto/workloads/sqlite/com.arm.wlauto.uiauto.sqlite.jar diff --git a/wlauto/workloads/sqlite/uiauto/build.sh b/wlauto/workloads/sqlite/uiauto/build.sh new file mode 100755 index 00000000..b8bcdf89 --- /dev/null +++ b/wlauto/workloads/sqlite/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.sqlite.jar ]]; then + cp bin/com.arm.wlauto.uiauto.sqlite.jar .. +fi diff --git a/wlauto/workloads/sqlite/uiauto/build.xml b/wlauto/workloads/sqlite/uiauto/build.xml new file mode 100644 index 00000000..aa324270 --- /dev/null +++ b/wlauto/workloads/sqlite/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.sqlite" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/sqlite/uiauto/project.properties b/wlauto/workloads/sqlite/uiauto/project.properties new file mode 100644 index 00000000..ce39f2d0 --- /dev/null +++ b/wlauto/workloads/sqlite/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-18 diff --git a/wlauto/workloads/sqlite/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/sqlite/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..c11725e3 --- /dev/null +++ b/wlauto/workloads/sqlite/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,103 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.sqlite; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.view.KeyEvent; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "sqlite"; + + public void runUiAutomation() throws Exception { + Bundle status = new Bundle(); + status.putString("product", getUiDevice().getProductName()); + UiSelector selector = new UiSelector(); + + UiObject text_start = new UiObject(selector.text("Start") + .className("android.widget.Button")); + text_start.click(); + + try { + UiObject stop_text = new UiObject(selector.textContains("Stop") + .className("android.widget.Button")); + waitUntilNoObject(stop_text, 600); + + sleep(2); + this.extractResults(); + } finally { + } + } + + public void extractResults() throws UiObjectNotFoundException{ + UiSelector selector = new UiSelector(); + UiScrollable resultList = new UiScrollable(selector.className("android.widget.ScrollView")); + resultList.scrollToBeginning(5); + selector = resultList.getSelector(); + int index = 0; + while (true){ + UiObject lastEntry = new UiObject(selector.childSelector(new UiSelector() + .className("android.widget.LinearLayout") + .childSelector(new UiSelector() + .index(index) + .childSelector(new UiSelector() + .className("android.widget.LinearLayout"))))); + if (lastEntry.exists()){ + UiObject value = new UiObject(selector.childSelector(new UiSelector() + .className("android.widget.LinearLayout") + .childSelector(new UiSelector() + .index(index) + .childSelector(new UiSelector() + .resourceIdMatches(".*test_result.*"))))); + Log.v("sqlite", "Overall = " + value.getText().replace("\n", " ")); + break; + } + + UiObject label = new UiObject(selector.childSelector(new UiSelector() + .className("android.widget.LinearLayout") + .childSelector(new UiSelector() + .index(index) + .childSelector(new UiSelector() + .index(0))))); + UiObject value = new UiObject(selector.childSelector(new UiSelector() + .className("android.widget.LinearLayout") + .childSelector(new UiSelector() + .index(index) + .childSelector(new UiSelector() + .index(1))))); + index++; + if (!label.exists()){ + resultList.scrollForward(); + index--; + sleep(1); + continue; + } + Log.v("sqlite", label.getText() + " = " + value.getText().replace("\n", " ")); + } + } +} diff --git a/wlauto/workloads/sysbench/__init__.py b/wlauto/workloads/sysbench/__init__.py new file mode 100644 index 00000000..b97a8512 --- /dev/null +++ b/wlauto/workloads/sysbench/__init__.py @@ -0,0 +1,111 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,W0201,E0203 + +import os + +from wlauto import Workload, Parameter, File +from wlauto.exceptions import WorkloadError +from wlauto.utils.misc import parse_value + + +class Sysbench(Workload): + + name = 'sysbench' + description = """ + SysBench is a modular, cross-platform and multi-threaded benchmark tool + for evaluating OS parameters that are important for a system running a + database under intensive load. + + The idea of this benchmark suite is to quickly get an impression about + system performance without setting up complex database benchmarks or + even without installing a database at all. + + **Features of SysBench** + + * file I/O performance + * scheduler performance + * memory allocation and transfer speed + * POSIX threads implementation performance + * database server performance + + + See: http://sysbench.sourceforge.net/docs/ + + """ + + parameters = [ + Parameter('timeout', kind=int, default=300), + Parameter('test', kind=str, default='cpu'), + Parameter('num_threads', kind=int, default=8), + Parameter('max_requests', kind=int, default=2000), + ] + + def __init__(self, device, **kwargs): + super(Sysbench, self).__init__(device) + self.command = self._build_command(test=self.test, + num_threads=self.num_threads, + max_requests=self.max_requests) + self.results_file = self.device.path.join(self.device.working_directory, 'sysbench_result.txt') + + def setup(self, context): + self._check_executable(context) + self.device.execute('am start -n com.android.browser/.BrowserActivity about:blank ') + + def run(self, context): + self.device.execute(self.command, timeout=self.timeout) + + def update_result(self, context): + host_results_file = os.path.join(context.output_directory, 'sysbench_result.txt') + self.device.pull_file(self.results_file, host_results_file) + + with open(host_results_file) as fh: + in_summary = False + metric_prefix = '' + for line in fh: + if line.startswith('Test execution summary:'): + in_summary = True + elif in_summary: + if not line.strip(): + break # end of summary section + parts = [p.strip() for p in line.split(':') if p.strip()] + if len(parts) == 2: + metric = metric_prefix + parts[0] + value, units = parse_value(parts[1]) + context.result.add_metric(metric, value, units) + elif len(parts) == 1: + metric_prefix = line.strip() + ' ' + else: + self.logger.warn('Could not parse line: "{}"'.format(line.rstrip('\n'))) + context.add_iteration_artifact('sysbench_output', kind='raw', path='sysbench_result.txt') + + def teardown(self, context): + self.device.execute('am force-stop com.android.browser') + self.device.delete_file(self.results_file) + + def _check_executable(self, context): + if self.device.is_installed('sysbench'): + return + path = context.resolver.get(File(owner=self, path='sysbench')) + if not path: + raise WorkloadError('sysbench binary is not installed on the device, and it does not found in dependencies on the host.') + self.device.install(path) + + def _build_command(self, **parameters): + param_strings = ['--{}={}'.format(k.replace('_', '-'), v) + for k, v in parameters.iteritems()] + sysbench_command = 'sysbench {} run'.format(' '.join(param_strings)) + return 'cd {} && {} > sysbench_result.txt'.format(self.device.working_directory, sysbench_command) diff --git a/wlauto/workloads/sysbench/sysbench b/wlauto/workloads/sysbench/sysbench Binary files differnew file mode 100644 index 00000000..094cad31 --- /dev/null +++ b/wlauto/workloads/sysbench/sysbench diff --git a/wlauto/workloads/templerun/__init__.py b/wlauto/workloads/templerun/__init__.py new file mode 100644 index 00000000..f967a49d --- /dev/null +++ b/wlauto/workloads/templerun/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from wlauto import GameWorkload + + +class Templerun(GameWorkload): + + name = 'templerun' + description = """ + Templerun game. + + """ + package = 'com.imangi.templerun' + activity = 'com.unity3d.player.UnityPlayerProxyActivity' + install_timeout = 500 diff --git a/wlauto/workloads/templerun/revent_files/.empty b/wlauto/workloads/templerun/revent_files/.empty new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wlauto/workloads/templerun/revent_files/.empty diff --git a/wlauto/workloads/templerun/revent_files/Nexus10.run.revent b/wlauto/workloads/templerun/revent_files/Nexus10.run.revent Binary files differnew file mode 100644 index 00000000..c1019cf7 --- /dev/null +++ b/wlauto/workloads/templerun/revent_files/Nexus10.run.revent diff --git a/wlauto/workloads/templerun/revent_files/Nexus10.setup.revent b/wlauto/workloads/templerun/revent_files/Nexus10.setup.revent Binary files differnew file mode 100644 index 00000000..90b1cb43 --- /dev/null +++ b/wlauto/workloads/templerun/revent_files/Nexus10.setup.revent diff --git a/wlauto/workloads/thechase/__init__.py b/wlauto/workloads/thechase/__init__.py new file mode 100755 index 00000000..48c94efa --- /dev/null +++ b/wlauto/workloads/thechase/__init__.py @@ -0,0 +1,46 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101 +import time + +from wlauto import ApkWorkload, Parameter + + +class TheChase(ApkWorkload): + + name = 'thechase' + description = """ + The Chase demo showcasing the capabilities of Unity game engine. + + This demo, is a static video-like game demo, that demonstrates advanced features + of the unity game engine. It loops continuously until terminated. + + """ + + package = 'com.unity3d.TheChase' + activity = 'com.unity3d.player.UnityPlayerNativeActivity' + install_timeout = 200 + view = 'SurfaceView' + + parameters = [ + Parameter('duration', kind=int, default=70, + description=('Duration, in seconds, note that the demo loops the same (roughly) 60 ' + 'second sceene until stopped.')), + ] + + def run(self, context): + time.sleep(self.duration) + diff --git a/wlauto/workloads/truckerparking3d/__init__.py b/wlauto/workloads/truckerparking3d/__init__.py new file mode 100644 index 00000000..8180d4fd --- /dev/null +++ b/wlauto/workloads/truckerparking3d/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from wlauto import GameWorkload + + +class TruckerParking3D(GameWorkload): + + name = 'truckerparking3d' + description = """ + Trucker Parking 3D game. + + (yes, apparently that's a thing...) + """ + package = 'com.tapinator.truck.parking.bus3d' + activity = 'com.tapinator.truck.parking.bus3d.GCMNotificationActivity' diff --git a/wlauto/workloads/vellamo/__init__.py b/wlauto/workloads/vellamo/__init__.py new file mode 100644 index 00000000..1c70a958 --- /dev/null +++ b/wlauto/workloads/vellamo/__init__.py @@ -0,0 +1,215 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import logging +from HTMLParser import HTMLParser +from collections import defaultdict, OrderedDict + +from wlauto import AndroidUiAutoBenchmark, Parameter +from wlauto.utils.types import list_of_strs, numeric +from wlauto.exceptions import WorkloadError + + +#pylint: disable=no-member +class Vellamo(AndroidUiAutoBenchmark): + + name = 'vellamo' + description = """ + Android benchmark designed by Qualcomm. + + Vellamo began as a mobile web benchmarking tool that today has expanded + to include three primary chapters. The Browser Chapter evaluates mobile + web browser performance, the Multicore chapter measures the synergy of + multiple CPU cores, and the Metal Chapter measures the CPU subsystem + performance of mobile processors. Through click-and-go test suites, + organized by chapter, Vellamo is designed to evaluate: UX, 3D graphics, + and memory read/write and peak bandwidth performance, and much more! + + Note: Vellamo v3.0 fails to run on Juno + + """ + package = 'com.quicinc.vellamo' + run_timeout = 15 * 60 + benchmark_types = { + '2.0.3': ['html5', 'metal'], + '3.0': ['Browser', 'Metal', 'Multi'], + } + valid_versions = benchmark_types.keys() + summary_metrics = None + + parameters = [ + Parameter('version', kind=str, allowed_values=valid_versions, default=sorted(benchmark_types, reverse=True)[0], + description=('Specify the version of Vellamo to be run. ' + 'If not specified, the latest available version will be used.')), + Parameter('benchmarks', kind=list_of_strs, allowed_values=benchmark_types['3.0'], default=benchmark_types['3.0'], + description=('Specify which benchmark sections of Vellamo to be run. Only valid on version 3.0 and newer.' + '\nNOTE: Browser benchmark can be problematic and seem to hang,' + 'just wait and it will progress after ~5 minutes')), + Parameter('browser', kind=int, default=1, + description=('Specify which of the installed browsers will be used for the tests. The number refers to ' + 'the order in which browsers are listed by Vellamo. E.g. ``1`` will select the first browser ' + 'listed, ``2`` -- the second, etc. Only valid for version ``3.0``.')) + ] + + def __init__(self, device, **kwargs): + super(Vellamo, self).__init__(device, **kwargs) + if self.version == '2.0.3': + self.activity = 'com.quicinc.vellamo.VellamoActivity' + if self.version == '3.0': + self.activity = 'com.quicinc.vellamo.main.MainActivity' + self.summary_metrics = self.benchmark_types[self.version] + + def setup(self, context): + self.uiauto_params['version'] = self.version + self.uiauto_params['browserToUse'] = self.browser + self.uiauto_params['metal'] = 'Metal' in self.benchmarks + self.uiauto_params['browser'] = 'Browser' in self.benchmarks + self.uiauto_params['multicore'] = 'Multi' in self.benchmarks + super(Vellamo, self).setup(context) + + def validate(self): + super(Vellamo, self).validate() + if self.version == '2.0.3' or not self.benchmarks or self.benchmarks == []: # pylint: disable=access-member-before-definition + self.benchmarks = self.benchmark_types[self.version] # pylint: disable=attribute-defined-outside-init + else: + for benchmark in self.benchmarks: + if benchmark not in self.benchmark_types[self.version]: + raise WorkloadError('Version {} does not support {} benchmarks'.format(self.version, benchmark)) + + def update_result(self, context): + super(Vellamo, self).update_result(context) + + # Get total scores from logcat + self.non_root_update_result(context) + + if not self.device.is_rooted: + return + + for test in self.benchmarks: # Get all scores from HTML files + filename = None + if test == "Browser": + result_folder = self.device.path.join(self.device.package_data_directory, self.package, 'files') + for result_file in self.device.listdir(result_folder, as_root=True): + if result_file.startswith("Browser"): + filename = result_file + else: + filename = '{}_results.html'.format(test) + + device_file = self.device.path.join(self.device.package_data_directory, self.package, 'files', filename) + host_file = os.path.join(context.output_directory, filename) + self.device.pull_file(device_file, host_file, as_root=True) + with open(host_file) as fh: + parser = VellamoResultParser() + parser.feed(fh.read()) + for benchmark in parser.benchmarks: + benchmark.name = benchmark.name.replace(' ', '_') + context.result.add_metric('{}_Total'.format(benchmark.name), benchmark.score) + for name, score in benchmark.metrics.items(): + name = name.replace(' ', '_') + context.result.add_metric('{}_{}'.format(benchmark.name, name), score) + context.add_iteration_artifact('vellamo_output', kind='raw', path=filename) + + def non_root_update_result(self, context): + failed = [] + with open(self.logcat_log) as logcat: + metrics = OrderedDict() + for line in logcat: + if 'VELLAMO RESULT:' in line: + info = line.split(':') + parts = info[2].split(" ") + metric = parts[1].strip() + value = int(parts[2].strip()) + metrics[metric] = value + if 'VELLAMO ERROR:' in line: + self.logger.warning("Browser crashed during benchmark, results may not be accurate") + for key, value in metrics.iteritems(): + key = key.replace(' ', '_') + context.result.add_metric(key, value) + if value == 0: + failed.append(key) + if failed: + raise WorkloadError("The following benchmark groups failed: {}".format(", ".join(failed))) + + +class VellamoResult(object): + + def __init__(self, name): + self.name = name + self.score = None + self.metrics = {} + + def add_metric(self, data): + split_data = data.split(":") + name = split_data[0].strip() + score = split_data[1].strip() + + if name in self.metrics: + raise KeyError("A metric of that name is already present") + self.metrics[name] = float(score) + + +class VellamoResultParser(HTMLParser): + + class StopParsingException(Exception): + pass + + def __init__(self): + HTMLParser.__init__(self) + self.inside_div = False + self.inside_span = 0 + self.inside_li = False + self.got_data = False + self.failed = False + self.benchmarks = [] + + def feed(self, text): + try: + HTMLParser.feed(self, text) + except self.StopParsingException: + pass + + def handle_starttag(self, tag, attrs): + if tag == 'div': + self.inside_div = True + if tag == 'span': + self.inside_span += 1 + if tag == 'li': + self.inside_li = True + + def handle_endtag(self, tag): + if tag == 'div': + self.inside_div = False + self.inside_span = 0 + self.got_data = False + self.failed = False + if tag == 'li': + self.inside_li = False + + def handle_data(self, data): + if self.inside_div and not self.failed: + if "Problem" in data: + self.failed = True + elif self.inside_span == 1: + self.benchmarks.append(VellamoResult(data)) + elif self.inside_span == 3 and not self.got_data: + self.benchmarks[-1].score = int(data) + self.got_data = True + elif self.inside_li and self.got_data: + if 'failed' not in data: + self.benchmarks[-1].add_metric(data) + else: + self.failed = True + diff --git a/wlauto/workloads/vellamo/com.arm.wlauto.uiauto.vellamo.jar b/wlauto/workloads/vellamo/com.arm.wlauto.uiauto.vellamo.jar Binary files differnew file mode 100644 index 00000000..9e71e7eb --- /dev/null +++ b/wlauto/workloads/vellamo/com.arm.wlauto.uiauto.vellamo.jar diff --git a/wlauto/workloads/vellamo/uiauto/build.sh b/wlauto/workloads/vellamo/uiauto/build.sh new file mode 100755 index 00000000..4c3ad807 --- /dev/null +++ b/wlauto/workloads/vellamo/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.vellamo.jar ]]; then + cp bin/com.arm.wlauto.uiauto.vellamo.jar .. +fi diff --git a/wlauto/workloads/vellamo/uiauto/build.xml b/wlauto/workloads/vellamo/uiauto/build.xml new file mode 100644 index 00000000..c137d62d --- /dev/null +++ b/wlauto/workloads/vellamo/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.vellamo" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/vellamo/uiauto/project.properties b/wlauto/workloads/vellamo/uiauto/project.properties new file mode 100644 index 00000000..ce39f2d0 --- /dev/null +++ b/wlauto/workloads/vellamo/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-18 diff --git a/wlauto/workloads/vellamo/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/vellamo/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..a49a18ee --- /dev/null +++ b/wlauto/workloads/vellamo/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,260 @@ +/* Copyright 2014-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.vellamo; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; +import android.view.KeyEvent; +import java.util.concurrent.TimeUnit; +import java.util.ArrayList; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.core.UiDevice; +import com.android.uiautomator.core.UiWatcher; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "vellamo"; + public static ArrayList<String> scores = new ArrayList(); + public static Boolean wasError = false; + + public void runUiAutomation() throws Exception { + Bundle parameters = getParams(); + String version = parameters.getString("version"); + Boolean browser = Boolean.parseBoolean(parameters.getString("browser")); + Boolean metal = Boolean.parseBoolean(parameters.getString("metal")); + Boolean multicore = Boolean.parseBoolean(parameters.getString("multicore")); + Integer browserToUse = Integer.parseInt(parameters.getString("browserToUse")) - 1; + + dismissEULA(); + + if (version.equals("2.0.3")) { + dissmissWelcomebanner(); + startTest(); + dismissNetworkConnectionDialogIfNecessary(); + dismissExplanationDialogIfNecessary(); + waitForTestCompletion(15 * 60, "com.quicinc.vellamo:id/act_ba_results_btn_no"); + getScore("html5", "com.quicinc.vellamo:id/act_ba_results_img_0"); + getScore("metal", "com.quicinc.vellamo:id/act_ba_results_img_1"); + } + + else { + dismissLetsRoll(); + if (browser) { + startBrowserTest(browserToUse); + proccessTest("Browser"); + } + if (multicore) { + startTestV3(1); + proccessTest("Multicore"); + + } + if (metal) { + startTestV3(2); + proccessTest("Metal"); + } + } + for(String result : scores){ + Log.v(TAG, String.format("VELLAMO RESULT: %s", result)); + } + if (wasError) Log.v("vellamoWatcher", "VELLAMO ERROR: Something crashed while running browser benchmark"); + } + + public void startTest() throws Exception { + UiSelector selector = new UiSelector(); + UiObject runButton = new UiObject(selector.textContains("Run All Chapters")); + + if (!runButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) { + UiObject pager = new UiObject(selector.className("android.support.v4.view.ViewPager")); + pager.swipeLeft(2); + if (!runButton.exists()) { + throw new UiObjectNotFoundException("Could not find \"Run All Chapters\" button."); + } + } + runButton.click(); + } + + public void startBrowserTest(int browserToUse) throws Exception { + //Ensure chrome is selected as "browser" fails to run the benchmark + UiSelector selector = new UiSelector(); + UiObject browserToUseButton = new UiObject(selector.className("android.widget.ImageButton") + .longClickable(true).instance(browserToUse)); + UiObject browserButton = new UiObject(selector.className("android.widget.ImageButton") + .longClickable(true).selected(true)); + //Disable browsers + while(browserButton.exists()) browserButton.click(); + if (browserToUseButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) { + if (browserToUseButton.exists()) { + browserToUseButton.click(); + } + } + + //enable a watcher to dismiss browser dialogs + UiWatcher stoppedWorkingDialogWatcher = new UiWatcher() { + @Override + public boolean checkForCondition() { + UiObject stoppedWorkingDialog = new UiObject(new UiSelector().textStartsWith("Unfortunately")); + if(stoppedWorkingDialog.exists()){ + wasError = true; + UiObject okButton = new UiObject(new UiSelector().className("android.widget.Button").text("OK")); + try { + okButton.click(); + } catch (UiObjectNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return (stoppedWorkingDialog.waitUntilGone(25000)); + } + return false; + } + }; + // Register watcher + UiDevice.getInstance().registerWatcher("stoppedWorkingDialogWatcher", stoppedWorkingDialogWatcher); + + // Run watcher + UiDevice.getInstance().runWatchers(); + + startTestV3(0); + } + + public void startTestV3(int run) throws Exception { + UiSelector selector = new UiSelector(); + + UiObject thirdRunButton = new UiObject(selector.resourceId("com.quicinc.vellamo:id/card_launcher_run_button").instance(run)); + if (!thirdRunButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) { + if (!thirdRunButton.exists()) { + throw new UiObjectNotFoundException("Could not find three \"Run\" buttons."); + } + } + + //Run benchmarks + UiObject runButton = new UiObject(selector.resourceId("com.quicinc.vellamo:id/card_launcher_run_button").instance(run)); + if (!runButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) { + if (!runButton.exists()) { + throw new UiObjectNotFoundException("Could not find correct \"Run\" button."); + } + } + runButton.click(); + + //Skip tutorial screens + UiObject swipeScreen = new UiObject(selector.textContains("Swipe left to continue")); + if (!swipeScreen.waitForExists(TimeUnit.SECONDS.toMillis(5))) { + if (!swipeScreen.exists()) { + throw new UiObjectNotFoundException("Could not find \"Swipe screen\"."); + } + } + sleep(1); + swipeScreen.swipeLeft(2); + sleep(1); + swipeScreen.swipeLeft(2); + + } + + public void proccessTest(String metric) throws Exception{ + waitForTestCompletion(15 * 60, "com.quicinc.vellamo:id/button_no"); + + //Remove watcher + UiDevice.getInstance().removeWatcher("stoppedWorkingDialogWatcher"); + + getScore(metric, "com.quicinc.vellamo:id/card_score_score"); + getUiDevice().pressBack(); + getUiDevice().pressBack(); + getUiDevice().pressBack(); + } + + public void getScore(String metric, String resourceID) throws Exception { + UiSelector selector = new UiSelector(); + UiObject score = new UiObject(selector.resourceId(resourceID)); + if (!score.waitForExists(TimeUnit.SECONDS.toMillis(5))) { + if (!score.exists()) { + throw new UiObjectNotFoundException("Could not find score on screen."); + } + } + scores.add(metric + " " + score.getText().trim()); + } + + public void waitForTestCompletion(int timeout, String resourceID) throws Exception { + UiSelector selector = new UiSelector(); + UiObject resultsNoButton = new UiObject(selector.resourceId(resourceID)); + if (!resultsNoButton.waitForExists(TimeUnit.SECONDS.toMillis(timeout))) { + throw new UiObjectNotFoundException("Did not see results screen."); + } + + } + + public void dismissEULA() throws Exception { + UiSelector selector = new UiSelector(); + waitText("Vellamo EULA"); + UiObject acceptButton = new UiObject(selector.text("Accept") + .className("android.widget.Button")); + if (acceptButton.exists()) { + acceptButton.click(); + } + } + + public void dissmissWelcomebanner() throws Exception { + UiSelector selector = new UiSelector(); + UiObject welcomeBanner = new UiObject(selector.textContains("WELCOME")); + if (welcomeBanner.waitForExists(TimeUnit.SECONDS.toMillis(5))) { + UiObject pager = new UiObject(selector.className("android.support.v4.view.ViewPager")); + pager.swipeLeft(2); + pager.swipeLeft(2); + } + } + + public void dismissLetsRoll() throws Exception { + UiSelector selector = new UiSelector(); + UiObject letsRollButton = new UiObject(selector.className("android.widget.Button") + .textContains("Let's Roll")); + if (!letsRollButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) { + if (!letsRollButton.exists()) { + throw new UiObjectNotFoundException("Could not find \"Let's Roll\" button."); + } + } + letsRollButton.click(); + } + + public void dismissNetworkConnectionDialogIfNecessary() throws Exception { + UiSelector selector = new UiSelector(); + UiObject dialog = new UiObject(selector.className("android.widget.TextView") + .textContains("No Network Connection")); + if (dialog.exists()) { + UiObject yesButton = new UiObject(selector.className("android.widget.Button") + .text("Yes")); + yesButton.click(); + } + } + + public void dismissExplanationDialogIfNecessary() throws Exception { + UiSelector selector = new UiSelector(); + UiObject dialog = new UiObject(selector.className("android.widget.TextView") + .textContains("Benchmarks Explanation")); + if (dialog.exists()) { + UiObject noButton = new UiObject(selector.className("android.widget.Button") + .text("No")); + noButton.click(); + } + } +} diff --git a/wlauto/workloads/video/__init__.py b/wlauto/workloads/video/__init__.py new file mode 100644 index 00000000..711a7b82 --- /dev/null +++ b/wlauto/workloads/video/__init__.py @@ -0,0 +1,137 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,E0203,W0201 + +import os +import time +import urllib +from collections import defaultdict + +from wlauto import Workload, settings, Parameter, Alias +from wlauto.exceptions import ConfigError, WorkloadError +from wlauto.utils.misc import ensure_directory_exists as _d +from wlauto.utils.types import boolean + +DOWNLOAD_URLS = { + '1080p': 'http://download.blender.org/peach/bigbuckbunny_movies/big_buck_bunny_1080p_surround.avi', + '720p': 'http://download.blender.org/peach/bigbuckbunny_movies/big_buck_bunny_720p_surround.avi', + '480p': 'http://download.blender.org/peach/bigbuckbunny_movies/big_buck_bunny_480p_surround-fix.avi' +} + + +class VideoWorkload(Workload): + name = 'video' + description = """ + Plays a video file using the standard android video player for a predetermined duration. + + The video can be specified either using ``resolution`` workload parameter, in which case + `Big Buck Bunny`_ MP4 video of that resolution will be downloaded and used, or using + ``filename`` parameter, in which case the video file specified will be used. + + + .. _Big Buck Bunny: http://www.bigbuckbunny.org/ + + """ + + parameters = [ + Parameter('play_duration', kind=int, default=20, + description='Playback duration of the video file. This become the duration of the workload.'), + Parameter('resolution', default='720p', allowed_values=['480p', '720p', '1080p'], + description='Specifies which resolution video file to play.'), + Parameter('filename', + description=""" + The name of the video file to play. This can be either a path + to the file anywhere on your file system, or it could be just a + name, in which case, the workload will look for it in + ``~/.workloads_automation/dependency/video`` + *Note*: either resolution or filename should be specified, but not both! + """), + Parameter('force_dependency_push', kind=boolean, default=False, + description=""" + If true, video will always be pushed to device, regardless + of whether the file is already on the device. Default is ``False``. + """), + ] + + aliases = [ + Alias('video_720p', resolution='720p'), + Alias('video_1080p', resolution='1080p'), + ] + + @property + def host_video_file(self): + if not self._selected_file: + if self.filename: + if self.filename[0] in './' or len(self.filename) > 1 and self.filename[1] == ':': + filepath = os.path.abspath(self.filename) + else: + filepath = os.path.join(self.video_directory, self.filename) + if not os.path.isfile(filepath): + raise WorkloadError('{} does not exist.'.format(filepath)) + self._selected_file = filepath + else: + files = self.video_files[self.resolution] + if not files: + url = DOWNLOAD_URLS[self.resolution] + filepath = os.path.join(self.video_directory, os.path.basename(url)) + self.logger.debug('Downloading {}...'.format(filepath)) + urllib.urlretrieve(url, filepath) + self._selected_file = filepath + else: + self._selected_file = files[0] + if len(files) > 1: + self.logger.warn('Multiple files for 720p found. Using {}.'.format(self._selected_file)) + self.logger.warn('Use \'filename\'parameter instead of \'resolution\' to specify a different file.') + return self._selected_file + + def init_resources(self, context): + self.video_directory = _d(os.path.join(settings.dependencies_directory, 'video')) + self.video_files = defaultdict(list) + self.enum_video_files() + self._selected_file = None + + def setup(self, context): + on_device_video_file = os.path.join(self.device.working_directory, os.path.basename(self.host_video_file)) + if self.force_dependency_push or not self.device.file_exists(on_device_video_file): + self.logger.debug('Copying {} to device.'.format(self.host_video_file)) + self.device.push_file(self.host_video_file, on_device_video_file, timeout=120) + self.device.execute('am start -n com.android.browser/.BrowserActivity about:blank') + time.sleep(5) + self.device.execute('am force-stop com.android.browser') + time.sleep(5) + self.device.clear_logcat() + command = 'am start -W -S -n com.android.gallery3d/.app.MovieActivity -d {}'.format(on_device_video_file) + self.device.execute(command) + + def run(self, context): + time.sleep(self.play_duration) + + def update_result(self, context): + self.device.execute('am force-stop com.android.gallery3d') + + def teardown(self, context): + pass + + def validate(self): + if (self.resolution and self.filename) and (self.resolution != self.parameters['resolution'].default): + raise ConfigError('Ether resolution *or* filename must be specified; but not both.') + + def enum_video_files(self): + for filename in os.listdir(self.video_directory): + for resolution in self.parameters['resolution'].allowed_values: + if resolution in filename: + self.video_files[resolution].append(os.path.join(self.video_directory, filename)) + diff --git a/wlauto/workloads/videostreaming/__init__.py b/wlauto/workloads/videostreaming/__init__.py new file mode 100644 index 00000000..1374e67b --- /dev/null +++ b/wlauto/workloads/videostreaming/__init__.py @@ -0,0 +1,73 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,E0203,W0201 +import os + +from wlauto import AndroidUiAutoBenchmark, Parameter +import wlauto.common.android.resources + + +class Videostreaming(AndroidUiAutoBenchmark): + name = 'videostreaming' + description = """ + Uses the FREEdi video player to search, stream and play the specified + video content from YouTube. + + """ + name = 'videostreaming' + package = 'tw.com.freedi.youtube.player' + activity = '.MainActivity' + + parameters = [ + Parameter('video_name', kind=str, + description='Name of the video to be played.'), + Parameter('resolution', kind=str, default='320p', allowed_values=['320p', '720p', '1080p'], + description='Resolution of the video to be played. If video_name is set' + 'this setting will be ignored'), + Parameter('sampling_interval', kind=int, default=20, + description=""" + Time interval, in seconds, after which the status of the video playback to + be monitoreThe elapsed time of the video playback is + monitored after after every ``sampling_interval`` seconds and + compared against the actual time elapsed and the previous + sampling point. If the video elapsed time is less that + (sampling time - ``tolerance``) , then the playback is aborted as + the video has not been playing continuously. + """), + Parameter('tolerance', kind=int, default=3, + description=""" + Specifies the amount, in seconds, by which sampling time is + allowed to deviate from elapsed video playback time. If the delta + is greater than this value (which could happen due to poor network + connection), workload result will be invalidated. + """), + Parameter('run_timeout', kind=int, default=200, + description='The duration in second for which to play the video'), + ] + + def init_resources(self, context): + self.uiauto_params['tolerance'] = self.tolerance + self.uiauto_params['sampling_interval'] = self.sampling_interval + if self.video_name and self.video_name != "": + self.uiauto_params['video_name'] = self.video_name.replace(" ", "0space0") # hack to get around uiautomator limitation + else: + self.uiauto_params['video_name'] = "abkk sathe {}".format(self.resolution).replace(" ", "0space0") + self.apk_file = context.resolver.get(wlauto.common.android.resources.ApkFile(self)) + self.uiauto_file = context.resolver.get(wlauto.common.android.resources.JarFile(self)) + self.device_uiauto_file = self.device.path.join(self.device.working_directory, + os.path.basename(self.uiauto_file)) + if not self.uiauto_package: + self.uiauto_package = os.path.splitext(os.path.basename(self.uiauto_file))[0] diff --git a/wlauto/workloads/videostreaming/com.arm.wlauto.uiauto.videostreaming.jar b/wlauto/workloads/videostreaming/com.arm.wlauto.uiauto.videostreaming.jar Binary files differnew file mode 100644 index 00000000..beb6790b --- /dev/null +++ b/wlauto/workloads/videostreaming/com.arm.wlauto.uiauto.videostreaming.jar diff --git a/wlauto/workloads/videostreaming/uiauto/build.sh b/wlauto/workloads/videostreaming/uiauto/build.sh new file mode 100755 index 00000000..07e2131e --- /dev/null +++ b/wlauto/workloads/videostreaming/uiauto/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + +class_dir=bin/classes/com/arm/wlauto/uiauto +base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wlauto.uiauto.videostreaming.jar ]]; then + cp bin/com.arm.wlauto.uiauto.videostreaming.jar .. +fi diff --git a/wlauto/workloads/videostreaming/uiauto/build.xml b/wlauto/workloads/videostreaming/uiauto/build.xml new file mode 100644 index 00000000..e897fec2 --- /dev/null +++ b/wlauto/workloads/videostreaming/uiauto/build.xml @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project name="com.arm.wlauto.uiauto.videostreaming" default="help"> + + <!-- The local.properties file is created and updated by the 'android' tool. + It contains the path to the SDK. It should *NOT* be checked into + Version Control Systems. --> + <property file="local.properties" /> + + <!-- The ant.properties file can be created by you. It is only edited by the + 'android' tool to add properties to it. + This is the place to change some Ant specific build properties. + Here are some properties you may want to change/update: + + source.dir + The name of the source directory. Default is 'src'. + out.dir + The name of the output directory. Default is 'bin'. + + For other overridable properties, look at the beginning of the rules + files in the SDK, at tools/ant/build.xml + + Properties related to the SDK location or the project target should + be updated using the 'android' tool with the 'update' action. + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. + + --> + <property file="ant.properties" /> + + <!-- if sdk.dir was not set from one of the property file, then + get it from the ANDROID_HOME env var. + This must be done before we load project.properties since + the proguard config can use sdk.dir --> + <property environment="env" /> + <condition property="sdk.dir" value="${env.ANDROID_HOME}"> + <isset property="env.ANDROID_HOME" /> + </condition> + + <!-- The project.properties file is created and updated by the 'android' + tool, as well as ADT. + + This contains project specific properties such as project target, and library + dependencies. Lower level build properties are stored in ant.properties + (or in .classpath for Eclipse projects). + + This file is an integral part of the build system for your + application and should be checked into Version Control Systems. --> + <loadproperties srcFile="project.properties" /> + + <!-- quick check on sdk.dir --> + <fail + message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable." + unless="sdk.dir" + /> + + <!-- + Import per project custom build rules if present at the root of the project. + This is the place to put custom intermediary targets such as: + -pre-build + -pre-compile + -post-compile (This is typically used for code obfuscation. + Compiled code location: ${out.classes.absolute.dir} + If this is not done in place, override ${out.dex.input.absolute.dir}) + -post-package + -post-build + -pre-clean + --> + <import file="custom_rules.xml" optional="true" /> + + <!-- Import the actual build file. + + To customize existing targets, there are two options: + - Customize only one target: + - copy/paste the target into this file, *before* the + <import> task. + - customize it to your needs. + - Customize the whole content of build.xml + - copy/paste the content of the rules files (minus the top node) + into this file, replacing the <import> task. + - customize to your needs. + + *********************** + ****** IMPORTANT ****** + *********************** + In all cases you must update the value of version-tag below to read 'custom' instead of an integer, + in order to avoid having your file be overridden by tools such as "android update project" + --> + <!-- version-tag: VERSION_TAG --> + <import file="${sdk.dir}/tools/ant/uibuild.xml" /> + +</project> diff --git a/wlauto/workloads/videostreaming/uiauto/project.properties b/wlauto/workloads/videostreaming/uiauto/project.properties new file mode 100644 index 00000000..ce39f2d0 --- /dev/null +++ b/wlauto/workloads/videostreaming/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-18 diff --git a/wlauto/workloads/videostreaming/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/videostreaming/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java new file mode 100644 index 00000000..e532b499 --- /dev/null +++ b/wlauto/workloads/videostreaming/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -0,0 +1,155 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wlauto.uiauto.videostreaming; + +import android.app.Activity; +import java.util.Date; +import android.os.Bundle; +import java.util.concurrent.TimeUnit; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wlauto.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "videostreaming"; + + /*function to convert time in string to sec*/ + public int computeTimeInSec(String time) { + final int seconds = 60; + if (!time.contains(":")) + return -1; + + int totalTime = 0, mulfactor = 1; + String [] strArr = time.split(":"); + + for (int j = strArr.length - 1; j >= 0; j--) { + totalTime += Integer.parseInt(strArr[j]) * (mulfactor); + mulfactor = mulfactor * seconds; + } + return totalTime; + } + + public void runUiAutomation() throws Exception { + final int timeout = 5; + int currentTime = 0, timeAfter20Sec = 0, videoTime = 0; + long timeBeforeGetText = 0, timeAfterGetText = 0, timeForGetText = 0; + Bundle status = new Bundle(); + + Bundle parameters = getParams(); + if (parameters.size() <= 0) + return; + + int tolerance = Integer.parseInt(parameters.getString("tolerance")); + int samplingInterval = Integer.parseInt(parameters + .getString("sampling_interval")); + String videoName = parameters.getString("video_name").replace("0space0", " "); //Hack to get around uiautomator limitation + + UiObject search = new UiObject(new UiSelector() + .className("android.widget.ImageButton").index(0)); + if (search.exists()) { + search.clickAndWaitForNewWindow(timeout); + } + + UiObject clickVideoTab = new UiObject(new UiSelector() + .className("android.widget.Button").text("Video")); + clickVideoTab.click(); + + UiObject enterKeyword = new UiObject(new UiSelector() + .className("android.widget.EditText") + .text("Please input the keywords")); + enterKeyword.clearTextField(); + enterKeyword.setText(videoName); + + UiSelector selector = new UiSelector(); + UiObject clickSearch = new UiObject(selector.resourceId("tw.com.freedi.youtube.player:id/startSearchBtn")); + clickSearch.clickAndWaitForNewWindow(timeout); + + UiObject clickVideo = new UiObject(new UiSelector().className("android.widget.TextView").textContains(videoName)); + if (!clickVideo.waitForExists(TimeUnit.SECONDS.toMillis(10))) { + if (!clickVideo.exists()) { + throw new UiObjectNotFoundException("Could not find video."); + } + } + + clickVideo.clickAndWaitForNewWindow(timeout); + + UiObject totalVideoTime = new UiObject(new UiSelector() + .className("android.widget.TextView").index(2)); + + UiObject rewind = new UiObject(new UiSelector() + .className("android.widget.RelativeLayout") + .index(0).childSelector(new UiSelector() + .className("android.widget.LinearLayout") + .index(1).childSelector(new UiSelector() + .className("android.widget.LinearLayout") + .index(1).childSelector(new UiSelector() + .className("android.widget.ImageButton") + .enabled(true).index(2))))); + rewind.click(); + + videoTime = computeTimeInSec(totalVideoTime.getText()); + + /** + * Measure the video elapsed time between sampling intervals and + * compare it against the actual time elapsed minus tolerance.If the + * video elapsed time is less than the (actual time elapsed - + * tolerance), raise the message. + */ + if (videoTime > samplingInterval) { + for (int i = 0; i < (videoTime / samplingInterval); i++) { + UiObject videoCurrentTime = new UiObject(new UiSelector() + .className("android.widget.TextView").index(0)); + + sleep(samplingInterval); + + // Handle the time taken by the getText function + timeBeforeGetText = new Date().getTime() / 1000; + timeAfter20Sec = computeTimeInSec(videoCurrentTime.getText()); + timeAfterGetText = new Date().getTime() / 1000; + timeForGetText = timeAfterGetText - timeBeforeGetText; + + if (timeAfter20Sec == -1) { + getUiDevice().pressHome(); + return; + } + + if ((timeAfter20Sec - (currentTime + timeForGetText)) < + (samplingInterval - tolerance)) { + getUiDevice().pressHome(); + + getAutomationSupport().sendStatus(Activity.RESULT_CANCELED, + status); + return; + } + currentTime = timeAfter20Sec; + + } + } else { + sleep(videoTime); + } + getUiDevice().pressBack(); + getUiDevice().pressHome(); + getAutomationSupport().sendStatus(Activity.RESULT_OK, status); + } +} |