diff options
author | Yongqin Liu <yongqin.liu@linaro.org> | 2015-03-10 18:32:54 +0800 |
---|---|---|
committer | Milosz Wasilewski <milosz.wasilewski@linaro.org> | 2015-03-24 10:12:15 +0000 |
commit | 2da0739ca4a1157d5fb5095b6a5c21ec7cc548b2 (patch) | |
tree | bd41c5d0a0f654aed207c8b2bd93f45f955ac280 /android | |
parent | c058de275e948c8fe1d3671a0efbd23527c5b3aa (diff) |
android: add tests of stringbench and libc-bench
and update the tests of bionic-benchmarks
Change-Id: I548b1620487802654c7f32dedd5a81221f71ad0f
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>
Diffstat (limited to 'android')
-rw-r--r-- | android/bionic-benchmark.yaml | 5 | ||||
-rw-r--r-- | android/libc-bench.yaml | 23 | ||||
-rwxr-xr-x | android/scripts/bionic-benchmarks.sh | 59 | ||||
-rw-r--r-- | android/scripts/common.sh | 292 | ||||
-rwxr-xr-x | android/scripts/libc-bench.sh | 60 | ||||
-rwxr-xr-x | android/scripts/stringbench.sh | 41 | ||||
-rw-r--r-- | android/stringbench.yaml | 24 |
7 files changed, 462 insertions, 42 deletions
diff --git a/android/bionic-benchmark.yaml b/android/bionic-benchmark.yaml index ac8b135..e6f522f 100644 --- a/android/bionic-benchmark.yaml +++ b/android/bionic-benchmark.yaml @@ -12,8 +12,9 @@ metadata: - juno params: - LOOP_TIMES: 1 + RECORD_RESULT_LOCAL: "FALSE" + LOOP_COUNT: "1" run: steps: - - ./android/scripts/bionic-benchmarks.sh $LOOP_TIMES + - ./android/scripts/bionic-benchmarks.sh --record-csv ${RECORD_RESULT_LOCAL} --loop-count ${LOOP_COUNT} diff --git a/android/libc-bench.yaml b/android/libc-bench.yaml new file mode 100644 index 0000000..fd64f49 --- /dev/null +++ b/android/libc-bench.yaml @@ -0,0 +1,23 @@ +metadata: + name: libc-bench + format: "Lava-Test-Shell Test Definition 1.0" + description: "Run libc-bench command to benchmark the + performance of bionic. + The source is here now: + git://android.git.linaro.org/platform/external/libc-bench" + maintainer: + - yongqin.liu@linaro.org + os: + - android + scope: + - performance + devices: + - juno + +params: + RECORD_RESULT_LOCAL: "FALSE" + LOOP_COUNT: "1" + +run: + steps: + - ./android/scripts/libc-bench.sh --record-csv ${RECORD_RESULT_LOCAL} --loop-count ${LOOP_COUNT} diff --git a/android/scripts/bionic-benchmarks.sh b/android/scripts/bionic-benchmarks.sh index 9839179..b3d27fb 100755 --- a/android/scripts/bionic-benchmarks.sh +++ b/android/scripts/bionic-benchmarks.sh @@ -1,75 +1,54 @@ #!/system/bin/sh -loop_times=$1 -if [ -z "${loop_times}" ]; then - loop_times=12 -fi -raw_data="/data/local/tmp/bionic-benchmarks-raw.csv" -rm -fr ${raw_data} - -add_lava_result_entry(){ - local cmd="lava-test-case" - if [ -n "$(which $cmd)" ];then - $cmd "$@" - else - echo "$cmd $@" - fi -} +local_file_path="$0" +local_file_parent=$(cd $(dirname ${local_file_path}); pwd) +. ${local_file_parent}/common.sh test_bionic_benchmark(){ - local arch=$1 && shift + local arch=$1 if [ "X$arch" != "X32" ] && [ "X$arch" != "X64" ]; then echo "The specified $arch is not specified!" return fi - local excluded_test=$1 && shift + local excluded_test=$2 local cmd="bionic-benchmarks${arch}" if [ -n "$(which ${cmd})" ]; then for line in $(${cmd} --help 2>&1|grep BM_); do if [ -n "$excluded_test" ]; then if echo "${excluded_test}"|grep -q ${line}; then - echo "${arch}_${line} skip" - add_lava_result_entry "${arch}_${line}" "--result" "skip" + output_test_result "${arch}_${line}" "skip" continue fi fi - echo "--------------start $line--------" local hasResult=false - for res_line in $(${cmd} ${line}|grep "BM_"|tr -s ' '|tr ' ' ','); do - echo "${arch}_$line pass" - add_lava_result_entry "${arch}_${line}" "--result" "pass" + for res_line in $(${cmd} "^${line}$"|grep "BM_"|tr -s ' '|tr ' ' ','); do + output_test_result "${arch}_${line}" "pass" local key=$(echo $res_line|cut -d, -f1|tr '/' '_') local iterations=$(echo $res_line|cut -d, -f2) local ns_time=$(echo $res_line|cut -d, -f3) local throughput=$(echo $res_line|cut -d, -f4) local throughput_units=$(echo $res_line|cut -d, -f5) - echo "${arch}_${key}_time pass $ns_time ns/op" - add_lava_result_entry "${arch}_${key}_time" "--result" "pass" '--measurement' "${ns_time}" '--units' "ns/op" - echo "${arch}_${key}_time,${ns_time}">>${raw_data} + output_test_result "${arch}_${key}_time" "pass" "${ns_time}" "ns/op" if [ -n "${throughput_units}" ];then - echo "${arch}_${key}_throught pass ${throughput} ${throughput_units}" - add_lava_result_entry "${arch}_${key}_throught" "--result" "pass" '--measurement' "${throughput}" '--units' "${throughput_units}" - echo "${arch}_${key}_throught,${throughput}">>${raw_data} + output_test_result "${arch}_${key}_throught" "pass" "${throughput}" "${throughput_units}" fi hasResult=true done if ! $hasResult; then - echo "${arch}_$line fail" - add_lava_result_entry "${arch}_${line}" "--result" "fail" + output_test_result "${arch}_${line}" "fail" fi - echo "--------------finished $line--------" done fi } -loop_index=0 -while [ ${loop_index} -lt ${loop_times} ]; do +test_func(){ test_bionic_benchmark "64" "BM_property_serial" test_bionic_benchmark "32" "BM_property_serial BM_property_read" - loop_index=$((loop_index + 1)) -done +} + +main(){ + var_test_func="test_func" + run_test "$@" +} -attach_cmd="lava-test-run-attach" -if [ -n "$(which ${attach_cmd})" ] && [ -f "${raw_data}" ]; then - ${attach_cmd} ${raw_data} text/plain -fi +main "$@" diff --git a/android/scripts/common.sh b/android/scripts/common.sh new file mode 100644 index 0000000..bc4a526 --- /dev/null +++ b/android/scripts/common.sh @@ -0,0 +1,292 @@ +#!/system/bin/sh + +G_LOOP_COUNT=12 +G_RECORD_LOCAL_CSV=TRUE +G_VERBOSE_OUTPUT=FALSE +F_RAW_DATA_CSV="/data/local/tmp/lava_test_shell_raw_data.csv" +F_STATISTIC_DATA_CSV="/data/local/tmp/lava_test_shell_statistic_data.csv" + +## Description: +## output the max value of the passed 2 parameters +## Usage: +## f_max "${val1}" "${val2}" +## Example: +## max=$(f_max "1.5" "2.0") +f_max(){ + local val1=$1 + local val2=$2 + [ -z "$val1" ] && echo $val2 + [ -z "$val2" ] && echo $val1 + + echo "$val1,$val2"|awk -F, '{if($1<$2) print $2; else print $1}' +} + +## Description: +## output the min value of the passed 2 parameters +## Usage: +## f_min "${val1}" "${val2}" +## Example: +## min=$(f_min "1.5" "2.0") +f_min(){ + local val1=$1 + local val2=$2 + [ -z "$val1" ] && echo $val1 + [ -z "$val2" ] && echo $val2 + + echo "$val1,$val2"|awk -F, '{if($1>$2) print $2; else print $1}' +} + +## Description: +## calculate the average value for specified csv file. +## The first field of that csv file should be the key/name of that line, +## Lines have the same key should be together. +## Usage: +## statistic "${csv_file_path}" "${file_number}" +## Example: +## statistic "$f_res_starttime" 2 +## Note: +## if less than 4 samples for that key/item there, average will be calculated as total/count +## if 4 or more samples for that key/item there, average will be calculated with max and min excluded +statistic(){ + local f_data=$1 + if ! [ -f "$f_data" ]; then + return + fi + local field_no=$2 + if [ -z "$field_no" ]; then + field_no=2 + fi + local total=0 + local max=0 + local min=0 + local old_key="" + local new_key="" + local count=0 + local units="" + sort ${f_data} >${f_data}.sort + for line in $(cat "${f_data}.sort" |tr ' ' '~'); do + if ! echo "$line"|grep -q ,; then + continue + fi + new_key=$(echo $line|cut -d, -f1) + local measurement_units=$(echo $line|cut -d, -f${field_no}) + if echo ${measurement_units}|grep -q '~'; then + value=$(echo ${measurement_units}|cut -d~ -f1) + else + value=${measurement_units} + fi + + if [ "X${new_key}" = "X${old_key}" ]; then + total=$(echo "${total},${value}"|awk -F, '{printf "%.2f",$1+$2;}') + count=$(echo "${count},1"|awk -F, '{printf $1+$2;}') + max=$(f_max "$max" "$value") + min=$(f_min "$min" "$value") + else + if [ "X${old_key}" != "X" ]; then + if [ $count -ge 4 ]; then + average=$(echo "${total},${max},${min},$count"|awk -F, '{printf "%.2f",($1-$2-$3)/($4-2);}') + else + average=$(echo "${total},$count"|awk -F, '{printf "%.2f",$1/$2;}') + fi + if [ -z "${units}" ]; then + echo "${old_key}=${average}" + else + echo "${old_key}=${average},${units}" + fi + fi + total="${value}" + max="${value}" + min="${value}" + old_key="${new_key}" + count=1 + if echo ${measurement_units}|grep -q '~'; then + units=$(echo ${measurement_units}|cut -d~ -f2) + else + units="" + fi + fi + done + if [ "X${new_key}" != "X" ]; then + if [ $count -ge 4 ]; then + average=$(echo "${total},${max},${min},$count"|awk -F, '{printf "%.2f",($1-$2-$3)/($4-2);}') + else + average=$(echo "${total},$count"|awk -F, '{printf "%.2f",$1/$2;}') + fi + if [ -z "${units}" ]; then + echo "${new_key}=${average}" + else + echo "${new_key}=${average},${units}" + fi + fi + rm "${f_data}.sort" +} + +## Description: +## output the test result to console and add for lava-test-shell, +## also write into one csv file for comparing manually +## Usage: +## output_test_result $test_name $result [ $measurement [ $units ] ] +## Note: +## G_RECORD_LOCAL_CSV: when this environment variant is set to "TRUE", +## the result will be recorded in a csv file in the following path: +## rawdata/final_result.csv +## G_VERBOSE_OUTPUT: when this environment variant is set to "TRUE", and only it is TRUE, +## the verbose informain about the result will be outputed +output_test_result(){ + local test_name=$1 + local result=$2 + local measurement=$3 + local units=$4 + + if [ -z "${test_name}" ] || [ -z "$result" ]; then + return + fi + local output="" + local lava_paras="" + local output_csv="" + if [ -z "$units" ]; then + units="points" + fi + if [ -z "${measurement}" ]; then + output="${test_name}=${result}" + lava_paras="${test_name} --result ${result}" + else + output="${test_name}=${measurement} ${units}" + lava_paras="${test_name} --result ${result} --measurement ${measurement} --units ${units}" + output_csv="${test_name},${measurement} ${units}" + fi + + if [ "X${G_VERBOSE_OUTPUT}" = "XTRUE" ];then + echo "${output}" + fi + + local cmd="lava-test-case" + if [ -n "$(which $cmd)" ];then + $cmd ${lava_paras} + elif [ "X${G_VERBOSE_OUTPUT}" = "XTRUE" ];then + echo "$cmd ${lava_paras}" + fi + if [ "X${G_RECORD_LOCAL_CSV}" = "XTRUE" ]; then + if [ -n "${output_csv}" ]; then + echo "${output_csv}">>${F_RAW_DATA_CSV} + fi + fi +} + +func_print_usage_common(){ + echo "$(basename $0) [--record-csv TRUE|others] [--loop-count LOOP_COUNT]" + echo " --record-csv: specify if record the result in csv format in file ${F_RAW_DATA_CSV}" + echo " Only record the file when TRUE is specified." + echo " --loop-count: specify the number that how many times should be run for each application to get the average result, default is 12" + echo " --verbose-output: output the result and lava-test-case command for each test case each time it is run" + echo "$(basename $0) [--help|-h]" + echo " print out this usage." +} + +func_parse_parameters_common(){ + local para_loop_count="" + while [ -n "$1" ]; do + case "X$1" in + X--record-csv) + para_record_csv=$2 + if [ -z "${para_record_csv}" ]; then + echo "Please specify the value for --record-csv option" + exit 1 + fi + shift 2 + ;; + X--verbose-output) + para_verbose_output=$2 + if [ -z "${para_verbose_output}" ]; then + echo "Please specify the value for --verbose-output option" + exit 1 + fi + shift 2 + ;; + X--loop-count) + para_loop_count=$2 + if [ -z "${para_loop_count}" ]; then + echo "Please specify the value for --loop-count option" + exit 1 + fi + shift 2 + ;; + X-h|X--help) + func_print_usage_common + exit 1 + ;; + X-*) + echo "Unknown option: $1" + func_print_usage_common + exit 1 + ;; + X*) + func_print_usage_common + exit 1 + ;; + esac + done + + if [ -n "${para_loop_count}" ]; then + local tmp_str=$(echo ${para_loop_count}|tr -d '[:digit:]') + if [ -z "${tmp_str}" ]; then + G_LOOP_COUNT=${para_loop_count} + else + echo "The specified LOOP_COUNT(${para_loop_count}) is not valid" + exit 1 + fi + fi + + if [ -n "${para_record_csv}" ] && [ "X${para_record_csv}" = "XTRUE" ];then + G_RECORD_LOCAL_CSV=TRUE + elif [ -n "${para_record_csv}" ];then + G_RECORD_LOCAL_CSV=FALSE + fi + + if [ -n "${para_verbose_output}" ] && [ "X${para_verbose_output}" = "XTRUE" ];then + G_VERBOSE_OUTPUT=TRUE + elif [ -n "${para_record_csv}" ];then + G_VERBOSE_OUTPUT=FALSE + fi +} + +## Description: +## common framework to run test multiple times +## also write result into one csv file for comparing manually +## Usage: +## output_test_result $test_name $result [ $measurement [ $units ] ] +## Note: +## RECORD_RESULT_LOCAL: when this environment variant is set to "TRUE", +## the result will be recorded in a csv file in the following path: +## rawdata/final_result.csv +run_test(){ + func_parse_parameters_common "$@" + if [ "X${G_RECORD_LOCAL_CSV}" = "XTRUE" ]; then + [ -f "${F_RAW_DATA_CSV}" ] && rm "${F_RAW_DATA_CSV}" + [ -f "${F_STATISTIC_DATA_CSV}" ] && rm "${F_STATISTIC_DATA_CSV}" + mkdir -p $(dirname ${F_RAW_DATA_CSV}) + fi + + loop_index=0 + while [ ${loop_index} -lt ${G_LOOP_COUNT} ]; do + if [ -n "${var_test_func}" ]; then + ${var_test_func} + fi + loop_index=$((loop_index + 1)) + done + + if [ "X${G_RECORD_LOCAL_CSV}" = "XTRUE" ]; then + statistic ${F_RAW_DATA_CSV} 2 |tee ${F_STATISTIC_DATA_CSV} + sed -i 's/=/,/' "${F_STATISTIC_DATA_CSV}" + + attach_cmd="lava-test-run-attach" + if [ -n "$(which ${attach_cmd})" ]; then + if [ -f "${F_RAW_DATA_CSV}" ]; then + ${attach_cmd} ${F_RAW_DATA_CSV} text/plain + fi + if [ -f "${F_STATISTIC_DATA_CSV}" ]; then + ${attach_cmd} ${F_STATISTIC_DATA_CSV} text/plain + fi + fi + fi +} diff --git a/android/scripts/libc-bench.sh b/android/scripts/libc-bench.sh new file mode 100755 index 0000000..961a743 --- /dev/null +++ b/android/scripts/libc-bench.sh @@ -0,0 +1,60 @@ +#!/system/bin/sh + +local_file_path="$0" +local_file_parent=$(cd $(dirname ${local_file_path}); pwd) +. ${local_file_parent}/common.sh + +test_stringbench(){ + local cmd=$1 && shift + local prefix=$1 && shift + if [ -z "${cmd}" ]; then + return + fi + if [ ! -x "${cmd}" ]; then + return + fi + if [ -n "$prefix" ]; then + prefix="${prefix}_" + fi + + local oldkey="" + local newkey="" + local value_line=""= + for res_line in $(${cmd} |tr -d ' '); do + if echo "${res_line}"|grep -q '^b_'; then + if [ -n "${oldkey}" ] && [ -n "${value_line}" ]; then + local time_value=$(echo ${value_line}|cut -d, -f1|cut -d: -f2) + local virt_value=$(echo ${value_line}|cut -d, -f2|cut -d: -f2) + local res_value=$(echo ${value_line}|cut -d, -f3|cut -d: -f2) + local dirty_value=$(echo ${value_line}|cut -d, -f4|cut -d: -f2) + output_test_result "${prefix}${oldkey}_time" "pass" "${time_value}" "seconds" + output_test_result "${prefix}${oldkey}_virt" "pass" "${virt_value}" "kB" + output_test_result "${prefix}${oldkey}_res" "pass" "${res_value}" "kB" + output_test_result "${prefix}${oldkey}_dirty" "pass" "${dirty_value}" "kB" + fi + newkey=$(echo $res_line|tr -c '[:alnum:]' '_'|tr -s '_' |sed 's/_$//') + value_line="" + oldkey=${newkey} + continue + fi + + if echo "${res_line}"|grep -q '^time:'; then + value_line="${res_line}" + continue + fi + done +} + +test_func(){ + stringbench="/system/xbin/libcbench" + stringbench64="/system/xbin/libcbench64" + test_stringbench "${stringbench}" "32Bit" + test_stringbench "${stringbench64}" "64Bit" +} + +main(){ + var_test_func="test_func" + run_test "$@" +} + +main "$@" diff --git a/android/scripts/stringbench.sh b/android/scripts/stringbench.sh new file mode 100755 index 0000000..34ee40f --- /dev/null +++ b/android/scripts/stringbench.sh @@ -0,0 +1,41 @@ +#!/system/bin/sh + +local_file_path="$0" +local_file_parent=$(cd $(dirname ${local_file_path}); pwd) +. ${local_file_parent}/common.sh + +test_stringbench(){ + local cmd=$1 && shift + local prefix=$1 && shift + if [ -z "${cmd}" ]; then + return + fi + if [ ! -x "${cmd}" ]; then + return + fi + if [ -n "$prefix" ]; then + prefix="${prefix}_" + fi + + for res_line in $(${cmd} |tr ' ' '_'); do + local test_case_id=$(echo $res_line|cut -d: -f1|tr -c '[:alnum:]:.' '_'|tr -s '_' |sed 's/_$//') + local measurement_units=$(echo $res_line|cut -d: -f2) + local measurement=$(echo ${measurement_units}|cut -d_ -f2) + local units=$(echo ${measurement_units}|cut -d_ -f2) + output_test_result "${prefix}${test_case_id}" "pass" "${measurement}" "seconds" + done +} + +test_func(){ + stringbench="/system/xbin/stringbench" + stringbench64="/system/xbin/stringbench64" + test_stringbench "${stringbench}" "32Bit" + test_stringbench "${stringbench64}" "64Bit" +} + +main(){ + var_test_func="test_func" + run_test "$@" +} + +main "$@" diff --git a/android/stringbench.yaml b/android/stringbench.yaml new file mode 100644 index 0000000..9c4ee1a --- /dev/null +++ b/android/stringbench.yaml @@ -0,0 +1,24 @@ +metadata: + name: stringbench + format: "Lava-Test-Shell Test Definition 1.0" + description: "Run stringbench command to benchmark the + performance of string relevant feature of bionic. + In the feature this test should be upstream to bionic-benchmarks. + The source is here now: + git://android.git.linaro.org/platform/external/stringbench" + maintainer: + - yongqin.liu@linaro.org + os: + - android + scope: + - performance + devices: + - juno + +params: + RECORD_RESULT_LOCAL: "FALSE" + LOOP_COUNT: "1" + +run: + steps: + - ./android/scripts/stringbench.sh --record-csv ${RECORD_RESULT_LOCAL} --loop-count ${LOOP_COUNT} |