rt-tests: Enable JSON test report

All rt-tests learned to output all results in JSON format. This makes
it possible the replace the really awkward log parser (it has to
handle subtle differences how a rt-tests logs on the console) and
just rely on a generic JSON parser implementation.

Signed-off-by: Daniel Wagner <dwagner@suse.de>

10 files changed, 50 insertions, 126 deletions

diff --git a/automated/lib/parse_rt_tests_results.py b/automated/lib/parse_rt_tests_results.py
index 3f248703..7ce9980d 100755
--- a/automated/lib/parse_rt_tests_results.py
+++ b/automated/lib/parse_rt_tests_results.py
@@ -24,106 +24,46 @@
 # SOFTWARE.
 #
 
-import os
-import re
 import sys
+import json
 
 
-def print_res(res, key):
-    print("t{}-{}-latency pass {} us".format(res["t"], key, res[key]))
+def print_res(t, res, key):
+    print("t{}-{}-latency pass {} us".format(t, key, res[key]))
 
 
-def get_block(filename):
-    # Fetch a text block from the file iterating backwards. Each block
-    # starts with an escape sequence which starts with '\x1b'.
-    with open(filename, "rb") as f:
-        try:
-            f.seek(0, os.SEEK_END)
-            while True:
-                pe = f.tell()
+def parse_threads(rawdata):
+    num_threads = int(rawdata["num_threads"])
+    for thread_id in range(num_threads):
+        tid = str(thread_id)
+        if "receiver" in rawdata["thread"][tid]:
+            data = rawdata["thread"][tid]["receiver"]
+        else:
+            data = rawdata["thread"][tid]
 
-                f.seek(-2, os.SEEK_CUR)
-                while f.read(1) != b"\x1b":
-                    f.seek(-2, os.SEEK_CUR)
-                    pa = f.tell()
+        for k in ["min", "avg", "max"]:
+            print_res(tid, data, k)
 
-                blk = f.read(pe - pa)
 
-                # Remove escape sequence at the start of the block
-                # The control sequence ends in 'A'
-                i = blk.find("A") + 1
-                yield blk[i:]
+def parse_json(testname, filename):
+    with open(filename) as file:
+        rawdata = json.load(file)
 
-                # Jump back to next block
-                f.seek(pa - 1, os.SEEK_SET)
-        except IOError:
-            # No escape sequence found
-            f.seek(0, os.SEEK_SET)
-            yield f.read()
+    if "num_threads" in rawdata:
+        # most rt-tests have generic per thread results
+        parse_threads(rawdata)
+    elif "inversions" in rawdata:
+        # pi_stress
+        print("inversion {}\n".format(rawdata("inversions")))
 
-
-def get_lastlines(filename):
-    for b in get_block(filename):
-        # Ignore empty blocks
-        if len(b.strip("\n")) == 0:
-            continue
-
-        return b.split("\n")
-
-
-def parse_cyclictest(filename):
-    fields = ["t", "min", "avg", "max"]
-
-    r = re.compile("[ :\n]+")
-    for line in get_lastlines(filename):
-        if not line.startswith("T:"):
-            continue
-
-        data = [x.lower() for x in r.split(line)]
-        res = {}
-        it = iter(data)
-        for e in it:
-            if e in fields:
-                res[e] = next(it)
-
-        print_res(res, "min")
-        print_res(res, "avg")
-        print_res(res, "max")
-
-
-def parse_pmqtest(filename):
-    fields = ["min", "avg", "max"]
-
-    rl = re.compile("[ ,:\n]+")
-    rt = re.compile("[ ,#]+")
-    for line in get_lastlines(filename):
-        data = [x.lower() for x in rl.split(line)]
-        res = {}
-        it = iter(data)
-        for e in it:
-            if e in fields:
-                res[e] = next(it)
-
-        if not res:
-            continue
-
-        # The id is constructed from the '#FROM -> #TO' output, e.g.
-        # #1 -> #0, Min    1, Cur    3, Avg    4, Max  119
-        data = rt.split(line)
-        res["t"] = "{}-{}".format(data[1], data[3])
-
-        print_res(res, "min")
-        print_res(res, "avg")
-        print_res(res, "max")
+    if int(rawdata["return_code"]) == 0:
+        print("{} pass".format(testname))
+    else:
+        print("{} fail".format(testname))
 
 
 def main():
-    tool = sys.argv[1]
-    logfile = sys.argv[2]
-    if tool in ["cyclictest", "signaltest", "cyclicdeadline"]:
-        parse_cyclictest(logfile)
-    elif tool in ["pmqtest", "ptsematest", "sigwaittest", "svsematest"]:
-        parse_pmqtest(logfile)
+    parse_json(sys.argv[1], sys.argv[2])
 
 
 if __name__ == "__main__":
diff --git a/automated/linux/cyclicdeadline/cyclicdeadline.sh b/automated/linux/cyclicdeadline/cyclicdeadline.sh
index 4a689c6e..20ab72e1 100755
--- a/automated/linux/cyclicdeadline/cyclicdeadline.sh
+++ b/automated/linux/cyclicdeadline/cyclicdeadline.sh
@@ -7,7 +7,7 @@
 . ../../lib/sh-test-lib
 
 OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/cyclicdeadline.txt"
+LOGFILE="${OUTPUT}/cyclicdeadline.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 
 INTERVAL="1000"
@@ -45,7 +45,7 @@ fi
 background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 
 "${binary}" -q -i "${INTERVAL}" -s "${STEP}" -t "${THREADS}" \
-	-D "${DURATION}" | tee "${LOGFILE}"
+	-D "${DURATION}" --json="${LOGFILE}"
 
 background_process_stop bgcmd
 
diff --git a/automated/linux/cyclictest/cyclictest.sh b/automated/linux/cyclictest/cyclictest.sh
index 31ffb25e..ab1484a7 100755
--- a/automated/linux/cyclictest/cyclictest.sh
+++ b/automated/linux/cyclictest/cyclictest.sh
@@ -7,7 +7,7 @@
 . ../../lib/sh-test-lib
 
 OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/cyclictest.txt"
+LOGFILE="${OUTPUT}/cyclictest.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 
 PRIORITY="98"
@@ -47,7 +47,7 @@ fi
 background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 
 "${binary}" -q -p "${PRIORITY}" -i "${INTERVAL}" -t "${THREADS}" -a "${AFFINITY}" \
-    -D "${DURATION}" -m | tee "${LOGFILE}"
+    -D "${DURATION}" -m --json="${LOGFILE}"
 
 background_process_stop bgcmd
 
diff --git a/automated/linux/pi-stress/pi-stress.sh b/automated/linux/pi-stress/pi-stress.sh
index 637fc968..9a71eb38 100755
--- a/automated/linux/pi-stress/pi-stress.sh
+++ b/automated/linux/pi-stress/pi-stress.sh
@@ -7,7 +7,7 @@
 . ../../lib/sh-test-lib
 
 OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/pi-stress.txt"
+LOGFILE="${OUTPUT}/pi-stress.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 export RESULT_FILE
 
@@ -56,17 +56,10 @@ background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 # pi_stress will send SIGTERM when test fails. The single will terminate the
 # test script. Catch and ignore it with trap.
 trap '' TERM
-"${binary}" -q --duration "${DURATION}" "${MLOCKALL}" "${RR}" | tee "${LOGFILE}"
+"${binary}" -q --duration "${DURATION}" "${MLOCKALL}" "${RR}" --json="${LOGFILE}"
 
 background_process_stop bgcmd
 
-# shellcheck disable=SC2181
-if [ "$?" -ne "0" ]; then
-    report_fail "pi-stress"
-elif grep -q -e "^ERROR:" -e "is deadlocked!" "${LOGFILE}"; then
-    report_fail "pi-stress"
-elif ! grep -q -e "Current Inversions:" "${LOGFILE}"; then
-    report_fail "pi-stress"
-else
-    report_pass "pi-stress"
-fi
+# Parse test log.
+../../lib/parse_rt_tests_results.py pi-stress "${LOGFILE}" \
+    | tee -a "${RESULT_FILE}"
diff --git a/automated/linux/pmqtest/pmqtest.sh b/automated/linux/pmqtest/pmqtest.sh
index c18f1335..57ecfba5 100755
--- a/automated/linux/pmqtest/pmqtest.sh
+++ b/automated/linux/pmqtest/pmqtest.sh
@@ -6,7 +6,7 @@
 
 TEST_DIR=$(dirname "$(realpath "$0")")
 OUTPUT="${TEST_DIR}/output"
-LOGFILE="${OUTPUT}/pmqtest.log"
+LOGFILE="${OUTPUT}/pmqtest.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 DURATION="5m"
 BACKGROUND_CMD=""
@@ -38,7 +38,7 @@ fi
 
 background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 
-"${binary}" -q -S -p 98 -D "${DURATION}" | tee "${LOGFILE}"
+"${binary}" -q -S -p 98 -D "${DURATION}" --json="${LOGFILE}"
 
 background_process_stop bgcmd
 
diff --git a/automated/linux/ptsematest/ptsematest.sh b/automated/linux/ptsematest/ptsematest.sh
index d3996b35..e98a473d 100755
--- a/automated/linux/ptsematest/ptsematest.sh
+++ b/automated/linux/ptsematest/ptsematest.sh
@@ -5,7 +5,7 @@
 
 TEST_DIR=$(dirname "$(realpath "$0")")
 OUTPUT="${TEST_DIR}/output"
-LOGFILE="${OUTPUT}/ptsematest.log"
+LOGFILE="${OUTPUT}/ptsematest.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 DURATION="5m"
 BACKGROUND_CMD=""
@@ -37,7 +37,7 @@ fi
 
 background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 
-"${binary}" -q -S -p 98 -D "${DURATION}" | tee "${LOGFILE}"
+"${binary}" -q -S -p 98 -D "${DURATION}" --json="${LOGFILE}"
 
 background_process_stop bgcmd
 
diff --git a/automated/linux/rt-migrate-test/rt-migrate-test.sh b/automated/linux/rt-migrate-test/rt-migrate-test.sh
index 5bd54e4e..281578de 100755
--- a/automated/linux/rt-migrate-test/rt-migrate-test.sh
+++ b/automated/linux/rt-migrate-test/rt-migrate-test.sh
@@ -5,7 +5,7 @@
 . ../../lib/sh-test-lib
 
 OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/rt-migrate-test.txt"
+LOGFILE="${OUTPUT}/rt-migrate-test.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 PRIORITY="51"
 DURATION="1m"
@@ -37,18 +37,10 @@ fi
 
 background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 
-"${binary}" -q -p "${PRIORITY}" -D "${DURATION}" -c | tee "${LOGFILE}"
+"${binary}" -q -p "${PRIORITY}" -D "${DURATION}" -c --json="${LOGFILE}"
 
 background_process_stop bgcmd
 
 # Parse test log.
-task_num=$(grep "Task" "${LOGFILE}" | tail -1 | awk '{print $2}')
-r=$(sed -n 's/Passed!/pass/p; s/Failed!/fail/p' "${LOGFILE}" | awk '{$1=$1;print}')
-for t in $(seq 0 "${task_num}"); do
-    # Get the priority of the task.
-    p=$(grep "Task $t" "${LOGFILE}" | awk '{print substr($4,1,length($4)-1)}')
-    sed -n "/Task $t/,/Avg/p" "${LOGFILE}" \
-        | grep -v "Task" \
-        | awk -v t="$t" -v p="$p" -v r="$r" '{printf("t%s-p%s-%s %s %s %s\n",t,p,tolower(substr($1, 1, length($1)-1)),r,$2,$3)}' \
-        | tee -a "${RESULT_FILE}"
-done
+../../lib/parse_rt_tests_results.py rt-migrate-test "${LOGFILE}" \
+    | tee -a "${RESULT_FILE}"
diff --git a/automated/linux/signaltest/signaltest.sh b/automated/linux/signaltest/signaltest.sh
index 5b534530..b40c1b9d 100755
--- a/automated/linux/signaltest/signaltest.sh
+++ b/automated/linux/signaltest/signaltest.sh
@@ -5,7 +5,7 @@
 . ../../lib/sh-test-lib
 
 OUTPUT="$(pwd)/output"
-LOGFILE="${OUTPUT}/signaltest.txt"
+LOGFILE="${OUTPUT}/signaltest.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 
 PRIORITY="98"
@@ -40,8 +40,7 @@ fi
 
 background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 
-"${binary}" -q -D "${DURATION}" -m -p "${PRIORITY}" -t "${THREADS}" \
-    | tee "${LOGFILE}"
+"${binary}" -q -D "${DURATION}" -m -p "${PRIORITY}" -t "${THREADS}" --json="${LOGFILE}"
 
 background_process_stop bgcmd
 
diff --git a/automated/linux/sigwaittest/sigwaittest.sh b/automated/linux/sigwaittest/sigwaittest.sh
index 7b3d066f..4e985198 100755
--- a/automated/linux/sigwaittest/sigwaittest.sh
+++ b/automated/linux/sigwaittest/sigwaittest.sh
@@ -7,7 +7,7 @@
 
 TEST_DIR=$(dirname "$(realpath "$0")")
 OUTPUT="${TEST_DIR}/output"
-LOGFILE="${OUTPUT}/sigwaittest.log"
+LOGFILE="${OUTPUT}/sigwaittest.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 DURATION="5m"
 BACKGROUND_CMD=""
@@ -39,7 +39,7 @@ fi
 
 background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 
-"${binary}" -q -t -a -p 98 -D "${DURATION}" | tee "${LOGFILE}"
+"${binary}" -q -t -a -p 98 -D "${DURATION}" --json="${LOGFILE}"
 
 background_process_stop bgcmd
 
diff --git a/automated/linux/svsematest/svsematest.sh b/automated/linux/svsematest/svsematest.sh
index c78df17e..d5a8e768 100755
--- a/automated/linux/svsematest/svsematest.sh
+++ b/automated/linux/svsematest/svsematest.sh
@@ -7,7 +7,7 @@
 
 TEST_DIR=$(dirname "$(realpath "$0")")
 OUTPUT="${TEST_DIR}/output"
-LOGFILE="${OUTPUT}/svsematest.log"
+LOGFILE="${OUTPUT}/svsematest.json"
 RESULT_FILE="${OUTPUT}/result.txt"
 DURATION="5m"
 BACKGROUND_CMD=""
@@ -39,7 +39,7 @@ fi
 
 background_process_start bgcmd --cmd "${BACKGROUND_CMD}"
 
-"${binary}" -q -t -a -p 98 -D "${DURATION}" | tee "${LOGFILE}"
+"${binary}" -q -t -a -p 98 -D "${DURATION}" --json="${LOGFILE}"
 
 background_process_stop bgcmd