diff options
Diffstat (limited to 'notify')
| -rw-r--r-- | notify/any.skipped | 1 | ||||
| -rw-r--r-- | notify/jira/comment-template.txt | 2 | ||||
| -rw-r--r-- | notify/lnt_report.json | 126 | ||||
| -rw-r--r-- | notify/mail-body.txt | 18 | ||||
| -rw-r--r-- | notify/mail-subject.txt | 2 | ||||
| -rw-r--r-- | notify/output-bmk-results.log | 162 |
6 files changed, 188 insertions, 123 deletions
diff --git a/notify/any.skipped b/notify/any.skipped deleted file mode 100644 index 95ed2ca..0000000 --- a/notify/any.skipped +++ /dev/null @@ -1 +0,0 @@ -538.imagick_r,libc.so.6,sped up by 33% - 538.imagick_r:libc.so.6,sped up by 33% - 538.imagick_r:libc.so.6 - from 3 to 2 perf samples
diff --git a/notify/jira/comment-template.txt b/notify/jira/comment-template.txt index 9ba0d9f..3292f4e 100644 --- a/notify/jira/comment-template.txt +++ b/notify/jira/comment-template.txt @@ -1,3 +1,3 @@ [LLVM-651] No change -Details: https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/105/artifact/artifacts/notify/mail-body.txt/*view*/ +Details: https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/106/artifact/artifacts/notify/mail-body.txt/*view*/ diff --git a/notify/lnt_report.json b/notify/lnt_report.json index 238358f..776d563 100644 --- a/notify/lnt_report.json +++ b/notify/lnt_report.json @@ -6,37 +6,37 @@ "Run": { "Info": { "tag": "tcwg_bmk-code_speed-cpu2017rate", - "run_order": "llvmorg-18-init-15685-g88548df0fc08", - "test_url": "https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/105/", + "run_order": "llvmorg-18-init-15814-g5c37e711df6b", + "test_url": "https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/106/", "llvm_url": "https://github.com/llvm/llvm-project.git", - "llvm_rev": "88548df0fc08364bd03148c936e36f0bb07dde8a", + "llvm_rev": "5c37e711df6b81c2b669fb8a0d8b15f7a79e6e4a", "__report_version__": "1" }, - "Start Time": "2023-12-22 10:07:23" + "Start Time": "2023-12-25 14:06:51" }, "Tests": [ { "Data": [ - 26624 + 116308 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.code_size" + "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.code_size" } , { "Data": [ - 1315751 + 139527 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.code_size" + "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.code_size" } , { "Data": [ - 139527 + 1315751 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.code_size" + "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.code_size" } , { @@ -49,6 +49,14 @@ , { "Data": [ + 26624 + ], + "Info": {}, + "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.code_size" + } + , + { + "Data": [ 3613454 ], "Info": {}, @@ -57,10 +65,10 @@ , { "Data": [ - 116308 + 11286 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.code_size" + "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.code_size" } , { @@ -81,7 +89,7 @@ , { "Data": [ - 9801 + 9791 ], "Info": {}, "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.exec" @@ -89,26 +97,34 @@ , { "Data": [ - 11865 + 2122 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.exec" + "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.exec" } , { "Data": [ - 2158 + 11777 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.exec" + "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.exec" + } + , + { + "Data": [ + 1 + ], + "Info": {}, + "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.exec" } , { "Data": [ - 11737 + 11876 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.exec" + "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.exec" } , { @@ -116,15 +132,15 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.compile_status" } , { "Data": [ - 0 + 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.execution_status" } , { @@ -132,7 +148,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.compile_status" } , { @@ -140,7 +156,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.execution_status" } , { @@ -148,15 +164,15 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.544.nab_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.compile_status" } , { "Data": [ - 1 + 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.544.nab_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.execution_status" } , { @@ -164,7 +180,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.544.nab_r.compile_status" } , { @@ -172,7 +188,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.544.nab_r.execution_status" } , { @@ -180,7 +196,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.510.parest_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.compile_status" } , { @@ -188,7 +204,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.510.parest_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.execution_status" } , { @@ -196,7 +212,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.511.povray_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.525.x264_r.compile_status" } , { @@ -204,7 +220,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.511.povray_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.525.x264_r.execution_status" } , { @@ -212,7 +228,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.510.parest_r.compile_status" } , { @@ -220,7 +236,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.510.parest_r.execution_status" } , { @@ -228,7 +244,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.520.omnetpp_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.511.povray_r.compile_status" } , { @@ -236,7 +252,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.520.omnetpp_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.511.povray_r.execution_status" } , { @@ -244,7 +260,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.compile_status" } , { @@ -252,7 +268,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.execution_status" } , { @@ -260,7 +276,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.500.perlbench_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.526.blender_r.compile_status" } , { @@ -268,7 +284,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.500.perlbench_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.526.blender_r.execution_status" } , { @@ -276,7 +292,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.502.gcc_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.500.perlbench_r.compile_status" } , { @@ -284,7 +300,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.502.gcc_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.500.perlbench_r.execution_status" } , { @@ -292,7 +308,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.508.namd_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.502.gcc_r.compile_status" } , { @@ -300,7 +316,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.508.namd_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.502.gcc_r.execution_status" } , { @@ -308,15 +324,15 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.compile_status" } , { "Data": [ - 0 + 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.execution_status" } , { @@ -324,7 +340,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.520.omnetpp_r.compile_status" } , { @@ -332,7 +348,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.520.omnetpp_r.execution_status" } , { @@ -340,15 +356,15 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.525.x264_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.compile_status" } , { "Data": [ - 1 + 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.525.x264_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.execution_status" } , { @@ -356,7 +372,7 @@ 0 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.526.blender_r.compile_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.508.namd_r.compile_status" } , { @@ -364,7 +380,7 @@ 1 ], "Info": {}, - "Name": "tcwg_bmk-code_speed-cpu2017rate.526.blender_r.execution_status" + "Name": "tcwg_bmk-code_speed-cpu2017rate.508.namd_r.execution_status" } , { diff --git a/notify/mail-body.txt b/notify/mail-body.txt index a284c1c..52c0fe5 100644 --- a/notify/mail-body.txt +++ b/notify/mail-body.txt @@ -4,13 +4,13 @@ We appreciate that it might be difficult to find the necessary logs or reproduce In CI config tcwg_bmk-code_speed-cpu2017rate/llvm-arm-master-O3_LTO after: - | 82 commits in llvm - | 88548df0fc08 [lld][LoongArch] Support the R_LARCH_CALL36 relocation type (#73346) - | 03b774762aac [libc++][test] Move `abi.compile.pass.cpp` to `libcxx` subdirectory (#76349) - | fe21b3941df2 [Basic] Use range-based for loops (NFC) - | af8d0502860d [Target] Use range-based for loops (NFC) - | 41cb686d0f05 [CodeGen] Use range-based for loops (NFC) - | ... and 77 more commits in llvm + | 129 commits in llvm + | 5c37e711df6b [Frontend] Use StringRef::drop_while (NFC) + | 7f69c8b3a6c0 [llvm-special-case-list-fuzzer] fix off-by-one read (#73888) + | 9d7b3573ac7b [ConstraintElim] Add pre-commit tests for PR76299. NFC. (#76512) + | e9bc4aaa7942 [mlir][gpu][docs] fix incorrect syntax for gpu.launch (#76381) + | dc1931a8c56d [mlgo] Fix post PR #76319 + | ... and 124 more commits in llvm No change @@ -30,6 +30,6 @@ This benchmarking CI is work-in-progress, and we welcome feedback and suggestion -----------------8<--------------------------8<--------------------------8<-------------------------- The information below can be used to reproduce a debug environment: -Current build : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/105/artifact/artifacts -Reference build : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/104/artifact/artifacts +Current build : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/106/artifact/artifacts +Reference build : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/105/artifact/artifacts diff --git a/notify/mail-subject.txt b/notify/mail-subject.txt index 99e770f..a4c03e6 100644 --- a/notify/mail-subject.txt +++ b/notify/mail-subject.txt @@ -1 +1 @@ -[Linaro-TCWG-CI] 82 commits in llvm: No change on arm O3 -LTO +[Linaro-TCWG-CI] 129 commits in llvm: No change on arm O3 -LTO diff --git a/notify/output-bmk-results.log b/notify/output-bmk-results.log index 805979f..82bb5b2 100644 --- a/notify/output-bmk-results.log +++ b/notify/output-bmk-results.log @@ -110,20 +110,35 @@ output-bmk-results.py(258): print(results_df) 5 510.parest_r parest_r_base.default ... failed-to-run failed-to-run 6 511.povray_r povray_r_base.default ... failed-to-run failed-to-run 7 519.lbm_r lbm_r_base.default ... failed-to-run failed-to-run -8 520.omnetpp_r omnetpp_r_base.default ... failed-to-run failed-to-run -9 523.xalancbmk_r cpuxalan_r_base.default ... failed-to-run failed-to-run +8 519.lbm_r lbm_r_base.default ... failed-to-run failed-to-run +9 520.omnetpp_r omnetpp_r_base.default ... failed-to-run failed-to-run 10 523.xalancbmk_r cpuxalan_r_base.default ... failed-to-run failed-to-run -11 525.x264_r x264_r_base.default ... failed-to-run failed-to-run -12 526.blender_r blender_r_base.default ... failed-to-run failed-to-run -13 531.deepsjeng_r deepsjeng_r_base.default ... success success -15 538.imagick_r imagick_r_base.default ... failed-to-run failed-to-run +11 523.xalancbmk_r cpuxalan_r_base.default ... failed-to-run failed-to-run +12 525.x264_r x264_r_base.default ... failed-to-run failed-to-run +13 526.blender_r blender_r_base.default ... failed-to-run failed-to-run +14 531.deepsjeng_r deepsjeng_r_base.default ... success success 16 538.imagick_r imagick_r_base.default ... failed-to-run failed-to-run -17 541.leela_r leela_r_base.default ... failed-to-run failed-to-run +17 538.imagick_r imagick_r_base.default ... failed-to-run failed-to-run 18 541.leela_r leela_r_base.default ... failed-to-run failed-to-run -20 544.nab_r nab_r_base.default ... failed-to-run failed-to-run -21 557.xz_r xz_r_base.default ... success success +19 541.leela_r leela_r_base.default ... failed-to-run failed-to-run +21 544.nab_r nab_r_base.default ... failed-to-run failed-to-run +22 557.xz_r xz_r_base.default ... success success -[19 rows x 20 columns] +[20 rows x 20 columns] +output-bmk-results.py(261): for index, row in results_df.iterrows(): +output-bmk-results.py(263): classif, short_diag = get_status_diag(row) + --- modulename: output-bmk-results, funcname: get_status_diag +output-bmk-results.py(113): bmk = row["benchmark"] +output-bmk-results.py(115): short_diag="" +output-bmk-results.py(116): classif="" +output-bmk-results.py(118): if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build": +output-bmk-results.py(121): elif row["status_x"]=="success" and row["status_y"]=="failed-to-run": +output-bmk-results.py(124): elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run": +output-bmk-results.py(127): elif row["status_x"]=="failed-to-run" and row["status_y"]=="success": +output-bmk-results.py(130): elif row["status_x"]=="failed-to-build" and row["status_y"]=="success": +output-bmk-results.py(134): return classif, short_diag +output-bmk-results.py(265): if classif != change_kind: +output-bmk-results.py(266): continue; output-bmk-results.py(261): for index, row in results_df.iterrows(): output-bmk-results.py(263): classif, short_diag = get_status_diag(row) --- modulename: output-bmk-results, funcname: get_status_diag @@ -415,20 +430,35 @@ output-bmk-results.py(258): print(results_df) 5 510.parest_r parest_r_base.default ... failed-to-run failed-to-run 6 511.povray_r povray_r_base.default ... failed-to-run failed-to-run 7 519.lbm_r lbm_r_base.default ... failed-to-run failed-to-run -8 520.omnetpp_r omnetpp_r_base.default ... failed-to-run failed-to-run -9 523.xalancbmk_r cpuxalan_r_base.default ... failed-to-run failed-to-run +8 519.lbm_r lbm_r_base.default ... failed-to-run failed-to-run +9 520.omnetpp_r omnetpp_r_base.default ... failed-to-run failed-to-run 10 523.xalancbmk_r cpuxalan_r_base.default ... failed-to-run failed-to-run -11 525.x264_r x264_r_base.default ... failed-to-run failed-to-run -12 526.blender_r blender_r_base.default ... failed-to-run failed-to-run -13 531.deepsjeng_r deepsjeng_r_base.default ... success success -15 538.imagick_r imagick_r_base.default ... failed-to-run failed-to-run +11 523.xalancbmk_r cpuxalan_r_base.default ... failed-to-run failed-to-run +12 525.x264_r x264_r_base.default ... failed-to-run failed-to-run +13 526.blender_r blender_r_base.default ... failed-to-run failed-to-run +14 531.deepsjeng_r deepsjeng_r_base.default ... success success 16 538.imagick_r imagick_r_base.default ... failed-to-run failed-to-run -17 541.leela_r leela_r_base.default ... failed-to-run failed-to-run +17 538.imagick_r imagick_r_base.default ... failed-to-run failed-to-run 18 541.leela_r leela_r_base.default ... failed-to-run failed-to-run -20 544.nab_r nab_r_base.default ... failed-to-run failed-to-run -21 557.xz_r xz_r_base.default ... success success +19 541.leela_r leela_r_base.default ... failed-to-run failed-to-run +21 544.nab_r nab_r_base.default ... failed-to-run failed-to-run +22 557.xz_r xz_r_base.default ... success success -[19 rows x 20 columns] +[20 rows x 20 columns] +output-bmk-results.py(261): for index, row in results_df.iterrows(): +output-bmk-results.py(263): classif, short_diag = get_status_diag(row) + --- modulename: output-bmk-results, funcname: get_status_diag +output-bmk-results.py(113): bmk = row["benchmark"] +output-bmk-results.py(115): short_diag="" +output-bmk-results.py(116): classif="" +output-bmk-results.py(118): if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build": +output-bmk-results.py(121): elif row["status_x"]=="success" and row["status_y"]=="failed-to-run": +output-bmk-results.py(124): elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run": +output-bmk-results.py(127): elif row["status_x"]=="failed-to-run" and row["status_y"]=="success": +output-bmk-results.py(130): elif row["status_x"]=="failed-to-build" and row["status_y"]=="success": +output-bmk-results.py(134): return classif, short_diag +output-bmk-results.py(265): if classif != change_kind: +output-bmk-results.py(266): continue; output-bmk-results.py(261): for index, row in results_df.iterrows(): output-bmk-results.py(263): classif, short_diag = get_status_diag(row) --- modulename: output-bmk-results, funcname: get_status_diag @@ -751,7 +781,7 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking exe.regression : 519.lbm_r,lbm_r_base.default : sample=0% (threshold=3%) +DEBUG: checking exe.regression : 523.xalancbmk_r,cpuxalan_r_base.default : sample=0% (threshold=3%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_regression output-bmk-results.py(183): if metric in metric_utils.higher_regress_metrics: @@ -773,7 +803,7 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking exe.regression : 523.xalancbmk_r,cpuxalan_r_base.default : sample=-2% (threshold=3%) +DEBUG: checking exe.regression : 523.xalancbmk_r,cpuxalan_r_base.default : sample=2% (threshold=3%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_regression output-bmk-results.py(183): if metric in metric_utils.higher_regress_metrics: @@ -795,7 +825,29 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking exe.regression : 523.xalancbmk_r,cpuxalan_r_base.default : sample=0% (threshold=3%) +DEBUG: checking exe.regression : 531.deepsjeng_r,deepsjeng_r_base.default : sample=0% (threshold=3%) +output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): + --- modulename: output-bmk-results, funcname: is_entry_regression +output-bmk-results.py(183): if metric in metric_utils.higher_regress_metrics: +output-bmk-results.py(184): return (result - 100 > threshold) +output-bmk-results.py(233): continue +output-bmk-results.py(224): for index, row in out_df.iterrows(): +output-bmk-results.py(226): threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"]) + --- modulename: output-bmk-results, funcname: get_threshold +output-bmk-results.py(98): if metric == "sample": +output-bmk-results.py(99): spec_thr = get_specific_thresholds(metric, mode, bmk, symb) + --- modulename: output-bmk-results, funcname: get_specific_thresholds +output-bmk-results.py(57): if specific_variability is None: +output-bmk-results.py(60): var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)] +output-bmk-results.py(61): if var.empty: +output-bmk-results.py(62): return np.nan +output-bmk-results.py(100): if not np.isnan(spec_thr): +output-bmk-results.py(107): if metric == "num_vect_loops" or metric == "num_sve_loops": +output-bmk-results.py(110): return default_threshold[(change_kind,metric,mode)] +output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ +output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) +output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ +DEBUG: checking exe.regression : 538.imagick_r,imagick_r_base.default : sample=0% (threshold=3%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_regression output-bmk-results.py(183): if metric in metric_utils.higher_regress_metrics: @@ -817,7 +869,7 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking exe.regression : 531.deepsjeng_r,deepsjeng_r_base.default : sample=1% (threshold=3%) +DEBUG: checking exe.regression : 538.imagick_r,imagick_r_base.default : sample=0% (threshold=3%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_regression output-bmk-results.py(183): if metric in metric_utils.higher_regress_metrics: @@ -945,7 +997,7 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking exe.improvement : 519.lbm_r,lbm_r_base.default : sample=0% (threshold=3%) +DEBUG: checking exe.improvement : 523.xalancbmk_r,cpuxalan_r_base.default : sample=0% (threshold=3%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_improvement output-bmk-results.py(192): if metric in metric_utils.higher_regress_metrics: @@ -967,7 +1019,7 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking exe.improvement : 523.xalancbmk_r,cpuxalan_r_base.default : sample=-2% (threshold=3%) +DEBUG: checking exe.improvement : 523.xalancbmk_r,cpuxalan_r_base.default : sample=2% (threshold=3%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_improvement output-bmk-results.py(192): if metric in metric_utils.higher_regress_metrics: @@ -989,7 +1041,29 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking exe.improvement : 523.xalancbmk_r,cpuxalan_r_base.default : sample=0% (threshold=3%) +DEBUG: checking exe.improvement : 531.deepsjeng_r,deepsjeng_r_base.default : sample=0% (threshold=3%) +output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): + --- modulename: output-bmk-results, funcname: is_entry_improvement +output-bmk-results.py(192): if metric in metric_utils.higher_regress_metrics: +output-bmk-results.py(193): return (100 - result > threshold) +output-bmk-results.py(233): continue +output-bmk-results.py(224): for index, row in out_df.iterrows(): +output-bmk-results.py(226): threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"]) + --- modulename: output-bmk-results, funcname: get_threshold +output-bmk-results.py(98): if metric == "sample": +output-bmk-results.py(99): spec_thr = get_specific_thresholds(metric, mode, bmk, symb) + --- modulename: output-bmk-results, funcname: get_specific_thresholds +output-bmk-results.py(57): if specific_variability is None: +output-bmk-results.py(60): var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)] +output-bmk-results.py(61): if var.empty: +output-bmk-results.py(62): return np.nan +output-bmk-results.py(100): if not np.isnan(spec_thr): +output-bmk-results.py(107): if metric == "num_vect_loops" or metric == "num_sve_loops": +output-bmk-results.py(110): return default_threshold[(change_kind,metric,mode)] +output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ +output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) +output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ +DEBUG: checking exe.improvement : 538.imagick_r,imagick_r_base.default : sample=0% (threshold=3%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_improvement output-bmk-results.py(192): if metric in metric_utils.higher_regress_metrics: @@ -1011,7 +1085,7 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking exe.improvement : 531.deepsjeng_r,deepsjeng_r_base.default : sample=1% (threshold=3%) +DEBUG: checking exe.improvement : 538.imagick_r,imagick_r_base.default : sample=0% (threshold=3%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_improvement output-bmk-results.py(192): if metric in metric_utils.higher_regress_metrics: @@ -1139,7 +1213,7 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking symbol.regression : 538.imagick_r,libc.so.6 : sample=33% (threshold=15%) +DEBUG: checking symbol.regression : 538.imagick_r,libc.so.6 : sample=0% (threshold=15%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_regression output-bmk-results.py(183): if metric in metric_utils.higher_regress_metrics: @@ -1223,37 +1297,12 @@ output-bmk-results.py(110): return default_threshold[(change_kind,metric,mod output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ output-bmk-results.py(229): .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold)) output-bmk-results.py(228): print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\ -DEBUG: checking symbol.improvement : 538.imagick_r,libc.so.6 : sample=33% (threshold=15%) +DEBUG: checking symbol.improvement : 538.imagick_r,libc.so.6 : sample=0% (threshold=15%) output-bmk-results.py(232): if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold): --- modulename: output-bmk-results, funcname: is_entry_improvement output-bmk-results.py(192): if metric in metric_utils.higher_regress_metrics: output-bmk-results.py(193): return (100 - result > threshold) -output-bmk-results.py(235): percent_change, short_diag, long_diag = get_short_long_diag(row, metric, sym_type, change_kind) - --- modulename: output-bmk-results, funcname: get_short_long_diag -output-bmk-results.py(137): bmk = row["benchmark"] -output-bmk-results.py(139): rel_value = row["rel_" + metric] -output-bmk-results.py(140): prev_value = row[metric + "_x"] -output-bmk-results.py(141): curr_value = row[metric + "_y"] -output-bmk-results.py(142): if metric == "num_vect_loops" or metric == "num_sve_loops": -output-bmk-results.py(152): suffix = "" -output-bmk-results.py(153): if metric == "sample": -output-bmk-results.py(154): prefix_regression = "slowed down by" -output-bmk-results.py(155): prefix_improvement = "sped up by" -output-bmk-results.py(156): suffix = "perf samples" -output-bmk-results.py(167): if sym_type=="symbol": -output-bmk-results.py(168): item=bmk+":"+row["symbol"] -output-bmk-results.py(172): short_diag = "{1} {2}% - {0}".format(item, locals()["prefix_" + change_kind], abs(rel_value - 100)) -output-bmk-results.py(173): long_diag = "{0} - from {1} to {2} {3}".format(short_diag, prev_value, curr_value, suffix) -output-bmk-results.py(174): return abs(rel_value - 100), short_diag, long_diag -output-bmk-results.py(239): if metric == "sample" \ -output-bmk-results.py(240): and row['symbol_md5sum_x'] == row['symbol_md5sum_y'] \ -output-bmk-results.py(241): and row['symbol_md5sum_x'] != "-1" \ -output-bmk-results.py(242): and row['symbol_md5sum_x'] != "d41d8cd98f00b204e9800998ecf8427e": -output-bmk-results.py(243): f_skip.write_csv((row["benchmark"], row["symbol"], short_diag, long_diag)) - --- modulename: output-bmk-results, funcname: write_csv -output-bmk-results.py(41): if not self.predicate or not self.csvwriter: -output-bmk-results.py(43): self.csvwriter.writerow(arr) -output-bmk-results.py(244): continue +output-bmk-results.py(233): continue output-bmk-results.py(224): for index, row in out_df.iterrows(): output-bmk-results.py(226): threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"]) --- modulename: output-bmk-results, funcname: get_threshold @@ -1292,6 +1341,7 @@ output-bmk-results.py(305): f_skip.close() output-bmk-results.py(29): if not self.outf: output-bmk-results.py(31): self.outf.close() output-bmk-results.py(32): if os.stat(self.filename).st_size == 0: +output-bmk-results.py(33): os.remove(self.filename) output-bmk-results.py(306): f_regr.close() --- modulename: output-bmk-results, funcname: close output-bmk-results.py(29): if not self.outf: |