onsuccess: #96: 1: [TCWG CI] https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/96/

Results :
  | # reset_artifacts:
  | -10
  | # build_bmk_llvm:
  | -3
  | # benchmark -- -O3_LTO_marm:
  | 1

check_regression status : 0

6 files changed, 370 insertions, 187 deletions

diff --git a/notify/any.skipped b/notify/any.skipped
deleted file mode 100644
index 95ed2ca..0000000
--- a/notify/any.skipped
+++ /dev/null
@@ -1 +0,0 @@
-538.imagick_r,libc.so.6,sped up by 33% - 538.imagick_r:libc.so.6,sped up by 33% - 538.imagick_r:libc.so.6 - from 3 to 2 perf samples

diff --git a/notify/jira/comment-template.txt b/notify/jira/comment-template.txt
index e78e537..74b66a7 100644
--- a/notify/jira/comment-template.txt
+++ b/notify/jira/comment-template.txt
@@ -1,3 +1,3 @@
 [LLVM-651]
 No change
-Details: https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/94/artifact/artifacts/notify/mail-body.txt/*view*/
+Details: https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/96/artifact/artifacts/notify/mail-body.txt/*view*/
diff --git a/notify/lnt_report.json b/notify/lnt_report.json
index d5d0c65..908af82 100644
--- a/notify/lnt_report.json
+++ b/notify/lnt_report.json
@@ -5,15 +5,49 @@
     },
     "Run": {
         "Info": {
-            "__report_version__": "1",
-            "run_order": "llvmorg-18-init-12457-g1dd387e10624",
-            "tag": "tcwg_bmk-code_speed-cpu2017rate"
+            "tag": "tcwg_bmk-code_speed-cpu2017rate",
+            "run_order": "llvmorg-18-init-12903-g08a6968127f0",
+            "test_url": "https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/96/",
+            "git_llvm": "https://github.com/llvm/llvm-project/commit/08a6968127f04a40d767bf3fe296bc36667c89eb",
+            "__report_version__": "1"
         },
-        "Start Time": "2023-11-22 21:03:23"
+        "Start Time": "2023-11-28 10:06:20"
     },
     "Tests": [
         {
             "Data": [
+                139047
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.code_size"
+        }
+	   ,
+        {
+            "Data": [
+                3614694
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.code_size"
+        }
+	   ,
+        {
+            "Data": [
+                11286
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.code_size"
+        }
+	   ,
+        {
+            "Data": [
+                26640
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.code_size"
+        }
+	   ,
+        {
+            "Data": [
                 1325439
             ],
             "Info": {},
@@ -30,42 +64,58 @@
 	   ,
         {
             "Data": [
-                11286
+                116396
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.code_size"
         }
 	   ,
         {
             "Data": [
-                115816
+                2541
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.exec"
         }
 	   ,
         {
             "Data": [
-                139047
+                11744
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.exec"
         }
 	   ,
         {
             "Data": [
-                26640
+                1
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.exec"
         }
 	   ,
         {
             "Data": [
-                3615014
+                2
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.exec"
+        }
+	   ,
+        {
+            "Data": [
+                9780
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.exec"
+        }
+	   ,
+        {
+            "Data": [
+                11808
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.exec"
         }
 	   ,
         {
@@ -78,18 +128,26 @@
 	   ,
         {
             "Data": [
-                11605
+                0
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.exec"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.compile_status"
         }
 	   ,
         {
             "Data": [
-                11749
+                1
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.exec"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.525.x264_r.compile_status"
         }
 	   ,
         {
@@ -97,31 +155,183 @@
                 1
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.exec"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.525.x264_r.execution_status"
         }
 	   ,
         {
             "Data": [
-                2
+                0
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.exec"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.508.namd_r.compile_status"
         }
 	   ,
         {
             "Data": [
-                2523
+                1
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.exec"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.508.namd_r.execution_status"
         }
 	   ,
         {
             "Data": [
-                9790
+                0
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.exec"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.510.parest_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.510.parest_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.511.povray_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.511.povray_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.500.perlbench_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.500.perlbench_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.502.gcc_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.502.gcc_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.538.imagick_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.544.nab_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.544.nab_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.526.blender_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.526.blender_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.519.lbm_r.execution_status"
+        }
+	   ,
+        {
+            "Data": [
+                0
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.520.omnetpp_r.compile_status"
+        }
+	   ,
+        {
+            "Data": [
+                1
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.520.omnetpp_r.execution_status"
         }
     ]
 }
diff --git a/notify/mail-body.txt b/notify/mail-body.txt
index d31fdb4..e31cd8c 100644
--- a/notify/mail-body.txt
+++ b/notify/mail-body.txt
@@ -2,13 +2,13 @@ Dear contributor, our automatic CI has detected problems related to your patch(e
 
 In CI config tcwg_bmk-code_speed-cpu2017rate/llvm-arm-master-O3_LTO after:
 
-  | 305 commits in llvm
-  | 1dd387e10624 [mlir][sparse] change dim level type -> level type (#73058)
-  | e07fec10ac20 [Flang] Add partial support for lowering procedure pointer assignment. (#70461)
-  | a842430c20f1 [AArch64] Add check that prologue insertion doesn't clobber live regs. (#71826)
-  | 369c0eb55be9 [SLP][NFC]Use SmallVector instead of std::vector and remove unused includes, NFC.
-  | cae46f621029 [MCP] Enhance MCP copy Instruction removal for special case (#70778)
-  | ... and 300 more commits in llvm
+  | 446 commits in llvm
+  | 08a6968127f0 [UTC] Support arm64-apple-macosx in update_llc_test_checks.py. (#73568)
+  | 4a3d2088d61e [mlir][linalg] Add TransposeConv2D Transform Op (#68567)
+  | 06157a6d865e [gn build] Port b0af8a1ede89
+  | b0af8a1ede89 Revert "[lldb] [mostly NFC] Large WP foundation: WatchpointResources (#68845)"
+  | 1459c627f0bc [lldb][PDB] Fix message order in test case
+  | ... and 441 more commits in llvm
 
 No change
 
@@ -28,6 +28,6 @@ This benchmarking CI is work-in-progress, and we welcome feedback and suggestion
 -----------------8<--------------------------8<--------------------------8<--------------------------
 The information below can be used to reproduce a debug environment:
 
-Current build   : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/94/artifact/artifacts
-Reference build : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/93/artifact/artifacts
+Current build   : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/96/artifact/artifacts
+Reference build : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O3_LTO-build/94/artifact/artifacts
 
diff --git a/notify/mail-subject.txt b/notify/mail-subject.txt
index 95dca44..8b00914 100644
--- a/notify/mail-subject.txt
+++ b/notify/mail-subject.txt
@@ -1 +1 @@
-[Linaro-TCWG-CI] 305 commits in llvm: No change on arm O3 -LTO
+[Linaro-TCWG-CI] 446 commits in llvm: No change on arm O3 -LTO
diff --git a/notify/output-bmk-results.log b/notify/output-bmk-results.log
index 7fdfa7c..7f92d69 100644
--- a/notify/output-bmk-results.log
+++ b/notify/output-bmk-results.log
@@ -107,66 +107,21 @@ output-bmk-results.py(258):     print(results_df)
 1         502.gcc_r     cpugcc_r_base.default  ...  failed-to-run  failed-to-run
 2         505.mcf_r        mcf_r_base.default  ...             -1             -1
 4        508.namd_r       namd_r_base.default  ...  failed-to-run  failed-to-run
-5      510.parest_r     parest_r_base.default  ...  failed-to-run  failed-to-run
-6      511.povray_r     povray_r_base.default  ...  failed-to-run  failed-to-run
-7         519.lbm_r        lbm_r_base.default  ...  failed-to-run             -1
-8         519.lbm_r        lbm_r_base.default  ...  failed-to-run  failed-to-run
-9     520.omnetpp_r    omnetpp_r_base.default  ...  failed-to-run  failed-to-run
-10  523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
-11  523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
-12       525.x264_r       x264_r_base.default  ...  failed-to-run  failed-to-run
-13    526.blender_r    blender_r_base.default  ...  failed-to-run  failed-to-run
-14  531.deepsjeng_r  deepsjeng_r_base.default  ...             -1             -1
-16    538.imagick_r    imagick_r_base.default  ...  failed-to-run  failed-to-run
-17    538.imagick_r    imagick_r_base.default  ...  failed-to-run  failed-to-run
-18      541.leela_r      leela_r_base.default  ...  failed-to-run  failed-to-run
-19      541.leela_r      leela_r_base.default  ...  failed-to-run  failed-to-run
-21        544.nab_r        nab_r_base.default  ...  failed-to-run  failed-to-run
-22         557.xz_r         xz_r_base.default  ...             -1             -1
+5         519.lbm_r        lbm_r_base.default  ...             -1  failed-to-run
+6         519.lbm_r        lbm_r_base.default  ...             -1  failed-to-run
+7   523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
+8   523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
+9        525.x264_r       x264_r_base.default  ...  failed-to-run  failed-to-run
+10    526.blender_r    blender_r_base.default  ...  failed-to-run  failed-to-run
+11  531.deepsjeng_r  deepsjeng_r_base.default  ...             -1             -1
+13    538.imagick_r    imagick_r_base.default  ...  failed-to-run  failed-to-run
+14    538.imagick_r    imagick_r_base.default  ...  failed-to-run  failed-to-run
+15      541.leela_r      leela_r_base.default  ...  failed-to-run  failed-to-run
+16      541.leela_r      leela_r_base.default  ...  failed-to-run  failed-to-run
+18        544.nab_r        nab_r_base.default  ...  failed-to-run  failed-to-run
+19         557.xz_r         xz_r_base.default  ...             -1             -1
 
-[20 rows x 20 columns]
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
+[17 rows x 20 columns]
 output-bmk-results.py(261):     for index, row in results_df.iterrows():
 output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
  --- modulename: output-bmk-results, funcname: get_status_diag
@@ -427,66 +382,21 @@ output-bmk-results.py(258):     print(results_df)
 1         502.gcc_r     cpugcc_r_base.default  ...  failed-to-run  failed-to-run
 2         505.mcf_r        mcf_r_base.default  ...             -1             -1
 4        508.namd_r       namd_r_base.default  ...  failed-to-run  failed-to-run
-5      510.parest_r     parest_r_base.default  ...  failed-to-run  failed-to-run
-6      511.povray_r     povray_r_base.default  ...  failed-to-run  failed-to-run
-7         519.lbm_r        lbm_r_base.default  ...  failed-to-run             -1
-8         519.lbm_r        lbm_r_base.default  ...  failed-to-run  failed-to-run
-9     520.omnetpp_r    omnetpp_r_base.default  ...  failed-to-run  failed-to-run
-10  523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
-11  523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
-12       525.x264_r       x264_r_base.default  ...  failed-to-run  failed-to-run
-13    526.blender_r    blender_r_base.default  ...  failed-to-run  failed-to-run
-14  531.deepsjeng_r  deepsjeng_r_base.default  ...             -1             -1
-16    538.imagick_r    imagick_r_base.default  ...  failed-to-run  failed-to-run
-17    538.imagick_r    imagick_r_base.default  ...  failed-to-run  failed-to-run
-18      541.leela_r      leela_r_base.default  ...  failed-to-run  failed-to-run
-19      541.leela_r      leela_r_base.default  ...  failed-to-run  failed-to-run
-21        544.nab_r        nab_r_base.default  ...  failed-to-run  failed-to-run
-22         557.xz_r         xz_r_base.default  ...             -1             -1
+5         519.lbm_r        lbm_r_base.default  ...             -1  failed-to-run
+6         519.lbm_r        lbm_r_base.default  ...             -1  failed-to-run
+7   523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
+8   523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
+9        525.x264_r       x264_r_base.default  ...  failed-to-run  failed-to-run
+10    526.blender_r    blender_r_base.default  ...  failed-to-run  failed-to-run
+11  531.deepsjeng_r  deepsjeng_r_base.default  ...             -1             -1
+13    538.imagick_r    imagick_r_base.default  ...  failed-to-run  failed-to-run
+14    538.imagick_r    imagick_r_base.default  ...  failed-to-run  failed-to-run
+15      541.leela_r      leela_r_base.default  ...  failed-to-run  failed-to-run
+16      541.leela_r      leela_r_base.default  ...  failed-to-run  failed-to-run
+18        544.nab_r        nab_r_base.default  ...  failed-to-run  failed-to-run
+19         557.xz_r         xz_r_base.default  ...             -1             -1
 
-[20 rows x 20 columns]
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
+[17 rows x 20 columns]
 output-bmk-results.py(261):     for index, row in results_df.iterrows():
 output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
  --- modulename: output-bmk-results, funcname: get_status_diag
@@ -825,7 +735,7 @@ output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mod
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking exe.regression : 523.xalancbmk_r,cpuxalan_r_base.default : sample=0% (threshold=3%)
+DEBUG: checking exe.regression : 523.xalancbmk_r,cpuxalan_r_base.default : sample=-1% (threshold=3%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -869,7 +779,51 @@ output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mod
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking exe.regression : 531.deepsjeng_r,deepsjeng_r_base.default : sample=0% (threshold=3%)
+DEBUG: checking exe.regression : 531.deepsjeng_r,deepsjeng_r_base.default : sample=-2% (threshold=3%)
+output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
+ --- modulename: output-bmk-results, funcname: is_entry_regression
+output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
+output-bmk-results.py(184):         return (result - 100 > threshold)
+output-bmk-results.py(233):             continue
+output-bmk-results.py(224):     for index, row in out_df.iterrows():
+output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
+ --- modulename: output-bmk-results, funcname: get_threshold
+output-bmk-results.py(98):     if metric == "sample":
+output-bmk-results.py(99):        spec_thr = get_specific_thresholds(metric, mode, bmk, symb)
+ --- modulename: output-bmk-results, funcname: get_specific_thresholds
+output-bmk-results.py(57):     if specific_variability is None:
+output-bmk-results.py(60):     var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)]
+output-bmk-results.py(61):     if var.empty:
+output-bmk-results.py(62):         return np.nan
+output-bmk-results.py(100):        if not np.isnan(spec_thr):
+output-bmk-results.py(107):     if metric == "num_vect_loops" or metric == "num_sve_loops":
+output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mode)]
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+DEBUG: checking exe.regression : 538.imagick_r,imagick_r_base.default : sample=0% (threshold=3%)
+output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
+ --- modulename: output-bmk-results, funcname: is_entry_regression
+output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
+output-bmk-results.py(184):         return (result - 100 > threshold)
+output-bmk-results.py(233):             continue
+output-bmk-results.py(224):     for index, row in out_df.iterrows():
+output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
+ --- modulename: output-bmk-results, funcname: get_threshold
+output-bmk-results.py(98):     if metric == "sample":
+output-bmk-results.py(99):        spec_thr = get_specific_thresholds(metric, mode, bmk, symb)
+ --- modulename: output-bmk-results, funcname: get_specific_thresholds
+output-bmk-results.py(57):     if specific_variability is None:
+output-bmk-results.py(60):     var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)]
+output-bmk-results.py(61):     if var.empty:
+output-bmk-results.py(62):         return np.nan
+output-bmk-results.py(100):        if not np.isnan(spec_thr):
+output-bmk-results.py(107):     if metric == "num_vect_loops" or metric == "num_sve_loops":
+output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mode)]
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+DEBUG: checking exe.regression : 538.imagick_r,imagick_r_base.default : sample=0% (threshold=3%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1041,7 +995,7 @@ output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mod
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking exe.improvement : 523.xalancbmk_r,cpuxalan_r_base.default : sample=0% (threshold=3%)
+DEBUG: checking exe.improvement : 523.xalancbmk_r,cpuxalan_r_base.default : sample=-1% (threshold=3%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1085,7 +1039,51 @@ output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mod
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking exe.improvement : 531.deepsjeng_r,deepsjeng_r_base.default : sample=0% (threshold=3%)
+DEBUG: checking exe.improvement : 531.deepsjeng_r,deepsjeng_r_base.default : sample=-2% (threshold=3%)
+output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
+ --- modulename: output-bmk-results, funcname: is_entry_improvement
+output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
+output-bmk-results.py(193):         return (100 - result > threshold)
+output-bmk-results.py(233):             continue
+output-bmk-results.py(224):     for index, row in out_df.iterrows():
+output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
+ --- modulename: output-bmk-results, funcname: get_threshold
+output-bmk-results.py(98):     if metric == "sample":
+output-bmk-results.py(99):        spec_thr = get_specific_thresholds(metric, mode, bmk, symb)
+ --- modulename: output-bmk-results, funcname: get_specific_thresholds
+output-bmk-results.py(57):     if specific_variability is None:
+output-bmk-results.py(60):     var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)]
+output-bmk-results.py(61):     if var.empty:
+output-bmk-results.py(62):         return np.nan
+output-bmk-results.py(100):        if not np.isnan(spec_thr):
+output-bmk-results.py(107):     if metric == "num_vect_loops" or metric == "num_sve_loops":
+output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mode)]
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+DEBUG: checking exe.improvement : 538.imagick_r,imagick_r_base.default : sample=0% (threshold=3%)
+output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
+ --- modulename: output-bmk-results, funcname: is_entry_improvement
+output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
+output-bmk-results.py(193):         return (100 - result > threshold)
+output-bmk-results.py(233):             continue
+output-bmk-results.py(224):     for index, row in out_df.iterrows():
+output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
+ --- modulename: output-bmk-results, funcname: get_threshold
+output-bmk-results.py(98):     if metric == "sample":
+output-bmk-results.py(99):        spec_thr = get_specific_thresholds(metric, mode, bmk, symb)
+ --- modulename: output-bmk-results, funcname: get_specific_thresholds
+output-bmk-results.py(57):     if specific_variability is None:
+output-bmk-results.py(60):     var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)]
+output-bmk-results.py(61):     if var.empty:
+output-bmk-results.py(62):         return np.nan
+output-bmk-results.py(100):        if not np.isnan(spec_thr):
+output-bmk-results.py(107):     if metric == "num_vect_loops" or metric == "num_sve_loops":
+output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mode)]
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+DEBUG: checking exe.improvement : 538.imagick_r,imagick_r_base.default : sample=0% (threshold=3%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1213,7 +1211,7 @@ output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mod
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 538.imagick_r,libc.so.6 : sample=33% (threshold=15%)
+DEBUG: checking symbol.regression : 538.imagick_r,libc.so.6 : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1297,37 +1295,12 @@ output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mod
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 538.imagick_r,libc.so.6 : sample=33% (threshold=15%)
+DEBUG: checking symbol.improvement : 538.imagick_r,libc.so.6 : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
 output-bmk-results.py(193):         return (100 - result > threshold)
-output-bmk-results.py(235):         percent_change, short_diag, long_diag = get_short_long_diag(row, metric, sym_type, change_kind)
- --- modulename: output-bmk-results, funcname: get_short_long_diag
-output-bmk-results.py(137):     bmk = row["benchmark"]
-output-bmk-results.py(139):     rel_value = row["rel_" + metric]
-output-bmk-results.py(140):     prev_value = row[metric + "_x"]
-output-bmk-results.py(141):     curr_value = row[metric + "_y"]
-output-bmk-results.py(142):     if metric == "num_vect_loops" or metric == "num_sve_loops":
-output-bmk-results.py(152):     suffix = ""
-output-bmk-results.py(153):     if metric == "sample":
-output-bmk-results.py(154):         prefix_regression = "slowed down by"
-output-bmk-results.py(155):         prefix_improvement = "sped up by"
-output-bmk-results.py(156):         suffix = "perf samples"
-output-bmk-results.py(167):     if sym_type=="symbol":
-output-bmk-results.py(168):       item=bmk+":"+row["symbol"]
-output-bmk-results.py(172):     short_diag = "{1} {2}% - {0}".format(item, locals()["prefix_" + change_kind], abs(rel_value - 100))
-output-bmk-results.py(173):     long_diag = "{0} - from {1} to {2} {3}".format(short_diag, prev_value, curr_value, suffix)
-output-bmk-results.py(174):     return abs(rel_value - 100), short_diag, long_diag
-output-bmk-results.py(239):         if metric == "sample" \
-output-bmk-results.py(240):           and row['symbol_md5sum_x'] == row['symbol_md5sum_y'] \
-output-bmk-results.py(241):           and row['symbol_md5sum_x'] != "-1" \
-output-bmk-results.py(242):           and row['symbol_md5sum_x'] != "d41d8cd98f00b204e9800998ecf8427e":
-output-bmk-results.py(243):             f_skip.write_csv((row["benchmark"], row["symbol"], short_diag, long_diag))
- --- modulename: output-bmk-results, funcname: write_csv
-output-bmk-results.py(41):     if not self.predicate or not self.csvwriter:
-output-bmk-results.py(43):     self.csvwriter.writerow(arr)
-output-bmk-results.py(244):             continue
+output-bmk-results.py(233):             continue
 output-bmk-results.py(224):     for index, row in out_df.iterrows():
 output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
  --- modulename: output-bmk-results, funcname: get_threshold
@@ -1366,6 +1339,7 @@ output-bmk-results.py(305):     f_skip.close()
 output-bmk-results.py(29):     if not self.outf:
 output-bmk-results.py(31):     self.outf.close()
 output-bmk-results.py(32):     if os.stat(self.filename).st_size == 0:
+output-bmk-results.py(33):        os.remove(self.filename)
 output-bmk-results.py(306):     f_regr.close()
  --- modulename: output-bmk-results, funcname: close
 output-bmk-results.py(29):     if not self.outf: