force: #27: 1: [TCWG CI] https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O2-build/27/

Results :
  | # reset_artifacts:
  | -10
  | # build_bmk_llvm:
  | -3
  | # benchmark -- -O2_marm:
  | 1

check_regression status : 0

7 files changed, 157 insertions, 127 deletions

diff --git a/notify/jira/comment-template.txt b/notify/jira/comment-template.txt
index 2ba8bd0..eb47599 100644
--- a/notify/jira/comment-template.txt
+++ b/notify/jira/comment-template.txt
@@ -1,3 +1,3 @@
 [LLVM-651]
 No change
-Details: https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O2-build/25/artifact/artifacts/notify/mail-body.txt/*view*/
+Details: https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O2-build/27/artifact/artifacts/notify/mail-body.txt/*view*/
diff --git a/notify/lnt_report.json b/notify/lnt_report.json
index c889262..2f1f1fa 100644
--- a/notify/lnt_report.json
+++ b/notify/lnt_report.json
@@ -9,31 +9,31 @@
             "run_order": "llvmorg-17-init-09924-g52882de0e641",
             "tag": "tcwg_bmk-code_speed-cpu2017rate"
         },
-        "Start Time": "2023-11-22 23:46:31"
+        "Start Time": "2023-11-22 23:52:48"
     },
     "Tests": [
         {
             "Data": [
-                91568
+                29232
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.code_size"
         }
 	   ,
         {
             "Data": [
-                3879428
+                135917
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.code_size"
         }
 	   ,
         {
             "Data": [
-                29232
+                91568
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.code_size"
         }
 	   ,
         {
@@ -46,18 +46,18 @@
 	   ,
         {
             "Data": [
-                177991
+                3879428
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.code_size"
         }
 	   ,
         {
             "Data": [
-                135917
+                177991
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.541.leela_r.code_size"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.code_size"
         }
 	   ,
         {
@@ -70,26 +70,34 @@
 	   ,
         {
             "Data": [
-                10156
+                2548
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.exec"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.523.xalancbmk_r.exec"
         }
 	   ,
         {
             "Data": [
-                10303
+                10149
             ],
             "Info": {},
-            "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.exec"
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.557.xz_r.exec"
         }
 	   ,
         {
             "Data": [
-                14047
+                14062
             ],
             "Info": {},
             "Name": "tcwg_bmk-code_speed-cpu2017rate.505.mcf_r.exec"
         }
+	   ,
+        {
+            "Data": [
+                10253
+            ],
+            "Info": {},
+            "Name": "tcwg_bmk-code_speed-cpu2017rate.531.deepsjeng_r.exec"
+        }
     ]
 }
diff --git a/notify/mail-body.txt b/notify/mail-body.txt
index 06c2405..61de103 100644
--- a/notify/mail-body.txt
+++ b/notify/mail-body.txt
@@ -2,13 +2,7 @@ Dear contributor, our automatic CI has detected problems related to your patch(e
 
 In CI config tcwg_bmk-code_speed-cpu2017rate/llvm-arm-master-O2 after:
 
-  | 568 commits in llvm
-  | 52882de0e641 [lldb][test] TestCPP20Standard.py: make it a libc++ test
-  | e15d6b520e1e [lldb][DWARFExpression] Fix DW_OP_div to use signed division
-  | 2cdb6b84c157 [clang][dataflow] Expose DataflowAnalysisContext from DataflowEnvironment.
-  | 084ca632ac81 [EarlyCSE] Only combine metadata for load CSE
-  | a67a21bf41da [EarlyCSE] Add additional metadata preservation test (NFC)
-  | ... and 563 more commits in llvm
+  | baseline build
 
 No change
 
@@ -28,6 +22,6 @@ This benchmarking CI is work-in-progress, and we welcome feedback and suggestion
 -----------------8<--------------------------8<--------------------------8<--------------------------
 The information below can be used to reproduce a debug environment:
 
-Current build   : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O2-build/25/artifact/artifacts
-Reference build : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O2-build/24/artifact/artifacts
+Current build   : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O2-build/27/artifact/artifacts
+Reference build : https://ci.linaro.org/job/tcwg_bmk-code_speed-cpu2017rate--llvm-arm-master-O2-build/25/artifact/artifacts
 
diff --git a/notify/mail-recipients.txt b/notify/mail-recipients.txt
index 56b65bb..aa219ef 100644
--- a/notify/mail-recipients.txt
+++ b/notify/mail-recipients.txt
@@ -1 +1 @@
-bcc:tcwg-validation@linaro.org,maxim.kuvyrkov@linaro.org
+bcc:tcwg-validation@linaro.org
diff --git a/notify/mail-subject.txt b/notify/mail-subject.txt
index e006f2a..704fbbe 100644
--- a/notify/mail-subject.txt
+++ b/notify/mail-subject.txt
@@ -1 +1 @@
-[Linaro-TCWG-CI] 568 commits in llvm: No change on arm O2
+[Linaro-TCWG-CI] baseline build: No change on arm O2
diff --git a/notify/output-bmk-results.log b/notify/output-bmk-results.log
index 2312872..b66ec5d 100644
--- a/notify/output-bmk-results.log
+++ b/notify/output-bmk-results.log
@@ -107,11 +107,9 @@ output-bmk-results.py(258):     print(results_df)
 1         502.gcc_r     cpugcc_r_base.default  ...  failed-to-run  failed-to-run
 2         505.mcf_r        mcf_r_base.default  ...        success        success
 7        508.namd_r       namd_r_base.default  ...  failed-to-run  failed-to-run
-8      510.parest_r     parest_r_base.default  ...  failed-to-run  failed-to-run
-9      511.povray_r     povray_r_base.default  ...  failed-to-run  failed-to-run
-10        519.lbm_r        lbm_r_base.default  ...  failed-to-run        success
-12        519.lbm_r        lbm_r_base.default  ...  failed-to-run  failed-to-run
-13    520.omnetpp_r    omnetpp_r_base.default  ...  failed-to-run  failed-to-run
+8         519.lbm_r        lbm_r_base.default  ...        success        success
+10    520.omnetpp_r    omnetpp_r_base.default  ...  failed-to-run  failed-to-run
+11  523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run             -1
 14  523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
 15       525.x264_r       x264_r_base.default  ...  failed-to-run  failed-to-run
 16    526.blender_r    blender_r_base.default  ...  failed-to-run  failed-to-run
@@ -121,7 +119,7 @@ output-bmk-results.py(258):     print(results_df)
 24        544.nab_r        nab_r_base.default  ...  failed-to-run  failed-to-run
 25         557.xz_r         xz_r_base.default  ...        success        success
 
-[17 rows x 20 columns]
+[15 rows x 20 columns]
 output-bmk-results.py(261):     for index, row in results_df.iterrows():
 output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
  --- modulename: output-bmk-results, funcname: get_status_diag
@@ -216,35 +214,6 @@ output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["s
 output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
 output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
 output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(128):         short_diag = "{0} run now OK".format(bmk)
-output-bmk-results.py(129):         classif="improvement"
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
 output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
 output-bmk-results.py(134):     return classif, short_diag
 output-bmk-results.py(265):         if classif != change_kind:
@@ -383,11 +352,9 @@ output-bmk-results.py(258):     print(results_df)
 1         502.gcc_r     cpugcc_r_base.default  ...  failed-to-run  failed-to-run
 2         505.mcf_r        mcf_r_base.default  ...        success        success
 7        508.namd_r       namd_r_base.default  ...  failed-to-run  failed-to-run
-8      510.parest_r     parest_r_base.default  ...  failed-to-run  failed-to-run
-9      511.povray_r     povray_r_base.default  ...  failed-to-run  failed-to-run
-10        519.lbm_r        lbm_r_base.default  ...  failed-to-run        success
-12        519.lbm_r        lbm_r_base.default  ...  failed-to-run  failed-to-run
-13    520.omnetpp_r    omnetpp_r_base.default  ...  failed-to-run  failed-to-run
+8         519.lbm_r        lbm_r_base.default  ...        success        success
+10    520.omnetpp_r    omnetpp_r_base.default  ...  failed-to-run  failed-to-run
+11  523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run             -1
 14  523.xalancbmk_r   cpuxalan_r_base.default  ...  failed-to-run  failed-to-run
 15       525.x264_r       x264_r_base.default  ...  failed-to-run  failed-to-run
 16    526.blender_r    blender_r_base.default  ...  failed-to-run  failed-to-run
@@ -397,7 +364,7 @@ output-bmk-results.py(258):     print(results_df)
 24        544.nab_r        nab_r_base.default  ...  failed-to-run  failed-to-run
 25         557.xz_r         xz_r_base.default  ...        success        success
 
-[17 rows x 20 columns]
+[15 rows x 20 columns]
 output-bmk-results.py(261):     for index, row in results_df.iterrows():
 output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
  --- modulename: output-bmk-results, funcname: get_status_diag
@@ -492,41 +459,6 @@ output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["s
 output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
 output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
 output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(128):         short_diag = "{0} run now OK".format(bmk)
-output-bmk-results.py(129):         classif="improvement"
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(268):         print("DEBUG: *** {0},{1} : {2}".format(row["benchmark"], row["symbol"], short_diag))
-DEBUG: *** 519.lbm_r,lbm_r_base.default : 519.lbm_r run now OK
-output-bmk-results.py(270):         f_out.write_csv((100, row["benchmark"], row["symbol"], short_diag, short_diag))
- --- modulename: output-bmk-results, funcname: write_csv
-output-bmk-results.py(41):     if not self.predicate or not self.csvwriter:
-output-bmk-results.py(43):     self.csvwriter.writerow(arr)
-output-bmk-results.py(271):         if change_kind == "regression":
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
-output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
-output-bmk-results.py(134):     return classif, short_diag
-output-bmk-results.py(265):         if classif != change_kind:
-output-bmk-results.py(266):             continue;
-output-bmk-results.py(261):     for index, row in results_df.iterrows():
-output-bmk-results.py(263):         classif, short_diag = get_status_diag(row)
- --- modulename: output-bmk-results, funcname: get_status_diag
-output-bmk-results.py(113):     bmk = row["benchmark"]
-output-bmk-results.py(115):     short_diag=""
-output-bmk-results.py(116):     classif=""
-output-bmk-results.py(118):     if row["status_x"]!="failed-to-build" and row["status_y"]=="failed-to-build":
-output-bmk-results.py(121):     elif row["status_x"]=="success" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(124):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="failed-to-run":
-output-bmk-results.py(127):     elif row["status_x"]=="failed-to-run" and row["status_y"]=="success":
 output-bmk-results.py(130):     elif row["status_x"]=="failed-to-build" and row["status_y"]=="success":
 output-bmk-results.py(134):     return classif, short_diag
 output-bmk-results.py(265):         if classif != change_kind:
@@ -649,6 +581,7 @@ output-bmk-results.py(275):     f_out.close()
 output-bmk-results.py(29):     if not self.outf:
 output-bmk-results.py(31):     self.outf.close()
 output-bmk-results.py(32):     if os.stat(self.filename).st_size == 0:
+output-bmk-results.py(33):        os.remove(self.filename)
 output-bmk-results.py(297):     output_bmk_results_1(exe_df, "exe", "regression", f_regr, f_skip, f_ebp, run_step_artifacts, metric, mode, details)
  --- modulename: output-bmk-results, funcname: output_bmk_results_1
 output-bmk-results.py(218):     f_out = Outfile("{0}/{1}.{2}".format(run_step_artifacts, sym_type, change_kind), "w", predicate=(details=="verbose"))
@@ -698,6 +631,30 @@ output-bmk-results.py(60):     var = specific_variability[ (specific_variability
 output-bmk-results.py(61):     if var.empty:
 output-bmk-results.py(63):     elif len(var)>1:
 output-bmk-results.py(68):     if var.iloc[0]['sample_variation_average']>0 :
+output-bmk-results.py(83):     return np.nan
+output-bmk-results.py(100):        if not np.isnan(spec_thr):
+output-bmk-results.py(107):     if metric == "num_vect_loops" or metric == "num_sve_loops":
+output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mode)]
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+DEBUG: checking exe.regression : 519.lbm_r,lbm_r_base.default : sample=0% (threshold=3%)
+output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
+ --- modulename: output-bmk-results, funcname: is_entry_regression
+output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
+output-bmk-results.py(184):         return (result - 100 > threshold)
+output-bmk-results.py(233):             continue
+output-bmk-results.py(224):     for index, row in out_df.iterrows():
+output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
+ --- modulename: output-bmk-results, funcname: get_threshold
+output-bmk-results.py(98):     if metric == "sample":
+output-bmk-results.py(99):        spec_thr = get_specific_thresholds(metric, mode, bmk, symb)
+ --- modulename: output-bmk-results, funcname: get_specific_thresholds
+output-bmk-results.py(57):     if specific_variability is None:
+output-bmk-results.py(60):     var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)]
+output-bmk-results.py(61):     if var.empty:
+output-bmk-results.py(63):     elif len(var)>1:
+output-bmk-results.py(68):     if var.iloc[0]['sample_variation_average']>0 :
 output-bmk-results.py(69):        threshold = ( var.iloc[0]['sample_variation_average'] )
 output-bmk-results.py(70):        if mode == "build":
 output-bmk-results.py(74):           threshold *= 3
@@ -797,6 +754,30 @@ output-bmk-results.py(60):     var = specific_variability[ (specific_variability
 output-bmk-results.py(61):     if var.empty:
 output-bmk-results.py(63):     elif len(var)>1:
 output-bmk-results.py(68):     if var.iloc[0]['sample_variation_average']>0 :
+output-bmk-results.py(83):     return np.nan
+output-bmk-results.py(100):        if not np.isnan(spec_thr):
+output-bmk-results.py(107):     if metric == "num_vect_loops" or metric == "num_sve_loops":
+output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mode)]
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+DEBUG: checking exe.improvement : 519.lbm_r,lbm_r_base.default : sample=0% (threshold=3%)
+output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
+ --- modulename: output-bmk-results, funcname: is_entry_improvement
+output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
+output-bmk-results.py(193):         return (100 - result > threshold)
+output-bmk-results.py(233):             continue
+output-bmk-results.py(224):     for index, row in out_df.iterrows():
+output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
+ --- modulename: output-bmk-results, funcname: get_threshold
+output-bmk-results.py(98):     if metric == "sample":
+output-bmk-results.py(99):        spec_thr = get_specific_thresholds(metric, mode, bmk, symb)
+ --- modulename: output-bmk-results, funcname: get_specific_thresholds
+output-bmk-results.py(57):     if specific_variability is None:
+output-bmk-results.py(60):     var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)]
+output-bmk-results.py(61):     if var.empty:
+output-bmk-results.py(63):     elif len(var)>1:
+output-bmk-results.py(68):     if var.iloc[0]['sample_variation_average']>0 :
 output-bmk-results.py(69):        threshold = ( var.iloc[0]['sample_variation_average'] )
 output-bmk-results.py(70):        if mode == "build":
 output-bmk-results.py(74):           threshold *= 3
@@ -879,7 +860,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 505.mcf_r,[.] primal_bea_mpp : sample=0% (threshold=15%)
+DEBUG: checking symbol.regression : 505.mcf_r,[.] primal_bea_mpp : sample=1% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -906,7 +887,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 505.mcf_r,[.] price_out_impl : sample=-2% (threshold=15%)
+DEBUG: checking symbol.regression : 505.mcf_r,[.] price_out_impl : sample=-1% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -933,7 +914,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 505.mcf_r,[.] cost_compare : sample=0% (threshold=15%)
+DEBUG: checking symbol.regression : 505.mcf_r,[.] cost_compare : sample=-2% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -960,7 +941,31 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 505.mcf_r,[.] replace_weaker_arc : sample=1% (threshold=15%)
+DEBUG: checking symbol.regression : 505.mcf_r,[.] replace_weaker_arc : sample=2% (threshold=15%)
+output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
+ --- modulename: output-bmk-results, funcname: is_entry_regression
+output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
+output-bmk-results.py(184):         return (result - 100 > threshold)
+output-bmk-results.py(233):             continue
+output-bmk-results.py(224):     for index, row in out_df.iterrows():
+output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
+ --- modulename: output-bmk-results, funcname: get_threshold
+output-bmk-results.py(98):     if metric == "sample":
+output-bmk-results.py(99):        spec_thr = get_specific_thresholds(metric, mode, bmk, symb)
+ --- modulename: output-bmk-results, funcname: get_specific_thresholds
+output-bmk-results.py(57):     if specific_variability is None:
+output-bmk-results.py(60):     var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)]
+output-bmk-results.py(61):     if var.empty:
+output-bmk-results.py(63):     elif len(var)>1:
+output-bmk-results.py(68):     if var.iloc[0]['sample_variation_average']>0 :
+output-bmk-results.py(83):     return np.nan
+output-bmk-results.py(100):        if not np.isnan(spec_thr):
+output-bmk-results.py(107):     if metric == "num_vect_loops" or metric == "num_sve_loops":
+output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mode)]
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+DEBUG: checking symbol.regression : 519.lbm_r,[.] LBM_initializeGrid : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -987,7 +992,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 531.deepsjeng_r,[.] _Z5fevalP7state_tiP12t_eval_comps : sample=6% (threshold=15%)
+DEBUG: checking symbol.regression : 531.deepsjeng_r,[.] _Z5fevalP7state_tiP12t_eval_comps : sample=-3% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1014,7 +1019,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 531.deepsjeng_r,[.] _Z7ProbeTTP7state_tPiiiPjS1_S1_S1_S1_i : sample=1% (threshold=15%)
+DEBUG: checking symbol.regression : 531.deepsjeng_r,[.] _Z7ProbeTTP7state_tPiiiPjS1_S1_S1_S1_i : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1041,7 +1046,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 531.deepsjeng_r,[.] _Z4makeP7state_ti : sample=-5% (threshold=15%)
+DEBUG: checking symbol.regression : 531.deepsjeng_r,[.] _Z4makeP7state_ti : sample=2% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1068,7 +1073,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 531.deepsjeng_r,[.] _Z6searchP7state_tiiiii : sample=-3% (threshold=18.93%)
+DEBUG: checking symbol.regression : 531.deepsjeng_r,[.] _Z6searchP7state_tiiiii : sample=3% (threshold=16.799999999999997%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1095,7 +1100,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 557.xz_r,[.] lzma_mf_bt4_find : sample=1% (threshold=15%)
+DEBUG: checking symbol.regression : 557.xz_r,[.] lzma_mf_bt4_find : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1122,7 +1127,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 557.xz_r,[.] lzma_lzma_optimum_normal : sample=-2% (threshold=15%)
+DEBUG: checking symbol.regression : 557.xz_r,[.] lzma_lzma_optimum_normal : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1149,7 +1154,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.regression : 557.xz_r,[.] lzma_mf_bt4_skip : sample=-4% (threshold=15%)
+DEBUG: checking symbol.regression : 557.xz_r,[.] lzma_mf_bt4_skip : sample=2% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_regression
 output-bmk-results.py(183):     if metric in metric_utils.higher_regress_metrics:
@@ -1194,7 +1199,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 505.mcf_r,[.] primal_bea_mpp : sample=0% (threshold=15%)
+DEBUG: checking symbol.improvement : 505.mcf_r,[.] primal_bea_mpp : sample=1% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1221,7 +1226,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 505.mcf_r,[.] price_out_impl : sample=-2% (threshold=15%)
+DEBUG: checking symbol.improvement : 505.mcf_r,[.] price_out_impl : sample=-1% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1248,7 +1253,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 505.mcf_r,[.] cost_compare : sample=0% (threshold=15%)
+DEBUG: checking symbol.improvement : 505.mcf_r,[.] cost_compare : sample=-2% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1275,7 +1280,31 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 505.mcf_r,[.] replace_weaker_arc : sample=1% (threshold=15%)
+DEBUG: checking symbol.improvement : 505.mcf_r,[.] replace_weaker_arc : sample=2% (threshold=15%)
+output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
+ --- modulename: output-bmk-results, funcname: is_entry_improvement
+output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
+output-bmk-results.py(193):         return (100 - result > threshold)
+output-bmk-results.py(233):             continue
+output-bmk-results.py(224):     for index, row in out_df.iterrows():
+output-bmk-results.py(226):         threshold = get_threshold(sym_type, metric, mode, row["benchmark"], row["symbol"])
+ --- modulename: output-bmk-results, funcname: get_threshold
+output-bmk-results.py(98):     if metric == "sample":
+output-bmk-results.py(99):        spec_thr = get_specific_thresholds(metric, mode, bmk, symb)
+ --- modulename: output-bmk-results, funcname: get_specific_thresholds
+output-bmk-results.py(57):     if specific_variability is None:
+output-bmk-results.py(60):     var = specific_variability[ (specific_variability['benchmark'] == bmk) & (specific_variability['symbol'].str.strip() == symb)]
+output-bmk-results.py(61):     if var.empty:
+output-bmk-results.py(63):     elif len(var)>1:
+output-bmk-results.py(68):     if var.iloc[0]['sample_variation_average']>0 :
+output-bmk-results.py(83):     return np.nan
+output-bmk-results.py(100):        if not np.isnan(spec_thr):
+output-bmk-results.py(107):     if metric == "num_vect_loops" or metric == "num_sve_loops":
+output-bmk-results.py(110):     return default_threshold[(change_kind,metric,mode)]
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
+output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
+DEBUG: checking symbol.improvement : 519.lbm_r,[.] LBM_initializeGrid : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1302,7 +1331,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 531.deepsjeng_r,[.] _Z5fevalP7state_tiP12t_eval_comps : sample=6% (threshold=15%)
+DEBUG: checking symbol.improvement : 531.deepsjeng_r,[.] _Z5fevalP7state_tiP12t_eval_comps : sample=-3% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1329,7 +1358,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 531.deepsjeng_r,[.] _Z7ProbeTTP7state_tPiiiPjS1_S1_S1_S1_i : sample=1% (threshold=15%)
+DEBUG: checking symbol.improvement : 531.deepsjeng_r,[.] _Z7ProbeTTP7state_tPiiiPjS1_S1_S1_S1_i : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1356,7 +1385,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 531.deepsjeng_r,[.] _Z4makeP7state_ti : sample=-5% (threshold=15%)
+DEBUG: checking symbol.improvement : 531.deepsjeng_r,[.] _Z4makeP7state_ti : sample=2% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1383,7 +1412,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 531.deepsjeng_r,[.] _Z6searchP7state_tiiiii : sample=-3% (threshold=18.93%)
+DEBUG: checking symbol.improvement : 531.deepsjeng_r,[.] _Z6searchP7state_tiiiii : sample=3% (threshold=16.799999999999997%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1410,7 +1439,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 557.xz_r,[.] lzma_mf_bt4_find : sample=1% (threshold=15%)
+DEBUG: checking symbol.improvement : 557.xz_r,[.] lzma_mf_bt4_find : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1437,7 +1466,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 557.xz_r,[.] lzma_lzma_optimum_normal : sample=-2% (threshold=15%)
+DEBUG: checking symbol.improvement : 557.xz_r,[.] lzma_lzma_optimum_normal : sample=0% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
@@ -1464,7 +1493,7 @@ output-bmk-results.py(105):           return spec_thr
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
 output-bmk-results.py(229):              .format(sym_type, change_kind, row["benchmark"], row["symbol"], metric, 100-row["rel_" + metric], threshold))
 output-bmk-results.py(228):         print("DEBUG: checking {0}.{1} : {2},{3} : {4}={5}% (threshold={6}%)"\
-DEBUG: checking symbol.improvement : 557.xz_r,[.] lzma_mf_bt4_skip : sample=-4% (threshold=15%)
+DEBUG: checking symbol.improvement : 557.xz_r,[.] lzma_mf_bt4_skip : sample=2% (threshold=15%)
 output-bmk-results.py(232):         if not is_entry_xxx[change_kind](metric, row["rel_" + metric], threshold):
  --- modulename: output-bmk-results, funcname: is_entry_improvement
 output-bmk-results.py(192):     if metric in metric_utils.higher_regress_metrics:
diff --git a/notify/status.improvement b/notify/status.improvement
deleted file mode 100644
index 12aa99d..0000000
--- a/notify/status.improvement
+++ /dev/null
@@ -1 +0,0 @@
-100,519.lbm_r,lbm_r_base.default,519.lbm_r run now OK,519.lbm_r run now OK