diff options
| author | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2023-05-25 12:21:49 +0000 |
|---|---|---|
| committer | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2023-05-26 06:28:55 +0000 |
| commit | d58a7ec4c974a4b861795099e1b62771c43532b4 (patch) | |
| tree | 78f2e23da376ff9057875537225ce9e0f8555d69 | |
| parent | da1b241bccbaa57cb78274657f89521460766d34 (diff) | |
Add --failures-expiration-date optionnew-flaky
This option is then passed to validate_failures.py and helps us
re-detect/confirm flaky tests before their previous entries expire.
We set expiration date "now+1 month" for flaky entries in git history
and then tell ABE to force expiry of these entries 1 week before
we stop using them in test comparison in
tcwg_gnu-build.sh:no_regression_p(). This gives us 1 week to
re-detect/confirm flaky tests without no_regression_p() noticing
anything.
Change-Id: Iadf3a6e213ce167909ca9c9be3d55d23aab2f947
| -rwxr-xr-x | abe.sh | 10 | ||||
| -rw-r--r-- | lib/globals.sh | 1 | ||||
| -rw-r--r-- | lib/make.sh | 10 |
3 files changed, 19 insertions, 2 deletions
@@ -213,6 +213,11 @@ OPTIONS as known/baseline flaky failures, and after the run the file will contain only newly detected flaky tests. + --failures-expiration-date YYYYMMDD + Use provided date YYYYMMDD to decide whether entries in + provided --expected-failures and --flaky-failures lists + have expired or not. + --force Force download packages and force rebuild packages. --gcc-compare-results <dir> @@ -986,6 +991,11 @@ while test $# -gt 0; do flaky_failures="$2" shift ;; + --failures-expiration-date) + check_directive failures-expiration-date "$2" + failures_expiration_date="$2" + shift + ;; --gcc-compare-results) check_directive gcc-compare-results "$2" if ! [ -d "$2" ]; then diff --git a/lib/globals.sh b/lib/globals.sh index 17582041..f73058eb 100644 --- a/lib/globals.sh +++ b/lib/globals.sh @@ -111,6 +111,7 @@ rerun_failed_tests=false gcc_compare_results="" expected_failures="" flaky_failures="" +failures_expiration_date="" # source a user specific config file for commonly used configure options. # These overide any of the above values. diff --git a/lib/make.sh b/lib/make.sh index 9f992d6e..9b757829 100644 --- a/lib/make.sh +++ b/lib/make.sh @@ -1101,6 +1101,11 @@ make_check() cp "$expected_failures" "$prev_try_fails" fi + local -a expiry_date_opt=() + if [ "$failures_expiration_date" != "" ]; then + expiry_date_opt+=(--expiry_date "$failures_expiration_date") + fi + # Construct the initial $known_flaky_and_fails list. # # For the first iteration (try #0) we expect fails, passes and flaky tests @@ -1310,7 +1315,7 @@ EOF "$validate_failures" \ --manifest="$known_flaky_and_fails" \ --build_dir="${builddir}$dir" \ - --verbosity=1 \ + --verbosity=1 "${expiry_date_opt[@]}" \ > "$new_fails" & res_new_fails=0 && wait $! || res_new_fails=$? @@ -1319,7 +1324,8 @@ EOF "$validate_failures" \ --manifest="$known_flaky_and_fails" \ --build_dir="${builddir}$dir" \ - --inverse_match --verbosity=1 \ + --verbosity=1 "${expiry_date_opt[@]}" \ + --inverse_match \ > "$new_passes" & res_new_passes=0 && wait $! || res_new_passes=$? |