From 49de3effd3fd91959503e2daf10e2bfd854ed504 Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Fri, 11 May 2018 21:29:53 +0000 Subject: [libFuzzer] [NFC] Split fuzzer.test into smaller tests Keeping fuzzer.test together as a gigantic test has no advantages and multiple disadvantages: - Worse test parallelization (fuzzer.test is always blocking the test run on a machine with many cores) - Debugging test failures is made more difficult (not clear what fails from fuzzer.test failing) - Makes porting tests to new platforms more difficult (whenever fuzzer.test fails have to inspect the output to figure out what is going on, and then restart all of it) - Hides dead code (in our case, "Done1000000" FileCheck variable was never used, DISABLED: not %t-UninstrumentedTest-Uninstrumented was never compiled, and there was small amount of duplication vs. simple-cmp.test) - Makes correspondence between LIT .test files and .cpp files less straightforward Differential Revision: https://reviews.llvm.org/D46557 git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@332145 91177308-0d34-0410-b5e6-96231b3b80d8 --- test/fuzzer/bogus-initialize.test | 4 ++ test/fuzzer/buffer-overflow-on-input.test | 5 +++ test/fuzzer/counters.test | 7 ++++ test/fuzzer/dso.test | 7 ++++ test/fuzzer/full-coverage-set.test | 3 ++ test/fuzzer/fuzzer.test | 70 ------------------------------- test/fuzzer/initialize.test | 3 ++ test/fuzzer/not-instrumented.test | 4 ++ test/fuzzer/null-deref-on-empty.test | 4 ++ test/fuzzer/null-deref.test | 10 +++++ test/fuzzer/simple-cmp.test | 6 ++- test/fuzzer/simple.test | 7 ++++ test/fuzzer/strncmp-oob.test | 6 +++ 13 files changed, 65 insertions(+), 71 deletions(-) create mode 100644 test/fuzzer/bogus-initialize.test create mode 100644 test/fuzzer/buffer-overflow-on-input.test create mode 100644 test/fuzzer/counters.test create mode 100644 test/fuzzer/dso.test create mode 100644 test/fuzzer/full-coverage-set.test delete mode 100644 test/fuzzer/fuzzer.test create mode 100644 test/fuzzer/initialize.test create mode 100644 test/fuzzer/not-instrumented.test create mode 100644 test/fuzzer/null-deref-on-empty.test create mode 100644 test/fuzzer/null-deref.test create mode 100644 test/fuzzer/simple.test create mode 100644 test/fuzzer/strncmp-oob.test (limited to 'test') diff --git a/test/fuzzer/bogus-initialize.test b/test/fuzzer/bogus-initialize.test new file mode 100644 index 000000000..2dff2d5a2 --- /dev/null +++ b/test/fuzzer/bogus-initialize.test @@ -0,0 +1,4 @@ +RUN: %cpp_compiler %S/BogusInitializeTest.cpp -o %t-BogusInitializeTest + +RUN: not %run %t-BogusInitializeTest 2>&1 | FileCheck %s --check-prefix=BOGUS_INITIALIZE +BOGUS_INITIALIZE: argv[0] has been modified in LLVMFuzzerInitialize diff --git a/test/fuzzer/buffer-overflow-on-input.test b/test/fuzzer/buffer-overflow-on-input.test new file mode 100644 index 000000000..6e40b75d7 --- /dev/null +++ b/test/fuzzer/buffer-overflow-on-input.test @@ -0,0 +1,5 @@ +RUN: %cpp_compiler %S/BufferOverflowOnInput.cpp -o %t-BufferOverflowOnInput + +RUN: not %run %t-BufferOverflowOnInput 2>&1 | FileCheck %s --check-prefix=OOB +OOB: AddressSanitizer: heap-buffer-overflow +OOB: is located 0 bytes to the right of 3-byte region diff --git a/test/fuzzer/counters.test b/test/fuzzer/counters.test new file mode 100644 index 000000000..92b5e1e2c --- /dev/null +++ b/test/fuzzer/counters.test @@ -0,0 +1,7 @@ +RUN: %cpp_compiler %S/CounterTest.cpp -o %t-CounterTest +RUN: not %run %t-CounterTest -max_len=6 -seed=1 -timeout=15 2>&1 | FileCheck %s --check-prefix=COUNTERS + +COUNTERS: INITED {{.*}} {{bits:|ft:}} +COUNTERS: NEW {{.*}} {{bits:|ft:}} {{[1-9]*}} +COUNTERS: NEW {{.*}} {{bits:|ft:}} {{[1-9]*}} +COUNTERS: BINGO diff --git a/test/fuzzer/dso.test b/test/fuzzer/dso.test new file mode 100644 index 000000000..d17ffe8ed --- /dev/null +++ b/test/fuzzer/dso.test @@ -0,0 +1,7 @@ +RUN: %cpp_compiler %S/DSO1.cpp -fPIC -shared -o %t-DSO1.so +RUN: %cpp_compiler %S/DSO2.cpp -fPIC -shared -o %t-DSO2.so +RUN: %cpp_compiler %S/DSOTestMain.cpp %S/DSOTestExtra.cpp -L. %t-DSO1.so %t-DSO2.so -o %t-DSOTest + +RUN: not %run %t-DSOTest 2>&1 | FileCheck %s --check-prefix=DSO +DSO: INFO: Loaded 3 modules +DSO: BINGO diff --git a/test/fuzzer/full-coverage-set.test b/test/fuzzer/full-coverage-set.test new file mode 100644 index 000000000..629873d4e --- /dev/null +++ b/test/fuzzer/full-coverage-set.test @@ -0,0 +1,3 @@ +CHECK: BINGO +RUN: %cpp_compiler %S/FullCoverageSetTest.cpp -o %t-FullCoverageSetTest +#not %run %t-FullCoverageSetTest -timeout=15 -seed=1 -mutate_depth=2 -use_full_coverage_set=1 2>&1 | FileCheck %s diff --git a/test/fuzzer/fuzzer.test b/test/fuzzer/fuzzer.test deleted file mode 100644 index 8dcb82d2b..000000000 --- a/test/fuzzer/fuzzer.test +++ /dev/null @@ -1,70 +0,0 @@ -CHECK: BINGO -Done1000000: Done 1000000 runs in -RUN: %cpp_compiler %S/BogusInitializeTest.cpp -o %t-BogusInitializeTest -RUN: %cpp_compiler %S/BufferOverflowOnInput.cpp -o %t-BufferOverflowOnInput -RUN: %cpp_compiler %S/CounterTest.cpp -o %t-CounterTest -RUN: %cpp_compiler %S/DSO1.cpp -fPIC -shared -o %t-DSO1.so -RUN: %cpp_compiler %S/DSO2.cpp -fPIC -shared -o %t-DSO2.so -RUN: %cpp_compiler %S/DSOTestMain.cpp %S/DSOTestExtra.cpp -L. %t-DSO1.so %t-DSO2.so -o %t-DSOTest -RUN: %cpp_compiler %S/FullCoverageSetTest.cpp -o %t-FullCoverageSetTest -RUN: %cpp_compiler %S/InitializeTest.cpp -o %t-InitializeTest -RUN: %cpp_compiler %S/NotinstrumentedTest.cpp -fsanitize-coverage=0 -o %t-NotinstrumentedTest-NoCoverage -RUN: %cpp_compiler %S/NullDerefOnEmptyTest.cpp -o %t-NullDerefOnEmptyTest -RUN: %cpp_compiler %S/NullDerefTest.cpp -o %t-NullDerefTest -RUN: %cpp_compiler %S/SimpleCmpTest.cpp -o %t-SimpleCmpTest -RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-SimpleTest -RUN: %cpp_compiler %S/StrncmpOOBTest.cpp -o %t-StrncmpOOBTest - -RUN: not %run %t-SimpleTest 2>&1 | FileCheck %s - -# only_ascii mode. Will perform some minimal self-validation. -RUN: not %run %t-SimpleTest -only_ascii=1 2>&1 - -RUN: %run %t-SimpleCmpTest -max_total_time=1 -use_cmp=0 2>&1 | FileCheck %s --check-prefix=MaxTotalTime -MaxTotalTime: Done {{.*}} runs in {{.}} second(s) - -RUN: not %run %t-NullDerefTest 2>&1 | FileCheck %s --check-prefix=NullDerefTest -RUN: not %run %t-NullDerefTest -close_fd_mask=3 2>&1 | FileCheck %s --check-prefix=NullDerefTest -NullDerefTest: ERROR: AddressSanitizer: {{SEGV|access-violation}} on unknown address -NullDerefTest: Test unit written to ./crash- -RUN: not %run %t-NullDerefTest -artifact_prefix=ZZZ 2>&1 | FileCheck %s --check-prefix=NullDerefTestPrefix -NullDerefTestPrefix: Test unit written to ZZZcrash- -RUN: not %run %t-NullDerefTest -artifact_prefix=ZZZ -exact_artifact_path=FOOBAR 2>&1 | FileCheck %s --check-prefix=NullDerefTestExactPath -NullDerefTestExactPath: Test unit written to FOOBAR - -RUN: not %run %t-NullDerefOnEmptyTest -print_final_stats=1 2>&1 | FileCheck %s --check-prefix=NULL_DEREF_ON_EMPTY -NULL_DEREF_ON_EMPTY: stat::number_of_executed_units: - -#not %run %t-FullCoverageSetTest -timeout=15 -seed=1 -mutate_depth=2 -use_full_coverage_set=1 2>&1 | FileCheck %s - -RUN: not %run %t-CounterTest -max_len=6 -seed=1 -timeout=15 2>&1 | FileCheck %s --check-prefix=COUNTERS - -COUNTERS: INITED {{.*}} {{bits:|ft:}} -COUNTERS: NEW {{.*}} {{bits:|ft:}} {{[1-9]*}} -COUNTERS: NEW {{.*}} {{bits:|ft:}} {{[1-9]*}} -COUNTERS: BINGO - -# Don't run UninstrumentedTest for now since we build libFuzzer itself with asan. -DISABLED: not %run %t-UninstrumentedTest-Uninstrumented 2>&1 | FileCheck %s --check-prefix=UNINSTRUMENTED -UNINSTRUMENTED: ERROR: __sanitizer_set_death_callback is not defined. Exiting. - -RUN: not %run %t-NotinstrumentedTest-NoCoverage 2>&1 | FileCheck %s --check-prefix=NO_COVERAGE -NO_COVERAGE: ERROR: no interesting inputs were found. Is the code instrumented for coverage? Exiting - -RUN: not %run %t-BufferOverflowOnInput 2>&1 | FileCheck %s --check-prefix=OOB -OOB: AddressSanitizer: heap-buffer-overflow -OOB: is located 0 bytes to the right of 3-byte region - -RUN: not %run %t-InitializeTest -use_value_profile=1 2>&1 | FileCheck %s - -RUN: not %run %t-DSOTest 2>&1 | FileCheck %s --check-prefix=DSO -DSO: INFO: Loaded 3 modules -DSO: BINGO - -RUN: env ASAN_OPTIONS=strict_string_checks=1 not %run %t-StrncmpOOBTest -seed=1 -runs=1000000 2>&1 | FileCheck %s --check-prefix=STRNCMP -STRNCMP: AddressSanitizer: heap-buffer-overflow -STRNCMP-NOT: __sanitizer_weak_hook_strncmp -STRNCMP: in LLVMFuzzerTestOneInput - -RUN: not %run %t-BogusInitializeTest 2>&1 | FileCheck %s --check-prefix=BOGUS_INITIALIZE -BOGUS_INITIALIZE: argv[0] has been modified in LLVMFuzzerInitialize diff --git a/test/fuzzer/initialize.test b/test/fuzzer/initialize.test new file mode 100644 index 000000000..dc6e86975 --- /dev/null +++ b/test/fuzzer/initialize.test @@ -0,0 +1,3 @@ +CHECK: BINGO +RUN: %cpp_compiler %S/InitializeTest.cpp -o %t-InitializeTest +RUN: not %run %t-InitializeTest -use_value_profile=1 2>&1 | FileCheck %s diff --git a/test/fuzzer/not-instrumented.test b/test/fuzzer/not-instrumented.test new file mode 100644 index 000000000..2330c4770 --- /dev/null +++ b/test/fuzzer/not-instrumented.test @@ -0,0 +1,4 @@ +RUN: %cpp_compiler %S/NotinstrumentedTest.cpp -fsanitize-coverage=0 -o %t-NotinstrumentedTest-NoCoverage +RUN: not %run %t-NotinstrumentedTest-NoCoverage 2>&1 | FileCheck %s --check-prefix=NO_COVERAGE + +NO_COVERAGE: ERROR: no interesting inputs were found. Is the code instrumented for coverage? Exiting diff --git a/test/fuzzer/null-deref-on-empty.test b/test/fuzzer/null-deref-on-empty.test new file mode 100644 index 000000000..f159a79f4 --- /dev/null +++ b/test/fuzzer/null-deref-on-empty.test @@ -0,0 +1,4 @@ +RUN: %cpp_compiler %S/NullDerefOnEmptyTest.cpp -o %t-NullDerefOnEmptyTest + +RUN: not %run %t-NullDerefOnEmptyTest -print_final_stats=1 2>&1 | FileCheck %s --check-prefix=NULL_DEREF_ON_EMPTY +NULL_DEREF_ON_EMPTY: stat::number_of_executed_units: diff --git a/test/fuzzer/null-deref.test b/test/fuzzer/null-deref.test new file mode 100644 index 000000000..31eb5990d --- /dev/null +++ b/test/fuzzer/null-deref.test @@ -0,0 +1,10 @@ +RUN: %cpp_compiler %S/NullDerefTest.cpp -o %t-NullDerefTest + +RUN: not %run %t-NullDerefTest 2>&1 | FileCheck %s --check-prefix=NullDerefTest +RUN: not %run %t-NullDerefTest -close_fd_mask=3 2>&1 | FileCheck %s --check-prefix=NullDerefTest +NullDerefTest: ERROR: AddressSanitizer: {{SEGV|access-violation}} on unknown address +NullDerefTest: Test unit written to ./crash- +RUN: not %run %t-NullDerefTest -artifact_prefix=ZZZ 2>&1 | FileCheck %s --check-prefix=NullDerefTestPrefix +NullDerefTestPrefix: Test unit written to ZZZcrash- +RUN: not %run %t-NullDerefTest -artifact_prefix=ZZZ -exact_artifact_path=FOOBAR 2>&1 | FileCheck %s --check-prefix=NullDerefTestExactPath +NullDerefTestExactPath: Test unit written to FOOBAR diff --git a/test/fuzzer/simple-cmp.test b/test/fuzzer/simple-cmp.test index 3206e9179..e146379b2 100644 --- a/test/fuzzer/simple-cmp.test +++ b/test/fuzzer/simple-cmp.test @@ -1,3 +1,7 @@ RUN: %cpp_compiler %S/SimpleCmpTest.cpp -o %t-SimpleCmpTest -CHECK: BINGO + RUN: not %run %t-SimpleCmpTest -seed=1 -runs=100000000 2>&1 | FileCheck %s +RUN: %run %t-SimpleCmpTest -max_total_time=1 -use_cmp=0 2>&1 | FileCheck %s --check-prefix=MaxTotalTime +MaxTotalTime: Done {{.*}} runs in {{.}} second(s) + +CHECK: BINGO diff --git a/test/fuzzer/simple.test b/test/fuzzer/simple.test new file mode 100644 index 000000000..97a09be7c --- /dev/null +++ b/test/fuzzer/simple.test @@ -0,0 +1,7 @@ +CHECK: BINGO +RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-SimpleTest + +RUN: not %run %t-SimpleTest 2>&1 | FileCheck %s + +# only_ascii mode. Will perform some minimal self-validation. +RUN: not %run %t-SimpleTest -only_ascii=1 2>&1 diff --git a/test/fuzzer/strncmp-oob.test b/test/fuzzer/strncmp-oob.test new file mode 100644 index 000000000..a0365d961 --- /dev/null +++ b/test/fuzzer/strncmp-oob.test @@ -0,0 +1,6 @@ +RUN: %cpp_compiler %S/StrncmpOOBTest.cpp -o %t-StrncmpOOBTest + +RUN: env ASAN_OPTIONS=strict_string_checks=1 not %run %t-StrncmpOOBTest -seed=1 -runs=1000000 2>&1 | FileCheck %s --check-prefix=STRNCMP +STRNCMP: AddressSanitizer: heap-buffer-overflow +STRNCMP-NOT: __sanitizer_weak_hook_strncmp +STRNCMP: in LLVMFuzzerTestOneInput -- cgit v1.2.3