diff --git a/fuzzers/baby_fuzzer_swap_differential/Makefile.toml b/fuzzers/baby_fuzzer_swap_differential/Makefile.toml index 36cf1a2e9f..0290f2dadc 100644 --- a/fuzzers/baby_fuzzer_swap_differential/Makefile.toml +++ b/fuzzers/baby_fuzzer_swap_differential/Makefile.toml @@ -31,7 +31,13 @@ windows_alias = "unsupported" [tasks.test_unix] script_runner = "@shell" script=''' -timeout 10s ${CARGO_TARGET_DIR}/release/${FUZZER_NAME} +timeout 10s ${CARGO_TARGET_DIR}/release/${FUZZER_NAME} >fuzz_stdout.log || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/frida_libpng/Makefile.toml b/fuzzers/frida_libpng/Makefile.toml index 8d33b88c62..31b59d783f 100644 --- a/fuzzers/frida_libpng/Makefile.toml +++ b/fuzzers/frida_libpng/Makefile.toml @@ -109,7 +109,13 @@ windows_alias = "test_windows" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 10s ./${FUZZER_NAME} -F LLVMFuzzerTestOneInput -H ./libpng-harness.so -l ./libpng-harness.so & +timeout 10s ./${FUZZER_NAME} -F LLVMFuzzerTestOneInput -H ./libpng-harness.so -l ./libpng-harness.so >fuzz_stdout.log || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer", "harness" ] diff --git a/fuzzers/fuzzbench/Makefile.toml b/fuzzers/fuzzbench/Makefile.toml index ca64b0a9c3..2f2908b19a 100644 --- a/fuzzers/fuzzbench/Makefile.toml +++ b/fuzzers/fuzzbench/Makefile.toml @@ -80,7 +80,13 @@ rm -rf libafl_unix_shmem_server || true mkdir in || true echo a > in/a # Allow sigterm as exit code -timeout 11s ./${FUZZER_NAME} -o out -i in || [ $? -eq 124 ] +timeout 11s ./${FUZZER_NAME} -o out -i in >fuzz_stdout.log || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi rm -rf out || true rm -rf in || true ''' diff --git a/fuzzers/fuzzbench_fork_qemu/Makefile.toml b/fuzzers/fuzzbench_fork_qemu/Makefile.toml index 987b46a564..2f137c79ff 100644 --- a/fuzzers/fuzzbench_fork_qemu/Makefile.toml +++ b/fuzzers/fuzzbench_fork_qemu/Makefile.toml @@ -79,7 +79,13 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 10s cargo run --release -- --libafl-in ../libfuzzer_libpng/corpus --libafl-out ./out ./${FUZZER_NAME} & +timeout 10s ./target/release/fuzzbench_fork_qemu --libafl-in ../libfuzzer_libpng/corpus --libafl-out ./out ./${FUZZER_NAME} || true +if [ -z "$(ls -A out/)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "harness", "fuzzer" ] diff --git a/fuzzers/fuzzbench_qemu/Makefile.toml b/fuzzers/fuzzbench_qemu/Makefile.toml index db4a31fa42..997bc099dd 100644 --- a/fuzzers/fuzzbench_qemu/Makefile.toml +++ b/fuzzers/fuzzbench_qemu/Makefile.toml @@ -79,7 +79,13 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 10s cargo run --release -- --libafl-in ../libfuzzer_libpng/corpus --libafl-out ./out ./${FUZZER_NAME} & +timeout 10s cargo run --release -- --libafl-in ../libfuzzer_libpng/corpus --libafl-out ./out ./${FUZZER_NAME} || true +if [ -z "$(ls -A out/)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "harness", "fuzzer" ] diff --git a/fuzzers/fuzzbench_text/Makefile.toml b/fuzzers/fuzzbench_text/Makefile.toml index 2d3d85ea72..4c04660a0e 100644 --- a/fuzzers/fuzzbench_text/Makefile.toml +++ b/fuzzers/fuzzbench_text/Makefile.toml @@ -81,7 +81,13 @@ rm -rf libafl_unix_shmem_server || true mkdir in || true echo a > in/a # Allow sigterm as exit code -timeout 11s ./${FUZZER_NAME} -o out -i in || [ $? -eq 124 ] +timeout 11s ./${FUZZER_NAME} -o out -i in >fuzz_stdout.log || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi rm -rf out || true rm -rf in || true ''' diff --git a/fuzzers/libfuzzer_libmozjpeg/Makefile.toml b/fuzzers/libfuzzer_libmozjpeg/Makefile.toml index 81434cfbc7..a25c74bc61 100644 --- a/fuzzers/libfuzzer_libmozjpeg/Makefile.toml +++ b/fuzzers/libfuzzer_libmozjpeg/Makefile.toml @@ -97,9 +97,15 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} & +timeout 11s ./${FUZZER_NAME} >fuzz_stdout.log & sleep 0.2 -timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null & +timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_libpng/Makefile.toml b/fuzzers/libfuzzer_libpng/Makefile.toml index b4c6c5d653..dbd91bc25e 100644 --- a/fuzzers/libfuzzer_libpng/Makefile.toml +++ b/fuzzers/libfuzzer_libpng/Makefile.toml @@ -159,9 +159,15 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} & +timeout 11s ./${FUZZER_NAME} >fuzz_stdout.log & sleep 0.2 -timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null & +timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_libpng_accounting/Makefile.toml b/fuzzers/libfuzzer_libpng_accounting/Makefile.toml index 6f84428082..9035b2c407 100644 --- a/fuzzers/libfuzzer_libpng_accounting/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_accounting/Makefile.toml @@ -96,7 +96,13 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null & +timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus >fuzz_stdout.log 2>/dev/null || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_libpng_cmin/Makefile.toml b/fuzzers/libfuzzer_libpng_cmin/Makefile.toml index 110c46948e..7f11b6f1c0 100644 --- a/fuzzers/libfuzzer_libpng_cmin/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_cmin/Makefile.toml @@ -159,9 +159,15 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} & +timeout 11s ./${FUZZER_NAME} >fuzz_stdout.log & sleep 0.2 -timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null & +timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_libpng_ctx/Makefile.toml b/fuzzers/libfuzzer_libpng_ctx/Makefile.toml index 47aef9372a..066ce915ff 100644 --- a/fuzzers/libfuzzer_libpng_ctx/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_ctx/Makefile.toml @@ -5,6 +5,7 @@ CARGO_TARGET_DIR = { value = "${PROJECT_DIR}/target", condition = { env_not_set LIBAFL_CC = '${CARGO_TARGET_DIR}/release/libafl_cc' LIBAFL_CXX = '${CARGO_TARGET_DIR}/release/libafl_cxx' FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}' + PROJECT_DIR = { script = ["pwd"] } [tasks.unsupported] @@ -96,7 +97,13 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null & +timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus >fuzz_stdout.log 2>/dev/null || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_libpng_launcher/Makefile.toml b/fuzzers/libfuzzer_libpng_launcher/Makefile.toml index f3188d9f44..af52f871bc 100644 --- a/fuzzers/libfuzzer_libpng_launcher/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_launcher/Makefile.toml @@ -96,7 +96,13 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null & +timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null >fuzz_stdout.log || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_stb_image/Makefile.toml b/fuzzers/libfuzzer_stb_image/Makefile.toml index c4e5ca6f89..344e23ce5c 100644 --- a/fuzzers/libfuzzer_stb_image/Makefile.toml +++ b/fuzzers/libfuzzer_stb_image/Makefile.toml @@ -60,9 +60,15 @@ windows_alias = "test_windows" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} & +timeout 11s ./${FUZZER_NAME} >fuzz_stdout.log & sleep 0.2 -timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null & +timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml b/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml index 0206937e37..752c07bc88 100644 --- a/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml +++ b/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml @@ -58,7 +58,13 @@ windows_alias = "test_windows" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} 2>/dev/null & +timeout 11s ./${FUZZER_NAME} 2>/dev/null >fuzz_stdout.log || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/nautilus_sync/Makefile.toml b/fuzzers/nautilus_sync/Makefile.toml index a58d2e6fda..bc548bc615 100644 --- a/fuzzers/nautilus_sync/Makefile.toml +++ b/fuzzers/nautilus_sync/Makefile.toml @@ -104,7 +104,13 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s ./${FUZZER_NAME} --cores 0 2>/dev/null & +timeout 11s ./${FUZZER_NAME} --cores 0 >fuzz_stdout.log 2>/dev/null || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "fuzzer" ] diff --git a/fuzzers/qemu_arm_launcher/Makefile.toml b/fuzzers/qemu_arm_launcher/Makefile.toml index 2984ffe5e0..61856a12f6 100644 --- a/fuzzers/qemu_arm_launcher/Makefile.toml +++ b/fuzzers/qemu_arm_launcher/Makefile.toml @@ -149,7 +149,13 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s cargo run --release ./${FUZZER_NAME} 2>/dev/null & +timeout 11s ./target/release/qemu_arm_launcher ./{FUZZER_NAME} 2>/dev/null >fuzz_stdout.log || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "harness", "fuzzer" ] diff --git a/fuzzers/qemu_launcher/Makefile.toml b/fuzzers/qemu_launcher/Makefile.toml index d395454a7c..36b4ddad4c 100644 --- a/fuzzers/qemu_launcher/Makefile.toml +++ b/fuzzers/qemu_launcher/Makefile.toml @@ -77,7 +77,13 @@ windows_alias = "unsupported" script_runner = "@shell" script=''' rm -rf libafl_unix_shmem_server || true -timeout 11s cargo run --release ./${FUZZER_NAME} 2>/dev/null & +timeout 11s ./target/release/qemu_launcher ./${FUZZER_NAME} 2>/dev/null >fuzz_stdout.log || true +if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then + echo "Fuzzer does not generate any testcases or any crashes" + exit 1 +else + echo "Fuzzer is working" +fi ''' dependencies = [ "harness", "fuzzer" ]