Check CI result on cargo make test for available fuzzers (#1107)

* fix libfuzzer_libpng_cmin

* fix libfuzzer_libpng_ctx

* revert libfuzzer_libpng_cmin and check ci by grepping broker stdout result instead

* revert libfuzzer_libpng_ctx and check ci by grepping broker stdout result instead

* add check ci for fuzzers

* add check ci for fuzzers

* add check ci

* add fuzzbench test

* add validation for qemu fuzzer

---------

Co-authored-by: Dongjia "toka" Zhang <tokazerkje@outlook.com>

fuzzers/baby_fuzzer_swap_differential/Makefile.toml

@@ -31,7 +31,13 @@ windows_alias = "unsupported"
 [tasks.test_unix]
 script_runner = "@shell"
 script='''
-timeout 10s ${CARGO_TARGET_DIR}/release/${FUZZER_NAME}
+timeout 10s ${CARGO_TARGET_DIR}/release/${FUZZER_NAME} >fuzz_stdout.log || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/frida_libpng/Makefile.toml

@@ -109,7 +109,13 @@ windows_alias = "test_windows"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 10s ./${FUZZER_NAME} -F LLVMFuzzerTestOneInput -H ./libpng-harness.so -l ./libpng-harness.so &
+timeout 10s ./${FUZZER_NAME} -F LLVMFuzzerTestOneInput -H ./libpng-harness.so -l ./libpng-harness.so >fuzz_stdout.log || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer", "harness" ]
 

fuzzers/fuzzbench/Makefile.toml

@@ -80,7 +80,13 @@ rm -rf libafl_unix_shmem_server || true
 mkdir in || true
 echo a > in/a
 # Allow sigterm as exit code 
-timeout 11s ./${FUZZER_NAME} -o out -i in || [ $? -eq 124 ]
+timeout 11s ./${FUZZER_NAME} -o out -i in >fuzz_stdout.log || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 rm -rf out || true
 rm -rf in || true
 '''

fuzzers/fuzzbench_fork_qemu/Makefile.toml

@@ -79,7 +79,13 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 10s cargo run --release -- --libafl-in ../libfuzzer_libpng/corpus --libafl-out ./out ./${FUZZER_NAME} &
+timeout 10s ./target/release/fuzzbench_fork_qemu --libafl-in ../libfuzzer_libpng/corpus --libafl-out ./out ./${FUZZER_NAME} || true
+if [ -z "$(ls -A out/)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "harness", "fuzzer" ]
 

fuzzers/fuzzbench_qemu/Makefile.toml

@@ -79,7 +79,13 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 10s cargo run --release -- --libafl-in ../libfuzzer_libpng/corpus --libafl-out ./out ./${FUZZER_NAME} &
+timeout 10s cargo run --release -- --libafl-in ../libfuzzer_libpng/corpus --libafl-out ./out ./${FUZZER_NAME} || true
+if [ -z "$(ls -A out/)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "harness", "fuzzer" ]
 

fuzzers/fuzzbench_text/Makefile.toml

@@ -81,7 +81,13 @@ rm -rf libafl_unix_shmem_server || true
 mkdir in || true
 echo a > in/a
 # Allow sigterm as exit code 
-timeout 11s ./${FUZZER_NAME} -o out -i in || [ $? -eq 124 ]
+timeout 11s ./${FUZZER_NAME} -o out -i in >fuzz_stdout.log || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 rm -rf out || true
 rm -rf in || true
 '''

fuzzers/libfuzzer_libmozjpeg/Makefile.toml

@@ -97,9 +97,15 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} &
+timeout 11s ./${FUZZER_NAME} >fuzz_stdout.log &
 sleep 0.2
-timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null &
+timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/libfuzzer_libpng/Makefile.toml

@@ -159,9 +159,15 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} &
+timeout 11s ./${FUZZER_NAME} >fuzz_stdout.log &
 sleep 0.2
-timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null &
+timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/libfuzzer_libpng_accounting/Makefile.toml

@@ -96,7 +96,13 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null &
+timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus >fuzz_stdout.log 2>/dev/null || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/libfuzzer_libpng_cmin/Makefile.toml

@@ -159,9 +159,15 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} &
+timeout 11s ./${FUZZER_NAME} >fuzz_stdout.log &
 sleep 0.2
-timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null &
+timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/libfuzzer_libpng_ctx/Makefile.toml

@@ -5,6 +5,7 @@ CARGO_TARGET_DIR = { value = "${PROJECT_DIR}/target", condition = { env_not_set
 LIBAFL_CC = '${CARGO_TARGET_DIR}/release/libafl_cc'
 LIBAFL_CXX = '${CARGO_TARGET_DIR}/release/libafl_cxx'
 FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}'
+
 PROJECT_DIR = { script = ["pwd"] }
 
 [tasks.unsupported]
@@ -96,7 +97,13 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null &
+timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus >fuzz_stdout.log 2>/dev/null || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/libfuzzer_libpng_launcher/Makefile.toml

@@ -96,7 +96,13 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null &
+timeout 11s ./${FUZZER_NAME} --cores 0 --input ./corpus 2>/dev/null >fuzz_stdout.log || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/libfuzzer_stb_image/Makefile.toml

@@ -60,9 +60,15 @@ windows_alias = "test_windows"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} &
+timeout 11s ./${FUZZER_NAME} >fuzz_stdout.log &
 sleep 0.2
-timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null &
+timeout 10s ./${FUZZER_NAME} >/dev/null 2>/dev/null || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/libfuzzer_stb_image_sugar/Makefile.toml

@@ -58,7 +58,13 @@ windows_alias = "test_windows"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} 2>/dev/null &
+timeout 11s ./${FUZZER_NAME} 2>/dev/null >fuzz_stdout.log || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/nautilus_sync/Makefile.toml

@@ -104,7 +104,13 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s ./${FUZZER_NAME} --cores 0 2>/dev/null &
+timeout 11s ./${FUZZER_NAME} --cores 0 >fuzz_stdout.log 2>/dev/null || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "fuzzer" ]
 

fuzzers/qemu_arm_launcher/Makefile.toml

@@ -149,7 +149,13 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s cargo run --release ./${FUZZER_NAME} 2>/dev/null &
+timeout 11s ./target/release/qemu_arm_launcher ./{FUZZER_NAME} 2>/dev/null >fuzz_stdout.log || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "harness", "fuzzer" ]
 

fuzzers/qemu_launcher/Makefile.toml

@@ -77,7 +77,13 @@ windows_alias = "unsupported"
 script_runner = "@shell"
 script='''
 rm -rf libafl_unix_shmem_server || true
-timeout 11s cargo run --release ./${FUZZER_NAME} 2>/dev/null &
+timeout 11s ./target/release/qemu_launcher ./${FUZZER_NAME} 2>/dev/null >fuzz_stdout.log || true
+if [ -z "$(grep "corpus: 30" fuzz_stdout.log)" ]; then
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+else
+    echo "Fuzzer is working"
+fi
 '''
 dependencies = [ "harness", "fuzzer" ]