cargo-fuzz testing (#2081)

* Build cargo-fuzz end to end

* Workaroudn failing CI

* Fix toml

* Build

* Exclude from workspace

* Fix version

* Fix workspace

* Fix workspace

* FIx workspace

* Fix path

* Set path correctly

* Install llvm-tools

* Test with distro llvm installed

* Use build

* complete

* remove from includes

* add back fuzzer tests, format

* add test for whether all fuzzers are actually tested in pipeline

* whoops

* I hate CI I hate CI I hate CI

* revert removal of llvm

* sh compatibility

* sigh

---------

Co-authored-by: Max Ammann <max@maxammann.org>

.github/workflows/build_and_test.yml

@@ -2,9 +2,9 @@ name: build and test
 
 on:
   push:
-    branches: [main, "pr/**"]
+    branches: [ main, "pr/**" ]
   pull_request:
-    branches: [main]
+    branches: [ main ]
   workflow_dispatch:
   merge_group:
 env:
@@ -17,7 +17,7 @@ jobs:
   common:
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ ubuntu-latest, windows-latest, macOS-latest ]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Install mimetype
@@ -70,7 +70,7 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        llvm-version: ["11", "12", "13", "14", "16", "17"]
+        llvm-version: [ "11", "12", "13", "14", "16", "17" ]
     steps:
       - name: Remove Dotnet & Haskell
         run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
@@ -318,12 +318,22 @@ jobs:
       - name: Format Check
         run: cargo fmt -- --check
 
+  fuzzers-preflight:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Fuzzer in CI Check
+        run: ./scripts/check_tested_fuzzers.sh
+
   fuzzers:
-    needs: ubuntu
+    needs:
+      - ubuntu
+      - fuzzers-preflight
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ ubuntu-latest ]
         fuzzer:
+          - ./fuzzers/cargo_fuzz
           - ./fuzzers/fuzzbench_fork_qemu
           - ./fuzzers/libfuzzer_stb_image_sugar
           - ./fuzzers/nyx_libxml2_standalone

fuzzers/cargo_fuzz/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "cargo_fuzz_test"
+edition = "2021"
+version = "0.0.0"
+description = "test"
+authors = ["Andrea Fioraldi <andreafioraldi@gmail.com>", "Dominik Maier <domenukk@gmail.com>"]
+repository = "https://github.com/AFLplusplus/LibAFL/"
+keywords = ["fuzzing", "testing", "compiler"]
+categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

fuzzers/cargo_fuzz/Makefile.toml

@@ -0,0 +1,44 @@
+[env]
+
+[tasks.unsupported]
+script_runner = "@shell"
+script = '''
+echo "Cargo-make not integrated yet on this"
+'''
+
+[tasks.install_llvm_tools]
+command = "rustup"
+args = ["toolchain", "install", "nightly", "--component", "llvm-tools-preview"]
+
+
+[tasks.install_cargo_fuzz]
+command = "cargo"
+args = ["install", "cargo-fuzz"]
+
+# Fuzzer
+[tasks.build]
+command = "cargo"
+args = ["+nightly", "fuzz", "build", "fuzz_target_1"]
+dependencies = ["install_cargo_fuzz", "install_llvm_tools"]
+
+[tasks.test]
+linux_alias = "test_unix"
+mac_alias = "unsupported"
+windows_alias = "unsupported"
+
+[tasks.test_unix]
+script = '''
+timeout 30s cargo +nightly fuzz run fuzz_target_1 2>&1 | tee fuzz_stdout.log || true
+if grep -qa "objectives: 1" fuzz_stdout.log; then
+    echo "Fuzzer is working"
+else
+    echo "Fuzzer does not generate any testcases or any crashes"
+    exit 1
+fi
+'''
+dependencies = ["build"]
+
+# Clean
+[tasks.clean]
+command = "rm "
+args = ["-rf", "fuzz/target"]

fuzzers/cargo_fuzz/README.md

@@ -0,0 +1,3 @@
+# cargo-fuzz
+
+This is a minimalistic example how to use LibAFL with cargo-fuzz. It uses the `libafl_libfuzzer` comatability layer to be libFuzzer compatiable.

fuzzers/cargo_fuzz/fuzz/.gitignore

@@ -0,0 +1,4 @@
+target
+corpus
+artifacts
+coverage

fuzzers/cargo_fuzz/fuzz/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "libafl-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[workspace]
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+
+[dependencies.cargo_fuzz_test]
+path = ".."
+
+[dependencies.libfuzzer-sys]
+path = "../../../libafl_libfuzzer"
+package = "libafl_libfuzzer"
+
+[[bin]]
+name = "fuzz_target_1"
+path = "fuzz_targets/fuzz_target_1.rs"
+test = false
+doc = false
+bench = false

fuzzers/cargo_fuzz/fuzz/fuzz_targets/fuzz_target_1.rs

@@ -0,0 +1,6 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use cargo_fuzz_test::do_thing;
+
+fuzz_target!(|data: &[u8]| do_thing(data));

fuzzers/cargo_fuzz/src/lib.rs

@@ -0,0 +1,11 @@
+pub fn do_thing(data: &[u8]) {
+    if data.get(0) == Some(&b'a') {
+        if data.get(1) == Some(&b'b') {
+            if data.get(2) == Some(&b'c') {
+                if data.get(3) == Some(&b'd') {
+                    panic!("We found the objective!");
+                }
+            }
+        }
+    }
+}

scripts/check_tested_fuzzers.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+ret=0
+
+while read -r fuzzdir; do
+  if ! grep -qa "$fuzzdir" .github/workflows/build_and_test.yml; then
+    ret=1
+    echo "Missing fuzzer ${fuzzdir} in CI tests!"
+  fi
+  if grep -qa "# - $fuzzdir" .github/workflows/build_and_test.yml; then
+    echo "Fuzzer ${fuzzdir} is explicitly ignored"
+  fi
+done < <(
+          find ./fuzzers -mindepth 1 -maxdepth 1 -type d
+          find ./fuzzers/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d
+        )
+
+exit $ret