CI: Only test fuzzers with diffing deps (#940)

* build and test fuzzer crate

* diffing fuzzers ci only

* clippy

* clippy merda

* clippy merde

* improve it

* comment

* split ubuntu CI workflow

* fix

.github/workflows/build_and_test.yml

@@ -69,8 +69,6 @@ jobs:
         packages: llvm llvm-dev clang ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev
     - name: get clang version
       run: command -v llvm-config && clang -v
-    - name: Install cargo-hack
-      run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
     - name: Add nightly rustfmt and clippy
       run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade
     - uses: actions/checkout@v3
@@ -95,9 +93,35 @@ jobs:
     - name: Test Docs
       run: cargo +nightly test --doc --all-features
 
-    # ---- build and feature check ----
+    # ---- build normal and examples ----
     - name: Run a normal build
       run: cargo build --verbose
+    - name: Build examples
+      run: cargo build --examples --verbose
+
+  ubuntu-check:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Remove Dotnet & Haskell
+      run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
+    - uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+    - name: Install and cache deps
+      uses: awalsh128/cache-apt-pkgs-action@v1.1.0
+      with:
+        packages: llvm llvm-dev clang ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev
+    - name: get clang version
+      run: command -v llvm-config && clang -v
+    - name: Install cargo-hack
+      run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
+    - name: Add nightly
+      run: rustup toolchain install nightly --allow-downgrade
+    - uses: actions/checkout@v3
+    - uses: Swatinem/rust-cache@v2
+
+    # ---- build and feature check ----
     # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs
     - name: Check each feature
       # Skipping `python` as it has to be built with the `maturin` tool
@@ -106,8 +130,6 @@ jobs:
       run: cargo hack check --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode --no-dev-deps
     - name: Check nightly features
       run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus
-    - name: Build examples
-      run: cargo build --examples --verbose
 
   ubuntu-concolic:
     runs-on: ubuntu-latest
@@ -182,10 +204,10 @@ jobs:
     - uses: Swatinem/rust-cache@v2
     - name: Build and run example fuzzers (Linux)
       if: runner.os == 'Linux'
-      run: ./scripts/test_all_fuzzers.sh
+      run: RUN_ON_CI=1 ./scripts/test_all_fuzzers.sh
     - name: Build and run example fuzzers (macOS)
       if: runner.os == 'macOS' # use bash v4
-      run: /usr/local/bin/bash ./scripts/test_all_fuzzers.sh
+      run: /usr/local/bin/bash -c 'RUN_ON_CI=1 ./scripts/test_all_fuzzers.sh'
 
   nostd-build:
     runs-on: ubuntu-latest

Cargo.toml

@@ -16,6 +16,7 @@ members = [
     "utils/deexit",
     "utils/gramatron/construct_automata",
     "utils/libafl_benches",
+    "utils/build_and_test_fuzzers",
 ]
 default-members = [
     "libafl",

fuzzers/qemu_systemmode/src/fuzzer.rs

@@ -16,8 +16,7 @@ use libafl::{
     corpus::{Corpus, InMemoryCorpus, OnDiskCorpus},
     events::EventConfig,
     executors::{ExitKind, TimeoutExecutor},
-    feedback_or,
-    feedback_or_fast,
+    feedback_or, feedback_or_fast,
     feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
     fuzzer::{Fuzzer, StdFuzzer},
     inputs::{BytesInput, HasTargetBytes},
@@ -116,7 +115,7 @@ pub fn fuzz() {
                 let mut pcs = (0..emu.num_cpus())
                     .map(|i| emu.cpu_from_index(i))
                     .map(|cpu| -> Result<u32, String> { cpu.read_reg(Regs::Pc) });
-                let _ret = match pcs
+                let ret = match pcs
                     .find(|pc| (breakpoint..breakpoint + 5).contains(pc.as_ref().unwrap_or(&0)))
                 {
                     Some(_) => ExitKind::Ok,

scripts/test_all_fuzzers.sh

@@ -5,8 +5,14 @@ cd "$SCRIPT_DIR/.." || exit 1
 
 # TODO: This should be rewritten in rust, a Makefile, or some platform-independent language
 
-fuzzers=$(find ./fuzzers -mindepth 1 -maxdepth 1 -type d)
-backtrace_fuzzers=$(find ./fuzzers/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d)
+if [[ -z "${RUN_ON_CI}" ]]; then
+  fuzzers=$(find ./fuzzers -mindepth 1 -maxdepth 1 -type d)
+  backtrace_fuzzers=$(find ./fuzzers/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d)
+else
+  cargo build -p build_and_test_fuzzers
+  fuzzers=$(cargo run -p build_and_test_fuzzers)
+  backtrace_fuzzers=""
+fi
 
 libafl=$(pwd)
 

utils/build_and_test_fuzzers/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "build_and_test_fuzzers"
+version = "0.1.0"
+edition = "2021"
+description = "Get diffing fuzzers from the last commit"
+repository = "https://github.com/AFLplusplus/LibAFL/"
+license = "MIT OR Apache-2.0"
+readme = "../../README.md"
+keywords = ["ci"]
+categories = ["development-tools::testing"]
+
+[dependencies]
+cargo_toml = "0.13"
+walkdir = "2"

utils/build_and_test_fuzzers/src/diffing.rs

@@ -0,0 +1,136 @@
+use std::{
+    collections::{HashMap, HashSet},
+    ffi::OsStr,
+    path::{Component, PathBuf},
+    process::{Command, Stdio},
+};
+
+use walkdir::WalkDir;
+
+#[must_use]
+pub fn run_git_diff(args: &[&str]) -> String {
+    let output = Command::new("git")
+        .stdout(Stdio::piped())
+        .arg("diff")
+        .args(args)
+        .output()
+        .expect("git diff failed");
+
+    String::from_utf8(output.stdout).unwrap()
+}
+
+#[must_use]
+pub fn get_diffing_files(commit_name: &str) -> Vec<PathBuf> {
+    let args = vec!["--name-only", commit_name];
+    let diff = run_git_diff(&args);
+    diff.lines()
+        .map(PathBuf::from)
+        .filter(|x| x.is_file())
+        .collect()
+}
+
+#[must_use]
+pub fn get_diffing_crates(diffing_files: &[PathBuf]) -> HashSet<PathBuf> {
+    // TODO maybe consider using a combination of this and https://docs.rs/cargo/0.28.0/cargo/sources/path/struct.PathSource.html
+    let mut crates = HashSet::default();
+    for file in diffing_files {
+        if let Some(dir) = file.parent() {
+            let manifest = dir.join("Cargo.toml");
+            if manifest.is_file()
+                && cargo_toml::Manifest::from_path(&manifest)
+                    .expect("cannot read manifest")
+                    .package
+                    .is_some()
+            {
+                crates.insert(manifest);
+            } else if let Some(dir1) = dir.parent() {
+                let manifest = dir1.join("Cargo.toml");
+                if manifest.is_file()
+                    && cargo_toml::Manifest::from_path(&manifest)
+                        .expect("cannot read manifest")
+                        .package
+                        .is_some()
+                {
+                    crates.insert(manifest);
+                } else if let Some(dir2) = dir1.parent() {
+                    let manifest = dir2.join("Cargo.toml");
+                    if manifest.is_file()
+                        && cargo_toml::Manifest::from_path(&manifest)
+                            .expect("cannot read manifest")
+                            .package
+                            .is_some()
+                    {
+                        crates.insert(manifest);
+                    }
+                }
+            }
+        }
+    }
+    crates
+}
+
+#[must_use]
+pub fn find_all_crates() -> HashSet<PathBuf> {
+    let mut crates = HashSet::default();
+    for entry in WalkDir::new(".")
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|e| {
+            !e.file_type().is_dir()
+                && e.path()
+                    .components()
+                    .filter(|c| *c == Component::Normal(OsStr::new("target")))
+                    .count()
+                    == 0
+        })
+    {
+        let file_name = String::from(entry.file_name().to_string_lossy());
+        if file_name == "Cargo.toml" {
+            crates.insert(entry.into_path());
+        }
+    }
+    crates
+}
+
+#[allow(clippy::implicit_hasher)]
+pub fn extend_diffing_crates_with_deps(
+    diffing_crates: &mut HashSet<PathBuf>,
+    all_crates: &HashSet<PathBuf>,
+) {
+    let mut crate_names = HashMap::<String, PathBuf>::new();
+    for file in all_crates {
+        let manifest = cargo_toml::Manifest::from_path(file).expect("cannot read manifest");
+        if let Some(package) = manifest.package {
+            crate_names.insert(package.name, file.clone());
+        }
+    }
+
+    let not_diffing = all_crates.difference(diffing_crates);
+    let mut deps_map = HashMap::<String, HashSet<_>>::new();
+
+    for file in not_diffing {
+        let manifest = cargo_toml::Manifest::from_path(file).expect("cannot read manifest");
+        if let Some(package) = manifest.package {
+            for (dep, _) in manifest.dependencies {
+                deps_map
+                    .entry(dep)
+                    .or_insert(HashSet::default())
+                    .insert(package.name.clone());
+            }
+        }
+    }
+
+    let diffing_crates_orig = diffing_crates.clone();
+    for file in diffing_crates_orig {
+        let manifest = cargo_toml::Manifest::from_path(file).expect("cannot read manifest");
+        if let Some(package) = manifest.package {
+            if let Some(names) = deps_map.get(&package.name) {
+                for name in names {
+                    if let Some(path) = crate_names.get(name) {
+                        diffing_crates.insert(path.clone());
+                    }
+                }
+            }
+        }
+    }
+}

utils/build_and_test_fuzzers/src/main.rs

@@ -0,0 +1,20 @@
+pub mod diffing;
+use std::env;
+
+pub use diffing::*;
+
+fn main() {
+    let args: Vec<String> = env::args().collect();
+    let commit = if args.len() > 1 { &args[1] } else { "HEAD^" };
+
+    let files = get_diffing_files(commit);
+    let mut diff_crates = get_diffing_crates(&files);
+    let all_crates = find_all_crates();
+    extend_diffing_crates_with_deps(&mut diff_crates, &all_crates);
+
+    for file in diff_crates {
+        if file.starts_with("./fuzzers") || file.starts_with("fuzzers") {
+            print!("{} ", file.parent().unwrap().display());
+        }
+    }
+}