Remove scripts/test_fuzzer.sh, Rewrite in Rust (#3205)

* no more shellscript

* metadatas

* clp

* clippo

* this language breaks everyday

* stop

* 2404?

* purge rust

* everything is broken

* test

* ci dead

* aa

* aa

* remove broken shit

* remove broken shit

* broken shit

* a

* lol

* aaaaa

* AI fix

* aa

* aaa

* yes?

* a

* FK

* tmate

* wtf who tf did this shit????

* aa

* useless printlnt

* for system too

* unset first

.github/workflows/build_and_test.yml

@@ -334,7 +334,7 @@ jobs:
       - name: Build and run example fuzzers (Linux)
         if: runner.os == 'Linux'
         shell: bash
-        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }}
+        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} cargo run --manifest-path ./utils/ci_runner/Cargo.toml -- ${{ matrix.fuzzer }}
 
   changes:
     runs-on: ubuntu-24.04
@@ -375,7 +375,7 @@ jobs:
       - name: Build and run example fuzzers (Linux)
         if: runner.os == 'Linux'
         shell: bash
-        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }}
+        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} cargo run --manifest-path ./utils/ci_runner/Cargo.toml -- ${{ matrix.fuzzer }}
 
   fuzzers-qemu-user:
     needs:
@@ -406,7 +406,7 @@ jobs:
       - name: Build and run example QEMU fuzzers (Linux)
         if: runner.os == 'Linux'
         shell: bash
-        run: ARCH=${{ matrix.arch }} RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }}
+        run: unset RUSTC && ARCH=${{ matrix.arch }} RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} cargo run --manifest-path ./utils/ci_runner/Cargo.toml --  ${{ matrix.fuzzer }}
 
   librasan-build:
       runs-on: ubuntu-24.04
@@ -466,7 +466,7 @@ jobs:
       - name: Build and run example QEMU fuzzers (Linux)
         if: runner.os == 'Linux'
         shell: bash
-        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }}
+        run: unset RUSTC && RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} cargo run --manifest-path ./utils/ci_runner/Cargo.toml --  ${{ matrix.fuzzer }}
 
   nostd-build:
     runs-on: ubuntu-24.04

.github/workflows/fuzzer-tester-prepare/action.yml

@@ -27,7 +27,6 @@ runs:
       uses: extractions/setup-just@v2
       with:
         just-version: '1.40.0'
-    
     - name: Add wasm target
       if: ${{ inputs.fuzzer-name == './fuzzers/fuzz_anything/baby_fuzzer_wasm' }}
       shell: bash

.github/workflows/qemu-fuzzer-tester-prepare/action.yml

@@ -9,7 +9,13 @@ runs:
         apt-get update 
         apt-get install -y qemu-utils sudo python3-msgpack python3-jinja2 curl python3-dev gcc-arm-none-eabi \
           gcc-arm-linux-gnueabi g++-arm-linux-gnueabi
+    - name: Remove old rust
+      shell: bash
+      run: sudo apt purge -y 'rust*' 'cargo*'
     - uses: dtolnay/rust-toolchain@stable
+      env:
+        RUSTUP_HOME: /usr/local/rustup
+        CARGO_HOME: /usr/local/cargo
     - name: enable mult-thread for `make`
       shell: bash
       run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)"

.github/workflows/ubuntu-prepare/action.yml

@@ -3,12 +3,16 @@ description: Sets up the Rust environment for the CI workflow
 runs:
   using: composite
   steps:
+    - name: Uninstall all currently installed rust
+      shell: bash
+      run: |
+        sudo apt purge -y 'cargo*' 'rust*'
     - name: Install and cache deps
       shell: bash
       run: |
         sudo apt-get update
         sudo apt-get install -y curl lsb-release wget software-properties-common gnupg shellcheck pax-utils libsqlite3-dev libpixman-1-dev libc6-dev gcc g++ build-essential libglib2.0-dev
-    - uses: dtolnay/rust-toolchain@stable
+    - uses: dtolnay/rust-toolchain@nightly
     - name: install just
       uses: extractions/setup-just@v2
       with:
@@ -22,9 +26,6 @@ runs:
     - name: Install cargo-hack
       shell: bash
       run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
-    - name: Add nightly
-      shell: bash
-      run: rustup toolchain install nightly --allow-downgrade
     - name: Default to nightly
       shell: bash
       run: rustup default nightly

Cargo.toml

@@ -26,6 +26,7 @@ members = [
   "utils/gramatron/construct_automata",
   "utils/libafl_benches",
   "utils/libafl_jumper",
+  "utils/ci_runner",
 ]
 
 default-members = [

Dockerfile

@@ -225,7 +225,5 @@ RUN cargo build && cargo build --release
 # Copy fuzzers over
 COPY fuzzers fuzzers
 
-# RUN ./scripts/test_fuzzer.sh --no-fmt
-
 ENTRYPOINT [ "/bin/bash", "-c" ]
 CMD ["/bin/bash"]

libafl/Cargo.toml

@@ -8,7 +8,7 @@ authors = [
 description = "Slot your own fuzzers together and extend their features using Rust"
 documentation = "https://docs.rs/libafl"
 repository = "https://github.com/AFLplusplus/LibAFL/"
-readme = "../README.md"
+readme = "../../README.md"
 license = "MIT OR Apache-2.0"
 keywords = ["fuzzing", "testing", "security"]
 edition = "2024"

scripts/test_fuzzer.sh

@@ -1,63 +0,0 @@
-#!/bin/bash
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-cd "$SCRIPT_DIR/.." || exit 1
-# TODO: This should be rewritten in rust, a Makefile, or some platform-independent language
-
-
-fuzzer_to_test="$1"
-export PROFILE=dev
-export PROFILE_DIR=debug
-
-echo "Testing" "$fuzzer_to_test"
-
-# override default profile settings for speed
-# export RUSTFLAGS="-C prefer-dynamic"
-for profile in DEV RELEASE; # loop for all profiles
-do
-    export CARGO_PROFILE_"$profile"_OPT_LEVEL=z # optimize for size
-    # runs into shared target dir bug:
-    # [pid 351769] openat(AT_FDCWD, "LibAFL/target/release/deps/libc-dbff77a14da5d893.libc.5deb7d4a-cgu.0.rcgu.dwo", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
-    # error: failed to build archive: No such file or directory
-    # export CARGO_PROFILE_"$profile"_SPLIT_DEBUGINFO=unpacked # minimize debug info
-    # export CARGO_PROFILE_"$profile"_PANIC=abort
-    export CARGO_PROFILE_"$profile"_INCREMENTAL=true
-done
-
-# shellcheck disable=SC2116
-for fuzzer in $(echo "$fuzzer_to_test");
-do
-    # skip nyx test on non-linux platforms
-    if [[ $fuzzer == *"nyx_"* ]] && [[ $(uname -s) != "Linux" ]]; then
-        continue
-    fi
-
-    (
-        cd "$fuzzer" || exit 1
-        # Clippy checks
-        if [ "$1" != "--no-clippy" ]; then
-            echo "[*] Running clippy for $fuzzer"
-            cargo clippy || exit 1
-        else
-            echo "[+] Skipping fmt and clippy for $fuzzer (--no-clippy specified)"
-        fi
-
-        if [ -e ./Makefile.toml ] && grep -qF "skip_core_tasks = true" Makefile.toml; then
-            echo "[*] Building $fuzzer (running tests is not supported in this context)"
-            just build || exit 1
-            echo "[+] Done building $fuzzer"
-        elif [ -e ./Makefile.toml ]; then
-            echo "[*] Testing $fuzzer"
-            just test || exit 1
-            echo "[+] Done testing $fuzzer"
-        elif [ -e ./Justfile ]; then
-            echo "[*] Testing $fuzzer"
-            just test || exit 1
-            echo "[+] Done testing $fuzzer"
-        else
-            echo "[*] Building $fuzzer"
-            cargo build || exit 1
-            echo "[+] Done building $fuzzer"
-        fi
-    )
-done

utils/ci_runner/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "ci_runner"
+edition = "2024"
+authors = ["Dongjia Zhang <tokazerkje@outlook.com>"]
+version.workspace = true
+license.workspace = true
+description = "libafl CI tools for testing fuzzers"
+repository = "https://github.com/AFLplusplus/LibAFL/"
+keywords = ["fuzzing", "testing", "security"]
+categories = [
+  "development-tools::testing",
+  "emulators",
+  "embedded",
+  "os",
+  "no-std",
+]
+readme = "../README.md"
+
+[dependencies]
+
+[lints]
+workspace = true

utils/ci_runner/src/main.rs

@@ -0,0 +1,100 @@
+use std::{
+    env, fs,
+    path::PathBuf,
+    process::{Command, exit},
+};
+
+fn run() -> Result<(), Box<dyn core::error::Error>> {
+    let me = env::args().next().ok_or("no argv[0]")?;
+    let exe_path = fs::canonicalize(&me)?;
+    let project_dir = exe_path
+        .parent()
+        .ok_or("failed to get libafl root dir")?
+        .parent()
+        .ok_or("failed to get libafl root dir")?
+        .parent()
+        .ok_or("failed to get libafl root dir")?;
+
+    env::set_current_dir(project_dir)?;
+    let args: Vec<String> = env::args().collect();
+    let mut fuzzers_to_test = Vec::new();
+
+    // take the first arg as the fuzzer name
+    if args.len() < 2 {
+        eprintln!("Expected fuzzer name as argument when RUN_ON_CI is set");
+        exit(1);
+    }
+    fuzzers_to_test.push(PathBuf::from(&args[1]));
+    unsafe {
+        env::set_var("PROFILE", "dev");
+        env::set_var("PROFILE_DIR", "debug");
+    }
+
+    for f in &fuzzers_to_test {
+        print!(" {}", f.to_string_lossy());
+    }
+    println!();
+
+    // 5) set cargo profile envs
+    for profile in &["DEV", "RELEASE"] {
+        unsafe {
+            env::set_var(format!("CARGO_PROFILE_{profile}_OPT_LEVEL"), "z");
+            env::set_var(format!("CARGO_PROFILE_{profile}_INCREMENTAL"), "true");
+        }
+    }
+
+    // 6) for each fuzzer
+    for fuzzer in fuzzers_to_test {
+        let fuzzer_name = fuzzer.to_string_lossy();
+        let name = fuzzer
+            .file_name()
+            .and_then(|n| n.to_str())
+            .unwrap_or_else(|| fuzzer_name.as_ref())
+            .to_string();
+
+        let path = project_dir.join(&fuzzer);
+
+        // Clippy
+        let do_clippy = args.get(1).is_none_or(|s| s != "--no-clippy");
+        if do_clippy {
+            println!("[*] Running clippy for {name}");
+            let status = Command::new("cargo")
+                .arg("clippy")
+                .current_dir(&path)
+                .status()?;
+            if !status.success() {
+                exit(1);
+            }
+        } else {
+            println!("[+] Skipping fmt and clippy for {name} (--no-clippy specified)");
+        }
+
+        if path.join("Justfile").exists() {
+            println!("[*] Testing {name}");
+            let status = Command::new("just")
+                .arg("test")
+                .current_dir(&path)
+                .status()?;
+            if !status.success() {
+                exit(1);
+            }
+            println!("[+] Done testing {name}");
+        } else {
+            println!("[*] Building {name}");
+            let status = Command::new("cargo")
+                .arg("build")
+                .current_dir(&path)
+                .status()?;
+            if !status.success() {
+                exit(1);
+            }
+            println!("[+] Done building {name}");
+        }
+    }
+
+    Ok(())
+}
+
+fn main() {
+    run().unwrap();
+}