Rusty LibAFL fmt (#2271)

Cargo.toml

@@ -35,6 +35,7 @@ exclude = [
     "libafl_qemu/libafl_qemu_sys",
     "utils/noaslr",
     "utils/gdb_qemu",
+    "utils/libafl_fmt",
     "scripts",
 ]
 

fuzzers/cargo_fuzz/fuzz/fuzz_targets/fuzz_target_1.rs

@@ -1,6 +1,6 @@
 #![no_main]
 
-use libfuzzer_sys::fuzz_target;
 use cargo_fuzz_test::do_thing;
+use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| do_thing(data));

scripts/fmt_all.sh

@@ -3,52 +3,8 @@
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 LIBAFL_DIR=$(realpath "$SCRIPT_DIR/..")
 
-# TODO: This should be rewritten in rust, a Makefile, or some platform-independent language
-
-if ! command -v parallel > /dev/null; then
-    echo "Parallel could not be found. Please install parallel (often found in the 'moreutils' package)."
-    exit 1
-fi
-
 if [ "$1" = "check" ]; then
-  CHECK=1
+  cargo run --manifest-path "$LIBAFL_DIR/utils/libafl_fmt/Cargo.toml" --release -- -c
-fi
-
-# Find main rust crates
-CRATES_TO_FMT=$(find "$LIBAFL_DIR" -type d \( -path "*/fuzzers/*" -o -path "*/target/*" -o -path "*/utils/noaslr" -o -path "*/utils/gdb_qemu" -o -path "*/docs/listings/baby_fuzzer/listing-*" \) -prune \
-  -o -name "Cargo.toml" -print \
-  | grep -v "$LIBAFL_DIR/Cargo.toml")$'\n'
-
-# Find fuzzer crates
-CRATES_TO_FMT+=$(find "$LIBAFL_DIR/fuzzers" "$LIBAFL_DIR/fuzzers/backtrace_baby_fuzzers" "$LIBAFL_DIR/libafl_libfuzzer/libafl_libfuzzer_runtime" -maxdepth 2 -name "Cargo.toml" -print)
-
-echo "Welcome to the happy fmt script. :)"
-
-if [ "$CHECK" ]; then
-  echo "Running fmt in check mode."
-  CARGO_FLAGS="--check"
-  CLANG_FLAGS="--dry-run"
-fi
-
-echo "[*] Formatting Rust crates..."
-if ! echo "$CRATES_TO_FMT" | parallel --halt-on-error 1 "echo '[*] Running fmt for {}'; cargo +nightly fmt $CARGO_FLAGS --manifest-path {}"
-then
-  echo "Rust format failed."
-  exit 1
-fi
-
-if command -v clang-format-18 > /dev/null; then
-  echo "[*] Formatting C(pp) files"
-
-  C_FILES=$(find "$LIBAFL_DIR" -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.c' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c' | grep -v 'QEMU-Nyx')
-  if ! clang-format-18 "$CLANG_FLAGS" -i --style=file "$C_FILES"
-  then
-    echo "C(pp) format failed."
-    exit 1
-  fi
-
 else
-  echo "Warning: clang-format-18 not found. C(pp) files formatting skipped."
+  cargo run --manifest-path "$LIBAFL_DIR/utils/libafl_fmt/Cargo.toml" --release
 fi
-
-echo "[*] Done :)"

utils/libafl_fmt/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "libafl_fmt"
+version = "0.1.0"
+edition = "2021"
+description = "Format the LibAFL repository"
+authors = ["Romain Malmain <romain.malmain@pm.me>"]
+license = "MIT OR Apache-2.0"
+
+[dependencies]
+project-root = "0.2"
+walkdir = "2.5"
+regex = "1.10"
+tokio = { version = "1.38", features = ["process", "rt", "rt-multi-thread", "macros"] }
+clap = { version = "4.5", features = ["derive"] }
+exitcode = "1.1"
+which = "6.0"

utils/libafl_fmt/src/main.rs

@@ -0,0 +1,185 @@
+use std::{io, io::ErrorKind, path::PathBuf, str::from_utf8};
+
+use clap::Parser;
+use regex::RegexSet;
+use tokio::{process::Command, task::JoinSet};
+use walkdir::WalkDir;
+use which::which;
+
+async fn run_cargo_fmt(path: PathBuf, is_check: bool, verbose: bool) -> io::Result<()> {
+    // Sanity Check
+    assert_eq!(path.file_name().unwrap().to_str().unwrap(), "Cargo.toml");
+
+    let task_str = if is_check { "Checking" } else { "Formatting" };
+
+    let mut fmt_command = Command::new("cargo");
+
+    fmt_command
+        .arg("+nightly")
+        .arg("fmt")
+        .arg("--manifest-path")
+        .arg(path.as_path());
+
+    if is_check {
+        fmt_command.arg("--check");
+    }
+
+    if verbose {
+        println!("[*] {} {}...", task_str, path.as_path().display());
+    }
+
+    let res = fmt_command.output().await?;
+
+    if !res.status.success() {
+        println!("{}", from_utf8(&res.stdout).unwrap());
+        return Err(io::Error::new(ErrorKind::Other, "Cargo fmt failed."));
+    }
+
+    Ok(())
+}
+
+async fn run_clang_fmt(
+    path: PathBuf,
+    clang: &str,
+    is_check: bool,
+    verbose: bool,
+) -> io::Result<()> {
+    let task_str = if is_check { "Checking" } else { "Formatting" };
+
+    let mut fmt_command = Command::new(clang);
+
+    fmt_command
+        .arg("-i")
+        .arg("--style")
+        .arg("file")
+        .arg(path.as_path());
+
+    if is_check {
+        fmt_command.arg("-Werror").arg("--dry-run");
+    }
+
+    fmt_command.arg(path.as_path());
+
+    if verbose {
+        println!("[*] {} {}...", task_str, path.as_path().display());
+    }
+
+    let res = fmt_command.output().await?;
+
+    if !res.status.success() {
+        println!("{}", from_utf8(&res.stderr).unwrap());
+        return Err(io::Error::new(
+            ErrorKind::Other,
+            format!("{} failed.", clang),
+        ));
+    }
+
+    Ok(())
+}
+
+#[derive(Parser)]
+struct Cli {
+    #[arg(short, long)]
+    check: bool,
+    #[arg(short, long)]
+    verbose: bool,
+}
+
+#[tokio::main]
+async fn main() -> io::Result<()> {
+    let cli = Cli::parse();
+    let libafl_root_dir = project_root::get_project_root().expect("Could not locate project root.");
+
+    let rust_excluded_directories = RegexSet::new([
+        r".*target.*",
+        r".*utils/noaslr.*",
+        r".*utils/gdb_qemu.*",
+        r".*docs/listings/baby_fuzzer/listing-.*",
+        r".*LibAFL/Cargo.toml.*",
+    ])
+    .expect("Could not create the regex set from the given regex");
+
+    let c_excluded_directories = RegexSet::new([
+        r".*target.*",
+        r".*libpng-1\.6.*",
+        r".*stb_image\.h$",
+        r".*dlmalloc\.c$",
+        r".*QEMU-Nyx.*",
+    ])
+    .expect("Could not create the regex set from the given regex");
+
+    let c_file_to_format = RegexSet::new([
+        r".*\.cpp$",
+        r".*\.hpp$",
+        r".*\.cc$",
+        r".*\.cxx$",
+        r".*\.c$",
+        r".*\.h$",
+    ])
+    .expect("Could not create the regex set from the given regex");
+
+    let rust_projects_to_fmt: Vec<PathBuf> = WalkDir::new(&libafl_root_dir)
+        .into_iter()
+        .filter_map(|entry| entry.ok())
+        .filter(|e| !rust_excluded_directories.is_match(e.path().as_os_str().to_str().unwrap()))
+        .filter(|e| e.file_name() == "Cargo.toml")
+        .map(|e| e.into_path())
+        .collect();
+
+    let mut tokio_joinset = JoinSet::new();
+
+    for project in rust_projects_to_fmt {
+        tokio_joinset.spawn(run_cargo_fmt(project, cli.check, cli.verbose));
+    }
+
+    let (clang, warning) = if which("clang-format-18").is_ok() {
+        (Some("clang-format-18"), None)
+    } else if which("clang-format").is_ok() {
+        (
+            Some("clang-format"),
+            Some("using clang-format, could provide a different result from clang-format-18"),
+        )
+    } else {
+        (
+            None,
+            Some("clang-format not found. Skipping C formatting..."),
+        )
+    };
+
+    if let Some(clang) = clang {
+        let c_files_to_fmt: Vec<PathBuf> = WalkDir::new(&libafl_root_dir)
+            .into_iter()
+            .filter_map(|entry| entry.ok())
+            .filter(|e| !c_excluded_directories.is_match(e.path().as_os_str().to_str().unwrap()))
+            .filter(|e| e.file_type().is_file())
+            .filter(|e| c_file_to_format.is_match(e.file_name().to_str().unwrap()))
+            .map(|e| e.into_path())
+            .collect();
+
+        for c_file in c_files_to_fmt {
+            tokio_joinset.spawn(run_clang_fmt(c_file, clang, cli.check, cli.verbose));
+        }
+    }
+
+    while let Some(res) = tokio_joinset.join_next().await {
+        match res? {
+            Ok(_) => {}
+            Err(err) => {
+                println!("Error: {}", err);
+                std::process::exit(exitcode::IOERR)
+            }
+        }
+    }
+
+    if let Some(warning) = warning {
+        println!("Warning: {}", warning);
+    }
+
+    if cli.check {
+        println!("[*] Check finished successfully.")
+    } else {
+        println!("[*] Formatting finished successfully.")
+    }
+
+    Ok(())
+}