Template out libafl_libfuzzer (#2398)

* template out libafl_libfuzzer

* fix some final path oddities

* missed a spot

Cargo.toml

@@ -36,6 +36,7 @@ exclude = [
     "libafl_concolic/symcc_libafl",
     "libafl_frida",
     "libafl_libfuzzer",
+    "libafl_libfuzzer_runtime",
     "libafl_nyx",
     "libafl_qemu",
     "libafl_tinyinst",

Dockerfile

@@ -131,8 +131,9 @@ COPY libafl_concolic/symcc_runtime libafl_concolic/symcc_runtime
 COPY libafl_concolic/test libafl_concolic/test
 COPY libafl_nyx/src libafl_nyx/src
 RUN touch libafl_nyx/src/lib.rs
+COPY libafl_libfuzzer_runtime libafl_libfuzzer_runtime
 COPY libafl_libfuzzer/src libafl_libfuzzer/src
-COPY libafl_libfuzzer/libafl_libfuzzer_runtime libafl_libfuzzer/libafl_libfuzzer_runtime
+COPY libafl_libfuzzer/runtime libafl_libfuzzer/runtime
 COPY libafl_libfuzzer/build.rs libafl_libfuzzer/build.rs
 RUN touch libafl_libfuzzer/src/lib.rs
 RUN cargo build && cargo build --release

libafl_libfuzzer/Cargo.toml

@@ -13,8 +13,7 @@ include = [
     "/src",
     "/Cargo.toml",
     "/build.rs",
-    "/libafl_libfuzzer_runtime",
-    "!/libafl_libfuzzer_runtime/target",
+    "/runtime",
 ]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -22,6 +21,7 @@ include = [
 [build-dependencies]
 cc = "1.0"
 rustversion = "1.0"
+toml = { version = "0.8.14", features = ["preserve_order"] }
 
 [features]
 default = ["fork"]
@@ -43,6 +43,9 @@ embed-runtime = []
 ## 🐇
 rabbit = []
 
+## For testing and publishing purposes only: enforce that the runtime uses versions rather than paths
+libafl-libfuzzer-use-version = []
+
 [dependencies]
 libfuzzer-sys = { version = "0.4.7", default-features = false }
 document-features = { version = "0.2", optional = true }

libafl_libfuzzer/README.md

@@ -83,13 +83,13 @@ CXXFLAGS='-fsanitize=fuzzer-no-link'
 The runtime for `libafl_libfuzzer` may be used standalone as a direct replacement for libFuzzer with other targets as
 well.
 To do so, [ensure a recent nightly version of Rust is installed](https://rustup.rs/), then enter the
-[`libafl_libfuzzer_runtime`](libafl_libfuzzer_runtime) folder and build the runtime with the following command:
+[`libafl_libfuzzer_runtime`](../libafl_libfuzzer_runtime) folder and build the runtime with the following command:
 
 ```bash
 ./build.sh
 ```
 
-The static library will be available at `libFuzzer.a` in the [`libafl_libfuzzer_runtime`](libafl_libfuzzer_runtime)
+The static library will be available at `libFuzzer.a` in the [`libafl_libfuzzer_runtime`](../libafl_libfuzzer_runtime)
 directory.
 If you encounter build failures without clear error outputs that help you resolve the issue, please [submit an issue].
 
@@ -144,7 +144,8 @@ to partial support of libfuzzer flags, `libafl_libfuzzer` offers:
 - `-fork` and `-jobs`
     - in `libafl_libfuzzer`, these are synonymous
 - `-ignore_crashes`, `-ignore_ooms`, and `-ignore_timeouts`
-    - note that setting `-tui=1` enables these flags by default, so you'll need to explicitly mention `-ignore_...=0` to disable them
+    - note that setting `-tui=1` enables these flags by default, so you'll need to explicitly mention `-ignore_...=0` to
+      disable them
 - `-rss_limit_mb` and `-malloc_limit_mb`
 - `-ignore_remaining_args`
 - `-shrink`
@@ -152,7 +153,11 @@ to partial support of libfuzzer flags, `libafl_libfuzzer` offers:
 - `-close_fd_mask`
 
 [libFuzzer]: https://llvm.org/docs/LibFuzzer.html
+
 [`libfuzzer-sys`]: https://docs.rs/libfuzzer-sys/
+
 [de-facto deprecation of libFuzzer]: https://llvm.org/docs/LibFuzzer.html#status
+
 [submit an issue]: https://github.com/AFLplusplus/LibAFL/issues/new/choose
+
 [grimoire]: https://www.usenix.org/conference/usenixsecurity19/presentation/blazytko

libafl_libfuzzer/build.rs

@@ -1,4 +1,6 @@
 use std::{
+    error::Error,
+    fs,
     fs::File,
     io::{BufRead, BufReader, BufWriter, Write},
     path::{Path, PathBuf},
@@ -12,25 +14,25 @@ const NAMESPACE: &str = "__libafl";
 const NAMESPACE_LEN: usize = NAMESPACE.as_bytes().len();
 
 #[allow(clippy::too_many_lines)]
-fn main() {
+fn main() -> Result<(), Box<dyn Error>> {
     if cfg!(any(clippy, docsrs)) {
-        return; // skip when clippy or docs is running
+        return Ok(()); // skip when clippy or docs is running
     }
 
     if cfg!(not(any(target_os = "linux", target_os = "macos"))) {
         println!(
             "cargo:warning=The libafl_libfuzzer runtime may only be built for linux or macos; failing fast."
         );
-        return;
+        return Ok(());
     }
 
     println!("cargo:rerun-if-changed=libafl_libfuzzer_runtime/src");
-    println!("cargo:rerun-if-changed=libafl_libfuzzer_runtime/Cargo.toml");
     println!("cargo:rerun-if-changed=libafl_libfuzzer_runtime/build.rs");
 
     let custom_lib_dir =
         AsRef::<Path>::as_ref(&std::env::var_os("OUT_DIR").unwrap()).join("libafl_libfuzzer");
-    std::fs::create_dir_all(&custom_lib_dir)
+    let custom_lib_target = custom_lib_dir.join("target");
+    fs::create_dir_all(&custom_lib_target)
         .expect("Couldn't create the output directory for the fuzzer runtime build");
 
     let lib_src: PathBuf = AsRef::<Path>::as_ref(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap())
@@ -51,8 +53,6 @@ fn main() {
         .env("PATH", std::env::var_os("PATH").unwrap())
         .current_dir(&lib_src);
 
-    let _ = std::fs::rename(lib_src.join("Cargo.toml.orig"), lib_src.join("Cargo.toml"));
-
     command.arg("build");
 
     let mut features = vec![];
@@ -69,143 +69,203 @@ fn main() {
         .arg("--release")
         .arg("--no-default-features")
         .arg("--target-dir")
-        .arg(&custom_lib_dir)
+        .arg(&custom_lib_target)
         .arg("--target")
         .arg(std::env::var_os("TARGET").unwrap());
 
+    // detect if we are a version or path/git dep, or testing version-based behavior
+    if fs::exists("../libafl_libfuzzer_runtime")? && !cfg!(feature = "libafl-libfuzzer-use-version")
+    {
+        command.current_dir("../libafl_libfuzzer_runtime");
+    } else {
+        // we are being used as a version dep; we need to create the package virtually
+
+        // remove old files; we need to trigger a rebuild if our path changes!
+        let _ = fs::remove_file(custom_lib_dir.join("src"));
+        let _ = fs::remove_dir_all(custom_lib_dir.join("src")); // maybe a dir in windows
+        let _ = fs::remove_file(custom_lib_dir.join("build.rs"));
+        let _ = fs::remove_file(custom_lib_dir.join("Cargo.toml"));
+
+        #[cfg(unix)]
+        {
+            // create symlinks for all the source files
+            use std::os::unix::fs::symlink;
+
+            // canonicalize can theoretically fail if we are within a non-executable directory?
+            symlink(fs::canonicalize("runtime/src")?, custom_lib_dir.join("src"))?;
+            symlink(
+                fs::canonicalize("runtime/build.rs")?,
+                custom_lib_dir.join("build.rs"),
+            )?;
+        }
+        #[cfg(not(unix))]
+        {
+            todo!("copy all the source files"); // we don't support libafl_libfuzzer for others rn
+        }
+        let mut template: toml::Value =
+            toml::from_str(&fs::read_to_string("runtime/Cargo.toml.template")?)?;
+        let toml::Value::Table(root) = &mut template else {
+            unreachable!("Invalid Cargo.toml");
+        };
+        root.insert(
+            "workspace".to_string(),
+            toml::Value::Table(toml::Table::new()),
+        );
+        let Some(toml::Value::Table(deps)) = root.get_mut("dependencies") else {
+            unreachable!("Invalid Cargo.toml");
+        };
+        let version = env!("CARGO_PKG_VERSION");
+        for (_name, spec) in deps {
+            if let toml::Value::Table(spec) = spec {
+                // replace all path deps with version deps
+                if spec.remove("path").is_some() {
+                    spec.insert(
+                        "version".to_string(),
+                        toml::Value::String(version.to_string()),
+                    );
+                }
+            }
+        }
+
+        let serialized = toml::to_string(&template)?;
+        fs::write(custom_lib_dir.join("Cargo.toml"), serialized)?;
+
+        // build in this filled out template
+        command.current_dir(custom_lib_dir);
+    }
+
     assert!(
         command.status().map_or(false, |s| s.success()),
         "Couldn't build runtime crate! Did you remember to use nightly? (`rustup default nightly` to install)"
     );
 
-    let mut archive_path = custom_lib_dir.join(std::env::var_os("TARGET").unwrap());
+    let mut archive_path = custom_lib_target.join(std::env::var_os("TARGET").unwrap());
     archive_path.push("release");
 
-    if cfg!(unix) {
-        archive_path.push("libafl_libfuzzer_runtime.a");
-        let target_libdir = Command::new("rustc")
-            .args(["--print", "target-libdir"])
-            .output()
-            .expect("Couldn't find rustc's target-libdir");
-        let target_libdir = String::from_utf8(target_libdir.stdout).unwrap();
-        let target_libdir = Path::new(target_libdir.trim());
+    archive_path.push("libafl_libfuzzer_runtime.a");
+    let target_libdir = Command::new("rustc")
+        .args(["--print", "target-libdir"])
+        .output()
+        .expect("Couldn't find rustc's target-libdir");
+    let target_libdir = String::from_utf8(target_libdir.stdout).unwrap();
+    let target_libdir = Path::new(target_libdir.trim());
 
-        // NOTE: depends on llvm-tools
-        let rust_objcopy = target_libdir.join("../bin/llvm-objcopy");
-        let nm = target_libdir.join("../bin/llvm-nm");
+    // NOTE: depends on llvm-tools
+    let rust_objcopy = target_libdir.join("../bin/llvm-objcopy");
+    let nm = target_libdir.join("../bin/llvm-nm");
 
-        let redefined_archive_path = custom_lib_dir.join("libFuzzer.a");
-        let redefined_symbols = custom_lib_dir.join("redefs.txt");
+    let redefined_archive_path = custom_lib_target.join("libFuzzer.a");
+    let redefined_symbols = custom_lib_target.join("redefs.txt");
 
-        let mut nm_child = Command::new(nm)
-            .arg(&archive_path)
-            .stdout(Stdio::piped())
-            .spawn()
-            .expect("llvm-nm works (are you using nightly?)");
+    let mut nm_child = Command::new(nm)
+        .arg(&archive_path)
+        .stdout(Stdio::piped())
+        .spawn()
+        .expect("llvm-nm works (are you using nightly?)");
 
-        let mut redefinitions_file = BufWriter::new(File::create(&redefined_symbols).unwrap());
+    let mut redefinitions_file = BufWriter::new(File::create(&redefined_symbols).unwrap());
 
-        let zn_prefix = if cfg!(target_os = "macos") {
-            // macOS symbols have an extra `_`
-            "__ZN"
-        } else {
-            "_ZN"
-        };
+    let zn_prefix = if cfg!(target_os = "macos") {
+        // macOS symbols have an extra `_`
+        "__ZN"
+    } else {
+        "_ZN"
+    };
 
-        let replacement = format!("{zn_prefix}{NAMESPACE_LEN}{NAMESPACE}");
+    let replacement = format!("{zn_prefix}{NAMESPACE_LEN}{NAMESPACE}");
 
-        // redefine all the rust-mangled symbols we can
-        // TODO this will break when v0 mangling is stabilised
-        for line in BufReader::new(nm_child.stdout.take().unwrap()).lines() {
-            let line = line.unwrap();
+    // redefine all the rust-mangled symbols we can
+    // TODO this will break when v0 mangling is stabilised
+    for line in BufReader::new(nm_child.stdout.take().unwrap()).lines() {
+        let line = line.unwrap();
 
-            // Skip headers
-            if line.ends_with(':') || line.is_empty() {
-                continue;
-            }
-            let (_, symbol) = line.rsplit_once(' ').unwrap();
-
-            if symbol.starts_with(zn_prefix) {
-                writeln!(
-                    redefinitions_file,
-                    "{} {}",
-                    symbol,
-                    symbol.replacen(zn_prefix, &replacement, 1)
-                )
-                .unwrap();
-            }
+        // Skip headers
+        if line.ends_with(':') || line.is_empty() {
+            continue;
         }
-        redefinitions_file.flush().unwrap();
-        drop(redefinitions_file);
+        let (_, symbol) = line.rsplit_once(' ').unwrap();
 
-        assert!(
+        if symbol.starts_with(zn_prefix) {
+            writeln!(
+                redefinitions_file,
+                "{} {}",
+                symbol,
+                symbol.replacen(zn_prefix, &replacement, 1)
+            )
+            .unwrap();
+        }
+    }
+    redefinitions_file.flush().unwrap();
+    drop(redefinitions_file);
+
+    assert!(
             nm_child.wait().map_or(false, |s| s.success()),
             "Couldn't link runtime crate! Do you have the llvm-tools component installed? (`rustup component add llvm-tools-preview` to install)"
         );
 
-        let mut objcopy_command = Command::new(rust_objcopy);
+    let mut objcopy_command = Command::new(rust_objcopy);
 
-        for symbol in [
-            "__rust_drop_panic",
-            "__rust_foreign_exception",
-            "rust_begin_unwind",
-            "rust_panic",
-            "rust_eh_personality",
-            "__rg_oom",
-            "__rdl_oom",
-            "__rdl_alloc",
-            "__rust_alloc",
-            "__rdl_dealloc",
-            "__rust_dealloc",
-            "__rdl_realloc",
-            "__rust_realloc",
-            "__rdl_alloc_zeroed",
-            "__rust_alloc_zeroed",
-            "__rust_alloc_error_handler",
-            "__rust_no_alloc_shim_is_unstable",
-            "__rust_alloc_error_handler_should_panic",
-        ] {
-            let mut symbol = symbol.to_string();
-            // macOS symbols have an extra `_`
-            if cfg!(target_os = "macos") {
-                symbol.insert(0, '_');
-            }
-
-            objcopy_command
-                .arg("--redefine-sym")
-                .arg(format!("{symbol}={symbol}_libafl_libfuzzer_runtime"));
+    for symbol in [
+        "__rust_drop_panic",
+        "__rust_foreign_exception",
+        "rust_begin_unwind",
+        "rust_panic",
+        "rust_eh_personality",
+        "__rg_oom",
+        "__rdl_oom",
+        "__rdl_alloc",
+        "__rust_alloc",
+        "__rdl_dealloc",
+        "__rust_dealloc",
+        "__rdl_realloc",
+        "__rust_realloc",
+        "__rdl_alloc_zeroed",
+        "__rust_alloc_zeroed",
+        "__rust_alloc_error_handler",
+        "__rust_no_alloc_shim_is_unstable",
+        "__rust_alloc_error_handler_should_panic",
+    ] {
+        let mut symbol = symbol.to_string();
+        // macOS symbols have an extra `_`
+        if cfg!(target_os = "macos") {
+            symbol.insert(0, '_');
         }
 
         objcopy_command
-            .arg("--redefine-syms")
-            .arg(redefined_symbols)
-            .args([&archive_path, &redefined_archive_path]);
+            .arg("--redefine-sym")
+            .arg(format!("{symbol}={symbol}_libafl_libfuzzer_runtime"));
+    }
 
-        assert!(
+    objcopy_command
+        .arg("--redefine-syms")
+        .arg(redefined_symbols)
+        .args([&archive_path, &redefined_archive_path]);
+
+    assert!(
             objcopy_command.status().map_or(false, |s| s.success()),
             "Couldn't rename allocators in the runtime crate! Do you have the llvm-tools component installed? (`rustup component add llvm-tools-preview` to install)"
         );
 
-        #[cfg(feature = "embed-runtime")]
-        {
-            // NOTE: lib, .a are added always on unix-like systems as described in:
-            // https://gist.github.com/novafacing/1389cbb2f0a362d7eb103e67b4468e2b
-            println!(
-                "cargo:rustc-env=LIBAFL_LIBFUZZER_RUNTIME_PATH={}",
-                redefined_archive_path.display()
-            );
-        }
-
+    #[cfg(feature = "embed-runtime")]
+    {
+        // NOTE: lib, .a are added always on unix-like systems as described in:
+        // https://gist.github.com/novafacing/1389cbb2f0a362d7eb103e67b4468e2b
         println!(
-            "cargo:rustc-link-search=native={}",
-            custom_lib_dir.to_str().unwrap()
+            "cargo:rustc-env=LIBAFL_LIBFUZZER_RUNTIME_PATH={}",
+            redefined_archive_path.display()
         );
-        println!("cargo:rustc-link-lib=static=Fuzzer");
-
-        if cfg!(target_os = "macos") {
-            println!("cargo:rustc-link-lib=c++");
-        } else {
-            println!("cargo:rustc-link-lib=stdc++");
-        }
     }
+
+    println!(
+        "cargo:rustc-link-search=native={}",
+        custom_lib_target.to_str().unwrap()
+    );
+    println!("cargo:rustc-link-lib=static=Fuzzer");
+
+    if cfg!(target_os = "macos") {
+        println!("cargo:rustc-link-lib=c++");
+    } else {
+        println!("cargo:rustc-link-lib=stdc++");
+    }
+    Ok(())
 }

libafl_libfuzzer/libafl_libfuzzer_runtime/rustfmt.toml

@@ -1,2 +0,0 @@
-group_imports = "StdExternalCrate"
-imports_granularity = "Crate"

libafl_libfuzzer/publish.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -x
-
-mv libafl_libfuzzer_runtime/Cargo.toml libafl_libfuzzer_runtime/Cargo.toml.orig
-cargo publish --allow-dirty --no-verify "$@"
-mv libafl_libfuzzer_runtime/Cargo.toml.orig libafl_libfuzzer_runtime/Cargo.toml

libafl_libfuzzer/runtime/Cargo.toml.template

@@ -24,16 +24,14 @@ inherits = "release"
 debug = false
 strip = true
 
-
 [lib]
-name = "afl_libfuzzer_runtime" # TODO fix name once cargo-fuzz stops stripping double-prefixes
-path = "src/lib.rs"
+name = "afl_libfuzzer_runtime" # historically, cargo-fuzz strips double-prefixes; maintain compat
 crate-type = ["staticlib", "rlib"]
 
 [dependencies]
-libafl = { path = "../../libafl", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "regex", "errors_backtrace", "serdeany_autoreg", "tui_monitor", "unicode"] }
-libafl_bolts = { path = "../../libafl_bolts", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "serdeany_autoreg", "errors_backtrace"] }
-libafl_targets = { path = "../../libafl_targets", features = ["sancov_8bit", "sancov_cmplog", "sancov_value_profile", "sancov_pcguard", "libfuzzer", "libfuzzer_oom", "libfuzzer_define_run_driver", "libfuzzer_interceptors", "sanitizers_flags", "whole_archive", "sanitizer_interfaces"] }
+libafl = { path = "../libafl", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "regex", "errors_backtrace", "serdeany_autoreg", "tui_monitor", "unicode"] }
+libafl_bolts = { path = "../libafl_bolts", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "serdeany_autoreg", "errors_backtrace"] }
+libafl_targets = { path = "../libafl_targets", features = ["sancov_8bit", "sancov_cmplog", "sancov_value_profile", "sancov_pcguard", "libfuzzer", "libfuzzer_oom", "libfuzzer_define_run_driver", "libfuzzer_interceptors", "sanitizers_flags", "whole_archive", "sanitizer_interfaces"] }
 
 ahash = { version = "0.8.3", default-features = false }
 libc = "0.2.1"
@@ -46,12 +44,8 @@ hashbrown = "0.14"
 
 # for identifying if we can grimoire-ify
 utf8-chars = "3.0.1"
-
 env_logger = "0.10"
 
-
 [build-dependencies]
 bindgen = "0.69.4"
 cc = { version = "1.0", features = ["parallel"] }
-
-[workspace]

libafl_libfuzzer_runtime/Cargo.toml

@@ -0,0 +1 @@
+../libafl_libfuzzer/runtime/Cargo.toml.template

libafl_libfuzzer_runtime/README.md

@@ -0,0 +1,6 @@
+# libafl_libfuzzer_runtime
+
+This is the runtime for `libafl_libfuzzer`.
+
+Please see the [`libafl_libfuzzer`](../libafl_libfuzzer) documentation for details.
+This crate should not be used alone except in very special circumstances.

libafl_libfuzzer_runtime/build.rs

@@ -0,0 +1 @@
+../libafl_libfuzzer/runtime/build.rs

libafl_libfuzzer_runtime/src

@@ -0,0 +1 @@
+../libafl_libfuzzer/runtime/src

scripts/clippy.sh

@@ -20,7 +20,7 @@ RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --no-deps --tests
    -A clippy::unreadable-literal
 
 if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-  cd libafl_libfuzzer/libafl_libfuzzer_runtime
+  cd libafl_libfuzzer_runtime
   RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --no-deps --tests --examples --benches -- -Z macro-backtrace \
      -D clippy::all \
      -D clippy::pedantic \

scripts/publish.sh

@@ -78,5 +78,5 @@ cargo publish "$@" --allow-dirty
 cd ../.. || exit 1
 
 cd libafl_libfuzzer
-./publish.sh "$@"
+cargo publish "$@"
 cd .. || exit 1