Template out libafl_libfuzzer (#2398)

* template out libafl_libfuzzer

* fix some final path oddities

* missed a spot

Cargo.toml

@@ -36,6 +36,7 @@ exclude = [
     "libafl_concolic/symcc_libafl",
     "libafl_frida",
     "libafl_libfuzzer",
+    "libafl_libfuzzer_runtime",
     "libafl_nyx",
     "libafl_qemu",
     "libafl_tinyinst",

Dockerfile

@@ -131,8 +131,9 @@ COPY libafl_concolic/symcc_runtime libafl_concolic/symcc_runtime
 COPY libafl_concolic/test libafl_concolic/test
 COPY libafl_nyx/src libafl_nyx/src
 RUN touch libafl_nyx/src/lib.rs
+COPY libafl_libfuzzer_runtime libafl_libfuzzer_runtime
 COPY libafl_libfuzzer/src libafl_libfuzzer/src
-COPY libafl_libfuzzer/libafl_libfuzzer_runtime libafl_libfuzzer/libafl_libfuzzer_runtime
+COPY libafl_libfuzzer/runtime libafl_libfuzzer/runtime
 COPY libafl_libfuzzer/build.rs libafl_libfuzzer/build.rs
 RUN touch libafl_libfuzzer/src/lib.rs
 RUN cargo build && cargo build --release

libafl_libfuzzer/Cargo.toml

@@ -13,8 +13,7 @@ include = [
     "/src",
     "/Cargo.toml",
     "/build.rs",
-    "/libafl_libfuzzer_runtime",
+    "/runtime",
-    "!/libafl_libfuzzer_runtime/target",
 ]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -22,6 +21,7 @@ include = [
 [build-dependencies]
 cc = "1.0"
 rustversion = "1.0"
+toml = { version = "0.8.14", features = ["preserve_order"] }
 
 [features]
 default = ["fork"]
@@ -43,6 +43,9 @@ embed-runtime = []
 ## 🐇
 rabbit = []
 
+## For testing and publishing purposes only: enforce that the runtime uses versions rather than paths
+libafl-libfuzzer-use-version = []
+
 [dependencies]
 libfuzzer-sys = { version = "0.4.7", default-features = false }
 document-features = { version = "0.2", optional = true }

libafl_libfuzzer/README.md

@@ -83,13 +83,13 @@ CXXFLAGS='-fsanitize=fuzzer-no-link'
 The runtime for `libafl_libfuzzer` may be used standalone as a direct replacement for libFuzzer with other targets as
 well.
 To do so, [ensure a recent nightly version of Rust is installed](https://rustup.rs/), then enter the
-[`libafl_libfuzzer_runtime`](libafl_libfuzzer_runtime) folder and build the runtime with the following command:
+[`libafl_libfuzzer_runtime`](../libafl_libfuzzer_runtime) folder and build the runtime with the following command:
 
 ```bash
 ./build.sh
 ```
 
-The static library will be available at `libFuzzer.a` in the [`libafl_libfuzzer_runtime`](libafl_libfuzzer_runtime)
+The static library will be available at `libFuzzer.a` in the [`libafl_libfuzzer_runtime`](../libafl_libfuzzer_runtime)
 directory.
 If you encounter build failures without clear error outputs that help you resolve the issue, please [submit an issue].
 
@@ -144,7 +144,8 @@ to partial support of libfuzzer flags, `libafl_libfuzzer` offers:
 - `-fork` and `-jobs`
     - in `libafl_libfuzzer`, these are synonymous
 - `-ignore_crashes`, `-ignore_ooms`, and `-ignore_timeouts`
-    - note that setting `-tui=1` enables these flags by default, so you'll need to explicitly mention `-ignore_...=0` to disable them
+    - note that setting `-tui=1` enables these flags by default, so you'll need to explicitly mention `-ignore_...=0` to
+      disable them
 - `-rss_limit_mb` and `-malloc_limit_mb`
 - `-ignore_remaining_args`
 - `-shrink`
@@ -152,7 +153,11 @@ to partial support of libfuzzer flags, `libafl_libfuzzer` offers:
 - `-close_fd_mask`
 
 [libFuzzer]: https://llvm.org/docs/LibFuzzer.html
+
 [`libfuzzer-sys`]: https://docs.rs/libfuzzer-sys/
+
 [de-facto deprecation of libFuzzer]: https://llvm.org/docs/LibFuzzer.html#status
+
 [submit an issue]: https://github.com/AFLplusplus/LibAFL/issues/new/choose
+
 [grimoire]: https://www.usenix.org/conference/usenixsecurity19/presentation/blazytko

libafl_libfuzzer/build.rs

@@ -1,4 +1,6 @@
 use std::{
+    error::Error,
+    fs,
     fs::File,
     io::{BufRead, BufReader, BufWriter, Write},
     path::{Path, PathBuf},
@@ -12,25 +14,25 @@ const NAMESPACE: &str = "__libafl";
 const NAMESPACE_LEN: usize = NAMESPACE.as_bytes().len();
 
 #[allow(clippy::too_many_lines)]
-fn main() {
+fn main() -> Result<(), Box<dyn Error>> {
     if cfg!(any(clippy, docsrs)) {
-        return; // skip when clippy or docs is running
+        return Ok(()); // skip when clippy or docs is running
     }
 
     if cfg!(not(any(target_os = "linux", target_os = "macos"))) {
         println!(
             "cargo:warning=The libafl_libfuzzer runtime may only be built for linux or macos; failing fast."
         );
-        return;
+        return Ok(());
     }
 
     println!("cargo:rerun-if-changed=libafl_libfuzzer_runtime/src");
-    println!("cargo:rerun-if-changed=libafl_libfuzzer_runtime/Cargo.toml");
     println!("cargo:rerun-if-changed=libafl_libfuzzer_runtime/build.rs");
 
     let custom_lib_dir =
         AsRef::<Path>::as_ref(&std::env::var_os("OUT_DIR").unwrap()).join("libafl_libfuzzer");
-    std::fs::create_dir_all(&custom_lib_dir)
+    let custom_lib_target = custom_lib_dir.join("target");
+    fs::create_dir_all(&custom_lib_target)
         .expect("Couldn't create the output directory for the fuzzer runtime build");
 
     let lib_src: PathBuf = AsRef::<Path>::as_ref(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap())
@@ -51,8 +53,6 @@ fn main() {
         .env("PATH", std::env::var_os("PATH").unwrap())
         .current_dir(&lib_src);
 
-    let _ = std::fs::rename(lib_src.join("Cargo.toml.orig"), lib_src.join("Cargo.toml"));
-
     command.arg("build");
 
     let mut features = vec![];
@@ -69,19 +69,79 @@ fn main() {
         .arg("--release")
         .arg("--no-default-features")
         .arg("--target-dir")
-        .arg(&custom_lib_dir)
+        .arg(&custom_lib_target)
         .arg("--target")
         .arg(std::env::var_os("TARGET").unwrap());
 
+    // detect if we are a version or path/git dep, or testing version-based behavior
+    if fs::exists("../libafl_libfuzzer_runtime")? && !cfg!(feature = "libafl-libfuzzer-use-version")
+    {
+        command.current_dir("../libafl_libfuzzer_runtime");
+    } else {
+        // we are being used as a version dep; we need to create the package virtually
+
+        // remove old files; we need to trigger a rebuild if our path changes!
+        let _ = fs::remove_file(custom_lib_dir.join("src"));
+        let _ = fs::remove_dir_all(custom_lib_dir.join("src")); // maybe a dir in windows
+        let _ = fs::remove_file(custom_lib_dir.join("build.rs"));
+        let _ = fs::remove_file(custom_lib_dir.join("Cargo.toml"));
+
+        #[cfg(unix)]
+        {
+            // create symlinks for all the source files
+            use std::os::unix::fs::symlink;
+
+            // canonicalize can theoretically fail if we are within a non-executable directory?
+            symlink(fs::canonicalize("runtime/src")?, custom_lib_dir.join("src"))?;
+            symlink(
+                fs::canonicalize("runtime/build.rs")?,
+                custom_lib_dir.join("build.rs"),
+            )?;
+        }
+        #[cfg(not(unix))]
+        {
+            todo!("copy all the source files"); // we don't support libafl_libfuzzer for others rn
+        }
+        let mut template: toml::Value =
+            toml::from_str(&fs::read_to_string("runtime/Cargo.toml.template")?)?;
+        let toml::Value::Table(root) = &mut template else {
+            unreachable!("Invalid Cargo.toml");
+        };
+        root.insert(
+            "workspace".to_string(),
+            toml::Value::Table(toml::Table::new()),
+        );
+        let Some(toml::Value::Table(deps)) = root.get_mut("dependencies") else {
+            unreachable!("Invalid Cargo.toml");
+        };
+        let version = env!("CARGO_PKG_VERSION");
+        for (_name, spec) in deps {
+            if let toml::Value::Table(spec) = spec {
+                // replace all path deps with version deps
+                if spec.remove("path").is_some() {
+                    spec.insert(
+                        "version".to_string(),
+                        toml::Value::String(version.to_string()),
+                    );
+                }
+            }
+        }
+
+        let serialized = toml::to_string(&template)?;
+        fs::write(custom_lib_dir.join("Cargo.toml"), serialized)?;
+
+        // build in this filled out template
+        command.current_dir(custom_lib_dir);
+    }
+
     assert!(
         command.status().map_or(false, |s| s.success()),
         "Couldn't build runtime crate! Did you remember to use nightly? (`rustup default nightly` to install)"
     );
 
-    let mut archive_path = custom_lib_dir.join(std::env::var_os("TARGET").unwrap());
+    let mut archive_path = custom_lib_target.join(std::env::var_os("TARGET").unwrap());
     archive_path.push("release");
 
-    if cfg!(unix) {
     archive_path.push("libafl_libfuzzer_runtime.a");
     let target_libdir = Command::new("rustc")
         .args(["--print", "target-libdir"])
@@ -94,8 +154,8 @@ fn main() {
     let rust_objcopy = target_libdir.join("../bin/llvm-objcopy");
     let nm = target_libdir.join("../bin/llvm-nm");
 
-        let redefined_archive_path = custom_lib_dir.join("libFuzzer.a");
+    let redefined_archive_path = custom_lib_target.join("libFuzzer.a");
-        let redefined_symbols = custom_lib_dir.join("redefs.txt");
+    let redefined_symbols = custom_lib_target.join("redefs.txt");
 
     let mut nm_child = Command::new(nm)
         .arg(&archive_path)
@@ -198,7 +258,7 @@ fn main() {
 
     println!(
         "cargo:rustc-link-search=native={}",
-            custom_lib_dir.to_str().unwrap()
+        custom_lib_target.to_str().unwrap()
     );
     println!("cargo:rustc-link-lib=static=Fuzzer");
 
@@ -207,5 +267,5 @@ fn main() {
     } else {
         println!("cargo:rustc-link-lib=stdc++");
     }
-    }
+    Ok(())
 }

libafl_libfuzzer/libafl_libfuzzer_runtime/rustfmt.toml

@@ -1,2 +0,0 @@
-group_imports = "StdExternalCrate"
-imports_granularity = "Crate"

libafl_libfuzzer/publish.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -x
-
-mv libafl_libfuzzer_runtime/Cargo.toml libafl_libfuzzer_runtime/Cargo.toml.orig
-cargo publish --allow-dirty --no-verify "$@"
-mv libafl_libfuzzer_runtime/Cargo.toml.orig libafl_libfuzzer_runtime/Cargo.toml

libafl_libfuzzer/runtime/Cargo.toml.template

@@ -24,16 +24,14 @@ inherits = "release"
 debug = false
 strip = true
 
-
 [lib]
-name = "afl_libfuzzer_runtime" # TODO fix name once cargo-fuzz stops stripping double-prefixes
+name = "afl_libfuzzer_runtime" # historically, cargo-fuzz strips double-prefixes; maintain compat
-path = "src/lib.rs"
 crate-type = ["staticlib", "rlib"]
 
 [dependencies]
-libafl = { path = "../../libafl", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "regex", "errors_backtrace", "serdeany_autoreg", "tui_monitor", "unicode"] }
+libafl = { path = "../libafl", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "regex", "errors_backtrace", "serdeany_autoreg", "tui_monitor", "unicode"] }
-libafl_bolts = { path = "../../libafl_bolts", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "serdeany_autoreg", "errors_backtrace"] }
+libafl_bolts = { path = "../libafl_bolts", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "serdeany_autoreg", "errors_backtrace"] }
-libafl_targets = { path = "../../libafl_targets", features = ["sancov_8bit", "sancov_cmplog", "sancov_value_profile", "sancov_pcguard", "libfuzzer", "libfuzzer_oom", "libfuzzer_define_run_driver", "libfuzzer_interceptors", "sanitizers_flags", "whole_archive", "sanitizer_interfaces"] }
+libafl_targets = { path = "../libafl_targets", features = ["sancov_8bit", "sancov_cmplog", "sancov_value_profile", "sancov_pcguard", "libfuzzer", "libfuzzer_oom", "libfuzzer_define_run_driver", "libfuzzer_interceptors", "sanitizers_flags", "whole_archive", "sanitizer_interfaces"] }
 
 ahash = { version = "0.8.3", default-features = false }
 libc = "0.2.1"
@@ -46,12 +44,8 @@ hashbrown = "0.14"
 
 # for identifying if we can grimoire-ify
 utf8-chars = "3.0.1"
-
 env_logger = "0.10"
 
-
 [build-dependencies]
 bindgen = "0.69.4"
 cc = { version = "1.0", features = ["parallel"] }
-
-[workspace]

libafl_libfuzzer_runtime/Cargo.toml

@@ -0,0 +1 @@
+../libafl_libfuzzer/runtime/Cargo.toml.template

libafl_libfuzzer_runtime/README.md

@@ -0,0 +1,6 @@
+# libafl_libfuzzer_runtime
+
+This is the runtime for `libafl_libfuzzer`.
+
+Please see the [`libafl_libfuzzer`](../libafl_libfuzzer) documentation for details.
+This crate should not be used alone except in very special circumstances.

libafl_libfuzzer_runtime/build.rs

@@ -0,0 +1 @@
+../libafl_libfuzzer/runtime/build.rs

libafl_libfuzzer_runtime/src

@@ -0,0 +1 @@
+../libafl_libfuzzer/runtime/src

scripts/clippy.sh

@@ -20,7 +20,7 @@ RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --no-deps --tests
    -A clippy::unreadable-literal
 
 if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-  cd libafl_libfuzzer/libafl_libfuzzer_runtime
+  cd libafl_libfuzzer_runtime
   RUST_BACKTRACE=full cargo +nightly clippy --all --all-features --no-deps --tests --examples --benches -- -Z macro-backtrace \
      -D clippy::all \
      -D clippy::pedantic \

scripts/publish.sh

@@ -78,5 +78,5 @@ cargo publish "$@" --allow-dirty
 cd ../.. || exit 1
 
 cd libafl_libfuzzer
-./publish.sh "$@"
+cargo publish "$@"
 cd .. || exit 1