Rework of libafl_qemu configuration (#2054)

* LibAFL QEMU can now be dynamically linked
* LibAFL QEMU reconfiguration happens less frequently (now using a signature check)
* Possibility to have custom rpath in QEMU

fuzzers/qemu_systemmode/Cargo.toml

@@ -12,6 +12,8 @@ classic = [] # The classic way to interact with LibAFL QEMU, with direct calls t
 breakpoint = [] # Uses the command system, with breakpoints
 sync_exit = [] # Uses the command system, with sync exit.
 
+shared = ["libafl_qemu/shared"]
+
 [profile.release]
 incremental = true
 debug = true
@@ -24,3 +26,6 @@ libafl_bolts = { path = "../../libafl_bolts/" }
 libafl_qemu = { path = "../../libafl_qemu/", features = ["arm", "systemmode"] }
 libafl_qemu_sys = { path = "../../libafl_qemu/libafl_qemu_sys", features = ["arm", "systemmode"] }
 env_logger = "*"
+
+[build-dependencies]
+libafl_qemu_build = { path = "../../libafl_qemu/libafl_qemu_build" }

fuzzers/qemu_systemmode/Makefile.toml

@@ -45,7 +45,6 @@ args = [
     "--no-default-features",
     "--features", "std,${FEATURE}",
     "--target-dir", "${TARGET_DIR}",
-    "-vv",
 ]
 dependencies = ["image"]
 

fuzzers/qemu_systemmode/build.rs

@@ -1,3 +1,5 @@
+use libafl_qemu_build::build_libafl_qemu;
+
 #[macro_export]
 macro_rules! assert_unique_feature {
     () => {};
@@ -12,4 +14,6 @@ macro_rules! assert_unique_feature {
 
 fn main() {
     assert_unique_feature!("classic", "breakpoint", "sync_exit");
+
+    build_libafl_qemu();
 }

fuzzers/qemu_systemmode/src/fuzzer_breakpoint.rs

@@ -13,7 +13,7 @@ use libafl::{
     inputs::BytesInput,
     monitors::MultiMonitor,
     mutators::scheduled::{havoc_mutations, StdScheduledMutator},
-    observers::{HitcountsMapObserver, TimeObserver, TrackingHinted, VariableMapObserver},
+    observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver},
     schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler},
     stages::{CalibrationStage, StdMutationalStage},
     state::{HasCorpus, StdState},

fuzzers/qemu_systemmode/src/fuzzer_classic.rs

@@ -13,7 +13,7 @@ use libafl::{
     inputs::{BytesInput, HasTargetBytes},
     monitors::MultiMonitor,
     mutators::scheduled::{havoc_mutations, StdScheduledMutator},
-    observers::{HitcountsMapObserver, TimeObserver, TrackingHinted, VariableMapObserver},
+    observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver},
     schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler},
     stages::StdMutationalStage,
     state::{HasCorpus, StdState},

fuzzers/qemu_systemmode/src/fuzzer_sync_exit.rs

@@ -13,7 +13,7 @@ use libafl::{
     inputs::BytesInput,
     monitors::MultiMonitor,
     mutators::scheduled::{havoc_mutations, StdScheduledMutator},
-    observers::{HitcountsMapObserver, TimeObserver, TrackingHinted, VariableMapObserver},
+    observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver},
     schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler},
     stages::{CalibrationStage, StdMutationalStage},
     state::{HasCorpus, StdState},

libafl_qemu/Cargo.toml

@@ -58,8 +58,8 @@ serdeany_autoreg = ["libafl_bolts/serdeany_autoreg"]
 
 slirp = [ "systemmode", "libafl_qemu_sys/slirp" ] # build qemu with host libslirp (for user networking)
 
-# disabled atm, enabled when fixed with dynamic list
+# Requires the binary's build.rs to call `build_libafl_qemu`
-# shared = [ "libafl_qemu_sys/shared" ]
+shared = [ "libafl_qemu_sys/shared" ]
 
 [dependencies]
 libafl = { path = "../libafl", version = "0.12.0", default-features = false, features = ["std", "derive", "regex"] }

libafl_qemu/libafl_qemu_build/src/build.rs

@@ -6,9 +6,11 @@ use std::{
 
 use which::which;
 
+use crate::cargo_add_rpath;
+
 const QEMU_URL: &str = "https://github.com/AFLplusplus/qemu-libafl-bridge";
 const QEMU_DIRNAME: &str = "qemu-libafl-bridge";
-const QEMU_REVISION: &str = "50b0c90e0aab07643ccb58cfbbef742bcfb8b7d1";
+const QEMU_REVISION: &str = "c9519ee8b6cb1ba54b7df1001f7f39f07218d514";
 
 #[allow(clippy::module_name_repetitions)]
 pub struct BuildResult {
@@ -22,138 +24,55 @@ fn build_dep_check(tools: &[&str]) {
     }
 }
 
-#[allow(clippy::too_many_lines, clippy::missing_panics_doc)]
+fn get_config_signature(config_cmd: &Command) -> String {
-#[must_use]
+    let mut signature_string = String::new();
-pub fn build(
+
-    cpu_target: &str,
+    // Command env
-    is_big_endian: bool,
+    let config_env: String = config_cmd
+        .get_envs()
+        .map(|(key, value)| {
+            format!(
+                "\t{}={}",
+                key.to_str().expect("Couldn't convert OsStr to str"),
+                if let Some(v) = value {
+                    v.to_str().expect("Could't convert OsStr to str")
+                } else {
+                    ""
+                }
+            )
+        })
+        .reduce(|acc, elt| format!("{acc}\n{elt}"))
+        .into_iter()
+        .collect();
+    signature_string += format!("Environment:\n{config_env}").as_str();
+
+    // Command args
+    let config_args: String = config_cmd
+        .get_args()
+        .map(|arg| format!("\t{}", arg.to_str().expect("Couldn't convert OsStr to str")))
+        .reduce(|acc, arg| format!("{acc}\n{arg}"))
+        .into_iter()
+        .collect();
+    signature_string += format!("\n\nArguments:\n{config_args}").as_str();
+
+    signature_string
+}
+
+#[allow(clippy::too_many_lines)]
+fn configure_qemu(
+    cc_compiler: &cc::Tool,
+    cpp_compiler: &cc::Tool,
+    qemu_path: &PathBuf,
+    build_dir: &Path,
     is_usermode: bool,
-    jobs: Option<u32>,
+    cpu_target: &String,
-) -> BuildResult {
+    target_suffix: &String,
-    let mut cpu_target = cpu_target.to_string();
+) -> Command {
-    // qemu-system-arm supports both big and little endian configurations and so
-    // therefore the "be" feature should ignored in this configuration. Also
-    // ignore the feature if we are running in clippy which enables all the
-    // features at once (disabling the check for mutually exclusive options)
-    // resulting in cpu_target being set to 'x86_64' above which obviously
-    // doesn't support BE.
-    if is_big_endian && cpu_target == "arm" && is_usermode && !cfg!(feature = "clippy") {
-        // We have told rustc which CPU target to use above (it doesn't need
-        // to make any changes for endianness), however, we need QEMU to be
-        // built for the right endian-ness, so we update the cpu_target for
-        // here on down
-        cpu_target += "eb";
-    }
-
-    if !is_big_endian && cpu_target == "mips" && !cfg!(feature = "clippy") {
-        cpu_target += "el";
-    }
-
-    let custom_qemu_dir = env::var_os("CUSTOM_QEMU_DIR").map(|x| x.to_string_lossy().to_string());
-    let custom_qemu_no_build = env::var("CUSTOM_QEMU_NO_BUILD").is_ok();
-    let custom_qemu_no_configure = env::var("CUSTOM_QEMU_NO_CONFIGURE").is_ok();
-    println!("cargo:rerun-if-env-changed=CUSTOM_QEMU_DIR");
-    println!("cargo:rerun-if-env-changed=CUSTOM_QEMU_NO_BUILD");
-    println!("cargo:rerun-if-env-changed=CUSTOM_QEMU_NO_CONFIGURE");
-
-    let out_dir = env::var_os("OUT_DIR").unwrap();
-    let out_dir = out_dir.to_string_lossy().to_string();
-    let out_dir_path = Path::new(&out_dir);
-    let mut target_dir = out_dir_path.to_path_buf();
-    target_dir.pop();
-    target_dir.pop();
-    target_dir.pop();
-
-    build_dep_check(&["git", "make"]);
-
-    let cc_compiler = cc::Build::new().cpp(false).get_compiler();
-    let cpp_compiler = cc::Build::new().cpp(true).get_compiler();
-
-    let qemu_path = if let Some(qemu_dir) = custom_qemu_dir.as_ref() {
-        Path::new(&qemu_dir).to_path_buf()
-    } else {
-        let qemu_path = target_dir.join(QEMU_DIRNAME);
-
-        let qemu_rev = target_dir.join("QEMU_REVISION");
-        if qemu_rev.exists()
-            && fs::read_to_string(&qemu_rev).expect("Failed to read QEMU_REVISION") != QEMU_REVISION
-        {
-            drop(fs::remove_dir_all(&qemu_path));
-        }
-
-        if !qemu_path.is_dir() {
-            println!("cargo:warning=Qemu not found, cloning with git ({QEMU_REVISION})...");
-            fs::create_dir_all(&qemu_path).unwrap();
-            assert!(Command::new("git")
-                .current_dir(&qemu_path)
-                .arg("init")
-                .status()
-                .unwrap()
-                .success());
-            assert!(Command::new("git")
-                .current_dir(&qemu_path)
-                .arg("remote")
-                .arg("add")
-                .arg("origin")
-                .arg(QEMU_URL)
-                .status()
-                .unwrap()
-                .success());
-            assert!(Command::new("git")
-                .current_dir(&qemu_path)
-                .arg("fetch")
-                .arg("--depth")
-                .arg("1")
-                .arg("origin")
-                .arg(QEMU_REVISION)
-                .status()
-                .unwrap()
-                .success());
-            assert!(Command::new("git")
-                .current_dir(&qemu_path)
-                .arg("checkout")
-                .arg("FETCH_HEAD")
-                .status()
-                .unwrap()
-                .success());
-            fs::write(&qemu_rev, QEMU_REVISION).unwrap();
-        }
-
-        qemu_path
-    };
-
-    let build_dir = qemu_path.join("build");
-
-    let target_suffix = if is_usermode {
-        "linux-user".to_string()
-    } else {
-        "softmmu".to_string()
-    };
-
-    let (output_lib, output_lib_link) = if is_usermode {
-        (
-            build_dir.join(format!("libqemu-{cpu_target}.so")),
-            format!("qemu-{cpu_target}"),
-        )
-    } else {
-        (
-            build_dir.join(format!("libqemu-system-{cpu_target}.so")),
-            format!("qemu-system-{cpu_target}"),
-        )
-    };
-
-    // println!("cargo:rerun-if-changed={}", output_lib.to_string_lossy());
-
-    if !output_lib.is_file() || (custom_qemu_dir.is_some() && !custom_qemu_no_build) {
-        /*drop(
-            Command::new("make")
-                .current_dir(&qemu_path)
-                .arg("distclean")
-                .status(),
-        );*/
     let mut cmd = Command::new("./configure");
-        cmd.current_dir(&qemu_path)
+
-            //.arg("--as-static-lib")
+    // Set common options for usermode and systemmode
+    cmd.current_dir(qemu_path)
+        .env("__LIBAFL_QEMU_CONFIGURE", "")
         .env("__LIBAFL_QEMU_BUILD_OUT", build_dir.join("linkinfo.json"))
         .env("__LIBAFL_QEMU_BUILD_CC", cc_compiler.path())
         .env("__LIBAFL_QEMU_BUILD_CXX", cpp_compiler.path())
@@ -166,18 +85,20 @@ pub fn build(
             qemu_path.join("linker_interceptor++.py").display()
         ))
         .arg("--as-shared-lib")
-            .arg(&format!("--target-list={cpu_target}-{target_suffix}"));
+        .arg(&format!("--target-list={cpu_target}-{target_suffix}"))
-        if cfg!(feature = "debug_assertions") {
+        .arg("--disable-bsd-user")
-            cmd.arg("--enable-debug");
+        .arg("--disable-capstone");
+
+    if cfg!(debug_assertions) {
+        // cmd.arg("--enable-debug");
+        // .arg("--enable-debug-tcg");
     }
-        if is_usermode && !custom_qemu_no_configure {
+
-            cmd.args([
+    if is_usermode {
-                "--disable-bsd-user",
+        // Usermode options
-                "--disable-fdt",
+        cmd.args(["--disable-fdt", "--disable-system"]);
-                "--disable-system",
+    } else {
-                "--disable-capstone",
+        // Systemmode options
-            ]);
-        } else if !custom_qemu_no_configure {
         cmd.arg(if cfg!(feature = "slirp") {
             "--enable-slirp"
         } else {
@@ -193,9 +114,7 @@ pub fn build(
         .arg("--disable-bochs")
         .arg("--disable-bpf")
         .arg("--disable-brlapi")
-            .arg("--disable-bsd-user")
         .arg("--disable-bzip2")
-            .arg("--disable-capstone")
         .arg("--disable-cap-ng")
         .arg("--disable-canokey")
         .arg("--disable-cloop")
@@ -294,12 +213,19 @@ pub fn build(
         .arg("--disable-tests");
     }
 
-        assert!(
+    cmd
-            cmd.status().expect("Invoking Configure failed").success(),
+}
-            "Configure didn't finish successfully"
+
-        );
+fn build_qemu(
+    cc_compiler: &cc::Tool,
+    cpp_compiler: &cc::Tool,
+    build_dir: &PathBuf,
+    jobs: Option<u32>,
+) -> Command {
     let mut cmd = Command::new("make");
-        cmd.current_dir(&build_dir)
+
+    cmd.current_dir(build_dir)
+        .env("__LIBAFL_QEMU_CONFIGURE", "")
         .env("__LIBAFL_QEMU_BUILD_OUT", build_dir.join("linkinfo.json"))
         .env("__LIBAFL_QEMU_BUILD_CC", cc_compiler.path())
         .env("__LIBAFL_QEMU_BUILD_CXX", cpp_compiler.path())
@@ -308,12 +234,187 @@ pub fn build(
     if let Some(j) = jobs {
         cmd.arg(&format!("{j}")).env("V", "1");
     }
+
+    cmd
+}
+
+#[allow(clippy::too_many_lines, clippy::missing_panics_doc)]
+#[must_use]
+pub fn build(
+    cpu_target: &str,
+    is_big_endian: bool,
+    is_usermode: bool,
+    jobs: Option<u32>,
+) -> BuildResult {
+    let mut cpu_target = cpu_target.to_string();
+    // qemu-system-arm supports both big and little endian configurations and so
+    // the "be" feature should be ignored in this configuration. Also
+    // ignore the feature if we are running in clippy which enables all the
+    // features at once (disabling the check for mutually exclusive options)
+    // resulting in cpu_target being set to 'x86_64' above which obviously
+    // doesn't support BE.
+    if is_big_endian && cpu_target == "arm" && is_usermode && !cfg!(feature = "clippy") {
+        // We have told rustc which CPU target to use above (it doesn't need
+        // to make any changes for endianness), however, we need QEMU to be
+        // built for the right endian-ness, so we update the cpu_target for
+        // here on down
+        cpu_target += "eb";
+    }
+
+    if !is_big_endian && cpu_target == "mips" && !cfg!(feature = "clippy") {
+        cpu_target += "el";
+    }
+
+    let libafl_qemu_dir = env::var_os("LIBAFL_QEMU_DIR").map(|x| x.to_string_lossy().to_string());
+    let libafl_qemu_force_configure = env::var("LIBAFL_QEMU_FORCE_CONFIGURE").is_ok();
+    let libafl_qemu_no_build = env::var("LIBAFL_QEMU_NO_BUILD").is_ok();
+
+    println!("cargo:rerun-if-env-changed=LIBAFL_QEMU_DIR");
+    println!("cargo:rerun-if-env-changed=LIBAFL_QEMU_FORCE_CONFIGURE");
+    println!("cargo:rerun-if-env-changed=LIBAFL_QEMU_NO_BUILD");
+
+    let out_dir = env::var_os("OUT_DIR").unwrap();
+    let out_dir = out_dir.to_string_lossy().to_string();
+    let out_dir_path = Path::new(&out_dir);
+    let mut target_dir = out_dir_path.to_path_buf();
+    target_dir.pop();
+    target_dir.pop();
+    target_dir.pop();
+
+    build_dep_check(&["git", "make"]);
+
+    let cc_compiler = cc::Build::new().cpp(false).get_compiler();
+    let cpp_compiler = cc::Build::new().cpp(true).get_compiler();
+
+    let qemu_path = if let Some(qemu_dir) = libafl_qemu_dir.as_ref() {
+        Path::new(&qemu_dir).to_path_buf()
+    } else {
+        let qemu_path = target_dir.join(QEMU_DIRNAME);
+
+        let qemu_rev = target_dir.join("QEMU_REVISION");
+        if qemu_rev.exists()
+            && fs::read_to_string(&qemu_rev).expect("Failed to read QEMU_REVISION") != QEMU_REVISION
+        {
+            drop(fs::remove_dir_all(&qemu_path));
+        }
+
+        if !qemu_path.is_dir() {
+            println!("cargo:warning=Qemu not found, cloning with git ({QEMU_REVISION})...");
+            fs::create_dir_all(&qemu_path).unwrap();
+            assert!(Command::new("git")
+                .current_dir(&qemu_path)
+                .arg("init")
+                .status()
+                .unwrap()
+                .success());
+            assert!(Command::new("git")
+                .current_dir(&qemu_path)
+                .arg("remote")
+                .arg("add")
+                .arg("origin")
+                .arg(QEMU_URL)
+                .status()
+                .unwrap()
+                .success());
+            assert!(Command::new("git")
+                .current_dir(&qemu_path)
+                .arg("fetch")
+                .arg("--depth")
+                .arg("1")
+                .arg("origin")
+                .arg(QEMU_REVISION)
+                .status()
+                .unwrap()
+                .success());
+            assert!(Command::new("git")
+                .current_dir(&qemu_path)
+                .arg("checkout")
+                .arg("FETCH_HEAD")
+                .status()
+                .unwrap()
+                .success());
+            fs::write(&qemu_rev, QEMU_REVISION).unwrap();
+        }
+
+        qemu_path
+    };
+
+    let qemu_build_dir = qemu_path.join("build");
+    let config_signature_path = qemu_build_dir.join("libafl_config");
+
+    let target_suffix = if is_usermode {
+        "linux-user".to_string()
+    } else {
+        "softmmu".to_string()
+    };
+
+    let (output_lib, output_lib_link) = if is_usermode {
+        (
+            qemu_build_dir.join(format!("libqemu-{cpu_target}.so")),
+            format!("qemu-{cpu_target}"),
+        )
+    } else {
+        (
+            qemu_build_dir.join(format!("libqemu-system-{cpu_target}.so")),
+            format!("qemu-system-{cpu_target}"),
+        )
+    };
+
+    let libafl_config_old_signature = fs::read_to_string(&config_signature_path);
+
+    let mut config_cmd = configure_qemu(
+        &cc_compiler,
+        &cpp_compiler,
+        &qemu_path,
+        &qemu_build_dir,
+        is_usermode,
+        &cpu_target,
+        &target_suffix,
+    );
+
+    let current_config_signature = get_config_signature(&config_cmd);
+    let must_reconfigure = if libafl_qemu_force_configure {
+        // If the user asked to reconfigure, do so
+        true
+    } else if let Ok(libafl_config_old_signature) = libafl_config_old_signature {
+        if libafl_config_old_signature == current_config_signature {
+            // Signature match, do not reconfigure
+            false
+        } else {
+            println!("cargo:warning=QEMU configuration is outdated. Reconfiguring...");
+            true
+        }
+    } else {
+        // In worst scenario, reconfigure
+        true
+    };
+
+    if must_reconfigure {
         assert!(
-            cmd.status().expect("Invoking Make Failed").success(),
+            config_cmd
+                .status()
+                .expect("Invoking Configure failed")
+                .success(),
+            "Configure didn't finish successfully"
+        );
+
+        // Config succeeded at this point, (over)write the signature file
+        fs::write(config_signature_path, current_config_signature)
+            .expect("Couldn't write config signature file.");
+    }
+
+    // Always build by default, make will detect if it is necessary to rebuild qemu
+    if !libafl_qemu_no_build {
+        let mut build_cmd = build_qemu(&cc_compiler, &cpp_compiler, &qemu_build_dir, jobs);
+
+        assert!(
+            build_cmd.status().expect("Invoking Make Failed").success(),
             "Make didn't finish successfully"
         );
     }
 
+    assert!(output_lib.is_file()); // Sanity check
+
     /*
     let mut objects = vec![];
     for dir in &[
@@ -338,13 +439,12 @@ pub fn build(
     */
 
     if cfg!(feature = "shared") {
-        println!(
+        let qemu_build_dir_str = qemu_build_dir.to_str().expect("Could not convert to str");
-            "cargo:rustc-link-search=native={}",
+        println!("cargo:rustc-link-search=native={qemu_build_dir_str}");
-            build_dir.to_string_lossy()
-        );
         println!("cargo:rustc-link-lib=dylib={output_lib_link}");
+        cargo_add_rpath(qemu_build_dir_str);
     } else {
-        let compile_commands_string = &fs::read_to_string(build_dir.join("linkinfo.json"))
+        let compile_commands_string = &fs::read_to_string(qemu_build_dir.join("linkinfo.json"))
             .expect("Failed to read linkinfo.json");
 
         let linkinfo = json::parse(compile_commands_string).expect("Failed to parse linkinfo.json");
@@ -357,16 +457,27 @@ pub fn build(
             );
         }
 
-        assert!(cpp_compiler
+        let mut link_command = cpp_compiler.to_command();
-            .to_command()
+
-            .current_dir(&build_dir)
+        link_command
+            .current_dir(&qemu_build_dir)
             .arg("-o")
             .arg("libqemu-partially-linked.o")
             .arg("-r")
-            .args(cmd)
+            .args(cmd);
-            .status()
+
-            .expect("Partial linked failure")
+        let link_str = format!("{link_command:?}");
-            .success());
+
+        let output = link_command.output().expect("Partial linked failure");
+
+        if !output.status.success() {
+            fs::write(qemu_build_dir.join("link.command"), link_str).expect("Link command failed.");
+            fs::write(qemu_build_dir.join("link.stdout"), &output.stdout)
+                .expect("Link stdout failed.");
+            fs::write(qemu_build_dir.join("link.stderr"), &output.stderr)
+                .expect("Link stderr failed.");
+            panic!("Linking failed.");
+        }
 
         /* // Old manual linking, kept here for reference and debugging
         if is_usermode {
@@ -456,7 +567,7 @@ pub fn build(
             .current_dir(out_dir_path)
             .arg("crs")
             .arg("libqemu-partially-linked.a")
-            .arg(build_dir.join("libqemu-partially-linked.o"))
+            .arg(qemu_build_dir.join("libqemu-partially-linked.o"))
             .status()
             .expect("Ar creation");
 
@@ -476,6 +587,21 @@ pub fn build(
                 .expect("linkinfo.json `libs` values must be strings");
             println!("cargo:rustc-link-lib={val}");
         }
+
+        for arg in linkinfo["rpath"].members() {
+            let val = arg
+                .as_str()
+                .expect("linkinfo.json `libs` values must be strings")
+                .to_string()
+                .replace(
+                    "$ORIGIN",
+                    qemu_build_dir
+                        .as_os_str()
+                        .to_str()
+                        .expect("Could not convert OsStr to str"),
+                );
+            cargo_add_rpath(&val);
+        }
     }
 
     /*
@@ -497,7 +623,7 @@ pub fn build(
         //}
 
         fs::create_dir_all(target_dir.join("pc-bios")).unwrap();
-        for path in fs::read_dir(build_dir.join("pc-bios")).unwrap() {
+        for path in fs::read_dir(qemu_build_dir.join("pc-bios")).unwrap() {
             let path = path.unwrap().path();
             if path.is_file() {
                 if let Some(name) = path.file_name() {
@@ -510,6 +636,6 @@ pub fn build(
 
     BuildResult {
         qemu_path,
-        build_dir,
+        build_dir: qemu_build_dir,
     }
 }

libafl_qemu/libafl_qemu_build/src/lib.rs

@@ -3,6 +3,7 @@ use std::{
     env, fs,
     path::{Path, PathBuf},
     process::Command,
+    ptr::addr_of_mut,
 };
 
 use regex::Regex;
@@ -15,6 +16,43 @@ pub use build::build;
 
 const LLVM_VERSION_MAX: i32 = 33;
 
+static mut CARGO_RPATH: Option<Vec<String>> = None;
+static CARGO_RPATH_SEPARATOR: &str = "|";
+
+pub fn cargo_add_rpath(rpath: &str) {
+    unsafe {
+        if let Some(rpaths) = &mut *addr_of_mut!(CARGO_RPATH) {
+            rpaths.push(rpath.to_string());
+        } else {
+            CARGO_RPATH = Some(vec![rpath.to_string()]);
+        }
+    }
+}
+
+pub fn cargo_propagate_rpath() {
+    unsafe {
+        if let Some(cargo_cmds) = &mut *addr_of_mut!(CARGO_RPATH) {
+            let rpath = cargo_cmds.join(CARGO_RPATH_SEPARATOR);
+            println!("cargo:rpath={rpath}");
+        }
+    }
+}
+
+/// Must be called from final binary crates
+pub fn build_libafl_qemu() {
+    // Add rpath if there are some
+    if let Some(rpaths) = env::var_os("DEP_QEMU_RPATH") {
+        let rpaths: Vec<&str> = rpaths
+            .to_str()
+            .expect("Cannot convert OsString to str")
+            .split(CARGO_RPATH_SEPARATOR)
+            .collect();
+        for rpath in rpaths {
+            println!("cargo:rustc-link-arg-bins=-Wl,-rpath,{rpath}");
+        }
+    }
+}
+
 pub fn build_with_bindings(
     cpu_target: &str,
     is_big_endian: bool,
@@ -42,6 +80,8 @@ pub fn build_with_bindings(
     let re = Regex::new("(Option<\\s*)unsafe( extern \"C\" fn\\(data: u64)").unwrap();
     let replaced = re.replace_all(&contents, "$1$2");
     fs::write(bindings_file, replaced.as_bytes()).expect("Unable to write file");
+
+    cargo_propagate_rpath();
 }
 
 // For bindgen, the llvm version must be >= of the rust llvm version

libafl_qemu/libafl_qemu_sys/Cargo.toml

@@ -10,6 +10,7 @@ license = "MIT OR Apache-2.0"
 keywords = ["fuzzing", "qemu", "instrumentation"]
 edition = "2021"
 categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"]
+links = "qemu"
 
 [package.metadata.docs.rs]
 features = ["x86_64", "usermode"]