Qemu features3 (#1538)

* Fix issue with libafl_qemu being repeatedly rebuilt

* Changes to make qemu_launcher a production ready fuzzer

* Remove _get prefix

* Don't collect DrCov data during the campaign

* Fix poor performance

* Better validation for core selection

* Changes to print debug when running in verbose mode

* Autofix

* Remove afl++-clang

* Fix build error on 32-bit

* Fix some clippy

* Fix OSX

* Set default version of clang/clang++

* Review changes

* Fix issue with fd sharing between processes

---------

Co-authored-by: Your Name <you@example.com>
Co-authored-by: Andrea Fioraldi <andreafioraldi@gmail.com>

fuzzers/qemu_launcher/Cargo.toml

@@ -29,4 +29,8 @@ clap = { version = "4.3.0", features = ["derive", "string"]}
 libafl = { path = "../../libafl/" }
 libafl_bolts = { path = "../../libafl_bolts/" }
 libafl_qemu = { path = "../../libafl_qemu/", features = ["usermode"] }
+log = {version = "0.4.20" }
+nix = { version = "0.26" }
 rangemap = { version = "1.3" }
+readonly = { version = "0.2.10" }
+typed-builder = { version = "0.15.1" }

fuzzers/qemu_launcher/Makefile.toml

@@ -194,6 +194,7 @@ args = [
 dependencies = ["build"]
 script_runner="@shell"
 script='''
+rm -f ${TARGET_DIR}/${PROFILE_DIR}/qemu_launcher-${CARGO_MAKE_PROFILE}
 mv ${TARGET_DIR}/${PROFILE_DIR}/qemu_launcher ${TARGET_DIR}/${PROFILE_DIR}/qemu_launcher-${CARGO_MAKE_PROFILE}
 '''
 
@@ -219,6 +220,27 @@ ${CROSS_CXX} \
 '''
 dependencies = [ "libpng" ]
 
+[tasks.debug]
+linux_alias = "debug_unix"
+mac_alias = "unsupported"
+windows_alias = "unsupported"
+
+[tasks.debug_unix]
+command = "${TARGET_DIR}/${PROFILE_DIR}/qemu_launcher-${CARGO_MAKE_PROFILE}"
+args = [
+    "--input", "./corpus",
+    "--output", "${TARGET_DIR}/output/",
+    "--log", "${TARGET_DIR}/output/log.txt",
+    "--cores", "0-7",
+    "--asan-cores", "0-3",
+    "--cmplog-cores", "2-5",
+    "--iterations", "100000",
+    "--verbose",
+    "--",
+    "${TARGET_DIR}/libpng-harness-${CARGO_MAKE_PROFILE}",
+]
+dependencies = [ "harness", "fuzzer" ]
+
 [tasks.run]
 linux_alias = "run_unix"
 mac_alias = "unsupported"
@@ -227,9 +249,31 @@ windows_alias = "unsupported"
 [tasks.run_unix]
 command = "${TARGET_DIR}/${PROFILE_DIR}/qemu_launcher-${CARGO_MAKE_PROFILE}"
 args = [
-    "--coverage", "${TARGET_DIR}/drcov.log",
     "--input", "./corpus",
     "--output", "${TARGET_DIR}/output/",
+    "--log", "${TARGET_DIR}/output/log.txt",
+    "--cores", "0-7",
+    "--asan-cores", "0-3",
+    "--cmplog-cores", "2-5",
+    "--iterations", "1000000",
+    "--tui",
+    "--",
+    "${TARGET_DIR}/libpng-harness-${CARGO_MAKE_PROFILE}",
+]
+dependencies = [ "harness", "fuzzer" ]
+
+[tasks.single]
+linux_alias = "single_unix"
+mac_alias = "unsupported"
+windows_alias = "unsupported"
+
+[tasks.single_unix]
+command = "${TARGET_DIR}/${PROFILE_DIR}/qemu_launcher-${CARGO_MAKE_PROFILE}"
+args = [
+    "--input", "./corpus",
+    "--output", "${TARGET_DIR}/output/",
+    "--log", "${TARGET_DIR}/output/log.txt",
+    "--cores", "0",
     "--",
     "${TARGET_DIR}/libpng-harness-${CARGO_MAKE_PROFILE}",
 ]

fuzzers/qemu_launcher/src/client.rs

@@ -0,0 +1,153 @@
+use std::{env, ops::Range};
+
+use libafl::{
+    corpus::{InMemoryOnDiskCorpus, OnDiskCorpus},
+    events::LlmpRestartingEventManager,
+    inputs::BytesInput,
+    state::StdState,
+    Error,
+};
+use libafl_bolts::{
+    core_affinity::CoreId, rands::StdRand, shmem::StdShMemProvider, tuples::tuple_list,
+};
+use libafl_qemu::{
+    asan::{init_with_asan, QemuAsanHelper},
+    cmplog::QemuCmpLogHelper,
+    edges::QemuEdgeCoverageHelper,
+    elf::EasyElf,
+    ArchExtras, Emulator, GuestAddr, QemuInstrumentationFilter,
+};
+
+use crate::{instance::Instance, options::FuzzerOptions};
+
+pub type ClientState =
+    StdState<BytesInput, InMemoryOnDiskCorpus<BytesInput>, StdRand, OnDiskCorpus<BytesInput>>;
+
+pub struct Client<'a> {
+    options: &'a FuzzerOptions,
+}
+
+impl<'a> Client<'a> {
+    pub fn new(options: &FuzzerOptions) -> Client {
+        Client { options }
+    }
+
+    fn args(&self) -> Result<Vec<String>, Error> {
+        let program = env::args()
+            .next()
+            .ok_or_else(|| Error::empty_optional("Failed to read program name"))?;
+
+        let mut args = self.options.args.clone();
+        args.insert(0, program);
+        Ok(args)
+    }
+
+    fn env(&self) -> Result<Vec<(String, String)>, Error> {
+        let env = env::vars()
+            .filter(|(k, _v)| k != "LD_LIBRARY_PATH")
+            .collect::<Vec<(String, String)>>();
+        Ok(env)
+    }
+
+    fn start_pc(emu: &Emulator) -> Result<GuestAddr, Error> {
+        let mut elf_buffer = Vec::new();
+        let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer)?;
+
+        let start_pc = elf
+            .resolve_symbol("LLVMFuzzerTestOneInput", emu.load_addr())
+            .ok_or_else(|| Error::empty_optional("Symbol LLVMFuzzerTestOneInput not found"))?;
+        Ok(start_pc)
+    }
+
+    fn coverage_filter(&self, emu: &Emulator) -> Result<QemuInstrumentationFilter, Error> {
+        /* Conversion is required on 32-bit targets, but not on 64-bit ones */
+        if let Some(includes) = &self.options.include {
+            #[cfg_attr(target_pointer_width = "64", allow(clippy::useless_conversion))]
+            let rules = includes
+                .iter()
+                .map(|x| Range {
+                    start: x.start.into(),
+                    end: x.end.into(),
+                })
+                .collect::<Vec<Range<GuestAddr>>>();
+            Ok(QemuInstrumentationFilter::AllowList(rules))
+        } else if let Some(excludes) = &self.options.exclude {
+            #[cfg_attr(target_pointer_width = "64", allow(clippy::useless_conversion))]
+            let rules = excludes
+                .iter()
+                .map(|x| Range {
+                    start: x.start.into(),
+                    end: x.end.into(),
+                })
+                .collect::<Vec<Range<GuestAddr>>>();
+            Ok(QemuInstrumentationFilter::DenyList(rules))
+        } else {
+            let mut elf_buffer = Vec::new();
+            let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer)?;
+            let range = elf
+                .get_section(".text", emu.load_addr())
+                .ok_or_else(|| Error::key_not_found("Failed to find .text section"))?;
+            Ok(QemuInstrumentationFilter::AllowList(vec![range]))
+        }
+    }
+
+    pub fn run(
+        &self,
+        state: Option<ClientState>,
+        mgr: LlmpRestartingEventManager<ClientState, StdShMemProvider>,
+        core_id: CoreId,
+    ) -> Result<(), Error> {
+        let mut args = self.args()?;
+        log::debug!("ARGS: {:#?}", args);
+
+        let mut env = self.env()?;
+        log::debug!("ENV: {:#?}", env);
+
+        let emu = {
+            if self.options.is_asan_core(core_id) {
+                init_with_asan(&mut args, &mut env)?
+            } else {
+                Emulator::new(&args, &env)?
+            }
+        };
+
+        let start_pc = Self::start_pc(&emu)?;
+        log::debug!("start_pc @ {start_pc:#x}");
+
+        emu.entry_break(start_pc);
+
+        let ret_addr: GuestAddr = emu
+            .read_return_address()
+            .map_err(|e| Error::unknown(format!("Failed to read return address: {e:}")))?;
+        log::debug!("ret_addr = {ret_addr:#x}");
+        emu.set_breakpoint(ret_addr);
+
+        let is_asan = self.options.is_asan_core(core_id);
+        let is_cmplog = self.options.is_cmplog_core(core_id);
+
+        let edge_coverage_helper = QemuEdgeCoverageHelper::new(self.coverage_filter(&emu)?);
+
+        let instance = Instance::builder()
+            .options(self.options)
+            .emu(&emu)
+            .mgr(mgr)
+            .core_id(core_id);
+        if is_asan && is_cmplog {
+            let helpers = tuple_list!(
+                edge_coverage_helper,
+                QemuCmpLogHelper::default(),
+                QemuAsanHelper::default(),
+            );
+            instance.build().run(helpers, state)
+        } else if is_asan {
+            let helpers = tuple_list!(edge_coverage_helper, QemuAsanHelper::default(),);
+            instance.build().run(helpers, state)
+        } else if is_cmplog {
+            let helpers = tuple_list!(edge_coverage_helper, QemuCmpLogHelper::default(),);
+            instance.build().run(helpers, state)
+        } else {
+            let helpers = tuple_list!(edge_coverage_helper,);
+            instance.build().run(helpers, state)
+        }
+    }
+}

fuzzers/qemu_launcher/src/fuzzer.rs

@@ -1,315 +1,113 @@
-//! A libfuzzer-like fuzzer using qemu for binary-only coverage
+use std::{
-//!
+    cell::RefCell,
-use core::{ptr::addr_of_mut, time::Duration};
+    fs::{File, OpenOptions},
-use std::{env, path::PathBuf, process};
+    io::{self, Write},
+};
 
-use clap::{builder::Str, Parser};
+use clap::Parser;
 use libafl::{
-    corpus::{Corpus, InMemoryCorpus, OnDiskCorpus},
+    events::{EventConfig, Launcher},
-    events::{launcher::Launcher, EventConfig, LlmpRestartingEventManager},
+    monitors::{
-    executors::{ExitKind, TimeoutExecutor},
+        tui::{ui::TuiUI, TuiMonitor},
-    feedback_or, feedback_or_fast,
+        Monitor, MultiMonitor,
-    feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
+    },
-    fuzzer::{Fuzzer, StdFuzzer},
-    inputs::{BytesInput, HasTargetBytes},
-    monitors::MultiMonitor,
-    mutators::scheduled::{havoc_mutations, StdScheduledMutator},
-    observers::{HitcountsMapObserver, TimeObserver, VariableMapObserver},
-    schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler},
-    stages::StdMutationalStage,
-    state::{HasCorpus, StdState},
     Error,
 };
 use libafl_bolts::{
-    core_affinity::Cores,
+    current_time,
-    current_nanos,
-    rands::StdRand,
     shmem::{ShMemProvider, StdShMemProvider},
-    tuples::tuple_list,
-    AsSlice,
 };
-use libafl_qemu::{
+#[cfg(unix)]
-    drcov::QemuDrCovHelper,
+use {
-    edges::{edges_map_mut_slice, QemuEdgeCoverageHelper, MAX_EDGES_NUM},
+    nix::unistd::dup,
-    elf::EasyElf,
+    std::os::unix::io::{AsRawFd, FromRawFd},
-    emu::Emulator,
-    ArchExtras, CallingConvention, GuestAddr, GuestReg, MmapPerms, QemuExecutor, QemuHooks,
-    QemuInstrumentationFilter, Regs,
 };
-use rangemap::RangeMap;
 
-#[derive(Default)]
+use crate::{client::Client, options::FuzzerOptions};
-pub struct Version;
 
-impl From<Version> for Str {
+pub struct Fuzzer {
-    fn from(_: Version) -> Str {
+    options: FuzzerOptions,
-        let version = [
-            ("Architecture:", env!("CPU_TARGET")),
-            ("Build Timestamp:", env!("VERGEN_BUILD_TIMESTAMP")),
-            ("Describe:", env!("VERGEN_GIT_DESCRIBE")),
-            ("Commit SHA:", env!("VERGEN_GIT_SHA")),
-            ("Commit Date:", env!("VERGEN_RUSTC_COMMIT_DATE")),
-            ("Commit Branch:", env!("VERGEN_GIT_BRANCH")),
-            ("Rustc Version:", env!("VERGEN_RUSTC_SEMVER")),
-            ("Rustc Channel:", env!("VERGEN_RUSTC_CHANNEL")),
-            ("Rustc Host Triple:", env!("VERGEN_RUSTC_HOST_TRIPLE")),
-            ("Rustc Commit SHA:", env!("VERGEN_RUSTC_COMMIT_HASH")),
-            ("Cargo Target Triple", env!("VERGEN_CARGO_TARGET_TRIPLE")),
-        ]
-        .iter()
-        .map(|(k, v)| format!("{k:25}: {v}\n"))
-        .collect::<String>();
-
-        format!("\n{version:}").into()
-    }
 }
 
-#[derive(Parser, Debug)]
+impl Fuzzer {
-#[clap(author, version, about, long_about = None)]
+    pub fn new() -> Fuzzer {
-#[command(
+        let options = FuzzerOptions::parse();
-    name = format!("qemu_coverage-{}",env!("CPU_TARGET")),
+        options.validate();
-    version = Version::default(),
+        Fuzzer { options }
-    about,
-    long_about = "Tool for generating DrCov coverage data using QEMU instrumentation"
-)]
-pub struct FuzzerOptions {
-    #[arg(long, help = "Coverage file")]
-    coverage: String,
-
-    #[arg(long, help = "Input directory")]
-    input: String,
-
-    #[arg(long, help = "Output directory")]
-    output: String,
-
-    #[arg(long, help = "Timeout in milli-seconds", default_value = "1000", value_parser = FuzzerOptions::parse_timeout)]
-    timeout: Duration,
-
-    #[arg(long = "port", help = "Broker port", default_value_t = 1337_u16)]
-    port: u16,
-
-    #[arg(long, help = "Cpu cores to use", default_value = "all", value_parser = Cores::from_cmdline)]
-    cores: Cores,
-
-    #[clap(short, long, help = "Enable output from the fuzzer clients")]
-    verbose: bool,
-
-    #[arg(last = true, help = "Arguments passed to the target")]
-    args: Vec<String>,
-}
-
-impl FuzzerOptions {
-    fn parse_timeout(src: &str) -> Result<Duration, Error> {
-        Ok(Duration::from_millis(src.parse()?))
-    }
-}
-
-pub fn fuzz() {
-    let mut options = FuzzerOptions::parse();
-
-    let output_dir = PathBuf::from(options.output);
-    let corpus_dirs = [PathBuf::from(options.input)];
-
-    let program = env::args().next().unwrap();
-    println!("Program: {program:}");
-
-    options.args.insert(0, program);
-    println!("ARGS: {:#?}", options.args);
-
-    env::remove_var("LD_LIBRARY_PATH");
-    let env: Vec<(String, String)> = env::vars().collect();
-    let emu = Emulator::new(&options.args, &env).unwrap();
-
-    let mut elf_buffer = Vec::new();
-    let elf = EasyElf::from_file(emu.binary_path(), &mut elf_buffer).unwrap();
-
-    let test_one_input_ptr = elf
-        .resolve_symbol("LLVMFuzzerTestOneInput", emu.load_addr())
-        .expect("Symbol LLVMFuzzerTestOneInput not found");
-    println!("LLVMFuzzerTestOneInput @ {test_one_input_ptr:#x}");
-
-    emu.set_breakpoint(test_one_input_ptr);
-    unsafe { emu.run() };
-
-    for m in emu.mappings() {
-        println!(
-            "Mapping: 0x{:016x}-0x{:016x}, {}",
-            m.start(),
-            m.end(),
-            m.path().unwrap_or("<EMPTY>")
-        );
     }
 
-    let pc: GuestReg = emu.read_reg(Regs::Pc).unwrap();
+    pub fn fuzz(&self) -> Result<(), Error> {
-    println!("Break at {pc:#x}");
+        if self.options.tui {
+            let ui =
+                TuiUI::with_version(String::from("QEMU Launcher"), String::from("0.10.1"), true);
+            let monitor = TuiMonitor::new(ui);
+            self.launch(monitor)
+        } else {
+            let log = self.options.log.as_ref().and_then(|l| {
+                OpenOptions::new()
+                    .append(true)
+                    .create(true)
+                    .open(l)
+                    .ok()
+                    .map(RefCell::new)
+            });
 
-    let ret_addr: GuestAddr = emu.read_return_address().unwrap();
+            #[cfg(unix)]
-    println!("Return address = {ret_addr:#x}");
+            let stdout_cpy = RefCell::new(unsafe {
+                let new_fd = dup(io::stdout().as_raw_fd())?;
+                File::from_raw_fd(new_fd)
+            });
 
-    emu.remove_breakpoint(test_one_input_ptr);
+            // The stats reporter for the broker
-    emu.set_breakpoint(ret_addr);
+            let monitor = MultiMonitor::new(|s| {
+                #[cfg(unix)]
+                writeln!(stdout_cpy.borrow_mut(), "{s}").unwrap();
+                #[cfg(windows)]
+                println!("{s}");
 
-    let input_addr = emu.map_private(0, 4096, MmapPerms::ReadWrite).unwrap();
+                if let Some(log) = &log {
-    println!("Placing input at {input_addr:#x}");
+                    writeln!(log.borrow_mut(), "{:?} {}", current_time(), s).unwrap();
-
+                }
-    let stack_ptr: GuestAddr = emu.read_reg(Regs::Sp).unwrap();
+            });
-
+            self.launch(monitor)
-    let reset = |buf: &[u8], len: GuestReg| -> Result<(), String> {
-        unsafe {
-            emu.write_mem(input_addr, buf);
-            emu.write_reg(Regs::Pc, test_one_input_ptr)?;
-            emu.write_reg(Regs::Sp, stack_ptr)?;
-            emu.write_return_address(ret_addr)?;
-            emu.write_function_argument(CallingConvention::Cdecl, 0, input_addr)?;
-            emu.write_function_argument(CallingConvention::Cdecl, 1, len)?;
-            emu.run();
-            Ok(())
         }
-    };
+    }
 
-    let mut harness = |input: &BytesInput| {
+    fn launch<M>(&self, monitor: M) -> Result<(), Error>
-        let target = input.target_bytes();
+    where
-        let buf = target
+        M: Monitor + Clone,
-            .as_slice()
+    {
-            .chunks(4096)
+        // The shared memory allocator
-            .next()
+        let shmem_provider = StdShMemProvider::new()?;
-            .expect("Failed to get chunk");
-        let len = buf.len() as GuestReg;
-        reset(buf, len).unwrap();
-        ExitKind::Ok
-    };
 
-    let mut run_client = |state: Option<_>, mut mgr: LlmpRestartingEventManager<_, _>, _core_id| {
+        /* If we are running in verbose, don't provide a replacement stdout, otherwise, use /dev/null */
-        // Create an observation channel using the coverage map
+        let stdout = if self.options.verbose {
-        let edges_observer = unsafe {
+            None
-            HitcountsMapObserver::new(VariableMapObserver::from_mut_slice(
+        } else {
-                "edges",
+            Some("/dev/null")
-                edges_map_mut_slice(),
-                addr_of_mut!(MAX_EDGES_NUM),
-            ))
         };
 
-        // Create an observation channel to keep track of the execution time
+        let client = Client::new(&self.options);
-        let time_observer = TimeObserver::new("time");
 
-        // Feedback to rate the interestingness of an input
+        // Build and run a Launcher
-        // This one is composed by two Feedbacks in OR
+        match Launcher::builder()
-        let mut feedback = feedback_or!(
+            .shmem_provider(shmem_provider)
-            // New maximization map feedback linked to the edges observer and the feedback state
+            .broker_port(self.options.port)
-            MaxMapFeedback::tracking(&edges_observer, true, false),
+            .configuration(EventConfig::from_build_id())
-            // Time feedback, this one does not need a feedback state
+            .monitor(monitor)
-            TimeFeedback::with_observer(&time_observer)
+            .run_client(|s, m, c| client.run(s, m, c))
-        );
+            .cores(&self.options.cores)
-
+            .stdout_file(stdout)
-        // A feedback to choose if an input is a solution or not
+            .stderr_file(stdout)
-        let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new());
+            .build()
-
+            .launch()
-        // If not restarting, create a State from scratch
+        {
-        let mut state = state.unwrap_or_else(|| {
+            Ok(()) => Ok(()),
-            StdState::new(
+            Err(Error::ShuttingDown) => {
-                // RNG
+                println!("Fuzzing stopped by user. Good bye.");
-                StdRand::with_seed(current_nanos()),
+                Ok(())
-                // Corpus that will be evolved, we keep it in memory for performance
+            }
-                InMemoryCorpus::new(),
+            Err(err) => Err(err),
-                // Corpus in which we store solutions (crashes in this example),
-                // on disk so the user can get them after stopping the fuzzer
-                OnDiskCorpus::new(output_dir.clone()).unwrap(),
-                // States of the feedbacks.
-                // The feedbacks can report the data that should persist in the State.
-                &mut feedback,
-                // Same for objective feedbacks
-                &mut objective,
-            )
-            .unwrap()
-        });
-
-        // A minimization+queue policy to get testcasess from the corpus
-        let scheduler = IndexesLenTimeMinimizerScheduler::new(QueueScheduler::new());
-
-        // A fuzzer with feedbacks and a corpus scheduler
-        let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective);
-
-        let rangemap = emu
-            .mappings()
-            .filter_map(|m| {
-                m.path()
-                    .map(|p| ((m.start() as usize)..(m.end() as usize), p.to_string()))
-                    .filter(|(_, p)| !p.is_empty())
-            })
-            .enumerate()
-            .fold(
-                RangeMap::<usize, (u16, String)>::new(),
-                |mut rm, (i, (r, p))| {
-                    rm.insert(r, (i as u16, p));
-                    rm
-                },
-            );
-
-        let mut hooks = QemuHooks::new(
-            &emu,
-            tuple_list!(
-                QemuEdgeCoverageHelper::default(),
-                QemuDrCovHelper::new(
-                    QemuInstrumentationFilter::None,
-                    rangemap,
-                    PathBuf::from(&options.coverage),
-                    false,
-                )
-            ),
-        );
-
-        // Create a QEMU in-process executor
-        let executor = QemuExecutor::new(
-            &mut hooks,
-            &mut harness,
-            tuple_list!(edges_observer, time_observer),
-            &mut fuzzer,
-            &mut state,
-            &mut mgr,
-        )
-        .expect("Failed to create QemuExecutor");
-
-        // Wrap the executor to keep track of the timeout
-        let mut executor = TimeoutExecutor::new(executor, options.timeout);
-
-        if state.must_load_initial_inputs() {
-            state
-                .load_initial_inputs(&mut fuzzer, &mut executor, &mut mgr, &corpus_dirs)
-                .unwrap_or_else(|_| {
-                    println!("Failed to load initial corpus at {:?}", &corpus_dirs);
-                    process::exit(0);
-                });
-            println!("We imported {} inputs from disk.", state.corpus().count());
         }
-
-        // Setup an havoc mutator with a mutational stage
-        let mutator = StdScheduledMutator::new(havoc_mutations());
-        let mut stages = tuple_list!(StdMutationalStage::new(mutator));
-
-        fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr)?;
-        Ok(())
-    };
-
-    // The shared memory allocator
-    let shmem_provider = StdShMemProvider::new().expect("Failed to init shared memory");
-
-    // The stats reporter for the broker
-    let monitor = MultiMonitor::new(|s| println!("{s}"));
-
-    // Build and run a Launcher
-    match Launcher::builder()
-        .shmem_provider(shmem_provider)
-        .broker_port(options.port)
-        .configuration(EventConfig::from_build_id())
-        .monitor(monitor)
-        .run_client(&mut run_client)
-        .cores(&options.cores)
-        .stdout_file(Some("/dev/null"))
-        .build()
-        .launch()
-    {
-        Ok(()) => (),
-        Err(Error::ShuttingDown) => println!("Fuzzing stopped by user. Good bye."),
-        Err(err) => panic!("Failed to run launcher: {err:?}"),
     }
 }

fuzzers/qemu_launcher/src/harness.rs

@@ -0,0 +1,85 @@
+use libafl::{
+    executors::ExitKind,
+    inputs::{BytesInput, HasTargetBytes},
+    Error,
+};
+use libafl_bolts::AsSlice;
+use libafl_qemu::{ArchExtras, CallingConvention, Emulator, GuestAddr, GuestReg, MmapPerms, Regs};
+
+pub struct Harness<'a> {
+    emu: &'a Emulator,
+    input_addr: GuestAddr,
+    pc: GuestAddr,
+    stack_ptr: GuestAddr,
+    ret_addr: GuestAddr,
+}
+
+pub const MAX_INPUT_SIZE: usize = 1048576; // 1MB
+
+impl<'a> Harness<'a> {
+    pub fn new(emu: &Emulator) -> Result<Harness, Error> {
+        let input_addr = emu
+            .map_private(0, MAX_INPUT_SIZE, MmapPerms::ReadWrite)
+            .map_err(|e| Error::unknown(format!("Failed to map input buffer: {e:}")))?;
+
+        let pc: GuestReg = emu
+            .read_reg(Regs::Pc)
+            .map_err(|e| Error::unknown(format!("Failed to read PC: {e:}")))?;
+
+        let stack_ptr: GuestAddr = emu
+            .read_reg(Regs::Sp)
+            .map_err(|e| Error::unknown(format!("Failed to read stack pointer: {e:}")))?;
+
+        let ret_addr: GuestAddr = emu
+            .read_return_address()
+            .map_err(|e| Error::unknown(format!("Failed to read return address: {e:}")))?;
+
+        Ok(Harness {
+            emu,
+            input_addr,
+            pc,
+            stack_ptr,
+            ret_addr,
+        })
+    }
+
+    pub fn run(&self, input: &BytesInput) -> ExitKind {
+        self.reset(input).unwrap();
+        ExitKind::Ok
+    }
+
+    fn reset(&self, input: &BytesInput) -> Result<(), Error> {
+        let target = input.target_bytes();
+        let mut buf = target.as_slice();
+        let mut len = buf.len();
+        if len > MAX_INPUT_SIZE {
+            buf = &buf[0..MAX_INPUT_SIZE];
+            len = MAX_INPUT_SIZE;
+        }
+        let len = len as GuestReg;
+
+        unsafe { self.emu.write_mem(self.input_addr, buf) };
+
+        self.emu
+            .write_reg(Regs::Pc, self.pc)
+            .map_err(|e| Error::unknown(format!("Failed to write PC: {e:}")))?;
+
+        self.emu
+            .write_reg(Regs::Sp, self.stack_ptr)
+            .map_err(|e| Error::unknown(format!("Failed to write SP: {e:}")))?;
+
+        self.emu
+            .write_return_address(self.ret_addr)
+            .map_err(|e| Error::unknown(format!("Failed to write return address: {e:}")))?;
+
+        self.emu
+            .write_function_argument(CallingConvention::Cdecl, 0, self.input_addr)
+            .map_err(|e| Error::unknown(format!("Failed to write argument 0: {e:}")))?;
+
+        self.emu
+            .write_function_argument(CallingConvention::Cdecl, 1, len)
+            .map_err(|e| Error::unknown(format!("Failed to write argument 1: {e:}")))?;
+        unsafe { self.emu.run() };
+        Ok(())
+    }
+}

fuzzers/qemu_launcher/src/instance.rs

@@ -0,0 +1,234 @@
+use core::ptr::addr_of_mut;
+use std::process;
+
+use libafl::{
+    corpus::{Corpus, InMemoryOnDiskCorpus, OnDiskCorpus},
+    events::{EventRestarter, LlmpRestartingEventManager},
+    executors::{ShadowExecutor, TimeoutExecutor},
+    feedback_or, feedback_or_fast,
+    feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
+    fuzzer::{Evaluator, Fuzzer, StdFuzzer},
+    inputs::BytesInput,
+    mutators::{
+        scheduled::havoc_mutations, token_mutations::I2SRandReplace, tokens_mutations,
+        StdMOptMutator, StdScheduledMutator, Tokens,
+    },
+    observers::{HitcountsMapObserver, TimeObserver, VariableMapObserver},
+    schedulers::{
+        powersched::PowerSchedule, IndexesLenTimeMinimizerScheduler, PowerQueueScheduler,
+    },
+    stages::{
+        calibrate::CalibrationStage, power::StdPowerMutationalStage, ShadowTracingStage,
+        StagesTuple, StdMutationalStage,
+    },
+    state::{HasCorpus, HasMetadata, StdState, UsesState},
+    Error,
+};
+use libafl_bolts::{
+    core_affinity::CoreId,
+    current_nanos,
+    rands::StdRand,
+    shmem::StdShMemProvider,
+    tuples::{tuple_list, Merge},
+};
+use libafl_qemu::{
+    cmplog::CmpLogObserver,
+    edges::{edges_map_mut_slice, MAX_EDGES_NUM},
+    helper::QemuHelperTuple,
+    Emulator, QemuExecutor, QemuHooks,
+};
+use typed_builder::TypedBuilder;
+
+use crate::{harness::Harness, options::FuzzerOptions};
+
+pub type ClientState =
+    StdState<BytesInput, InMemoryOnDiskCorpus<BytesInput>, StdRand, OnDiskCorpus<BytesInput>>;
+
+pub type ClientMgr = LlmpRestartingEventManager<ClientState, StdShMemProvider>;
+
+#[derive(TypedBuilder)]
+pub struct Instance<'a> {
+    options: &'a FuzzerOptions,
+    emu: &'a Emulator,
+    mgr: ClientMgr,
+    core_id: CoreId,
+}
+
+impl<'a> Instance<'a> {
+    pub fn run<QT>(&mut self, helpers: QT, state: Option<ClientState>) -> Result<(), Error>
+    where
+        QT: QemuHelperTuple<ClientState>,
+    {
+        let mut hooks = QemuHooks::new(self.emu, helpers);
+
+        // Create an observation channel using the coverage map
+        let edges_observer = unsafe {
+            HitcountsMapObserver::new(VariableMapObserver::from_mut_slice(
+                "edges",
+                edges_map_mut_slice(),
+                addr_of_mut!(MAX_EDGES_NUM),
+            ))
+        };
+
+        // Create an observation channel to keep track of the execution time
+        let time_observer = TimeObserver::new("time");
+
+        let map_feedback = MaxMapFeedback::tracking(&edges_observer, true, false);
+
+        let calibration = CalibrationStage::new(&map_feedback);
+
+        // Feedback to rate the interestingness of an input
+        // This one is composed by two Feedbacks in OR
+        let mut feedback = feedback_or!(
+            // New maximization map feedback linked to the edges observer and the feedback state
+            map_feedback,
+            // Time feedback, this one does not need a feedback state
+            TimeFeedback::with_observer(&time_observer)
+        );
+
+        // A feedback to choose if an input is a solution or not
+        let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new());
+
+        // // If not restarting, create a State from scratch
+        let mut state = match state {
+            Some(x) => x,
+            None => {
+                StdState::new(
+                    // RNG
+                    StdRand::with_seed(current_nanos()),
+                    // Corpus that will be evolved, we keep it in memory for performance
+                    InMemoryOnDiskCorpus::no_meta(self.options.queue_dir(self.core_id))?,
+                    // Corpus in which we store solutions (crashes in this example),
+                    // on disk so the user can get them after stopping the fuzzer
+                    OnDiskCorpus::new(self.options.crashes_dir(self.core_id))?,
+                    // States of the feedbacks.
+                    // The feedbacks can report the data that should persist in the State.
+                    &mut feedback,
+                    // Same for objective feedbacks
+                    &mut objective,
+                )?
+            }
+        };
+
+        // A minimization+queue policy to get testcasess from the corpus
+        let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new(
+            &mut state,
+            &edges_observer,
+            PowerSchedule::FAST,
+        ));
+
+        let observers = tuple_list!(edges_observer, time_observer);
+
+        if let Some(tokenfile) = &self.options.tokens {
+            if state.metadata_map().get::<Tokens>().is_none() {
+                state.add_metadata(Tokens::from_file(tokenfile)?);
+            }
+        }
+
+        let harness = Harness::new(self.emu)?;
+        let mut harness = |input: &BytesInput| harness.run(input);
+
+        // A fuzzer with feedbacks and a corpus scheduler
+        let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective);
+
+        if self.options.is_cmplog_core(self.core_id) {
+            // Create a QEMU in-process executor
+            let executor = QemuExecutor::new(
+                &mut hooks,
+                &mut harness,
+                observers,
+                &mut fuzzer,
+                &mut state,
+                &mut self.mgr,
+            )?;
+
+            // Wrap the executor to keep track of the timeout
+            let executor = TimeoutExecutor::new(executor, self.options.timeout);
+
+            // Create an observation channel using cmplog map
+            let cmplog_observer = CmpLogObserver::new("cmplog", true);
+
+            let mut executor = ShadowExecutor::new(executor, tuple_list!(cmplog_observer));
+
+            let tracing = ShadowTracingStage::new(&mut executor);
+
+            // Setup a randomic Input2State stage
+            let i2s = StdMutationalStage::new(StdScheduledMutator::new(tuple_list!(
+                I2SRandReplace::new()
+            )));
+
+            // Setup a MOPT mutator
+            let mutator = StdMOptMutator::new(
+                &mut state,
+                havoc_mutations().merge(tokens_mutations()),
+                7,
+                5,
+            )?;
+
+            let power = StdPowerMutationalStage::new(mutator);
+
+            // The order of the stages matter!
+            let mut stages = tuple_list!(calibration, tracing, i2s, power);
+
+            self.fuzz(&mut state, &mut fuzzer, &mut executor, &mut stages)
+        } else {
+            // Create a QEMU in-process executor
+            let executor = QemuExecutor::new(
+                &mut hooks,
+                &mut harness,
+                observers,
+                &mut fuzzer,
+                &mut state,
+                &mut self.mgr,
+            )?;
+
+            // Wrap the executor to keep track of the timeout
+            let mut executor = TimeoutExecutor::new(executor, self.options.timeout);
+
+            // Setup an havoc mutator with a mutational stage
+            let mutator = StdScheduledMutator::new(havoc_mutations());
+            let mut stages = tuple_list!(StdMutationalStage::new(mutator));
+
+            self.fuzz(&mut state, &mut fuzzer, &mut executor, &mut stages)
+        }
+    }
+
+    fn fuzz<Z, E, ST>(
+        &mut self,
+        state: &mut ClientState,
+        fuzzer: &mut Z,
+        executor: &mut E,
+        stages: &mut ST,
+    ) -> Result<(), Error>
+    where
+        Z: Fuzzer<E, ClientMgr, ST>
+            + UsesState<State = ClientState>
+            + Evaluator<E, ClientMgr, State = ClientState>,
+        E: UsesState<State = ClientState>,
+        ST: StagesTuple<E, ClientMgr, ClientState, Z>,
+    {
+        let corpus_dirs = [self.options.input_dir()];
+
+        if state.must_load_initial_inputs() {
+            state
+                .load_initial_inputs(fuzzer, executor, &mut self.mgr, &corpus_dirs)
+                .unwrap_or_else(|_| {
+                    println!("Failed to load initial corpus at {:?}", corpus_dirs);
+                    process::exit(0);
+                });
+            println!("We imported {} inputs from disk.", state.corpus().count());
+        }
+
+        if let Some(iters) = self.options.iterations {
+            fuzzer.fuzz_loop_for(stages, executor, state, &mut self.mgr, iters)?;
+
+            // It's important, that we store the state before restarting!
+            // Else, the parent will not respawn a new child and quit.
+            self.mgr.on_restart(state)?;
+        } else {
+            fuzzer.fuzz_loop(stages, executor, state, &mut self.mgr)?;
+        }
+
+        Ok(())
+    }
+}

fuzzers/qemu_launcher/src/main.rs

@@ -1,10 +1,23 @@
 //! A libfuzzer-like fuzzer using qemu for binary-only coverage
 #[cfg(target_os = "linux")]
+mod client;
+#[cfg(target_os = "linux")]
 mod fuzzer;
+#[cfg(target_os = "linux")]
+mod harness;
+#[cfg(target_os = "linux")]
+mod instance;
+#[cfg(target_os = "linux")]
+mod options;
+#[cfg(target_os = "linux")]
+mod version;
+
+#[cfg(target_os = "linux")]
+use crate::fuzzer::Fuzzer;
 
 #[cfg(target_os = "linux")]
 pub fn main() {
-    fuzzer::fuzz();
+    Fuzzer::new().fuzz().unwrap();
 }
 
 #[cfg(not(target_os = "linux"))]

fuzzers/qemu_launcher/src/options.rs

@@ -0,0 +1,168 @@
+use core::time::Duration;
+use std::{env, ops::Range, path::PathBuf};
+
+use clap::{error::ErrorKind, CommandFactory, Parser};
+use libafl::Error;
+use libafl_bolts::core_affinity::{CoreId, Cores};
+use libafl_qemu::GuestAddr;
+
+use crate::version::Version;
+
+#[readonly::make]
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+#[command(
+    name = format!("qemu_coverage-{}",env!("CPU_TARGET")),
+    version = Version::default(),
+    about,
+    long_about = "Binary fuzzer using QEMU binary instrumentation"
+)]
+pub struct FuzzerOptions {
+    #[arg(long, help = "Input directory")]
+    pub input: String,
+
+    #[arg(long, help = "Output directory")]
+    pub output: String,
+
+    #[arg(long, help = "Tokens file")]
+    pub tokens: Option<String>,
+
+    #[arg(long, help = "Log file")]
+    pub log: Option<String>,
+
+    #[arg(long, help = "Timeout in milli-seconds", default_value = "1000", value_parser = FuzzerOptions::parse_timeout)]
+    pub timeout: Duration,
+
+    #[arg(long = "port", help = "Broker port", default_value_t = 1337_u16)]
+    pub port: u16,
+
+    #[arg(long, help = "Cpu cores to use", default_value = "all", value_parser = Cores::from_cmdline)]
+    pub cores: Cores,
+
+    #[arg(long, help = "Cpu cores to use to use for ASAN", value_parser = Cores::from_cmdline)]
+    pub asan_cores: Option<Cores>,
+
+    #[arg(long, help = "Cpu cores to use to use for CmpLog", value_parser = Cores::from_cmdline)]
+    pub cmplog_cores: Option<Cores>,
+
+    #[clap(short, long, help = "Enable output from the fuzzer clients")]
+    pub verbose: bool,
+
+    #[clap(long, help = "Enable AFL++ style output", conflicts_with = "verbose")]
+    pub tui: bool,
+
+    #[arg(long = "iterations", help = "Maximum numer of iterations")]
+    pub iterations: Option<u64>,
+
+    #[arg(long = "include", help="Include address ranges", value_parser = FuzzerOptions::parse_ranges)]
+    pub include: Option<Vec<Range<GuestAddr>>>,
+
+    #[arg(long = "exclude", help="Exclude address ranges", value_parser = FuzzerOptions::parse_ranges, conflicts_with="include")]
+    pub exclude: Option<Vec<Range<GuestAddr>>>,
+
+    #[arg(last = true, help = "Arguments passed to the target")]
+    pub args: Vec<String>,
+}
+
+impl FuzzerOptions {
+    fn parse_timeout(src: &str) -> Result<Duration, Error> {
+        Ok(Duration::from_millis(src.parse()?))
+    }
+
+    fn parse_ranges(src: &str) -> Result<Vec<Range<GuestAddr>>, Error> {
+        src.split(',')
+            .map(|r| {
+                let parts = r.split('-').collect::<Vec<&str>>();
+                if parts.len() == 2 {
+                    let start = GuestAddr::from_str_radix(parts[0].trim_start_matches("0x"), 16)
+                        .map_err(|e| {
+                            Error::illegal_argument(format!(
+                                "Invalid start address: {} ({e:})",
+                                parts[0]
+                            ))
+                        })?;
+                    let end = GuestAddr::from_str_radix(parts[1].trim_start_matches("0x"), 16)
+                        .map_err(|e| {
+                            Error::illegal_argument(format!(
+                                "Invalid end address: {} ({e:})",
+                                parts[1]
+                            ))
+                        })?;
+                    Ok(Range { start, end })
+                } else {
+                    Err(Error::illegal_argument(format!(
+                        "Invalid range provided: {r:}"
+                    )))
+                }
+            })
+            .collect::<Result<Vec<Range<GuestAddr>>, Error>>()
+    }
+
+    pub fn is_asan_core(&self, core_id: CoreId) -> bool {
+        self.asan_cores
+            .as_ref()
+            .map_or(false, |c| c.contains(core_id))
+    }
+
+    pub fn is_cmplog_core(&self, core_id: CoreId) -> bool {
+        self.cmplog_cores
+            .as_ref()
+            .map_or(false, |c| c.contains(core_id))
+    }
+
+    pub fn input_dir(&self) -> PathBuf {
+        PathBuf::from(&self.input)
+    }
+
+    pub fn output_dir(&self, core_id: CoreId) -> PathBuf {
+        let mut dir = PathBuf::from(&self.output);
+        dir.push(format!("cpu_{:03}", core_id.0));
+        dir
+    }
+
+    pub fn queue_dir(&self, core_id: CoreId) -> PathBuf {
+        let mut dir = self.output_dir(core_id).clone();
+        dir.push("queue");
+        dir
+    }
+
+    pub fn crashes_dir(&self, core_id: CoreId) -> PathBuf {
+        let mut dir = self.output_dir(core_id).clone();
+        dir.push("crashes");
+        dir
+    }
+
+    pub fn validate(&self) {
+        if let Some(asan_cores) = &self.asan_cores {
+            for id in &asan_cores.ids {
+                if !self.cores.contains(*id) {
+                    let mut cmd = FuzzerOptions::command();
+                    cmd.error(
+                        ErrorKind::ValueValidation,
+                        format!(
+                            "Cmplog cores ({}) must be a subset of total cores ({})",
+                            asan_cores.cmdline, self.cores.cmdline
+                        ),
+                    )
+                    .exit();
+                }
+            }
+        }
+
+        if let Some(cmplog_cores) = &self.cmplog_cores {
+            for id in &cmplog_cores.ids {
+                if !self.cores.contains(*id) {
+                    let mut cmd = FuzzerOptions::command();
+                    cmd.error(
+                        ErrorKind::ValueValidation,
+                        format!(
+                            "Cmplog cores ({}) must be a subset of total cores ({})",
+                            cmplog_cores.cmdline, self.cores.cmdline
+                        ),
+                    )
+                    .exit();
+                }
+            }
+        }
+    }
+}

fuzzers/qemu_launcher/src/version.rs

@@ -0,0 +1,29 @@
+use std::env;
+
+use clap::builder::Str;
+
+#[derive(Default)]
+pub struct Version;
+
+impl From<Version> for Str {
+    fn from(_: Version) -> Str {
+        let version = [
+            ("Architecture:", env!("CPU_TARGET")),
+            ("Build Timestamp:", env!("VERGEN_BUILD_TIMESTAMP")),
+            ("Describe:", env!("VERGEN_GIT_DESCRIBE")),
+            ("Commit SHA:", env!("VERGEN_GIT_SHA")),
+            ("Commit Date:", env!("VERGEN_RUSTC_COMMIT_DATE")),
+            ("Commit Branch:", env!("VERGEN_GIT_BRANCH")),
+            ("Rustc Version:", env!("VERGEN_RUSTC_SEMVER")),
+            ("Rustc Channel:", env!("VERGEN_RUSTC_CHANNEL")),
+            ("Rustc Host Triple:", env!("VERGEN_RUSTC_HOST_TRIPLE")),
+            ("Rustc Commit SHA:", env!("VERGEN_RUSTC_COMMIT_HASH")),
+            ("Cargo Target Triple", env!("VERGEN_CARGO_TARGET_TRIPLE")),
+        ]
+        .iter()
+        .map(|(k, v)| format!("{k:25}: {v}\n"))
+        .collect::<String>();
+
+        format!("\n{version:}").into()
+    }
+}

libafl_qemu/libafl_qemu_build/src/lib.rs

@@ -16,8 +16,6 @@ pub fn build_with_bindings(
     jobs: Option<u32>,
     bindings_file: &Path,
 ) {
-    println!("cargo:rerun-if-changed={}", bindings_file.display());
-
     let (qemu_dir, build_dir) = build::build(cpu_target, is_big_endian, is_usermode, jobs);
     let clang_args = qemu_bindgen_clang_args(&qemu_dir, &build_dir, cpu_target, is_usermode);
 

libafl_qemu/src/elf.rs

@@ -1,6 +1,6 @@
 //! Utilities to parse and process ELFs
 
-use std::{convert::AsRef, fs::File, io::Read, path::Path, str};
+use std::{convert::AsRef, fs::File, io::Read, ops::Range, path::Path, str};
 
 use goblin::elf::{header::ET_DYN, Elf};
 use libafl::Error;
@@ -67,6 +67,33 @@ impl<'a> EasyElf<'a> {
         None
     }
 
+    #[must_use]
+    pub fn get_section(&self, name: &str, load_addr: GuestAddr) -> Option<Range<GuestAddr>> {
+        for section in &self.elf.section_headers {
+            if let Some(section_name) = self.elf.shdr_strtab.get_at(section.sh_name) {
+                log::debug!(
+                    "section_name: {section_name:}, sh_addr: 0x{:x}, sh_size: 0x{:x}",
+                    section.sh_addr,
+                    section.sh_size
+                );
+                if section_name == name {
+                    return if section.sh_addr == 0 {
+                        None
+                    } else if self.is_pic() {
+                        let start = section.sh_addr as GuestAddr + load_addr;
+                        let end = start + section.sh_size as GuestAddr;
+                        Some(Range { start, end })
+                    } else {
+                        let start = section.sh_addr as GuestAddr;
+                        let end = start + section.sh_size as GuestAddr;
+                        Some(Range { start, end })
+                    };
+                }
+            }
+        }
+        None
+    }
+
     fn is_pic(&self) -> bool {
         self.elf.header.e_type == ET_DYN
     }

libafl_qemu/src/emu.rs

@@ -981,6 +981,14 @@ impl Emulator {
         }
     }
 
+    pub fn entry_break(&self, addr: GuestAddr) {
+        self.set_breakpoint(addr);
+        unsafe {
+            self.run();
+        }
+        self.remove_breakpoint(addr);
+    }
+
     pub fn set_hook(
         &self,
         addr: GuestAddr,
@@ -1427,6 +1435,10 @@ pub mod pybind {
             self.emu.set_breakpoint(addr);
         }
 
+        fn entry_break(&self, addr: GuestAddr) {
+            self.emu.entry_break(addr);
+        }
+
         fn remove_breakpoint(&self, addr: GuestAddr) {
             self.emu.remove_breakpoint(addr);
         }