diff --git a/Cargo.toml b/Cargo.toml index 26062f8e42..44dce7bd42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,8 +160,8 @@ std_instead_of_core = "deny" cargo = { level = "warn", priority = -1 } # Allow -negative_feature_names = "allow" # TODO: turn into 'warn' when working -multiple_crate_versions = "allow" # TODO: turn into `warn` when working +negative_feature_names = "allow" # TODO: turn into 'warn' when working +multiple_crate_versions = "allow" # TODO: turn into `warn` when working unreadable_literal = "allow" type_repetition_in_bounds = "allow" missing_errors_doc = "allow" @@ -173,9 +173,8 @@ module_name_repetitions = "allow" unsafe_derive_deserialize = "allow" similar_names = "allow" too_many_lines = "allow" -comparison_chain = "allow" # This lint makes **ZERO** sense -unnecessary_debug_formatting = "allow" # :thumbsdown: :thumbsdown: :thumbsdown: :thumbsdown: :thumbsdown: :thumbsdown: -struct_field_names = "allow" # ???? +comparison_chain = "allow" # This lint makes **ZERO** sense +struct_field_names = "allow" # ???? [workspace.lints.rustdoc] # Deny diff --git a/bindings/pylibafl/test.py b/bindings/pylibafl/test.py index 1ad3ef8c1e..92fb1cc964 100644 --- a/bindings/pylibafl/test.py +++ b/bindings/pylibafl/test.py @@ -3,7 +3,7 @@ import ctypes import platform print("Starting to fuzz from python!") -fuzzer = sugar.InMemoryBytesCoverageSugar( +fuzzer = sugar.InProcessBytesCoverageSugar( input_dirs=["./in"], output_dir="out", broker_port=1337, cores=[0, 1] ) fuzzer.run(lambda b: print("foo")) diff --git a/docs/src/DEBUGGING.md b/docs/src/DEBUGGING.md index 7da5c2f0d0..3d6166e2d0 100644 --- a/docs/src/DEBUGGING.md +++ b/docs/src/DEBUGGING.md @@ -1,40 +1,53 @@ # General debugging tips -This file answers some commmon questions that arise when you are writing a fuzzer using LibAFL. -## Q. My fuzzer crashed but the stack trace is useless. +This file answers some common questions that arise when you are writing a fuzzer using LibAFL. + +## Q. My fuzzer crashed but the stack trace is useless + You can enable the `errors_backtrace` feature of the `libafl` crate. With this the stacktrace is meaningful. -## Q. I started the fuzzer but the corpus count is 0. -Unless the initial corpus is loaded with the "load_initial_inputs_forced" function, we only store the interesting inputs, which is the inputs that triggered the feedback. So this usually means that your input was not interesting or your target was simply not properly implemented. -Either way, what you can do is attach to the executable with gdb and set a breakpoint at where the new edges should be reported. If no instrumentation code is executed, then the the problem is in the instrumentation. If the instrumentation code is hit, but still the your input is not instrumented, then the problem could be that you are not passign the observer/feedback correctly to the fuzzer. +## Q. I started the fuzzer but the corpus count is 0 + +Unless the initial corpus is loaded with the "load_initial_inputs_forced" function, we only store the interesting inputs, which is the inputs that triggered the feedback. So this usually means that your input was not interesting, or your target was simply not properly implemented. +Either way, what you can do is attach to the executable with gdb and set a breakpoint at where the new edges should be reported. If no instrumentation code is executed, then the problem is in the instrumentation. If the instrumentation code is hit, but still your input is not deemed interesting/stored, then the problem could be that you are not passing the observer/feedback correctly to the fuzzer. + +## Q. I started the fuzzer but the coverage is 0 -## Q. I started the fuzzer but the coverage is 0. This could mean two things. Perhaps your target was not properly instrumented, or you are not using the correct observer, feedback feature. In this case, again, what usually should do is to run the fuzzer with gdb and set a breakpoint at where the coverage is recorded (e.g. __sanitizer_coverage_trace_pcguard), and validate that the target is giving the feedback to the fuzzer. -## Q. I started the fuzzer but there's no output. +## Q. I started the fuzzer but there's no output + First, verify that your stdout and stderr are not redirected to `/dev/null`. If you get the log, then it should either fall into the previous 2 cases. Either the fuzzer crashed because you didn't have the initial seeds, or the coverage feedback is not working. ## Q. I don't see any output from my fuzzer (println!() or logging) + First, check that you are not redirecting things to `/dev/null` else you will see nothing. To see the log that you added with `log::trace!();`, you need to initialize the logger (any logger, `env_logger` or `SimpleStdoutLogger` from `libafl_bolts`) before the fuzzer starts. Also you have to make sure that you are runing with `RUST_LOG=` and you are *NOT* using `release_max_level_info` feature of `log` crate in your `Cargo.toml` of your fuzzer -## Q. My fuzzer is slow. +## Q. My fuzzer is slow + Try running the fuzzer with the `introspection` feature of the `libafl`. This will show how much time is spent on each module of your fuzzer. Also you might be using a wrong size of the coverage map. If you see `2621440` for the size of the coverage map, you are doing it wrong. One possible mistake is the misuse of `libafl_targets::coverage::EDGES_MAP` -``` + +```rust let map = StdMapObserver::from_mut_ptr("edges", EDGES_MAP.as_mut_ptr(), EDGES_MAP.len()); ``` + You should *never* use the `EDGES_MAP`'s size as this is just the size of the allocated size of the coverage map. Consider using something smaller or our default value `libafl_targets::LIBAFL_EDGES_MAP_DEFAULT_SIZE`. -## Q. I still have problems with my fuzzer. +## Q. I still have problems with my fuzzer + Finally, if you really have no idea what is going on, run your fuzzer with logging enabled. (You can use `env_logger`, `SimpleStdoutLogger`, `SimpleStderrLogger` from `libafl_bolts`. `fuzzbench_text` has an example to show how to use it.) (Don't forget to enable stdout and stderr), and you can open an issue or ask us in Discord. -## Q. My fuzzer died of `Storing state in crashed fuzzer instance did not work`. +## Q. My fuzzer died of `Storing state in crashed fuzzer instance did not work` + If the exit code is zero, then this is because either your harness exited or you are using fuzzer_loop_for and forgot to add `mgr.on_restart` at the end of the fuzzer. In the first case, you should patch your harness not to exit. (or use `utils/deexit`). ## Q. I can't leave the TUI screen -Type `q` then you leave TUI. -## Q. I see `QEMU internal SIGSEGV {code=MAPERR, addr=0x48}` and my QEMU fuzzer doesn't run. -Are you running QEMU fuzzer on WSL? You have to enable vsyscall https://github.com/microsoft/WSL/issues/4694#issuecomment-556095344. \ No newline at end of file +Type `q` to leave TUI. + +## Q. I see `QEMU internal SIGSEGV {code=MAPERR, addr=0x48}` and my QEMU fuzzer doesn't run + +Are you running QEMU fuzzer on WSL? You have to enable vsyscall . diff --git a/docs/src/core_concepts/executor.md b/docs/src/core_concepts/executor.md index 6f42286ae0..bb5c62f057 100644 --- a/docs/src/core_concepts/executor.md +++ b/docs/src/core_concepts/executor.md @@ -1,7 +1,7 @@ # Executor In different fuzzers, this concept of executing the program under test means each run is now always the same. -For instance, for in-memory fuzzers like libFuzzer an execution is a call to an harness function, for hypervisor-based fuzzers like [kAFL](https://github.com/IntelLabs/kAFL) instead an entire operating system is started from a snapshot for each run. +For instance, for in-process fuzzers like libFuzzer an execution is a call to an harness function, for hypervisor-based fuzzers like [kAFL](https://github.com/IntelLabs/kAFL) instead an entire operating system is started from a snapshot for each run. In our model, an Executor is the entity that defines not only how to execute the target, but all the volatile operations that are related to just a single run of the target. diff --git a/docs/src/message_passing/spawn_instances.md b/docs/src/message_passing/spawn_instances.md index 7cdbdb4e1c..cab189ca25 100644 --- a/docs/src/message_passing/spawn_instances.md +++ b/docs/src/message_passing/spawn_instances.md @@ -5,7 +5,7 @@ Multiple fuzzer instances can be spawned using different ways. ## Manually, via a TCP port The straightforward way to do Multi-Threading is to use the [`LlmpRestartingEventManager`](https://docs.rs/libafl/latest/libafl/events/llmp/restarting/struct.LlmpRestartingEventManager.html), specifically to use [`setup_restarting_mgr_std`](https://docs.rs/libafl/latest/libafl/events/llmp/restarting/fn.setup_restarting_mgr_std.html). -It abstracts away all the pesky details about restarts on crash handling (for in-memory fuzzers) and multi-threading. +It abstracts away all the pesky details about restarts on crash handling (for in-process fuzzers) and multi-threading. With it, every instance you launch manually tries to connect to a TCP port on the local machine. If the port is not yet bound, this instance becomes the broker, binding itself to the port to await new clients. diff --git a/fuzzers/forkserver/fuzzbench_forkserver_cmplog/src/main.rs b/fuzzers/forkserver/fuzzbench_forkserver_cmplog/src/main.rs index cbde2ba326..9570d5b522 100644 --- a/fuzzers/forkserver/fuzzbench_forkserver_cmplog/src/main.rs +++ b/fuzzers/forkserver/fuzzbench_forkserver_cmplog/src/main.rs @@ -367,6 +367,7 @@ fn fuzz( .shmem_provider(&mut shmem_provider) .parse_afl_cmdline(arguments) .is_persistent(true) + // increase timeouts for cmplog .timeout(timeout * 10) .kill_signal(signal) .build(tuple_list!(cmplog_observer)) diff --git a/fuzzers/forkserver/libafl-fuzz/src/fuzzer.rs b/fuzzers/forkserver/libafl-fuzz/src/fuzzer.rs index 7ae2402e00..46931e3031 100644 --- a/fuzzers/forkserver/libafl-fuzz/src/fuzzer.rs +++ b/fuzzers/forkserver/libafl-fuzz/src/fuzzer.rs @@ -176,7 +176,7 @@ define_run_client!(state, mgr, fuzzer_dir, core_id, opt, is_main_node, { let mut tokens = Tokens::new(); tokens = tokens.add_from_files(&opt.dicts)?; - // Create a AFLStatsStage; + // Create a AflStatsStage; let afl_stats_stage = AflStatsStage::builder() .stats_file(fuzzer_dir.join("fuzzer_stats")) .plot_file(fuzzer_dir.join("plot_data")) diff --git a/fuzzers/inprocess/libfuzzer_stb_image_sugar/src/main.rs b/fuzzers/inprocess/libfuzzer_stb_image_sugar/src/main.rs index 70722fdd53..96bc0ec9d4 100644 --- a/fuzzers/inprocess/libfuzzer_stb_image_sugar/src/main.rs +++ b/fuzzers/inprocess/libfuzzer_stb_image_sugar/src/main.rs @@ -7,7 +7,7 @@ static GLOBAL: MiMalloc = MiMalloc; use std::{env, path::PathBuf}; use libafl_bolts::core_affinity::Cores; -use libafl_sugar::InMemoryBytesCoverageSugar; +use libafl_sugar::InProcessBytesCoverageSugar; use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input}; pub fn main() { @@ -35,7 +35,7 @@ fn fuzz(input_dirs: &[PathBuf], output_dir: PathBuf, cores: &Cores, broker_port: println!("Warning: LLVMFuzzerInitialize failed with -1"); } - InMemoryBytesCoverageSugar::builder() + InProcessBytesCoverageSugar::builder() .input_dirs(input_dirs) .output_dir(output_dir) .cores(cores) diff --git a/libafl/src/generators/nautilus.rs b/libafl/src/generators/nautilus.rs index b0c76b74c4..642cbadbed 100644 --- a/libafl/src/generators/nautilus.rs +++ b/libafl/src/generators/nautilus.rs @@ -110,7 +110,8 @@ impl NautilusContext { let reader = BufReader::new(file); let rules: Vec> = serde_json::from_reader(reader).map_err(|err| { Error::illegal_argument(format!( - "Error loading context from json grammar file {grammar_file:?}: {err:?}" + "Error loading context from json grammar file {}: {err:?}", + grammar_file.display() )) })?; Ok(Self::new(tree_depth, &rules)) diff --git a/libafl/src/stages/sync.rs b/libafl/src/stages/sync.rs index 74e87d1e39..9ae63d2cf6 100644 --- a/libafl/src/stages/sync.rs +++ b/libafl/src/stages/sync.rs @@ -107,7 +107,7 @@ where let mut new_files = vec![]; for dir in &self.sync_dirs { - log::debug!("Syncing from dir: {dir:?}"); + log::debug!("Syncing from dir: {}", dir.display()); let new_dir_files = find_new_files_rec(dir, &last)?; new_files.extend(new_dir_files); } @@ -134,7 +134,7 @@ where .unwrap() .left_to_sync .retain(|p| p != &path); - log::debug!("Syncing and evaluating {path:?}"); + log::debug!("Syncing and evaluating {}", path.display()); fuzzer.evaluate_input(state, executor, manager, &input)?; } diff --git a/libafl/src/state/mod.rs b/libafl/src/state/mod.rs index 4d86e84760..60a88492c6 100644 --- a/libafl/src/state/mod.rs +++ b/libafl/src/state/mod.rs @@ -701,7 +701,7 @@ where fn load_file( &mut self, - path: &PathBuf, + path: &Path, manager: &mut EM, fuzzer: &mut Z, executor: &mut E, @@ -711,11 +711,14 @@ where EM: EventFirer, Z: Evaluator, { - log::info!("Loading file {path:?} ..."); + log::info!("Loading file {} ...", path.display()); let input = match (config.loader)(fuzzer, self, path) { Ok(input) => input, Err(err) => { - log::error!("Skipping input that we could not load from {path:?}: {err:?}"); + log::error!( + "Skipping input that we could not load from {}: {err:?}", + path.display() + ); return Ok(ExecuteInputResult::default()); } }; @@ -726,7 +729,10 @@ where let (res, _) = fuzzer.evaluate_input(self, executor, manager, &input)?; if !(res.is_corpus() || res.is_solution()) { fuzzer.add_disabled_input(self, input)?; - log::warn!("input {:?} was not interesting, adding as disabled.", &path); + log::warn!( + "Input {} was not interesting, adding as disabled.", + path.display() + ); } Ok(res) } diff --git a/libafl_bolts/src/serdeany.rs b/libafl_bolts/src/serdeany.rs index cea8976bcb..0c712f54fd 100644 --- a/libafl_bolts/src/serdeany.rs +++ b/libafl_bolts/src/serdeany.rs @@ -554,6 +554,8 @@ pub mod serdeany_registry { #[cfg(not(feature = "stable_anymap"))] let type_repr = &type_repr; + // Couldn't figure out how to make the suggested clippy fix work (even though it looks correct) + #[allow(clippy::manual_map)] match self.map.get(type_repr) { None => None, Some(h) => Some(h.values().map(|x| x.as_any().downcast_ref::().unwrap())), @@ -579,6 +581,8 @@ pub mod serdeany_registry { #[cfg(not(feature = "stable_anymap"))] let type_repr = &type_repr; + // Couldn't figure out how to make the suggested clippy fix work (even though it looks correct) + #[allow(clippy::manual_map)] match self.map.get_mut(type_repr) { None => None, Some(h) => Some( diff --git a/libafl_cc/build.rs b/libafl_cc/build.rs index 22392b1ac0..480b72d146 100644 --- a/libafl_cc/build.rs +++ b/libafl_cc/build.rs @@ -382,6 +382,9 @@ pub const LIBAFL_CC_LLVM_VERSION: Option = None; let llvm_version = find_llvm_version(); + // We want the paths quoted, and debug formatting does that - allow debug formatting. + #[allow(unknown_lints)] // not on stable yet + #[allow(clippy::unnecessary_debug_formatting)] write!( clang_constants_file, "// These constants are autogenerated by build.rs diff --git a/libafl_frida/src/helper.rs b/libafl_frida/src/helper.rs index 9846d740e5..2677d58495 100644 --- a/libafl_frida/src/helper.rs +++ b/libafl_frida/src/helper.rs @@ -303,10 +303,15 @@ impl FridaInstrumentationHelperBuilder { let name = path .file_name() .and_then(|name| name.to_str()) - .unwrap_or_else(|| panic!("Failed to get script file name from path: {path:?}")); + .unwrap_or_else(|| { + panic!( + "Failed to get script file name from path: {}", + path.display() + ) + }); let script_prefix = include_str!("script.js"); let file_contents = read_to_string(path) - .unwrap_or_else(|err| panic!("Failed to read script {path:?}: {err:?}")); + .unwrap_or_else(|err| panic!("Failed to read script {}: {err:?}", path.display())); let payload = script_prefix.to_string() + &file_contents; let gum = Gum::obtain(); let backend = match backend { diff --git a/libafl_qemu/src/modules/usermode/asan.rs b/libafl_qemu/src/modules/usermode/asan.rs index cef2513dd9..16b7947c3c 100644 --- a/libafl_qemu/src/modules/usermode/asan.rs +++ b/libafl_qemu/src/modules/usermode/asan.rs @@ -998,7 +998,8 @@ where assert!( asan_lib.as_path().exists(), - "The ASAN library doesn't exist: {asan_lib:#?}" + "The ASAN library doesn't exist: {}", + asan_lib.display() ); let asan_lib = asan_lib diff --git a/libafl_qemu/src/modules/usermode/asan_guest.rs b/libafl_qemu/src/modules/usermode/asan_guest.rs index 712a8c511d..491d267e8b 100644 --- a/libafl_qemu/src/modules/usermode/asan_guest.rs +++ b/libafl_qemu/src/modules/usermode/asan_guest.rs @@ -203,7 +203,8 @@ where assert!( asan_lib.as_path().exists(), - "The ASAN library doesn't exist: {asan_lib:#?}" + "The ASAN library doesn't exist: {}", + asan_lib.display() ); let asan_lib = asan_lib diff --git a/libafl_sugar/src/forkserver.rs b/libafl_sugar/src/forkserver.rs index f28c595b66..9771bbb8dc 100644 --- a/libafl_sugar/src/forkserver.rs +++ b/libafl_sugar/src/forkserver.rs @@ -6,31 +6,37 @@ use std::{fs, path::PathBuf}; use libafl::{ Error, HasMetadata, corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus}, - events::{EventConfig, EventRestarter, LlmpRestartingEventManager, launcher::Launcher}, - executors::forkserver::ForkserverExecutor, - feedback_or, feedback_or_fast, + events::{EventConfig, LlmpRestartingEventManager, launcher::Launcher}, + executors::forkserver::{ForkserverExecutor, SHM_CMPLOG_ENV_VAR}, + feedback_and_fast, feedback_or, feedback_or_fast, feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, fuzzer::{Fuzzer, StdFuzzer}, generators::RandBytesGenerator, + inputs::BytesInput, monitors::MultiMonitor, mutators::{ + I2SRandReplace, StdMOptMutator, havoc_mutations::havoc_mutations, scheduled::{HavocScheduledMutator, tokens_mutations}, token_mutations::Tokens, }, - observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, - schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, - stages::StdMutationalStage, + observers::{CanTrack, HitcountsMapObserver, StdCmpObserver, StdMapObserver, TimeObserver}, + schedulers::{ + IndexesLenTimeMinimizerScheduler, StdWeightedScheduler, powersched::PowerSchedule, + }, + stages::{CalibrationStage, StdMutationalStage, StdPowerMutationalStage, TracingStage}, state::{HasCorpus, StdState}, }; use libafl_bolts::{ AsSliceMut, TargetArgs, core_affinity::Cores, nonzero, + ownedref::OwnedRefMut, rands::StdRand, shmem::{ShMem, ShMemProvider, UnixShMemProvider}, tuples::{Merge, tuple_list}, }; +use libafl_targets::AFLppCmpLogMap; use typed_builder::TypedBuilder; use crate::{CORPUS_CACHE_SIZE, DEFAULT_TIMEOUT_SECS}; @@ -51,9 +57,9 @@ pub struct ForkserverBytesCoverageSugar<'a> { /// Dictionary #[builder(default = None)] tokens_file: Option, - // Flag if use CmpLog + // CmpLog binary to execute. If `None`, we will skip CmpLog. #[builder(default = None)] - use_cmplog: Option, + cmplog_binary: Option, #[builder(default = 1337_u16)] broker_port: u16, /// The list of cores to run on @@ -63,13 +69,10 @@ pub struct ForkserverBytesCoverageSugar<'a> { #[builder(default = None, setter(strip_option))] remote_broker_addr: Option, /// Path to program to execute - program: String, + binary: PathBuf, /// Arguments of the program to execute arguments: &'a [String], #[builder(default = false)] - /// Use shared mem testcase delivery - shmem_testcase: bool, - #[builder(default = false)] /// Print target program output debug_output: bool, /// Fuzz `iterations` number of times, instead of indefinitely; implies use of `fuzz_loop_for` @@ -91,10 +94,6 @@ impl ForkserverBytesCoverageSugar<'_> { None => EventConfig::AlwaysUnique, }; - if self.use_cmplog.unwrap_or(false) { - log::warn!("use of cmplog not currently supported, use_cmplog ignored."); - } - let timeout = Duration::from_secs(self.timeout.unwrap_or(DEFAULT_TIMEOUT_SECS)); let mut out_dir = self.output_dir.clone(); @@ -123,6 +122,9 @@ impl ForkserverBytesCoverageSugar<'_> { _core_id| { let time_observer = time_observer.clone(); + // Create an empty set of tokens, first populated by the target program + let mut tokens = Tokens::new(); + // Coverage map shared between target and fuzzer let mut shmem = shmem_provider_client.new_shmem(MAP_SIZE).unwrap(); unsafe { @@ -141,6 +143,13 @@ impl ForkserverBytesCoverageSugar<'_> { .track_indices() }; + // New maximization map feedback linked to the edges observer and the feedback state + let map_feedback = MaxMapFeedback::with_name("map_feedback", &edges_observer); + // Extra MapFeedback to deduplicate finds according to the cov map + let map_objective = MaxMapFeedback::with_name("map_objective", &edges_observer); + + let calibration = CalibrationStage::new(&map_feedback); + // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( @@ -151,7 +160,10 @@ impl ForkserverBytesCoverageSugar<'_> { ); // A feedback to choose if an input is a solution or not - let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); + let mut objective = feedback_and_fast!( + feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()), + map_objective + ); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { @@ -169,38 +181,40 @@ impl ForkserverBytesCoverageSugar<'_> { .unwrap() }); - // Create an empty set of tokens, first populated by the target program - let mut tokens = Tokens::new(); + // Setup a MOPT mutator + let mutator = StdMOptMutator::new( + &mut state, + havoc_mutations().merge(tokens_mutations()), + 7, + 5, + )?; + + let power: StdPowerMutationalStage<_, _, BytesInput, _, _, _> = + StdPowerMutationalStage::new(mutator); // A minimization+queue policy to get testcasess from the corpus - let scheduler = - IndexesLenTimeMinimizerScheduler::new(&edges_observer, QueueScheduler::new()); + let scheduler = IndexesLenTimeMinimizerScheduler::new( + &edges_observer, + StdWeightedScheduler::with_schedule( + &mut state, + &edges_observer, + Some(PowerSchedule::explore()), + ), + ); // A fuzzer with feedbacks and a corpus scheduler let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); - let forkserver = if self.shmem_testcase { - ForkserverExecutor::builder() - .program(self.program.clone()) - .parse_afl_cmdline(self.arguments) - .is_persistent(true) - .autotokens(&mut tokens) - .coverage_map_size(MAP_SIZE) - .timeout(timeout) - .debug_child(self.debug_output) - .shmem_provider(&mut shmem_provider_client) - .build_dynamic_map(edges_observer, tuple_list!(time_observer)) - } else { - ForkserverExecutor::builder() - .program(self.program.clone()) - .parse_afl_cmdline(self.arguments) - .is_persistent(true) - .autotokens(&mut tokens) - .coverage_map_size(MAP_SIZE) - .timeout(timeout) - .debug_child(self.debug_output) - .build_dynamic_map(edges_observer, tuple_list!(time_observer)) - }; + let forkserver = ForkserverExecutor::builder() + .program(self.binary.clone()) + .parse_afl_cmdline(self.arguments) + .is_persistent(true) + .autotokens(&mut tokens) + .coverage_map_size(MAP_SIZE) + .timeout(timeout) + .debug_child(self.debug_output) + .shmem_provider(&mut shmem_provider_client) + .build_dynamic_map(edges_observer, tuple_list!(time_observer)); let mut executor = forkserver.unwrap(); if let Some(tokens_file) = &self.tokens_file { @@ -245,14 +259,40 @@ impl ForkserverBytesCoverageSugar<'_> { } } - if self.tokens_file.is_some() { - // Setup a basic mutator - let mutator = - HavocScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); - let mutational = StdMutationalStage::new(mutator); + if let Some(exec) = &self.cmplog_binary { + // The cmplog map shared between observer and executor + let mut cmplog_shmem = shmem_provider_client + .uninit_on_shmem::() + .unwrap(); + // let the forkserver know the shmid + unsafe { + cmplog_shmem.write_to_env(SHM_CMPLOG_ENV_VAR).unwrap(); + } + let cmpmap = + unsafe { OwnedRefMut::::from_shmem(&mut cmplog_shmem) }; + + let cmplog_observer = StdCmpObserver::new("cmplog", cmpmap, true); + + let cmplog_executor = ForkserverExecutor::builder() + .program(exec) + .debug_child(self.debug_output) + .shmem_provider(&mut shmem_provider_client) + .parse_afl_cmdline(self.arguments) + .is_persistent(true) + // We give more time to the cmplog run + .timeout(timeout * 10) + .build(tuple_list!(cmplog_observer)) + .unwrap(); + + let tracing = TracingStage::new(cmplog_executor); + + // Setup a randomic Input2State stage + let i2s = StdMutationalStage::new(HavocScheduledMutator::new(tuple_list!( + I2SRandReplace::new() + ))); // The order of the stages matter! - let mut stages = tuple_list!(mutational); + let mut stages = tuple_list!(calibration, tracing, i2s, power); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( @@ -262,18 +302,12 @@ impl ForkserverBytesCoverageSugar<'_> { &mut mgr, iters, )?; - mgr.on_restart(&mut state)?; - std::process::exit(0); } else { fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr)?; } } else { - // Setup a basic mutator - let mutator = HavocScheduledMutator::new(havoc_mutations()); - let mutational = StdMutationalStage::new(mutator); - // The order of the stages matter! - let mut stages = tuple_list!(mutational); + let mut stages = tuple_list!(calibration, power); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( @@ -283,13 +317,10 @@ impl ForkserverBytesCoverageSugar<'_> { &mut mgr, iters, )?; - mgr.on_restart(&mut state)?; - std::process::exit(0); } else { fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr)?; } } - Ok(()) }; @@ -330,7 +361,6 @@ pub mod pybind { output_dir: PathBuf, broker_port: u16, cores: Cores, - use_cmplog: Option, iterations: Option, tokens_file: Option, timeout: Option, @@ -340,13 +370,11 @@ pub mod pybind { impl ForkserverBytesCoverageSugar { /// Create a new [`ForkserverBytesCoverageSugar`] #[new] - #[expect(clippy::too_many_arguments)] #[pyo3(signature = ( input_dirs, output_dir, broker_port, cores, - use_cmplog=None, iterations=None, tokens_file=None, timeout=None @@ -356,7 +384,6 @@ pub mod pybind { output_dir: PathBuf, broker_port: u16, cores: Vec, - use_cmplog: Option, iterations: Option, tokens_file: Option, timeout: Option, @@ -366,7 +393,6 @@ pub mod pybind { output_dir, broker_port, cores: cores.into(), - use_cmplog, iterations, tokens_file, timeout, @@ -375,15 +401,15 @@ pub mod pybind { /// Run the fuzzer #[expect(clippy::needless_pass_by_value)] - pub fn run(&self, program: String, arguments: Vec) { + pub fn run(&self, binary: String, arguments: Vec, cmplog_binary: Option) { forkserver::ForkserverBytesCoverageSugar::builder() .input_dirs(&self.input_dirs) .output_dir(self.output_dir.clone()) .broker_port(self.broker_port) .cores(&self.cores) - .program(program) + .binary(binary.into()) + .cmplog_binary(cmplog_binary.map(PathBuf::from)) .arguments(&arguments) - .use_cmplog(self.use_cmplog) .timeout(self.timeout) .tokens_file(self.tokens_file.clone()) .iterations(self.iterations) diff --git a/libafl_sugar/src/inmemory.rs b/libafl_sugar/src/inprocess.rs similarity index 88% rename from libafl_sugar/src/inmemory.rs rename to libafl_sugar/src/inprocess.rs index e2b6e59f10..060378d474 100644 --- a/libafl_sugar/src/inmemory.rs +++ b/libafl_sugar/src/inprocess.rs @@ -1,4 +1,4 @@ -//! In-Memory fuzzing made easy. +//! In-Process fuzzing made easy. //! Use this sugar for scaling `libfuzzer`-style fuzzers. use core::{ @@ -13,7 +13,7 @@ use libafl::{ corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus}, events::{EventConfig, EventRestarter, LlmpRestartingEventManager, launcher::Launcher}, executors::{ExitKind, ShadowExecutor, inprocess::InProcessExecutor}, - feedback_or, feedback_or_fast, + feedback_and_fast, feedback_or, feedback_or_fast, feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, fuzzer::{Fuzzer, StdFuzzer}, generators::RandBytesGenerator, @@ -26,7 +26,7 @@ use libafl::{ }, observers::{CanTrack, HitcountsMapObserver, StdMapObserver, TimeObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, - stages::{ShadowTracingStage, StdMutationalStage}, + stages::{CalibrationStage, ShadowTracingStage, StdMutationalStage}, state::{HasCorpus, StdState}, }; use libafl_bolts::{ @@ -38,15 +38,15 @@ use libafl_bolts::{ shmem::{ShMemProvider, StdShMemProvider}, tuples::{Merge, tuple_list}, }; -use libafl_targets::{CmpLogObserver, edges_map_mut_ptr}; +use libafl_targets::{CmpLogObserver, EDGES_MAP_ALLOCATED_SIZE, edges_map_mut_ptr}; use typed_builder::TypedBuilder; use crate::{CORPUS_CACHE_SIZE, DEFAULT_TIMEOUT_SECS}; -/// In-Memory fuzzing made easy. +/// In-Process fuzzing made easy. /// Use this sugar for scaling `libfuzzer`-style fuzzers. #[derive(TypedBuilder)] -pub struct InMemoryBytesCoverageSugar<'a, H> +pub struct InProcessBytesCoverageSugar<'a, H> where H: FnMut(&[u8]), { @@ -78,20 +78,17 @@ where /// Bytes harness #[builder(setter(strip_option))] harness: Option, - /// The map size used for the fuzzer - #[builder(default = 65536usize)] - map_size: usize, /// Fuzz `iterations` number of times, instead of indefinitely; implies use of `fuzz_loop_for` #[builder(default = None)] iterations: Option, } -impl Debug for InMemoryBytesCoverageSugar<'_, H> +impl Debug for InProcessBytesCoverageSugar<'_, H> where H: FnMut(&[u8]), { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("InMemoryBytesCoverageSugar") + f.debug_struct("InProcessBytesCoverageSugar") .field("configuration", &self.configuration) .field("timeout", &self.timeout) .field("input_dirs", &self.input_dirs) @@ -113,7 +110,7 @@ where } } -impl InMemoryBytesCoverageSugar<'_, H> +impl InProcessBytesCoverageSugar<'_, H> where H: FnMut(&[u8]), { @@ -129,11 +126,11 @@ where let mut out_dir = self.output_dir.clone(); if fs::create_dir(&out_dir).is_err() { - log::info!("Out dir at {:?} already exists.", &out_dir); + log::info!("Out dir at {} already exists.", out_dir.display()); assert!( out_dir.is_dir(), - "Out dir at {:?} is not a valid directory!", - &out_dir + "Out dir at {} is not a valid directory!", + out_dir.display() ); } let mut crashes = out_dir.clone(); @@ -158,24 +155,36 @@ where let edges_observer = HitcountsMapObserver::new(unsafe { StdMapObserver::from_mut_slice( "edges", - OwnedMutSlice::from_raw_parts_mut(edges_map_mut_ptr(), self.map_size), + OwnedMutSlice::from_raw_parts_mut( + edges_map_mut_ptr(), + EDGES_MAP_ALLOCATED_SIZE, + ), ) }) .track_indices(); let cmplog_observer = CmpLogObserver::new("cmplog", true); + // New maximization map feedback linked to the edges observer and the feedback state + let map_feedback = MaxMapFeedback::with_name("map_feedback", &edges_observer); + // Extra MapFeedback to deduplicate finds according to the cov map + let map_objective = MaxMapFeedback::with_name("map_objective", &edges_observer); + + let calibration = CalibrationStage::new(&map_feedback); + // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( - // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::new(&edges_observer), + map_feedback, // Time feedback, this one does not need a feedback state TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not - let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); + let mut objective = feedback_and_fast!( + feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()), + map_objective + ); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { @@ -276,7 +285,7 @@ where // The order of the stages matter! if self.use_cmplog.unwrap_or(false) { - let mut stages = tuple_list!(tracing, i2s, mutational); + let mut stages = tuple_list!(calibration, tracing, i2s, mutational); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( &mut stages, @@ -291,7 +300,7 @@ where fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr)?; } } else { - let mut stages = tuple_list!(mutational); + let mut stages = tuple_list!(calibration, mutational); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( &mut stages, @@ -313,7 +322,7 @@ where // The order of the stages matter! if self.use_cmplog.unwrap_or(false) { - let mut stages = tuple_list!(tracing, i2s, mutational); + let mut stages = tuple_list!(calibration, tracing, i2s, mutational); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( &mut stages, @@ -375,13 +384,13 @@ pub mod pybind { use libafl_bolts::core_affinity::Cores; use pyo3::{prelude::*, types::PyBytes}; - use crate::inmemory; + use crate::inprocess; - /// In-Memory fuzzing made easy. + /// In-Process fuzzing made easy. /// Use this sugar for scaling `libfuzzer`-style fuzzers. #[pyclass(unsendable)] #[derive(Debug)] - struct InMemoryBytesCoverageSugar { + struct InProcessBytesCoverageSugar { input_dirs: Vec, output_dir: PathBuf, broker_port: u16, @@ -393,8 +402,8 @@ pub mod pybind { } #[pymethods] - impl InMemoryBytesCoverageSugar { - /// Create a new [`InMemoryBytesCoverageSugar`] + impl InProcessBytesCoverageSugar { + /// Create a new [`InProcessBytesCoverageSugar`] #[new] #[expect(clippy::too_many_arguments)] #[pyo3(signature = ( @@ -432,7 +441,7 @@ pub mod pybind { /// Run the fuzzer #[expect(clippy::needless_pass_by_value)] pub fn run(&self, harness: PyObject) { - inmemory::InMemoryBytesCoverageSugar::builder() + inprocess::InProcessBytesCoverageSugar::builder() .input_dirs(&self.input_dirs) .output_dir(self.output_dir.clone()) .broker_port(self.broker_port) @@ -456,7 +465,7 @@ pub mod pybind { /// Register the module pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_class::()?; + m.add_class::()?; Ok(()) } } diff --git a/libafl_sugar/src/lib.rs b/libafl_sugar/src/lib.rs index c9839203f8..c358e7d6fb 100644 --- a/libafl_sugar/src/lib.rs +++ b/libafl_sugar/src/lib.rs @@ -41,8 +41,8 @@ ) )] -pub mod inmemory; -pub use inmemory::InMemoryBytesCoverageSugar; +pub mod inprocess; +pub use inprocess::InProcessBytesCoverageSugar; #[cfg(target_os = "linux")] pub mod qemu; @@ -68,7 +68,7 @@ use pyo3::prelude::*; #[pymodule] #[pyo3(name = "libafl_sugar")] pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> { - inmemory::pybind::register(m)?; + inprocess::pybind::register(m)?; #[cfg(target_os = "linux")] { qemu::pybind::register(m)?; diff --git a/libafl_sugar/src/qemu.rs b/libafl_sugar/src/qemu.rs index cef7772715..1020599f8f 100644 --- a/libafl_sugar/src/qemu.rs +++ b/libafl_sugar/src/qemu.rs @@ -1,4 +1,4 @@ -//! In-memory fuzzer with `QEMU`-based binary-only instrumentation +//! In-Process fuzzer with `QEMU`-based binary-only instrumentation use core::{ fmt::{self, Debug, Formatter}, net::SocketAddr, @@ -11,7 +11,7 @@ use libafl::{ corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus}, events::{EventConfig, EventRestarter, LlmpRestartingEventManager, launcher::Launcher}, executors::{ExitKind, ShadowExecutor}, - feedback_or, feedback_or_fast, + feedback_and_fast, feedback_or, feedback_or_fast, feedbacks::{CrashFeedback, MaxMapFeedback, TimeFeedback, TimeoutFeedback}, fuzzer::{Fuzzer, StdFuzzer}, generators::RandBytesGenerator, @@ -25,7 +25,7 @@ use libafl::{ }, observers::{CanTrack, HitcountsMapObserver, TimeObserver, VariableMapObserver}, schedulers::{IndexesLenTimeMinimizerScheduler, QueueScheduler}, - stages::{ShadowTracingStage, StdMutationalStage}, + stages::{CalibrationStage, ShadowTracingStage, StdMutationalStage}, state::{HasCorpus, StdState}, }; use libafl_bolts::{ @@ -169,17 +169,27 @@ where // Keep tracks of CMPs let cmplog_observer = CmpLogObserver::new("cmplog", true); + // New maximization map feedback linked to the edges observer and the feedback state + let map_feedback = MaxMapFeedback::with_name("map_feedback", &edges_observer); + // Extra MapFeedback to deduplicate finds according to the cov map + let map_objective = MaxMapFeedback::with_name("map_objective", &edges_observer); + + let calibration = CalibrationStage::new(&map_feedback); + let calibration_cmplog = CalibrationStage::new(&map_feedback); + // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( - // New maximization map feedback linked to the edges observer and the feedback state - MaxMapFeedback::new(&edges_observer), + map_feedback, // Time feedback, this one does not need a feedback state TimeFeedback::new(&time_observer) ); // A feedback to choose if an input is a solution or not - let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()); + let mut objective = feedback_and_fast!( + feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new()), + map_objective + ); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { @@ -313,7 +323,7 @@ where let mutational = StdMutationalStage::new(mutator); // The order of the stages matter! - let mut stages = tuple_list!(tracing, i2s, mutational); + let mut stages = tuple_list!(calibration_cmplog, tracing, i2s, mutational); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( @@ -334,7 +344,7 @@ where let mutational = StdMutationalStage::new(mutator); // The order of the stages matter! - let mut stages = tuple_list!(tracing, i2s, mutational); + let mut stages = tuple_list!(calibration_cmplog, tracing, i2s, mutational); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( @@ -427,7 +437,7 @@ where let mutational = StdMutationalStage::new(mutator); // The order of the stages matter! - let mut stages = tuple_list!(mutational); + let mut stages = tuple_list!(calibration, mutational); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( @@ -448,7 +458,7 @@ where let mutational = StdMutationalStage::new(mutator); // The order of the stages matter! - let mut stages = tuple_list!(mutational); + let mut stages = tuple_list!(calibration, mutational); if let Some(iters) = self.iterations { fuzzer.fuzz_loop_for( diff --git a/utils/drcov_utils/src/bin/drcov_dump_addrs.rs b/utils/drcov_utils/src/bin/drcov_dump_addrs.rs index 28a4e7ad6c..5217a0fd1b 100644 --- a/utils/drcov_utils/src/bin/drcov_dump_addrs.rs +++ b/utils/drcov_utils/src/bin/drcov_dump_addrs.rs @@ -43,7 +43,7 @@ pub struct Opt { fn process(opts: &Opt, input: &PathBuf) -> Result<(), std::io::Error> { let Ok(drcov) = DrCovReader::read(&input) - .map_err(|err| eprintln!("Ignored coverage file {input:?}, reason: {err:?}")) + .map_err(|err| eprintln!("Ignored coverage file {}, reason: {err:?}", input.display())) else { return Ok(()); }; @@ -63,12 +63,19 @@ fn process(opts: &Opt, input: &PathBuf) -> Result<(), std::io::Error> { ); let Ok(file) = File::create_new(&out_file).map_err(|err| { - eprintln!("Could not create file {out_file:?} - continuing: {err:?}"); + eprintln!( + "Could not create file {} - continuing: {err:?}", + out_file.display() + ); }) else { return Ok(()); }; - println!("Dumping traces from drcov file {input:?} to {out_file:?}",); + println!( + "Dumping traces from drcov file {} to {}", + input.display(), + out_file.display() + ); Box::new(file) } else { @@ -90,7 +97,12 @@ fn process(opts: &Opt, input: &PathBuf) -> Result<(), std::io::Error> { module.path.display() )?; } - writeln!(writer, "# {} Blocks covered in {input:?}.", blocks.len())?; + writeln!( + writer, + "# {} Blocks covered in {}.", + input.display(), + blocks.len() + )?; if opts.addrs { writeln!(writer)?; @@ -128,11 +140,15 @@ fn main() { if let Some(out_dir) = &opts.out_dir { if !out_dir.exists() { if let Err(err) = create_dir_all(out_dir) { - eprintln!("Failed to create dir {out_dir:?}: {err:?}"); + eprintln!("Failed to create dir {}: {err:?}", out_dir.display()); } } - assert!(out_dir.is_dir(), "Out_dir {out_dir:?} not a directory!"); + assert!( + out_dir.is_dir(), + "Out_dir {} not a directory!", + out_dir.display() + ); } for input in &opts.inputs { diff --git a/utils/drcov_utils/src/bin/drcov_merge.rs b/utils/drcov_utils/src/bin/drcov_merge.rs index f0ae0445f9..a6c688ec95 100644 --- a/utils/drcov_utils/src/bin/drcov_merge.rs +++ b/utils/drcov_utils/src/bin/drcov_merge.rs @@ -40,16 +40,22 @@ fn main() { println!("Unique block mode"); } - println!("Reading inital drcov file from {initial_input:?}"); + println!("Reading inital drcov file from {}", initial_input.display()); let mut main_drcov = DrCovReader::read(initial_input).expect("Failed to read fist input!"); for input in inputs { - if let Ok(current_drcov) = DrCovReader::read(input) - .map_err(|err| eprintln!("Warning: failed to read drcov file at {input:?}: {err:?}")) - { - println!("Merging {input:?}"); + if let Ok(current_drcov) = DrCovReader::read(input).map_err(|err| { + eprintln!( + "Warning: failed to read drcov file at {}: {err:?}", + input.display() + ) + }) { + println!("Merging {}", input.display()); if let Err(err) = main_drcov.merge(¤t_drcov, opts.unique) { - eprintln!("Warning: failed to merge drcov file at {input:?}: {err:?}"); + eprintln!( + "Warning: failed to merge drcov file at {}: {err:?}", + input.display() + ); } } }