From 6810e6085b02a1e255b2806d092eb730372cc48d Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Tue, 1 Feb 2022 10:10:47 +0100 Subject: [PATCH] Builder for CommandExecutor & Tokens Refactoring (#508) * builder for CommandExecutor * tokens api cleanup, clippy * fix doctest * cleanup * added testcase, remodelled * command executor builder fix * fix fuzzer(?) * implemented From for configurator * nits * clippy * unused * autotokens * cleanup * nits * Err instead of empty tokens * fix tokens fn * fix err * more error fixing * tokens remodelling * typo * recoverable fail on missing autotokens * clippy, nostd * asslice, into_iter, etc. for tokens * adapt fuzzers * iter * fixes, clippy * fix * more clippy * no_std * more fix * fixed typo * cmd_executor builds again * bring back ASAN stuff to Command Executor * forkserver speedup * no need to static * back to earlier --- fuzzers/baby_fuzzer_grimoire/src/main.rs | 7 +- .../command_executor/src/main.rs | 21 +- fuzzers/frida_libpng/src/fuzzer.rs | 2 +- fuzzers/fuzzbench/src/lib.rs | 11 +- fuzzers/fuzzbench_qemu/src/fuzzer.rs | 2 +- fuzzers/fuzzbench_text/src/lib.rs | 20 +- fuzzers/generic_inmemory/src/lib.rs | 2 +- fuzzers/libafl_atheris/src/lib.rs | 2 +- fuzzers/libfuzzer_libmozjpeg/src/lib.rs | 2 +- fuzzers/libfuzzer_libpng/src/lib.rs | 2 +- fuzzers/libfuzzer_libpng_ctx/src/lib.rs | 2 +- fuzzers/libfuzzer_libpng_launcher/src/lib.rs | 2 +- .../fuzzer/src/main.rs | 14 +- libafl/src/bolts/fs.rs | 87 ++- libafl/src/bolts/os/pipes.rs | 3 - libafl/src/bolts/shmem.rs | 2 +- libafl/src/bolts/tuples.rs | 18 +- libafl/src/executors/command.rs | 544 ++++++++++++++++-- libafl/src/executors/forkserver.rs | 76 +-- libafl/src/executors/mod.rs | 2 +- libafl/src/feedbacks/new_hash_feedback.rs | 2 +- libafl/src/inputs/encoded.rs | 1 + libafl/src/lib.rs | 24 +- libafl/src/mutators/token_mutations.rs | 255 ++++---- libafl/src/stages/push/mod.rs | 2 +- libafl_cc/build.rs | 4 +- libafl_concolic/symcc_runtime/build.rs | 22 +- libafl_sugar/src/forkserver.rs | 2 +- libafl_sugar/src/inmemory.rs | 2 +- libafl_sugar/src/qemu.rs | 2 +- libafl_targets/build.rs | 2 +- libafl_targets/src/coverage.rs | 24 +- 32 files changed, 857 insertions(+), 306 deletions(-) diff --git a/fuzzers/baby_fuzzer_grimoire/src/main.rs b/fuzzers/baby_fuzzer_grimoire/src/main.rs index 59dd8d725b..e344cf85e5 100644 --- a/fuzzers/baby_fuzzer_grimoire/src/main.rs +++ b/fuzzers/baby_fuzzer_grimoire/src/main.rs @@ -32,7 +32,7 @@ fn signals_set(idx: usize) { } fn is_sub(mut haystack: &[T], needle: &[T]) -> bool { - if needle.len() == 0 { + if needle.is_empty() { return true; } while !haystack.is_empty() { @@ -115,10 +115,7 @@ pub fn main() { ); if state.metadata().get::().is_none() { - state.add_metadata(Tokens::new(vec![ - "FOO".as_bytes().to_vec(), - "BAR".as_bytes().to_vec(), - ])); + state.add_metadata(Tokens::from([b"FOO".to_vec(), b"BAR".to_vec()])); } // The Monitor trait define how the fuzzer stats are reported to the user diff --git a/fuzzers/backtrace_baby_fuzzers/command_executor/src/main.rs b/fuzzers/backtrace_baby_fuzzers/command_executor/src/main.rs index 92544452c6..3b4219a106 100644 --- a/fuzzers/backtrace_baby_fuzzers/command_executor/src/main.rs +++ b/fuzzers/backtrace_baby_fuzzers/command_executor/src/main.rs @@ -1,8 +1,3 @@ -use std::path::PathBuf; - -#[cfg(windows)] -use std::ptr::write_volatile; - use libafl::{ bolts::{ current_nanos, @@ -13,6 +8,7 @@ use libafl::{ }, corpus::{InMemoryCorpus, OnDiskCorpus, QueueCorpusScheduler}, events::SimpleEventManager, + executors::command::CommandConfigurator, feedback_and, feedbacks::{ CrashFeedback, MapFeedbackState, MaxMapFeedback, NewHashFeedback, NewHashFeedbackState, @@ -25,10 +21,13 @@ use libafl::{ observers::{get_asan_runtime_flags, ASANBacktraceObserver, StdMapObserver}, stages::mutational::StdMutationalStage, state::StdState, + Error, }; -use libafl::{executors::command::CommandConfigurator, Error}; +#[cfg(windows)] +use std::ptr::write_volatile; use std::{ io::Write, + path::PathBuf, process::{Child, Command, Stdio}, }; @@ -93,14 +92,8 @@ pub fn main() { shmem_id: ShMemId, } - impl CommandConfigurator for MyExecutor { - fn spawn_child( - &mut self, - _fuzzer: &mut Z, - _state: &mut S, - _mgr: &mut EM, - input: &I, - ) -> Result { + impl CommandConfigurator for MyExecutor { + fn spawn_child(&mut self, input: &I) -> Result { let mut command = Command::new("./test_command"); let command = command diff --git a/fuzzers/frida_libpng/src/fuzzer.rs b/fuzzers/frida_libpng/src/fuzzer.rs index 42ff451c32..5c986a3a1b 100644 --- a/fuzzers/frida_libpng/src/fuzzer.rs +++ b/fuzzers/frida_libpng/src/fuzzer.rs @@ -271,7 +271,7 @@ unsafe fn fuzz( // Create a PNG dictionary if not existing if state.metadata().get::().is_none() { - state.add_metadata(Tokens::new(vec![ + state.add_metadata(Tokens::from([ vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header b"IHDR".to_vec(), b"IDAT".to_vec(), diff --git a/fuzzers/fuzzbench/src/lib.rs b/fuzzers/fuzzbench/src/lib.rs index ad9de01327..8e84dcf2a8 100644 --- a/fuzzers/fuzzbench/src/lib.rs +++ b/fuzzers/fuzzbench/src/lib.rs @@ -38,7 +38,7 @@ use libafl::{ monitors::SimpleMonitor, mutators::{ scheduled::havoc_mutations, token_mutations::I2SRandReplace, tokens_mutations, - StdMOptMutator, StdScheduledMutator, TokenSection, Tokens, + StdMOptMutator, StdScheduledMutator, Tokens, }, observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, stages::{ @@ -55,7 +55,7 @@ use libafl_targets::{ }; #[cfg(target_os = "linux")] -use libafl_targets::token_section; +use libafl_targets::autotokens; /// The fuzzer main (as `no_mangle` C function) #[no_mangle] @@ -358,15 +358,14 @@ fn fuzz( if state.metadata().get::().is_none() { let mut toks = Tokens::default(); if let Some(tokenfile) = tokenfile { - toks = toks.parse_tokens_file(vec![tokenfile])?; + toks.add_from_file(tokenfile)?; } #[cfg(target_os = "linux")] { - let token_section = TokenSection::new(token_section()); - toks = toks.parse_autotokens(token_section)?; + toks += autotokens()?; } - if !toks.tokens().is_empty() { + if !toks.is_empty() { state.add_metadata(toks); } } diff --git a/fuzzers/fuzzbench_qemu/src/fuzzer.rs b/fuzzers/fuzzbench_qemu/src/fuzzer.rs index effdca4e7b..9998d9dacd 100644 --- a/fuzzers/fuzzbench_qemu/src/fuzzer.rs +++ b/fuzzers/fuzzbench_qemu/src/fuzzer.rs @@ -346,7 +346,7 @@ fn fuzz( // Read tokens if let Some(tokenfile) = tokenfile { if state.metadata().get::().is_none() { - state.add_metadata(Tokens::from_tokens_file(tokenfile)?); + state.add_metadata(Tokens::from_file(tokenfile)?); } } diff --git a/fuzzers/fuzzbench_text/src/lib.rs b/fuzzers/fuzzbench_text/src/lib.rs index 4be2e33846..08caf50b29 100644 --- a/fuzzers/fuzzbench_text/src/lib.rs +++ b/fuzzers/fuzzbench_text/src/lib.rs @@ -44,7 +44,7 @@ use libafl::{ }, scheduled::havoc_mutations, token_mutations::I2SRandReplace, - tokens_mutations, StdMOptMutator, StdScheduledMutator, TokenSection, Tokens, + tokens_mutations, StdMOptMutator, StdScheduledMutator, Tokens, }, observers::{HitcountsMapObserver, StdMapObserver, TimeObserver}, stages::{ @@ -61,7 +61,7 @@ use libafl_targets::{ }; #[cfg(target_os = "linux")] -use libafl_targets::token_section; +use libafl_targets::autotokens; /// The fuzzer main (as `no_mangle` C function) #[no_mangle] @@ -222,7 +222,7 @@ fn check_if_textual(seeds_dir: &Path, tokenfile: &Option) -> bool { let (found, tot) = count_textual_inputs(&seeds_dir); let is_text = found * 100 / tot > 90; // 90% of text inputs if let Some(tokenfile) = tokenfile { - let toks = Tokens::from_tokens_file(tokenfile).unwrap(); + let toks = Tokens::from_file(tokenfile).unwrap(); if !toks.tokens().is_empty() { let mut cnt = 0; for t in toks.tokens() { @@ -419,15 +419,14 @@ fn fuzz_binary( if state.metadata().get::().is_none() { let mut toks = Tokens::default(); if let Some(tokenfile) = tokenfile { - toks = toks.parse_tokens_file(vec![tokenfile])?; + toks.add_from_file(tokenfile)?; } #[cfg(target_os = "linux")] { - let token_section = TokenSection::new(token_section()); - toks = toks.parse_autotokens(token_section)?; + toks += autotokens()?; } - if !toks.tokens().is_empty() { + if !toks.is_empty() { state.add_metadata(toks); } } @@ -639,15 +638,14 @@ fn fuzz_text( if state.metadata().get::().is_none() { let mut toks = Tokens::default(); if let Some(tokenfile) = tokenfile { - toks = toks.parse_tokens_file(vec![tokenfile])?; + toks.add_from_file(tokenfile)?; } #[cfg(target_os = "linux")] { - let token_section = TokenSection::new(token_section()); - toks = toks.parse_autotokens(token_section)?; + toks += autotokens()?; } - if !toks.tokens().is_empty() { + if !toks.is_empty() { state.add_metadata(toks); } } diff --git a/fuzzers/generic_inmemory/src/lib.rs b/fuzzers/generic_inmemory/src/lib.rs index 3a29204c61..c230bd1afb 100644 --- a/fuzzers/generic_inmemory/src/lib.rs +++ b/fuzzers/generic_inmemory/src/lib.rs @@ -195,7 +195,7 @@ pub fn libafl_main() { // Create a dictionary if not existing if state.metadata().get::().is_none() { for tokens_file in &token_files { - state.add_metadata(Tokens::from_tokens_file(tokens_file)?); + state.add_metadata(Tokens::from_file(tokens_file)?); } } diff --git a/fuzzers/libafl_atheris/src/lib.rs b/fuzzers/libafl_atheris/src/lib.rs index 8c0526546f..739f33e286 100644 --- a/fuzzers/libafl_atheris/src/lib.rs +++ b/fuzzers/libafl_atheris/src/lib.rs @@ -266,7 +266,7 @@ pub fn LLVMFuzzerRunDriver( // Create a dictionary if not existing if state.metadata().get::().is_none() { for tokens_file in &token_files { - state.add_metadata(Tokens::from_tokens_file(tokens_file)?); + state.add_metadata(Tokens::from_file(tokens_file)?); } } diff --git a/fuzzers/libfuzzer_libmozjpeg/src/lib.rs b/fuzzers/libfuzzer_libmozjpeg/src/lib.rs index 8504a86b7e..9b2b75ac82 100644 --- a/fuzzers/libfuzzer_libmozjpeg/src/lib.rs +++ b/fuzzers/libfuzzer_libmozjpeg/src/lib.rs @@ -128,7 +128,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // Add the JPEG tokens if not existing if state.metadata().get::().is_none() { - state.add_metadata(Tokens::from_tokens_file("./jpeg.dict")?); + state.add_metadata(Tokens::from_file("./jpeg.dict")?); } // Setup a basic mutator with a mutational stage diff --git a/fuzzers/libfuzzer_libpng/src/lib.rs b/fuzzers/libfuzzer_libpng/src/lib.rs index cd873b5ee6..fb84b8cd3d 100644 --- a/fuzzers/libfuzzer_libpng/src/lib.rs +++ b/fuzzers/libfuzzer_libpng/src/lib.rs @@ -119,7 +119,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re // Create a PNG dictionary if not existing if state.metadata().get::().is_none() { - state.add_metadata(Tokens::new(vec![ + state.add_metadata(Tokens::from([ vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header "IHDR".as_bytes().to_vec(), "IDAT".as_bytes().to_vec(), diff --git a/fuzzers/libfuzzer_libpng_ctx/src/lib.rs b/fuzzers/libfuzzer_libpng_ctx/src/lib.rs index d161d390aa..a01268af77 100644 --- a/fuzzers/libfuzzer_libpng_ctx/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_ctx/src/lib.rs @@ -187,7 +187,7 @@ pub fn libafl_main() { // Create a PNG dictionary if not existing if state.metadata().get::().is_none() { - state.add_metadata(Tokens::new(vec![ + state.add_metadata(Tokens::from([ vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header "IHDR".as_bytes().to_vec(), "IDAT".as_bytes().to_vec(), diff --git a/fuzzers/libfuzzer_libpng_launcher/src/lib.rs b/fuzzers/libfuzzer_libpng_launcher/src/lib.rs index 5c3e9f6e77..e5ba30da77 100644 --- a/fuzzers/libfuzzer_libpng_launcher/src/lib.rs +++ b/fuzzers/libfuzzer_libpng_launcher/src/lib.rs @@ -188,7 +188,7 @@ pub fn libafl_main() { // Create a PNG dictionary if not existing if state.metadata().get::().is_none() { - state.add_metadata(Tokens::new(vec![ + state.add_metadata(Tokens::from([ vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header "IHDR".as_bytes().to_vec(), "IDAT".as_bytes().to_vec(), diff --git a/fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs b/fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs index 54387b12d8..bf1f261646 100644 --- a/fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs +++ b/fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs @@ -245,19 +245,15 @@ fn fuzz( use std::process::{Child, Command, Stdio}; #[derive(Default)] -pub struct MyCommandConfigurator; +pub struct MyCommandConfigurator { + command: Option, +} -impl CommandConfigurator for MyCommandConfigurator +impl CommandConfigurator for MyCommandConfigurator where I: HasTargetBytes + Input, { - fn spawn_child( - &mut self, - _fuzzer: &mut Z, - _state: &mut S, - _mgr: &mut EM, - input: &I, - ) -> Result { + fn spawn_child(&mut self, input: &I) -> Result { input.to_file("cur_input")?; Ok(Command::new("./target_symcc.out") diff --git a/libafl/src/bolts/fs.rs b/libafl/src/bolts/fs.rs index 4fd755836b..970d85638e 100644 --- a/libafl/src/bolts/fs.rs +++ b/libafl/src/bolts/fs.rs @@ -1,13 +1,19 @@ //! `LibAFL` functionality for filesystem interaction use std::{ - fs::{self, OpenOptions}, - io::Write, - path::Path, + fs::{self, remove_file, File, OpenOptions}, + io::{Seek, SeekFrom, Write}, + path::{Path, PathBuf}, }; +#[cfg(unix)] +use std::os::unix::prelude::{AsRawFd, RawFd}; + use crate::Error; +/// The default filename to use to deliver testcases to the target +pub const DEFAULT_OUTFILE: &str = ".cur_input"; + /// Creates a `.{file_name}.tmp` file, and writes all bytes to it. /// After all bytes have been written, the tmp-file is moved to it's original `path`. /// This way, on the majority of operating systems, the final file will never be incomplete or racey. @@ -38,6 +44,81 @@ where inner(path.as_ref(), bytes) } +/// An [`OutFile`] to write fuzzer input to. +/// The target/forkserver will read from this file. +#[cfg(feature = "std")] +#[derive(Debug)] +pub struct OutFile { + /// The filename/path too this [`OutFile`] + pub path: PathBuf, + /// The underlying file that got created + pub file: File, +} + +impl Clone for OutFile { + fn clone(&self) -> Self { + Self { + path: self.path.clone(), + file: self.file.try_clone().unwrap(), + } + } +} + +#[cfg(feature = "std")] +impl OutFile { + /// Creates a new [`OutFile`] + pub fn create

(filename: P) -> Result + where + P: AsRef, + { + let f = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&filename)?; + f.set_len(0)?; + Ok(Self { + path: filename.as_ref().to_owned(), + file: f, + }) + } + + /// Gets the file as raw file descriptor + #[must_use] + #[cfg(unix)] + pub fn as_raw_fd(&self) -> RawFd { + self.file.as_raw_fd() + } + + /// Writes the given buffer to the file + pub fn write_buf(&mut self, buf: &[u8]) -> Result<(), Error> { + self.rewind()?; + self.file.write_all(buf)?; + self.file.set_len(buf.len() as u64)?; + self.file.flush()?; + // Rewind again otherwise the target will not read stdin from the beginning + self.rewind() + } + + /// Rewinds the file to the beginning + #[inline] + pub fn rewind(&mut self) -> Result<(), Error> { + if let Err(err) = self.file.seek(SeekFrom::Start(0)) { + Err(err.into()) + } else { + Ok(()) + } + } +} + +#[cfg(feature = "std")] +impl Drop for OutFile { + fn drop(&mut self) { + // try to remove the file, but ignore errors + drop(remove_file(&self.path)); + } +} + #[cfg(test)] mod test { use crate::bolts::fs::write_file_atomic; diff --git a/libafl/src/bolts/os/pipes.rs b/libafl/src/bolts/os/pipes.rs index 68dd69e73f..486e5a65d4 100644 --- a/libafl/src/bolts/os/pipes.rs +++ b/libafl/src/bolts/os/pipes.rs @@ -8,9 +8,6 @@ use std::{ os::unix::io::RawFd, }; -#[cfg(not(feature = "std"))] -type RawFd = i32; - /// A unix pipe wrapper for `LibAFL` #[cfg(feature = "std")] #[derive(Debug, Clone)] diff --git a/libafl/src/bolts/shmem.rs b/libafl/src/bolts/shmem.rs index b2b7591ae0..2ebc2c1da8 100644 --- a/libafl/src/bolts/shmem.rs +++ b/libafl/src/bolts/shmem.rs @@ -1180,7 +1180,7 @@ pub mod win32_shmem { } Ok(Self { - id: ShMemId::try_from_slice(&map_str_bytes).unwrap(), + id: ShMemId::try_from_slice(map_str_bytes).unwrap(), handle, map, map_size, diff --git a/libafl/src/bolts/tuples.rs b/libafl/src/bolts/tuples.rs index 4f9e5db0bf..5316047880 100644 --- a/libafl/src/bolts/tuples.rs +++ b/libafl/src/bolts/tuples.rs @@ -163,14 +163,16 @@ where /// Match by type pub trait MatchType { /// Match by type and call the passed `f` function with a borrow, if found - fn match_type(&self, f: fn(t: &T)); + fn match_type(&self, f: &mut FN); /// Match by type and call the passed `f` function with a mutable borrow, if found - fn match_type_mut(&mut self, f: fn(t: &mut T)); + fn match_type_mut(&mut self, f: &mut FN); } impl MatchType for () { - fn match_type(&self, _f: fn(t: &T)) {} - fn match_type_mut(&mut self, _f: fn(t: &mut T)) {} + /// Match by type and call the passed `f` function with a borrow, if found + fn match_type(&self, _: &mut FN) {} + /// Match by type and call the passed `f` function with a mutable borrow, if found + fn match_type_mut(&mut self, _: &mut FN) {} } impl MatchType for (Head, Tail) @@ -178,20 +180,20 @@ where Head: 'static, Tail: MatchType, { - fn match_type(&self, f: fn(t: &T)) { + fn match_type(&self, f: &mut FN) { // Switch this check to https://stackoverflow.com/a/60138532/7658998 when in stable and remove 'static if TypeId::of::() == TypeId::of::() { f(unsafe { (addr_of!(self.0) as *const T).as_ref() }.unwrap()); } - self.1.match_type::(f); + self.1.match_type::(f); } - fn match_type_mut(&mut self, f: fn(t: &mut T)) { + fn match_type_mut(&mut self, f: &mut FN) { // Switch this check to https://stackoverflow.com/a/60138532/7658998 when in stable and remove 'static if TypeId::of::() == TypeId::of::() { f(unsafe { (addr_of_mut!(self.0) as *mut T).as_mut() }.unwrap()); } - self.1.match_type_mut::(f); + self.1.match_type_mut::(f); } } diff --git a/libafl/src/executors/command.rs b/libafl/src/executors/command.rs index 1f2455db01..67dca47ac6 100644 --- a/libafl/src/executors/command.rs +++ b/libafl/src/executors/command.rs @@ -6,26 +6,161 @@ use core::{ #[cfg(feature = "std")] use std::process::Child; +use std::{ + ffi::{OsStr, OsString}, + io::Write, + os::unix::prelude::OsStringExt, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; -use crate::observers::ASANBacktraceObserver; +use crate::{ + bolts::{ + fs::{OutFile, DEFAULT_OUTFILE}, + tuples::MatchName, + AsSlice, + }, + inputs::HasTargetBytes, + observers::{ASANBacktraceObserver, ObserversTuple}, +}; #[cfg(feature = "std")] -use crate::{executors::HasObservers, inputs::Input, observers::ObserversTuple, Error}; +use crate::{inputs::Input, Error}; #[cfg(all(feature = "std", unix))] use crate::executors::{Executor, ExitKind}; #[cfg(all(feature = "std", unix))] use std::time::Duration; + +use super::HasObservers; + +/// How to deliver input to an external program +/// `StdIn`: The traget reads from stdin +/// `File`: The target reads from the specified [`OutFile`] +#[derive(Debug, Clone)] +pub enum InputLocation { + /// Mutate a commandline argument to deliver an input + Arg { + /// The offset of the argument to mutate + argnum: usize, + }, + /// Deliver input via `StdIn` + StdIn, + /// Deliver the iniput via the specified [`OutFile`] + /// You can use specify [`OutFile::create(DEFAULT_OUTFILE)`] to use a default filename. + File { + /// The fiel to write input to. The target should read input from this location. + out_file: OutFile, + }, +} + +/// Clones a [`Command`] (without stdio and stdout/stderr - they are not accesible) +fn clone_command(cmd: &Command) -> Command { + let mut new_cmd = Command::new(cmd.get_program()); + new_cmd.args(cmd.get_args()); + new_cmd.env_clear(); + new_cmd.envs( + cmd.get_envs() + .filter_map(|(key, value)| value.map(|value| (key, value))), + ); + if let Some(cwd) = cmd.get_current_dir() { + new_cmd.current_dir(cwd); + } + new_cmd +} + +/// A simple Configurator that takes the most common parameters +/// Writes the input either to stdio or to a file +#[derive(Debug)] +pub struct StdCommandConfigurator { + /// If set to true, the child output will remain visible + /// By default, the child output is hidden to increase execution speed + pub debug_child: bool, + /// true: input gets delivered via stdink + pub input_location: InputLocation, + /// The Command to execute + pub command: Command, +} + +impl CommandConfigurator for StdCommandConfigurator { + fn spawn_child(&mut self, input: &I) -> Result + where + I: Input + HasTargetBytes, + { + match &mut self.input_location { + InputLocation::Arg { argnum } => { + let args = self.command.get_args(); + let mut cmd = Command::new(self.command.get_program()); + + if !self.debug_child { + cmd.stdout(Stdio::null()); + cmd.stderr(Stdio::null()); + } + + for (i, arg) in args.enumerate() { + if i == *argnum { + cmd.arg(OsString::from_vec(input.target_bytes().as_slice().to_vec())); + } else { + cmd.arg(arg); + } + } + cmd.envs( + self.command + .get_envs() + .filter_map(|(key, value)| value.map(|value| (key, value))), + ); + if let Some(cwd) = self.command.get_current_dir() { + cmd.current_dir(cwd); + } + Ok(cmd.spawn()?) + } + InputLocation::StdIn => { + self.command.stdin(Stdio::piped()).spawn()?; + let mut handle = self.command.spawn()?; + let mut stdin = handle.stdin.take().unwrap(); + stdin.write_all(input.target_bytes().as_slice())?; + stdin.flush()?; + drop(stdin); + Ok(handle) + } + InputLocation::File { out_file } => { + out_file.write_buf(input.target_bytes().as_slice())?; + Ok(self.command.spawn()?) + } + } + } +} + /// A `CommandExecutor` is a wrapper around [`std::process::Command`] to execute a target as a child process. /// Construct a `CommandExecutor` by implementing [`CommandConfigurator`] for a type of your choice and calling [`CommandConfigurator::into_executor`] on it. -pub struct CommandExecutor { +/// Instead, you can use [`CommandExecutor::builder()`] to construct a [`CommandExecutor`] backed by a [`StdCommandConfigurator`]. +pub struct CommandExecutor +where + T: Debug, + OT: Debug, +{ inner: T, - /// [`crate::observers::Observer`]s for this executor observers: OT, + /// cache if the AsanBacktraceObserver is present + has_asan_observer: bool, phantom: PhantomData<(EM, I, S, Z)>, } -impl Debug for CommandExecutor { +impl CommandExecutor<(), (), (), (), (), ()> { + /// Creates a builder for a new [`CommandExecutor`], + /// backed by a [`StdCommandConfigurator`] + /// This is usually the easiest way to construct a [`CommandExecutor`]. + #[must_use] + pub fn builder() -> CommandExecutorBuilder { + CommandExecutorBuilder::new() + } +} + +impl Debug for CommandExecutor +where + T: Debug, + OT: Debug, +{ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("CommandExecutor") .field("inner", &self.inner) @@ -34,20 +169,117 @@ impl Debug for CommandExecutor CommandExecutor { +impl CommandExecutor +where + T: Debug, + OT: Debug, +{ /// Accesses the inner value pub fn inner(&mut self) -> &mut T { &mut self.inner } } +impl CommandExecutor +where + OT: MatchName + Debug, +{ + /// Creates a new `CommandExecutor`. + /// Instead of parsing the Command for `@@`, it will + pub fn from_cmd_with_file

( + cmd: &Command, + debug_child: bool, + observers: OT, + path: P, + ) -> Result + where + P: AsRef, + { + let mut command = clone_command(cmd); + if !debug_child { + command.stdout(Stdio::null()); + command.stderr(Stdio::null()); + } + command.stdin(Stdio::null()); + + let has_asan_observer = observers + .match_name::("ASANBacktraceObserver") + .is_some(); + if has_asan_observer { + command.stderr(Stdio::piped()); + } + + Ok(Self { + observers, + has_asan_observer, + inner: StdCommandConfigurator { + input_location: InputLocation::File { + out_file: OutFile::create(path)?, + }, + command, + debug_child, + }, + phantom: PhantomData, + }) + } + + /// Parses an AFL-like comandline, replacing `@@` with the input file. + /// If no `@@` was found, will use stdin for input. + /// The arg 0 is the program. + pub fn parse_afl_cmdline( + args: IT, + observers: OT, + debug_child: bool, + ) -> Result + where + IT: IntoIterator, + O: AsRef, + { + let mut atat_at = None; + let mut builder = CommandExecutorBuilder::new(); + builder.debug_child(debug_child); + let afl_delim = OsStr::new("@@"); + + for (pos, arg) in args.into_iter().enumerate() { + if pos == 0 { + if arg.as_ref() == afl_delim { + return Err(Error::IllegalArgument( + "The first argument must not be @@ but the program to execute".into(), + )); + } + builder.program(arg); + } else if arg.as_ref() == afl_delim { + if atat_at.is_some() { + return Err(Error::IllegalArgument( + "Multiple @@ in afl commandline are not permitted".into(), + )); + } + atat_at = Some(pos); + builder.input(InputLocation::File { + out_file: OutFile::create(DEFAULT_OUTFILE)?, + }); + builder.arg(DEFAULT_OUTFILE); + } else { + builder.arg(arg); + } + } + + if atat_at.is_none() { + builder.input(InputLocation::StdIn); + } + + builder.build(observers) + } +} + // this only works on unix because of the reliance on checking the process signal for detecting OOM #[cfg(all(feature = "std", unix))] -impl Executor for CommandExecutor +impl Executor for CommandExecutor where - I: Input, - T: CommandConfigurator, - OT: ObserversTuple, + I: Input + HasTargetBytes, + T: CommandConfigurator, + OT: Debug + MatchName, + T: Debug, { fn run_target( &mut self, @@ -59,7 +291,7 @@ where use std::os::unix::prelude::ExitStatusExt; use wait_timeout::ChildExt; - let mut child = self.inner.spawn_child(_fuzzer, _state, _mgr, input)?; + let mut child = self.inner.spawn_child(input)?; let res = match child .wait_timeout(Duration::from_secs(5)) @@ -80,37 +312,212 @@ where } }; - let stderr = child.stderr.as_mut().unwrap(); - if let Some(obs) = self - .observers - .match_name_mut::("ASANBacktraceObserver") - { - obs.parse_asan_output_from_childstderr(stderr); + if self.has_asan_observer { + let stderr = child.stderr.as_mut().ok_or_else(|| { + Error::IllegalState( + "Using ASANBacktraceObserver but stderr was not `Stdio::pipe` in CommandExecutor".into(), + ) + })?; + self.observers + .match_name_mut::("ASANBacktraceObserver") + .unwrap() + .parse_asan_output_from_childstderr(stderr); }; res } } -#[cfg(all(feature = "std", unix))] -impl HasObservers +impl, S, T: Debug, Z> HasObservers for CommandExecutor -where - I: Input, - OT: ObserversTuple, - T: CommandConfigurator, { - #[inline] fn observers(&self) -> &OT { &self.observers } - #[inline] fn observers_mut(&mut self) -> &mut OT { &mut self.observers } } +/// The builder for a default [`CommandExecutor`] that should fit most use-cases. +#[derive(Debug, Clone)] +pub struct CommandExecutorBuilder { + debug_child: bool, + program: Option, + args_before: Vec, + input_location: Option, + args_after: Vec, + cwd: Option, + envs: Vec<(OsString, OsString)>, +} + +impl Default for CommandExecutorBuilder { + fn default() -> Self { + Self::new() + } +} + +impl CommandExecutorBuilder { + /// Create a new [`CommandExecutorBuilder`] + #[must_use] + fn new() -> CommandExecutorBuilder { + CommandExecutorBuilder { + program: None, + args_before: vec![], + input_location: None, + args_after: vec![], + cwd: None, + envs: vec![], + debug_child: false, + } + } + + /// Set the binary to execute + /// This option is required. + pub fn program(&mut self, program: O) -> &mut Self + where + O: AsRef, + { + self.program = Some(program.as_ref().to_owned()); + self + } + + /// Set the input mode and location. + /// This option is mandatory, if not set, the `build` method will error. + pub fn input(&mut self, input: InputLocation) -> &mut Self { + // This is an error in the user code, no point in returning Err. + assert!( + self.input_location.is_none(), + "input location already set, cannot set it again" + ); + self.input_location = Some(input); + self + } + + /// Adds an argument to the program's commandline. + pub fn arg>(&mut self, arg: O) -> &mut CommandExecutorBuilder { + match self.input_location { + Some(InputLocation::StdIn) => self.args_before.push(arg.as_ref().to_owned()), + Some(_) | None => self.args_after.push(arg.as_ref().to_owned()), + }; + self + } + + /// Adds a range of arguments to the program's commandline. + pub fn args(&mut self, args: IT) -> &mut CommandExecutorBuilder + where + IT: IntoIterator, + O: AsRef, + { + for arg in args { + self.arg(arg.as_ref()); + } + self + } + + /// Adds a range of environment variables to the executed command. + pub fn envs(&mut self, vars: IT) -> &mut CommandExecutorBuilder + where + IT: IntoIterator, + K: AsRef, + V: AsRef, + { + for (ref key, ref val) in vars { + self.env(key.as_ref(), val.as_ref()); + } + self + } + + /// Adds an environment variable to the executed command. + pub fn env(&mut self, key: K, val: V) -> &mut CommandExecutorBuilder + where + K: AsRef, + V: AsRef, + { + self.envs + .push((key.as_ref().to_owned(), val.as_ref().to_owned())); + self + } + + /// Sets the working directory for the child process. + pub fn current_dir>(&mut self, dir: P) -> &mut CommandExecutorBuilder { + self.cwd = Some(dir.as_ref().to_owned()); + self + } + + /// If set to true, the child's output won't be redirecited to `/dev/null`. + /// Defaults to `false`. + pub fn debug_child(&mut self, debug_child: bool) -> &mut CommandExecutorBuilder { + self.debug_child = debug_child; + self + } + + /// Builds the `ComandExecutor` + pub fn build( + &self, + observers: OT, + ) -> Result, Error> + where + OT: Debug + MatchName, + { + let program = if let Some(program) = &self.program { + program + } else { + return Err(Error::IllegalArgument( + "ComandExecutor::builder: no program set!".into(), + )); + }; + let mut command = Command::new(program); + command.args(&self.args_before); + match &self.input_location { + Some(InputLocation::StdIn) => { + command.stdin(Stdio::piped()); + } + Some(InputLocation::File { out_file }) => { + command.stdin(Stdio::null()); + command.arg(&out_file.path); + } + Some(InputLocation::Arg { .. }) => { + command.stdin(Stdio::null()); + command.arg("DUMMY"); + } + None => { + return Err(Error::IllegalArgument( + "ComandExecutor::builder: no input_location set!".into(), + )) + } + } + command.args(&self.args_after); + command.envs( + self.envs + .iter() + .map(|(k, v)| (k.as_os_str(), v.as_os_str())), + ); + if let Some(cwd) = &self.cwd { + command.current_dir(cwd); + } + if !self.debug_child { + command.stdout(Stdio::null()); + command.stderr(Stdio::null()); + } + if observers + .match_name::("ASANBacktraceObserver") + .is_some() + { + // we need stderr for ASANBackt + command.stderr(Stdio::piped()); + } + + let configurator = StdCommandConfigurator { + debug_child: self.debug_child, + input_location: self.input_location.clone().unwrap(), + command, + }; + Ok(configurator.into_executor(observers)) + } +} + /// A `CommandConfigurator` takes care of creating and spawning a [`std::process::Command`] for the [`CommandExecutor`]. /// # Example /// ``` @@ -119,12 +526,9 @@ where /// #[derive(Debug)] /// struct MyExecutor; /// -/// impl CommandConfigurator for MyExecutor { -/// fn spawn_child( +/// impl CommandConfigurator for MyExecutor { +/// fn spawn_child( /// &mut self, -/// fuzzer: &mut Z, -/// state: &mut S, -/// mgr: &mut EM, /// input: &I, /// ) -> Result { /// let mut command = Command::new("../if"); @@ -145,25 +549,83 @@ where /// } /// ``` #[cfg(all(feature = "std", unix))] -pub trait CommandConfigurator: Sized + Debug { +pub trait CommandConfigurator: Sized + Debug { /// Spawns a new process with the given configuration. - fn spawn_child( - &mut self, - fuzzer: &mut Z, - state: &mut S, - mgr: &mut EM, - input: &I, - ) -> Result; + fn spawn_child(&mut self, input: &I) -> Result + where + I: Input + HasTargetBytes; /// Create an `Executor` from this `CommandConfigurator`. - fn into_executor(self, observers: OT) -> CommandExecutor + fn into_executor(self, observers: OT) -> CommandExecutor where - OT: ObserversTuple, + OT: Debug + MatchName, { + let has_asan_observer = observers + .match_name::("ASANBacktraceObserver") + .is_some(); + CommandExecutor { - inner: self, observers, + has_asan_observer, + inner: self, phantom: PhantomData, } } } + +#[cfg(test)] +mod tests { + use crate::{ + events::SimpleEventManager, + executors::{ + command::{CommandExecutor, InputLocation}, + Executor, + }, + inputs::BytesInput, + monitors::SimpleMonitor, + }; + + #[test] + #[cfg(unix)] + fn test_builder() { + let mut mgr = SimpleEventManager::::new(SimpleMonitor::new(|status| { + println!("{}", status); + })); + + let mut executor = CommandExecutor::builder(); + executor + .program("ls") + .input(InputLocation::Arg { argnum: 0 }); + let executor = executor.build(()); + let mut executor = executor.unwrap(); + + executor + .run_target( + &mut (), + &mut (), + &mut mgr, + &BytesInput::new(b"test".to_vec()), + ) + .unwrap(); + } + + #[test] + #[cfg(unix)] + fn test_parse_afl_cmdline() { + let mut mgr = SimpleEventManager::::new(SimpleMonitor::new(|status| { + println!("{}", status); + })); + + let mut executor = + CommandExecutor::parse_afl_cmdline(&["file".to_string(), "@@".to_string()], (), true) + .unwrap(); + executor + .run_target( + &mut (), + &mut (), + &mut mgr, + &BytesInput::new(b"test".to_vec()), + ) + .unwrap(); + } +} diff --git a/libafl/src/executors/forkserver.rs b/libafl/src/executors/forkserver.rs index 10af931bae..bad91a4b52 100644 --- a/libafl/src/executors/forkserver.rs +++ b/libafl/src/executors/forkserver.rs @@ -6,17 +6,14 @@ use core::{ time::Duration, }; use std::{ - fs::{File, OpenOptions}, - io::{self, prelude::*, ErrorKind, SeekFrom}, - os::unix::{ - io::{AsRawFd, RawFd}, - process::CommandExt, - }, + io::{self, prelude::*, ErrorKind}, + os::unix::{io::RawFd, process::CommandExt}, process::{Command, Stdio}, }; use crate::{ bolts::{ + fs::OutFile, os::{dup2, pipes::Pipe}, shmem::{ShMem, ShMemProvider, StdShMemProvider}, AsMutSlice, AsSlice, @@ -155,47 +152,6 @@ impl ConfigTarget for Command { } } -/// The [`OutFile`] to write input to. -/// The target/forkserver will read from this file. -#[derive(Debug)] -pub struct OutFile { - /// The file - file: File, -} - -impl OutFile { - /// Creates a new [`OutFile`] - pub fn new(file_name: &str) -> Result { - let f = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .open(file_name)?; - Ok(Self { file: f }) - } - - /// Gets the file as raw file descriptor - #[must_use] - pub fn as_raw_fd(&self) -> RawFd { - self.file.as_raw_fd() - } - - /// Writes the given buffer to the file - pub fn write_buf(&mut self, buf: &[u8]) { - self.rewind(); - self.file.write_all(buf).unwrap(); - self.file.set_len(buf.len() as u64).unwrap(); - self.file.flush().unwrap(); - // Rewind again otherwise the target will not read stdin from the beginning - self.rewind(); - } - - /// Rewinds the file to the beginning - pub fn rewind(&mut self) { - self.file.seek(SeekFrom::Start(0)).unwrap(); - } -} - /// The [`Forkserver`] is communication channel with a child process that forks on request of the fuzzer. /// The communication happens via pipe. #[derive(Debug)] @@ -434,7 +390,7 @@ where None => { self.executor .out_file_mut() - .write_buf(input.target_bytes().as_slice()); + .write_buf(input.target_bytes().as_slice())?; } } @@ -515,6 +471,8 @@ where observers: OT, map: Option, phantom: PhantomData<(I, S)>, + /// Cache that indicates if we have a asan observer registered. + has_asan_observer: Option, } impl Debug for ForkserverExecutor @@ -597,7 +555,7 @@ where } } - let out_file = OutFile::new(&out_filename)?; + let out_file = OutFile::create(&out_filename)?; let map = match shmem_provider { None => None, @@ -647,6 +605,7 @@ where } Ok(Self { + has_asan_observer: None, // initialized on first use target, args, out_file, @@ -706,7 +665,7 @@ where .copy_from_slice(target_bytes.as_slice()); } None => { - self.out_file.write_buf(input.target_bytes().as_slice()); + self.out_file.write_buf(input.target_bytes().as_slice())?; } } @@ -745,11 +704,18 @@ where if libc::WIFSIGNALED(self.forkserver.status()) { exit_kind = ExitKind::Crash; - if let Some(obs) = self - .observers_mut() - .match_name_mut::("ASANBacktraceObserver") - { - obs.parse_asan_output_from_asan_log_file(pid); + if self.has_asan_observer.is_none() { + self.has_asan_observer = Some( + self.observers() + .match_name::("ASANBacktraceObserver") + .is_some(), + ); + } + if self.has_asan_observer.unwrap() { + self.observers_mut() + .match_name_mut::("ASANBacktraceObserver") + .unwrap() + .parse_asan_output_from_asan_log_file(pid); } } diff --git a/libafl/src/executors/mod.rs b/libafl/src/executors/mod.rs index 56c146d2ed..37a23b88b5 100644 --- a/libafl/src/executors/mod.rs +++ b/libafl/src/executors/mod.rs @@ -15,7 +15,7 @@ pub use timeout::TimeoutExecutor; #[cfg(all(feature = "std", feature = "fork", unix))] pub mod forkserver; #[cfg(all(feature = "std", feature = "fork", unix))] -pub use forkserver::{Forkserver, ForkserverExecutor, OutFile, TimeoutForkserverExecutor}; +pub use forkserver::{Forkserver, ForkserverExecutor, TimeoutForkserverExecutor}; pub mod combined; pub use combined::CombinedExecutor; diff --git a/libafl/src/feedbacks/new_hash_feedback.rs b/libafl/src/feedbacks/new_hash_feedback.rs index 001c46b33c..50b22b131c 100644 --- a/libafl/src/feedbacks/new_hash_feedback.rs +++ b/libafl/src/feedbacks/new_hash_feedback.rs @@ -131,7 +131,7 @@ where let backtrace_state = _state .feedback_states_mut() - .match_name_mut::>(&self.observer_name.to_string()) + .match_name_mut::>(&self.observer_name) .unwrap(); match observer.hash() { diff --git a/libafl/src/inputs/encoded.rs b/libafl/src/inputs/encoded.rs index 3ab2e1dd09..c11c61eb6b 100644 --- a/libafl/src/inputs/encoded.rs +++ b/libafl/src/inputs/encoded.rs @@ -28,6 +28,7 @@ where /// Trait to decode encoded input to bytes pub trait InputDecoder { /// Decode encoded input to bytes + #[allow(clippy::ptr_arg)] // we reuse the alloced `Vec` fn decode(&self, input: &EncodedInput, bytes: &mut Vec) -> Result<(), Error>; } diff --git a/libafl/src/lib.rs b/libafl/src/lib.rs index 7fec940b4e..5b8d7e69af 100644 --- a/libafl/src/lib.rs +++ b/libafl/src/lib.rs @@ -93,31 +93,9 @@ pub mod stages; pub mod state; pub mod fuzzer; -pub use fuzzer::*; - -/// The `stats` module got renamed to [`monitors`]. -/// It monitors and displays the statistics of the fuzzing process. -#[deprecated(since = "0.7.0", note = "The `stats` module got renamed to `monitors`")] -pub mod stats { - #[deprecated( - since = "0.7.0", - note = "Use monitors::MultiMonitor instead of stats::MultiStats!" - )] - pub use crate::monitors::MultiMonitor as MultiStats; - #[deprecated( - since = "0.7.0", - note = "Use monitors::SimpleMonitor instead of stats::SimpleStats!" - )] - pub use crate::monitors::SimpleMonitor as SimpleStats; - #[deprecated( - since = "0.7.0", - note = "Use monitors::UserMonitor instead of stats::SimpleStats!" - )] - pub use crate::monitors::UserStats; -} - use alloc::string::{FromUtf8Error, String}; use core::{array::TryFromSliceError, fmt, num::ParseIntError, num::TryFromIntError}; +pub use fuzzer::*; #[cfg(feature = "std")] use std::{env::VarError, io}; diff --git a/libafl/src/mutators/token_mutations.rs b/libafl/src/mutators/token_mutations.rs index ba3a4d16c3..c72db2eb6e 100644 --- a/libafl/src/mutators/token_mutations.rs +++ b/libafl/src/mutators/token_mutations.rs @@ -1,11 +1,19 @@ //! Tokens are what afl calls extras or dictionaries. //! They may be inserted as part of mutations during fuzzing. -use alloc::vec::Vec; -use core::mem::size_of; -use serde::{Deserialize, Serialize}; - #[cfg(feature = "std")] use crate::mutators::str_decode; +#[cfg(target_os = "linux")] +use alloc::string::ToString; +use alloc::vec::Vec; +use core::slice::Iter; +use core::{ + mem::size_of, + ops::{Add, AddAssign}, +}; +#[cfg(target_os = "linux")] +use core::{ptr::null, slice::from_raw_parts}; +use hashbrown::HashSet; +use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use std::{ fs::File, @@ -14,7 +22,7 @@ use std::{ }; use crate::{ - bolts::rands::Rand, + bolts::{rands::Rand, AsSlice}, inputs::{HasBytesVec, Input}, mutators::{buffer_self_copy, mutations::buffer_copy, MutationResult, Mutator, Named}, observers::cmp::{CmpValues, CmpValuesMetadata}, @@ -22,29 +30,13 @@ use crate::{ Error, }; -#[derive(Debug, Clone, Copy)] -/// Struct for token start and end -pub struct TokenSection { - start: *const u8, - stop: *const u8, -} - -impl TokenSection { - /// Init - #[must_use] - pub fn new(section: (*const u8, *const u8)) -> Self { - Self { - start: section.0, - stop: section.1, - } - } -} - /// A state metadata holding a list of tokens -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] #[allow(clippy::unsafe_derive_deserialize)] pub struct Tokens { - token_vec: Vec>, + // We keep a vec and a set, set for faster deduplication, vec for access + tokens_vec: Vec>, + tokens_set: HashSet>, } crate::impl_serdeany!(Tokens); @@ -53,95 +45,90 @@ crate::impl_serdeany!(Tokens); impl Tokens { /// Creates a new tokens metadata (old-skool afl name: `dictornary`) #[must_use] - pub fn new(token_vec: Vec>) -> Self { - Self { token_vec } + pub fn new() -> Self { + Self { + ..Tokens::default() + } } - #[must_use] - /// Build tokens from vec - pub fn parse_vec(mut self, vec: Vec>) -> Self { - self.token_vec = vec; + /// Add tokens from a slice of Vecs of bytes + pub fn add_tokens(&mut self, tokens: IT) -> &mut Self + where + IT: IntoIterator, + V: AsRef>, + { + for token in tokens { + self.add_token(token.as_ref()); + } self } /// Build tokens from files #[cfg(feature = "std")] - pub fn parse_tokens_file

(mut self, files: Vec

) -> Result + pub fn add_from_files(mut self, files: IT) -> Result where + IT: IntoIterator, P: AsRef, { for file in files { - self.add_tokens_from_file(file)?; + self.add_from_file(file)?; } Ok(self) } - /// Build tokens from autotokens - pub fn parse_autotokens(mut self, autotoken: TokenSection) -> Result { - unsafe { - self.add_from_autotokens(autotoken)?; + /// Create a token section from a start and an end pointer + /// Reads from an autotokens section, returning the count of new entries read + #[must_use] + #[cfg(target_os = "linux")] + pub unsafe fn from_ptrs(token_start: *const u8, token_stop: *const u8) -> Result { + let mut ret = Self::default(); + if token_start == null() || token_stop == null() { + return Err(Error::IllegalArgument("token_start or token_stop is null. If you are using autotokens() you likely did not build your target with the \"AutoTokens\"-pass".to_string())); } - Ok(self) - } + if token_stop <= token_start { + return Err(Error::IllegalArgument(format!( + "Tried to create tokens from illegal section: stop < start ({:?} < {:?})", + token_stop, token_start + ))); + } + let section_size: usize = token_stop.offset_from(token_start).try_into().unwrap(); + // println!("size: {}", section_size); + let slice = from_raw_parts(token_start, section_size); - /// Reads from an autotokens section, returning the count of new entries read - pub unsafe fn add_from_autotokens(&mut self, autotoken: TokenSection) -> Result { - if cfg!(target_os = "linux") { - let mut entries = 0; - let token_start = autotoken.start; - let token_stop = autotoken.stop; - let section_size: usize = token_stop.offset_from(token_start).try_into().unwrap(); - // println!("size: {}", section_size); - let slice = core::slice::from_raw_parts(token_start, section_size); + let mut head = 0; - let mut head = 0; - - // Now we know the beginning and the end of the token section.. let's parse them into tokens - loop { - if head >= section_size { - // Sanity Check - assert!(head == section_size); - break; - } - let size = slice[head] as usize; - head += 1; - if size > 0 { - self.add_token(&slice[head..head + size].to_vec()); - #[cfg(feature = "std")] - println!( - "Token size: {} content: {:x?}", - size, - &slice[head..head + size].to_vec() - ); - head += size; - entries += 1; - } + // Now we know the beginning and the end of the token section.. let's parse them into tokens + loop { + if head >= section_size { + // Sanity Check + assert!(head == section_size); + break; + } + let size = slice[head] as usize; + head += 1; + if size > 0 { + ret.add_token(&slice[head..head + size].to_vec()); + /* #[cfg(feature = "std")] + println!( + "Token size: {} content: {:x?}", + size, + &slice[head..head + size].to_vec() + ); */ + head += size; } - - Ok(entries) - } else { - // TODO: Autodict for OSX and windows - Ok(0) } - } - /// Creates a new token from autotokens - pub fn from_autotokens(autotoken: TokenSection) -> Result { - let mut ret = Self::new(vec![]); - unsafe { - ret.add_from_autotokens(autotoken)?; - } Ok(ret) } /// Creates a new instance from a file #[cfg(feature = "std")] - pub fn from_tokens_file

(file: P) -> Result + pub fn from_file

(file: P) -> Result where P: AsRef, { - let mut ret = Self::new(vec![]); - ret.add_tokens_from_file(file)?; + let mut ret = Self::new(); + ret.add_from_file(file)?; Ok(ret) } @@ -149,21 +136,19 @@ impl Tokens { /// Returns `false` if the token was already present and did not get added. #[allow(clippy::ptr_arg)] pub fn add_token(&mut self, token: &Vec) -> bool { - if self.token_vec.contains(token) { + if !self.tokens_set.insert(token.clone()) { return false; } - self.token_vec.push(token.clone()); + self.tokens_vec.push(token.clone()); true } /// Reads a tokens file, returning the count of new entries read #[cfg(feature = "std")] - pub fn add_tokens_from_file

(&mut self, file: P) -> Result + pub fn add_from_file

(&mut self, file: P) -> Result<&mut Self, Error> where P: AsRef, { - let mut entries = 0; - // println!("Loading tokens file {:?} ...", file); let file = File::open(file)?; // panic if not found @@ -206,18 +191,96 @@ impl Tokens { }; // add - if self.add_token(&token) { - entries += 1; - } + self.add_token(&token); } - Ok(entries) + Ok(self) + } + + /// Returns the amount of tokens in this Tokens instance + #[inline] + #[must_use] + pub fn len(&self) -> usize { + self.tokens_vec.len() + } + + /// Returns if this tokens-instance is empty + #[inline] + #[must_use] + pub fn is_empty(&self) -> bool { + self.tokens_vec.is_empty() } /// Gets the tokens stored in this db #[must_use] pub fn tokens(&self) -> &[Vec] { - &self.token_vec + &self.tokens_vec + } +} + +impl AddAssign for Tokens { + fn add_assign(&mut self, other: Self) { + self.add_tokens(&other); + } +} + +impl AddAssign<&[Vec]> for Tokens { + fn add_assign(&mut self, other: &[Vec]) { + self.add_tokens(other); + } +} + +impl Add<&[Vec]> for Tokens { + type Output = Self; + fn add(self, other: &[Vec]) -> Self { + let mut ret = self; + ret.add_tokens(other); + ret + } +} + +impl Add for Tokens { + type Output = Self; + + fn add(self, other: Self) -> Self { + self.add(other.tokens_vec.as_slice()) + } +} + +impl From for Tokens +where + IT: IntoIterator, + V: AsRef>, +{ + fn from(tokens: IT) -> Self { + let mut ret = Self::default(); + ret.add_tokens(tokens); + ret + } +} + +impl AsSlice> for Tokens { + fn as_slice(&self) -> &[Vec] { + self.tokens() + } +} + +impl Add for &Tokens { + type Output = Tokens; + + fn add(self, other: Self) -> Tokens { + let mut ret: Tokens = self.clone(); + ret.add_tokens(other); + ret + } +} + +impl<'a, 'it> IntoIterator for &'it Tokens { + type Item = > as Iterator>::Item; + type IntoIter = Iter<'it, Vec>; + + fn into_iter(self) -> Self::IntoIter { + self.as_slice().iter() } } @@ -553,7 +616,7 @@ token1="A\x41A" token2="B" "###; fs::write("test.tkns", data).expect("Unable to write test.tkns"); - let tokens = Tokens::from_tokens_file(&"test.tkns").unwrap(); + let tokens = Tokens::from_file(&"test.tkns").unwrap(); #[cfg(feature = "std")] println!("Token file entries: {:?}", tokens.tokens()); assert_eq!(tokens.tokens().len(), 2); diff --git a/libafl/src/stages/push/mod.rs b/libafl/src/stages/push/mod.rs index 56a422a524..1e9c7f2612 100644 --- a/libafl/src/stages/push/mod.rs +++ b/libafl/src/stages/push/mod.rs @@ -138,7 +138,7 @@ where /// Sets the shared state for this helper (and all other helpers owning the same [`RefCell`]) #[inline] pub fn set_shared_state(&mut self, shared_state: PushStageSharedState) { - (&mut *self.shared_state.borrow_mut()).replace(shared_state); + (*self.shared_state.borrow_mut()).replace(shared_state); } /// Takes the shared state from this helper, replacing it with `None` diff --git a/libafl_cc/build.rs b/libafl_cc/build.rs index cecc1e5a10..55949a8213 100644 --- a/libafl_cc/build.rs +++ b/libafl_cc/build.rs @@ -101,7 +101,7 @@ fn main() { ); write!( - &mut clang_constants_file, + clang_constants_file, "// These constants are autogenerated by build.rs /// The path to the `clang` executable @@ -175,7 +175,7 @@ fn main() { .expect("Failed to compile autotokens-pass.cc"); } else { write!( - &mut clang_constants_file, + clang_constants_file, "// These constants are autogenerated by build.rs /// The path to the `clang` executable diff --git a/libafl_concolic/symcc_runtime/build.rs b/libafl_concolic/symcc_runtime/build.rs index 26b9e19582..7274e5dc82 100644 --- a/libafl_concolic/symcc_runtime/build.rs +++ b/libafl_concolic/symcc_runtime/build.rs @@ -81,7 +81,7 @@ fn main() { fn write_cpp_function_export_macro(out_path: &Path, cpp_bindings: &bindgen::Bindings) { let mut macro_file = File::create(out_path.join("cpp_exports_macro.rs")).unwrap(); writeln!( - &mut macro_file, + macro_file, "#[doc(hidden)] #[macro_export] macro_rules! export_cpp_runtime_functions {{ @@ -92,14 +92,14 @@ fn write_cpp_function_export_macro(out_path: &Path, cpp_bindings: &bindgen::Bind .captures_iter(&cpp_bindings.to_string()) .for_each(|captures| { writeln!( - &mut macro_file, + macro_file, " symcc_runtime::export_c_symbol!({});", &captures[1] ) .unwrap(); }); writeln!( - &mut macro_file, + macro_file, " }}; }}", ) @@ -149,7 +149,7 @@ fn write_rust_runtime_macro_file(out_path: &Path, symcc_src_path: &Path) { .expect("Unable to generate bindings"); let mut rust_runtime_macro = File::create(out_path.join("rust_exports_macro.rs")).unwrap(); writeln!( - &mut rust_runtime_macro, + rust_runtime_macro, "#[doc(hidden)] #[macro_export] macro_rules! invoke_macro_with_rust_runtime_exports {{ @@ -160,7 +160,7 @@ fn write_rust_runtime_macro_file(out_path: &Path, symcc_src_path: &Path) { .captures_iter(&rust_bindings.to_string()) .for_each(|captures| { writeln!( - &mut rust_runtime_macro, + rust_runtime_macro, " $macro!({},{}; $($extra_ident),*);", &captures[1].replace("_rsym_", ""), &FUNCTION_NAME_REGEX.captures(&captures[1]).unwrap()[1] @@ -168,7 +168,7 @@ fn write_rust_runtime_macro_file(out_path: &Path, symcc_src_path: &Path) { .unwrap(); }); writeln!( - &mut rust_runtime_macro, + rust_runtime_macro, " }}; }}", ) @@ -181,20 +181,20 @@ fn write_symcc_runtime_bindings_file(out_path: &Path, cpp_bindings: &bindgen::Bi if let Some(captures) = FUNCTION_NAME_REGEX.captures(l) { let function_name = &captures[1]; writeln!( - &mut bindings_file, + bindings_file, "#[link_name=\"{}{}\"]", SYMCC_RUNTIME_FUNCTION_NAME_PREFIX, function_name ) .unwrap(); } - writeln!(&mut bindings_file, "{}", l).unwrap(); + writeln!(bindings_file, "{}", l).unwrap(); }); } fn write_symcc_rename_header(rename_header_path: &Path, cpp_bindings: &bindgen::Bindings) { let mut rename_header_file = File::create(rename_header_path).unwrap(); writeln!( - &mut rename_header_file, + rename_header_file, "#ifndef PREFIX_EXPORTS_H #define PREFIX_EXPORTS_H", ) @@ -207,14 +207,14 @@ fn write_symcc_rename_header(rename_header_path: &Path, cpp_bindings: &bindgen:: .map(|captures| captures[1].to_string()) .for_each(|val| { writeln!( - &mut rename_header_file, + rename_header_file, "#define {} {}{}", &val, SYMCC_RUNTIME_FUNCTION_NAME_PREFIX, &val ) .unwrap(); }); - writeln!(&mut rename_header_file, "#endif").unwrap(); + writeln!(rename_header_file, "#endif").unwrap(); } fn build_and_link_symcc_runtime(symcc_src_path: &Path, rename_header_path: &Path) { diff --git a/libafl_sugar/src/forkserver.rs b/libafl_sugar/src/forkserver.rs index cfd777528a..b6d62fcaf6 100644 --- a/libafl_sugar/src/forkserver.rs +++ b/libafl_sugar/src/forkserver.rs @@ -166,7 +166,7 @@ impl<'a, const MAP_SIZE: usize> ForkserverBytesCoverageSugar<'a, MAP_SIZE> { // Create a dictionary if not existing if let Some(tokens_file) = &self.tokens_file { if state.metadata().get::().is_none() { - state.add_metadata(Tokens::from_tokens_file(tokens_file)?); + state.add_metadata(Tokens::from_file(tokens_file)?); } } diff --git a/libafl_sugar/src/inmemory.rs b/libafl_sugar/src/inmemory.rs index 85a0dbb1c5..076a4f82ea 100644 --- a/libafl_sugar/src/inmemory.rs +++ b/libafl_sugar/src/inmemory.rs @@ -187,7 +187,7 @@ where // Create a dictionary if not existing if let Some(tokens_file) = &self.tokens_file { if state.metadata().get::().is_none() { - state.add_metadata(Tokens::from_tokens_file(tokens_file)?); + state.add_metadata(Tokens::from_file(tokens_file)?); } } diff --git a/libafl_sugar/src/qemu.rs b/libafl_sugar/src/qemu.rs index 466b20c9d0..d577663642 100644 --- a/libafl_sugar/src/qemu.rs +++ b/libafl_sugar/src/qemu.rs @@ -191,7 +191,7 @@ where // Create a dictionary if not existing if let Some(tokens_file) = &self.tokens_file { if state.metadata().get::().is_none() { - state.add_metadata(Tokens::from_tokens_file(tokens_file)?); + state.add_metadata(Tokens::from_file(tokens_file)?); } } diff --git a/libafl_targets/build.rs b/libafl_targets/build.rs index 5c33d8341c..ac2a192cdd 100644 --- a/libafl_targets/build.rs +++ b/libafl_targets/build.rs @@ -25,7 +25,7 @@ fn main() { .expect("Could not parse LIBAFL_CMPLOG_MAP_H"); write!( - &mut constants_file, + constants_file, "// These constants are autogenerated by build.rs /// The size of the edges map diff --git a/libafl_targets/src/coverage.rs b/libafl_targets/src/coverage.rs index 58e3eb99d2..76e3fd28d6 100644 --- a/libafl_targets/src/coverage.rs +++ b/libafl_targets/src/coverage.rs @@ -1,6 +1,8 @@ //! Coverage maps as static mut array use crate::EDGES_MAP_SIZE; +#[cfg(target_os = "linux")] +use libafl::{mutators::Tokens, Error}; /// The map for edges. #[no_mangle] @@ -24,11 +26,27 @@ extern "C" { } pub use __afl_area_ptr as EDGES_MAP_PTR; -/// Return token section's start and end as a tuple +/// Return Tokens from the compile-time token section +/// Will return `Error::IllegalState` if no token section was found +/// In this case, the compilation probably did not include an `AutoTokens`-pass +/// +/// # Safety +/// +/// This fn is safe to call, as long as the compilation diid not break, previously #[cfg(target_os = "linux")] #[must_use] -pub fn token_section() -> (*const u8, *const u8) { - unsafe { (__token_start, __token_stop) } +pub fn autotokens() -> Result { + unsafe { + if __token_start.is_null() || __token_stop.is_null() { + Err(Error::IllegalState( + "AutoTokens section not found, likely the targe is not compiled with AutoTokens" + .into(), + )) + } else { + // we can safely unwrap + Tokens::from_ptrs(__token_start, __token_stop) + } + } } /// The size of the map for edges.