diff --git a/fuzzers/libfuzzer_libpng/src/fuzzer.rs b/fuzzers/libfuzzer_libpng/src/fuzzer.rs index 7a338a4282..0586583013 100644 --- a/fuzzers/libfuzzer_libpng/src/fuzzer.rs +++ b/fuzzers/libfuzzer_libpng/src/fuzzer.rs @@ -5,7 +5,10 @@ use std::{env, path::PathBuf}; use libafl::{ bolts::{shmem::UnixShMem, tuples::tuple_list}, - corpus::{Corpus, InMemoryCorpus, OnDiskCorpus, RandCorpusScheduler}, + corpus::{ + Corpus, InMemoryCorpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus, + RandCorpusScheduler, + }, events::setup_restarting_mgr, executors::{inprocess::InProcessExecutor, Executor, ExitKind}, feedbacks::{CrashFeedback, MaxMapFeedback}, @@ -101,7 +104,7 @@ fn fuzz(corpus_dirs: Vec, objective_dir: PathBuf, broker_port: u16) -> println!("We're a client, let's fuzz :)"); // Create a PNG dictionary if not existing - if state.metadata().get::().is_none() { + if state.metadatas().get::().is_none() { state.add_metadata(TokensMetadata::new(vec![ vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header "IHDR".as_bytes().to_vec(), @@ -115,8 +118,10 @@ fn fuzz(corpus_dirs: Vec, objective_dir: PathBuf, broker_port: u16) -> let mutator = HavocBytesMutator::default(); let stage = StdMutationalStage::new(mutator); - // A fuzzer with just one stage and a random policy to get testcasess from the corpus - let fuzzer = StdFuzzer::new(RandCorpusScheduler::new(), tuple_list!(stage)); + // A fuzzer with just one stage and a minimization+queue policy to get testcasess from the corpus + let scheduler = IndexesLenTimeMinimizerCorpusScheduler::new(RandCorpusScheduler::new()); + //let scheduler = RandCorpusScheduler::new(); + let fuzzer = StdFuzzer::new(scheduler, tuple_list!(stage)); // Create the executor for an in-process function with just one observer for edge coverage let mut executor = InProcessExecutor::new( diff --git a/libafl/src/corpus/minset.rs b/libafl/src/corpus/minimizer.rs similarity index 64% rename from libafl/src/corpus/minset.rs rename to libafl/src/corpus/minimizer.rs index ed3eeb0cf8..9c49aa3293 100644 --- a/libafl/src/corpus/minset.rs +++ b/libafl/src/corpus/minimizer.rs @@ -1,15 +1,19 @@ use crate::{ bolts::serdeany::SerdeAny, corpus::{Corpus, CorpusScheduler, Testcase}, + feedbacks::MapIndexesMetadata, inputs::{HasLen, Input}, - state::{HasCorpus, HasMetadata}, + state::{HasCorpus, HasMetadata, HasRand}, + utils::{AsSlice, Rand}, Error, }; -use core::{iter::IntoIterator, marker::PhantomData}; +use core::marker::PhantomData; use hashbrown::{HashMap, HashSet}; use serde::{Deserialize, Serialize}; +pub const DEFAULT_SKIP_NOT_FAV_PROB: u64 = 95; + /// A testcase metadata saying if a testcase is favored #[derive(Serialize, Deserialize)] pub struct IsFavoredMetadata {} @@ -56,29 +60,29 @@ where } } -pub struct MinimizerCorpusScheduler +pub struct MinimizerCorpusScheduler where CS: CorpusScheduler, F: FavFactor, I: Input, - IT: IntoIterator + SerdeAny, - for<'a> &'a IT: IntoIterator, + M: AsSlice + SerdeAny, S: HasCorpus + HasMetadata, C: Corpus, { base: CS, - phantom: PhantomData<(C, F, I, IT, S)>, + skip_not_fav_prob: u64, + phantom: PhantomData<(C, F, I, M, R, S)>, } -impl CorpusScheduler for MinimizerCorpusScheduler +impl CorpusScheduler for MinimizerCorpusScheduler where CS: CorpusScheduler, F: FavFactor, I: Input, - IT: IntoIterator + SerdeAny, - for<'a> &'a IT: IntoIterator, - S: HasCorpus + HasMetadata, + M: AsSlice + SerdeAny, + S: HasCorpus + HasMetadata + HasRand, C: Corpus, + R: Rand, { /// Add an entry to the corpus and return its index fn on_add(&self, state: &mut S, idx: usize) -> Result<(), Error> { @@ -91,7 +95,7 @@ where self.base.on_replace(state, idx, testcase) } - /// Removes an entry from the corpus, returning it if it was present. + /// Removes an entry from the corpus, returning M if M was present. fn on_remove( &self, state: &mut S, @@ -101,27 +105,38 @@ where self.base.on_remove(state, idx, testcase) } - // TODO: IntoIter /// Gets the next entry fn next(&self, state: &mut S) -> Result { self.cull(state)?; - self.base.next(state) + let mut idx = self.base.next(state)?; + while { + let has = !state + .corpus() + .get(idx)? + .borrow() + .has_metadata::(); + has + } && state.rand_mut().below(100) < self.skip_not_fav_prob + { + idx = self.base.next(state)?; + } + Ok(idx) } } -impl MinimizerCorpusScheduler +impl MinimizerCorpusScheduler where CS: CorpusScheduler, F: FavFactor, I: Input, - IT: IntoIterator + SerdeAny, - for<'a> &'a IT: IntoIterator, - S: HasCorpus + HasMetadata, + M: AsSlice + SerdeAny, + S: HasCorpus + HasMetadata + HasRand, C: Corpus, + R: Rand, { pub fn update_score(&self, state: &mut S, idx: usize) -> Result<(), Error> { // Create a new top rated meta if not existing - if state.metadata().get::().is_none() { + if state.metadatas().get::().is_none() { state.add_metadata(TopRatedsMetadata::new()); } @@ -129,31 +144,32 @@ where { let mut entry = state.corpus().get(idx)?.borrow_mut(); let factor = F::compute(&mut *entry)?; - for elem in entry.metadatas().get::().ok_or_else(|| { + let meta = entry.metadatas().get::().ok_or_else(|| { Error::KeyNotFound(format!( "Metadata needed for MinimizerCorpusScheduler not found in testcase #{}", idx )) - })? { + })?; + for elem in meta.as_slice() { if let Some(old_idx) = state - .metadata() + .metadatas() .get::() .unwrap() .map - .get(&elem) + .get(elem) { if factor > F::compute(&mut *state.corpus().get(*old_idx)?.borrow_mut())? { continue; } } - new_favoreds.push((elem, idx)); + new_favoreds.push((*elem, idx)); } } for pair in new_favoreds { state - .metadata_mut() + .metadatas_mut() .get_mut::() .unwrap() .map @@ -164,19 +180,20 @@ where pub fn cull(&self, state: &mut S) -> Result<(), Error> { let mut acc = HashSet::new(); - let top_rated = state.metadata().get::().unwrap(); + let top_rated = state.metadatas().get::().unwrap(); for key in top_rated.map.keys() { if !acc.contains(key) { let idx = top_rated.map.get(key).unwrap(); let mut entry = state.corpus().get(*idx)?.borrow_mut(); - for elem in entry.metadatas().get::().ok_or_else(|| { + let meta = entry.metadatas().get::().ok_or_else(|| { Error::KeyNotFound(format!( "Metadata needed for MinimizerCorpusScheduler not found in testcase #{}", idx )) - })? { - acc.insert(elem); + })?; + for elem in meta.as_slice() { + acc.insert(*elem); } entry.add_metadata(IsFavoredMetadata {}); @@ -189,7 +206,22 @@ where pub fn new(base: CS) -> Self { Self { base: base, + skip_not_fav_prob: DEFAULT_SKIP_NOT_FAV_PROB, + phantom: PhantomData, + } + } + + pub fn with_skip_prob(base: CS, skip_not_fav_prob: u64) -> Self { + Self { + base: base, + skip_not_fav_prob: skip_not_fav_prob, phantom: PhantomData, } } } + +pub type LenTimeMinimizerCorpusScheduler = + MinimizerCorpusScheduler, I, M, R, S>; + +pub type IndexesLenTimeMinimizerCorpusScheduler = + MinimizerCorpusScheduler, I, MapIndexesMetadata, R, S>; diff --git a/libafl/src/corpus/mod.rs b/libafl/src/corpus/mod.rs index 0707f3d9be..6b2fc082d5 100644 --- a/libafl/src/corpus/mod.rs +++ b/libafl/src/corpus/mod.rs @@ -14,7 +14,12 @@ pub use ondisk::OnDiskCorpus; pub mod queue; pub use queue::QueueCorpusScheduler; -pub mod minset; +pub mod minimizer; +pub use minimizer::{ + FavFactor, IndexesLenTimeMinimizerCorpusScheduler, IsFavoredMetadata, + LenTimeMinimizerCorpusScheduler, LenTimeMulFavFactor, MinimizerCorpusScheduler, + TopRatedsMetadata, +}; use alloc::borrow::ToOwned; use core::{cell::RefCell, marker::PhantomData}; @@ -82,7 +87,6 @@ where Ok(()) } - // TODO: IntoIter /// Gets the next entry fn next(&self, state: &mut S) -> Result; } diff --git a/libafl/src/corpus/testcase.rs b/libafl/src/corpus/testcase.rs index 8b8cad970a..0da3c6505a 100644 --- a/libafl/src/corpus/testcase.rs +++ b/libafl/src/corpus/testcase.rs @@ -6,8 +6,9 @@ use core::{convert::Into, default::Default, option::Option, time::Duration}; use serde::{Deserialize, Serialize}; use crate::{ - bolts::serdeany::{SerdeAny, SerdeAnyMap}, + bolts::serdeany::SerdeAnyMap, inputs::{HasLen, Input}, + state::HasMetadata, Error, }; @@ -32,6 +33,23 @@ where cached_len: Option, } +impl HasMetadata for Testcase +where + I: Input, +{ + /// Get all the metadatas into an HashMap + #[inline] + fn metadatas(&self) -> &SerdeAnyMap { + &self.metadatas + } + + /// Get all the metadatas into an HashMap (mutable) + #[inline] + fn metadatas_mut(&mut self) -> &mut SerdeAnyMap { + &mut self.metadatas + } +} + /// Impl of a testcase impl Testcase where @@ -120,27 +138,6 @@ where self.fitness = fitness; } - /// Get all the metadatas into an HashMap - #[inline] - pub fn metadatas(&self) -> &SerdeAnyMap { - &self.metadatas - } - - /// Get all the metadatas into an HashMap (mutable) - #[inline] - pub fn metadatas_mut(&mut self) -> &mut SerdeAnyMap { - &mut self.metadatas - } - - /// Add a metadata - #[inline] - pub fn add_metadata(&mut self, meta: M) - where - M: SerdeAny, - { - self.metadatas.insert(meta); - } - /// Get the execution time of the testcase pub fn exec_time(&self) -> &Option { &self.exec_time diff --git a/libafl/src/feedbacks/map.rs b/libafl/src/feedbacks/map.rs index 9e1ead65af..18daa5cd5a 100644 --- a/libafl/src/feedbacks/map.rs +++ b/libafl/src/feedbacks/map.rs @@ -13,15 +13,14 @@ use crate::{ feedbacks::Feedback, inputs::Input, observers::{MapObserver, Observer, ObserversTuple}, + state::HasMetadata, + utils::AsSlice, Error, }; pub type MaxMapFeedback = MapFeedback, O>; pub type MinMapFeedback = MapFeedback, O>; -//pub type MaxMapTrackerFeedback = MapFeedback, O>; -//pub type MinMapTrackerFeedback = MapFeedback, O>; - /// A Reducer function is used to aggregate values for the novelty search pub trait Reducer: Serialize + serde::de::DeserializeOwned + 'static where @@ -74,122 +73,6 @@ where } } -/* -/// The most common AFL-like feedback type -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(bound = "T: serde::de::DeserializeOwned")] -pub struct MapFeedback -where - T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned, - R: Reducer, - O: MapObserver, -{ - /// Contains information about untouched entries - history_map: Vec, - /// Name identifier of this instance - name: String, - /// Phantom Data of Reducer - phantom: PhantomData<(R, O)>, -} - -impl Feedback for MapFeedback -where - T: Integer - + Default - + Copy - + 'static - + serde::Serialize - + serde::de::DeserializeOwned - + core::fmt::Debug, - R: Reducer, - O: MapObserver, - I: Input, -{ - fn is_interesting( - &mut self, - _input: &I, - observers: &OT, - _exit_kind: ExitKind, - ) -> Result { - let mut interesting = 0; - // TODO optimize - let observer = observers.match_name_type::(&self.name).unwrap(); - let size = observer.usable_count(); - //println!("count: {:?}, map: {:?}, history: {:?}", size, observer.map(), &self.history_map); - for i in 0..size { - let history = self.history_map[i]; - let item = observer.map()[i]; - let reduced = R::reduce(history, item); - if history != reduced { - self.history_map[i] = reduced; - interesting += 1; - } - } - - //println!("..interesting: {:?}, new_history: {:?}\n", interesting, &self.history_map); - //std::thread::sleep(std::time::Duration::from_millis(100)); - - Ok(interesting) - } -} - -impl Named for MapFeedback -where - T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned, - R: Reducer, - O: MapObserver, -{ - #[inline] - fn name(&self) -> &str { - self.name.as_str() - } -} - -impl MapFeedback -where - T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned, - R: Reducer, - O: MapObserver + Observer, -{ - /// Create new MapFeedback - pub fn new(name: &'static str, map_size: usize) -> Self { - Self { - history_map: vec![T::default(); map_size], - phantom: PhantomData, - name: name.to_string(), - } - } - - /// Create new MapFeedback for the observer type. - /// Name should match that of the observer. - pub fn new_with_observer(name: &'static str, map_observer: &O) -> Self { - debug_assert_eq!(name, map_observer.name()); - Self { - history_map: vec![T::default(); map_observer.map().len()], - phantom: PhantomData, - name: name.to_string(), - } - } -} - -impl MapFeedback -where - T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned, - R: Reducer, - O: MapObserver, -{ - /// Create new MapFeedback using a map observer, and a map. - /// The map can be shared. - pub fn with_history_map(name: &'static str, history_map: Vec) -> Self { - Self { - history_map: history_map, - name: name.to_string(), - phantom: PhantomData, - } - } -} -*/ - /// A testcase metadata holding a list of indexes of a map #[derive(Serialize, Deserialize)] pub struct MapIndexesMetadata { @@ -198,12 +81,10 @@ pub struct MapIndexesMetadata { crate::impl_serdeany!(MapIndexesMetadata); -impl IntoIterator for MapIndexesMetadata { - type Item = usize; - type IntoIter = alloc::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.list.into_iter() +impl AsSlice for MapIndexesMetadata { + /// Convert to a slice + fn as_slice(&self) -> &[usize] { + self.list.as_slice() } } @@ -221,15 +102,12 @@ pub struct MapNoveltiesMetadata { crate::impl_serdeany!(MapNoveltiesMetadata); -impl IntoIterator for MapNoveltiesMetadata { - type Item = usize; - type IntoIter = alloc::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.list.into_iter() +impl AsSlice for MapNoveltiesMetadata { + /// Convert to a slice + fn as_slice(&self) -> &[usize] { + self.list.as_slice() } } - impl MapNoveltiesMetadata { pub fn new(list: Vec) -> Self { Self { list } diff --git a/libafl/src/mutators/token_mutations.rs b/libafl/src/mutators/token_mutations.rs index bfd1f1f82a..6702afeb80 100644 --- a/libafl/src/mutators/token_mutations.rs +++ b/libafl/src/mutators/token_mutations.rs @@ -37,7 +37,7 @@ where { let max_size = state.max_size(); let tokens_len = { - let meta = state.metadata().get::(); + let meta = state.metadatas().get::(); if meta.is_none() { return Ok(MutationResult::Skipped); } @@ -51,7 +51,7 @@ where let size = input.bytes().len(); let off = state.rand_mut().below((size + 1) as u64) as usize; - let meta = state.metadata().get::().unwrap(); + let meta = state.metadatas().get::().unwrap(); let token = &meta.tokens[token_idx]; let mut len = token.len(); @@ -83,7 +83,7 @@ where } let tokens_len = { - let meta = state.metadata().get::(); + let meta = state.metadatas().get::(); if meta.is_none() { return Ok(MutationResult::Skipped); } @@ -96,7 +96,7 @@ where let off = state.rand_mut().below(size as u64) as usize; - let meta = state.metadata().get::().unwrap(); + let meta = state.metadatas().get::().unwrap(); let token = &meta.tokens[token_idx]; let mut len = token.len(); if off + len > size { diff --git a/libafl/src/state/mod.rs b/libafl/src/state/mod.rs index b00bddd23e..ef028d0401 100644 --- a/libafl/src/state/mod.rs +++ b/libafl/src/state/mod.rs @@ -73,9 +73,9 @@ where /// Trait for elements offering metadata pub trait HasMetadata { /// A map, storing all metadata - fn metadata(&self) -> &SerdeAnyMap; + fn metadatas(&self) -> &SerdeAnyMap; /// A map, storing all metadata (mut) - fn metadata_mut(&mut self) -> &mut SerdeAnyMap; + fn metadatas_mut(&mut self) -> &mut SerdeAnyMap; /// Add a metadata to the metadata map #[inline] @@ -83,7 +83,7 @@ pub trait HasMetadata { where M: SerdeAny, { - self.metadata_mut().insert(meta); + self.metadatas_mut().insert(meta); } /// Check for a metadata @@ -92,7 +92,7 @@ pub trait HasMetadata { where M: SerdeAny, { - self.metadata().get::().is_some() + self.metadatas().get::().is_some() } } @@ -307,13 +307,13 @@ where { /// Get all the metadata into an HashMap #[inline] - fn metadata(&self) -> &SerdeAnyMap { + fn metadatas(&self) -> &SerdeAnyMap { &self.metadata } /// Get all the metadata into an HashMap (mutable) #[inline] - fn metadata_mut(&mut self) -> &mut SerdeAnyMap { + fn metadatas_mut(&mut self) -> &mut SerdeAnyMap { &mut self.metadata } } diff --git a/libafl/src/utils.rs b/libafl/src/utils.rs index 7484c81c25..072c86e3c0 100644 --- a/libafl/src/utils.rs +++ b/libafl/src/utils.rs @@ -7,6 +7,11 @@ use xxhash_rust::xxh3::xxh3_64_with_seed; #[cfg(feature = "std")] use std::time::{SystemTime, UNIX_EPOCH}; +pub trait AsSlice { + /// Convert to a slice + fn as_slice(&self) -> &[T]; +} + pub type StdRand = RomuTrioRand; /// Ways to get random around here