finish minimizer scheduler

fuzzers/libfuzzer_libpng/src/fuzzer.rs

@@ -5,7 +5,10 @@ use std::{env, path::PathBuf};
 
 use libafl::{
     bolts::{shmem::UnixShMem, tuples::tuple_list},
-    corpus::{Corpus, InMemoryCorpus, OnDiskCorpus, RandCorpusScheduler},
+    corpus::{
+        Corpus, InMemoryCorpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
+        RandCorpusScheduler,
+    },
     events::setup_restarting_mgr,
     executors::{inprocess::InProcessExecutor, Executor, ExitKind},
     feedbacks::{CrashFeedback, MaxMapFeedback},
@@ -101,7 +104,7 @@ fn fuzz(corpus_dirs: Vec<PathBuf>, objective_dir: PathBuf, broker_port: u16) ->
     println!("We're a client, let's fuzz :)");
 
     // Create a PNG dictionary if not existing
-    if state.metadata().get::<TokensMetadata>().is_none() {
+    if state.metadatas().get::<TokensMetadata>().is_none() {
         state.add_metadata(TokensMetadata::new(vec![
             vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header
             "IHDR".as_bytes().to_vec(),
@@ -115,8 +118,10 @@ fn fuzz(corpus_dirs: Vec<PathBuf>, objective_dir: PathBuf, broker_port: u16) ->
     let mutator = HavocBytesMutator::default();
     let stage = StdMutationalStage::new(mutator);
 
-    // A fuzzer with just one stage and a random policy to get testcasess from the corpus
+    // A fuzzer with just one stage and a minimization+queue policy to get testcasess from the corpus
-    let fuzzer = StdFuzzer::new(RandCorpusScheduler::new(), tuple_list!(stage));
+    let scheduler = IndexesLenTimeMinimizerCorpusScheduler::new(RandCorpusScheduler::new());
+    //let scheduler = RandCorpusScheduler::new();
+    let fuzzer = StdFuzzer::new(scheduler, tuple_list!(stage));
 
     // Create the executor for an in-process function with just one observer for edge coverage
     let mut executor = InProcessExecutor::new(

libafl/src/corpus/minimizer.rs

@@ -1,15 +1,19 @@
 use crate::{
     bolts::serdeany::SerdeAny,
     corpus::{Corpus, CorpusScheduler, Testcase},
+    feedbacks::MapIndexesMetadata,
     inputs::{HasLen, Input},
-    state::{HasCorpus, HasMetadata},
+    state::{HasCorpus, HasMetadata, HasRand},
+    utils::{AsSlice, Rand},
     Error,
 };
 
-use core::{iter::IntoIterator, marker::PhantomData};
+use core::marker::PhantomData;
 use hashbrown::{HashMap, HashSet};
 use serde::{Deserialize, Serialize};
 
+pub const DEFAULT_SKIP_NOT_FAV_PROB: u64 = 95;
+
 /// A testcase metadata saying if a testcase is favored
 #[derive(Serialize, Deserialize)]
 pub struct IsFavoredMetadata {}
@@ -56,29 +60,29 @@ where
     }
 }
 
-pub struct MinimizerCorpusScheduler<C, CS, F, I, IT, S>
+pub struct MinimizerCorpusScheduler<C, CS, F, I, M, R, S>
 where
     CS: CorpusScheduler<I, S>,
     F: FavFactor<I>,
     I: Input,
-    IT: IntoIterator<Item = usize> + SerdeAny,
+    M: AsSlice<usize> + SerdeAny,
-    for<'a> &'a IT: IntoIterator<Item = usize>,
     S: HasCorpus<C, I> + HasMetadata,
     C: Corpus<I>,
 {
     base: CS,
-    phantom: PhantomData<(C, F, I, IT, S)>,
+    skip_not_fav_prob: u64,
+    phantom: PhantomData<(C, F, I, M, R, S)>,
 }
 
-impl<C, CS, F, I, IT, S> CorpusScheduler<I, S> for MinimizerCorpusScheduler<C, CS, F, I, IT, S>
+impl<C, CS, F, I, M, R, S> CorpusScheduler<I, S> for MinimizerCorpusScheduler<C, CS, F, I, M, R, S>
 where
     CS: CorpusScheduler<I, S>,
     F: FavFactor<I>,
     I: Input,
-    IT: IntoIterator<Item = usize> + SerdeAny,
+    M: AsSlice<usize> + SerdeAny,
-    for<'a> &'a IT: IntoIterator<Item = usize>,
+    S: HasCorpus<C, I> + HasMetadata + HasRand<R>,
-    S: HasCorpus<C, I> + HasMetadata,
     C: Corpus<I>,
+    R: Rand,
 {
     /// Add an entry to the corpus and return its index
     fn on_add(&self, state: &mut S, idx: usize) -> Result<(), Error> {
@@ -91,7 +95,7 @@ where
         self.base.on_replace(state, idx, testcase)
     }
 
-    /// Removes an entry from the corpus, returning it if it was present.
+    /// Removes an entry from the corpus, returning M if M was present.
     fn on_remove(
         &self,
         state: &mut S,
@@ -101,27 +105,38 @@ where
         self.base.on_remove(state, idx, testcase)
     }
 
-    // TODO: IntoIter
     /// Gets the next entry
     fn next(&self, state: &mut S) -> Result<usize, Error> {
         self.cull(state)?;
-        self.base.next(state)
+        let mut idx = self.base.next(state)?;
+        while {
+            let has = !state
+                .corpus()
+                .get(idx)?
+                .borrow()
+                .has_metadata::<IsFavoredMetadata>();
+            has
+        } && state.rand_mut().below(100) < self.skip_not_fav_prob
+        {
+            idx = self.base.next(state)?;
+        }
+        Ok(idx)
     }
 }
 
-impl<C, CS, F, I, IT, S> MinimizerCorpusScheduler<C, CS, F, I, IT, S>
+impl<C, CS, F, I, M, R, S> MinimizerCorpusScheduler<C, CS, F, I, M, R, S>
 where
     CS: CorpusScheduler<I, S>,
     F: FavFactor<I>,
     I: Input,
-    IT: IntoIterator<Item = usize> + SerdeAny,
+    M: AsSlice<usize> + SerdeAny,
-    for<'a> &'a IT: IntoIterator<Item = usize>,
+    S: HasCorpus<C, I> + HasMetadata + HasRand<R>,
-    S: HasCorpus<C, I> + HasMetadata,
     C: Corpus<I>,
+    R: Rand,
 {
     pub fn update_score(&self, state: &mut S, idx: usize) -> Result<(), Error> {
         // Create a new top rated meta if not existing
-        if state.metadata().get::<TopRatedsMetadata>().is_none() {
+        if state.metadatas().get::<TopRatedsMetadata>().is_none() {
             state.add_metadata(TopRatedsMetadata::new());
         }
 
@@ -129,31 +144,32 @@ where
         {
             let mut entry = state.corpus().get(idx)?.borrow_mut();
             let factor = F::compute(&mut *entry)?;
-            for elem in entry.metadatas().get::<IT>().ok_or_else(|| {
+            let meta = entry.metadatas().get::<M>().ok_or_else(|| {
                 Error::KeyNotFound(format!(
                     "Metadata needed for MinimizerCorpusScheduler not found in testcase #{}",
                     idx
                 ))
-            })? {
+            })?;
+            for elem in meta.as_slice() {
                 if let Some(old_idx) = state
-                    .metadata()
+                    .metadatas()
                     .get::<TopRatedsMetadata>()
                     .unwrap()
                     .map
-                    .get(&elem)
+                    .get(elem)
                 {
                     if factor > F::compute(&mut *state.corpus().get(*old_idx)?.borrow_mut())? {
                         continue;
                     }
                 }
 
-                new_favoreds.push((elem, idx));
+                new_favoreds.push((*elem, idx));
             }
         }
 
         for pair in new_favoreds {
             state
-                .metadata_mut()
+                .metadatas_mut()
                 .get_mut::<TopRatedsMetadata>()
                 .unwrap()
                 .map
@@ -164,19 +180,20 @@ where
 
     pub fn cull(&self, state: &mut S) -> Result<(), Error> {
         let mut acc = HashSet::new();
-        let top_rated = state.metadata().get::<TopRatedsMetadata>().unwrap();
+        let top_rated = state.metadatas().get::<TopRatedsMetadata>().unwrap();
 
         for key in top_rated.map.keys() {
             if !acc.contains(key) {
                 let idx = top_rated.map.get(key).unwrap();
                 let mut entry = state.corpus().get(*idx)?.borrow_mut();
-                for elem in entry.metadatas().get::<IT>().ok_or_else(|| {
+                let meta = entry.metadatas().get::<M>().ok_or_else(|| {
                     Error::KeyNotFound(format!(
                         "Metadata needed for MinimizerCorpusScheduler not found in testcase #{}",
                         idx
                     ))
-                })? {
+                })?;
-                    acc.insert(elem);
+                for elem in meta.as_slice() {
+                    acc.insert(*elem);
                 }
 
                 entry.add_metadata(IsFavoredMetadata {});
@@ -189,7 +206,22 @@ where
     pub fn new(base: CS) -> Self {
         Self {
             base: base,
+            skip_not_fav_prob: DEFAULT_SKIP_NOT_FAV_PROB,
+            phantom: PhantomData,
+        }
+    }
+
+    pub fn with_skip_prob(base: CS, skip_not_fav_prob: u64) -> Self {
+        Self {
+            base: base,
+            skip_not_fav_prob: skip_not_fav_prob,
             phantom: PhantomData,
         }
     }
 }
+
+pub type LenTimeMinimizerCorpusScheduler<C, CS, I, M, R, S> =
+    MinimizerCorpusScheduler<C, CS, LenTimeMulFavFactor<I>, I, M, R, S>;
+
+pub type IndexesLenTimeMinimizerCorpusScheduler<C, CS, I, R, S> =
+    MinimizerCorpusScheduler<C, CS, LenTimeMulFavFactor<I>, I, MapIndexesMetadata, R, S>;

libafl/src/corpus/mod.rs

@@ -14,7 +14,12 @@ pub use ondisk::OnDiskCorpus;
 pub mod queue;
 pub use queue::QueueCorpusScheduler;
 
-pub mod minset;
+pub mod minimizer;
+pub use minimizer::{
+    FavFactor, IndexesLenTimeMinimizerCorpusScheduler, IsFavoredMetadata,
+    LenTimeMinimizerCorpusScheduler, LenTimeMulFavFactor, MinimizerCorpusScheduler,
+    TopRatedsMetadata,
+};
 
 use alloc::borrow::ToOwned;
 use core::{cell::RefCell, marker::PhantomData};
@@ -82,7 +87,6 @@ where
         Ok(())
     }
 
-    // TODO: IntoIter
     /// Gets the next entry
     fn next(&self, state: &mut S) -> Result<usize, Error>;
 }

libafl/src/corpus/testcase.rs

@@ -6,8 +6,9 @@ use core::{convert::Into, default::Default, option::Option, time::Duration};
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    bolts::serdeany::{SerdeAny, SerdeAnyMap},
+    bolts::serdeany::SerdeAnyMap,
     inputs::{HasLen, Input},
+    state::HasMetadata,
     Error,
 };
 
@@ -32,6 +33,23 @@ where
     cached_len: Option<usize>,
 }
 
+impl<I> HasMetadata for Testcase<I>
+where
+    I: Input,
+{
+    /// Get all the metadatas into an HashMap
+    #[inline]
+    fn metadatas(&self) -> &SerdeAnyMap {
+        &self.metadatas
+    }
+
+    /// Get all the metadatas into an HashMap (mutable)
+    #[inline]
+    fn metadatas_mut(&mut self) -> &mut SerdeAnyMap {
+        &mut self.metadatas
+    }
+}
+
 /// Impl of a testcase
 impl<I> Testcase<I>
 where
@@ -120,27 +138,6 @@ where
         self.fitness = fitness;
     }
 
-    /// Get all the metadatas into an HashMap
-    #[inline]
-    pub fn metadatas(&self) -> &SerdeAnyMap {
-        &self.metadatas
-    }
-
-    /// Get all the metadatas into an HashMap (mutable)
-    #[inline]
-    pub fn metadatas_mut(&mut self) -> &mut SerdeAnyMap {
-        &mut self.metadatas
-    }
-
-    /// Add a metadata
-    #[inline]
-    pub fn add_metadata<M>(&mut self, meta: M)
-    where
-        M: SerdeAny,
-    {
-        self.metadatas.insert(meta);
-    }
-
     /// Get the execution time of the testcase
     pub fn exec_time(&self) -> &Option<Duration> {
         &self.exec_time

libafl/src/feedbacks/map.rs

@@ -13,15 +13,14 @@ use crate::{
     feedbacks::Feedback,
     inputs::Input,
     observers::{MapObserver, Observer, ObserversTuple},
+    state::HasMetadata,
+    utils::AsSlice,
     Error,
 };
 
 pub type MaxMapFeedback<T, O> = MapFeedback<T, MaxReducer<T>, O>;
 pub type MinMapFeedback<T, O> = MapFeedback<T, MinReducer<T>, O>;
 
-//pub type MaxMapTrackerFeedback<T, O> = MapFeedback<T, MaxReducer<T>, O>;
-//pub type MinMapTrackerFeedback<T, O> = MapFeedback<T, MinReducer<T>, O>;
-
 /// A Reducer function is used to aggregate values for the novelty search
 pub trait Reducer<T>: Serialize + serde::de::DeserializeOwned + 'static
 where
@@ -74,122 +73,6 @@ where
     }
 }
 
-/*
-/// The most common AFL-like feedback type
-#[derive(Serialize, Deserialize, Clone, Debug)]
-#[serde(bound = "T: serde::de::DeserializeOwned")]
-pub struct MapFeedback<T, R, O>
-where
-    T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned,
-    R: Reducer<T>,
-    O: MapObserver<T>,
-{
-    /// Contains information about untouched entries
-    history_map: Vec<T>,
-    /// Name identifier of this instance
-    name: String,
-    /// Phantom Data of Reducer
-    phantom: PhantomData<(R, O)>,
-}
-
-impl<T, R, O, I> Feedback<I> for MapFeedback<T, R, O>
-where
-    T: Integer
-        + Default
-        + Copy
-        + 'static
-        + serde::Serialize
-        + serde::de::DeserializeOwned
-        + core::fmt::Debug,
-    R: Reducer<T>,
-    O: MapObserver<T>,
-    I: Input,
-{
-    fn is_interesting<OT: ObserversTuple>(
-        &mut self,
-        _input: &I,
-        observers: &OT,
-        _exit_kind: ExitKind,
-    ) -> Result<u32, Error> {
-        let mut interesting = 0;
-        // TODO optimize
-        let observer = observers.match_name_type::<O>(&self.name).unwrap();
-        let size = observer.usable_count();
-        //println!("count: {:?}, map: {:?}, history: {:?}", size, observer.map(), &self.history_map);
-        for i in 0..size {
-            let history = self.history_map[i];
-            let item = observer.map()[i];
-            let reduced = R::reduce(history, item);
-            if history != reduced {
-                self.history_map[i] = reduced;
-                interesting += 1;
-            }
-        }
-
-        //println!("..interesting: {:?}, new_history: {:?}\n", interesting, &self.history_map);
-        //std::thread::sleep(std::time::Duration::from_millis(100));
-
-        Ok(interesting)
-    }
-}
-
-impl<T, R, O> Named for MapFeedback<T, R, O>
-where
-    T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned,
-    R: Reducer<T>,
-    O: MapObserver<T>,
-{
-    #[inline]
-    fn name(&self) -> &str {
-        self.name.as_str()
-    }
-}
-
-impl<T, R, O> MapFeedback<T, R, O>
-where
-    T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned,
-    R: Reducer<T>,
-    O: MapObserver<T> + Observer,
-{
-    /// Create new MapFeedback
-    pub fn new(name: &'static str, map_size: usize) -> Self {
-        Self {
-            history_map: vec![T::default(); map_size],
-            phantom: PhantomData,
-            name: name.to_string(),
-        }
-    }
-
-    /// Create new MapFeedback for the observer type.
-    /// Name should match that of the observer.
-    pub fn new_with_observer(name: &'static str, map_observer: &O) -> Self {
-        debug_assert_eq!(name, map_observer.name());
-        Self {
-            history_map: vec![T::default(); map_observer.map().len()],
-            phantom: PhantomData,
-            name: name.to_string(),
-        }
-    }
-}
-
-impl<T, R, O> MapFeedback<T, R, O>
-where
-    T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned,
-    R: Reducer<T>,
-    O: MapObserver<T>,
-{
-    /// Create new MapFeedback using a map observer, and a map.
-    /// The map can be shared.
-    pub fn with_history_map(name: &'static str, history_map: Vec<T>) -> Self {
-        Self {
-            history_map: history_map,
-            name: name.to_string(),
-            phantom: PhantomData,
-        }
-    }
-}
-*/
-
 /// A testcase metadata holding a list of indexes of a map
 #[derive(Serialize, Deserialize)]
 pub struct MapIndexesMetadata {
@@ -198,12 +81,10 @@ pub struct MapIndexesMetadata {
 
 crate::impl_serdeany!(MapIndexesMetadata);
 
-impl IntoIterator for MapIndexesMetadata {
+impl AsSlice<usize> for MapIndexesMetadata {
-    type Item = usize;
+    /// Convert to a slice
-    type IntoIter = alloc::vec::IntoIter<Self::Item>;
+    fn as_slice(&self) -> &[usize] {
-
+        self.list.as_slice()
-    fn into_iter(self) -> Self::IntoIter {
-        self.list.into_iter()
     }
 }
 
@@ -221,15 +102,12 @@ pub struct MapNoveltiesMetadata {
 
 crate::impl_serdeany!(MapNoveltiesMetadata);
 
-impl IntoIterator for MapNoveltiesMetadata {
+impl AsSlice<usize> for MapNoveltiesMetadata {
-    type Item = usize;
+    /// Convert to a slice
-    type IntoIter = alloc::vec::IntoIter<Self::Item>;
+    fn as_slice(&self) -> &[usize] {
-
+        self.list.as_slice()
-    fn into_iter(self) -> Self::IntoIter {
-        self.list.into_iter()
     }
 }
-
 impl MapNoveltiesMetadata {
     pub fn new(list: Vec<usize>) -> Self {
         Self { list }

libafl/src/mutators/token_mutations.rs

@@ -37,7 +37,7 @@ where
 {
     let max_size = state.max_size();
     let tokens_len = {
-        let meta = state.metadata().get::<TokensMetadata>();
+        let meta = state.metadatas().get::<TokensMetadata>();
         if meta.is_none() {
             return Ok(MutationResult::Skipped);
         }
@@ -51,7 +51,7 @@ where
     let size = input.bytes().len();
     let off = state.rand_mut().below((size + 1) as u64) as usize;
 
-    let meta = state.metadata().get::<TokensMetadata>().unwrap();
+    let meta = state.metadatas().get::<TokensMetadata>().unwrap();
     let token = &meta.tokens[token_idx];
     let mut len = token.len();
 
@@ -83,7 +83,7 @@ where
     }
 
     let tokens_len = {
-        let meta = state.metadata().get::<TokensMetadata>();
+        let meta = state.metadatas().get::<TokensMetadata>();
         if meta.is_none() {
             return Ok(MutationResult::Skipped);
         }
@@ -96,7 +96,7 @@ where
 
     let off = state.rand_mut().below(size as u64) as usize;
 
-    let meta = state.metadata().get::<TokensMetadata>().unwrap();
+    let meta = state.metadatas().get::<TokensMetadata>().unwrap();
     let token = &meta.tokens[token_idx];
     let mut len = token.len();
     if off + len > size {

libafl/src/state/mod.rs

@@ -73,9 +73,9 @@ where
 /// Trait for elements offering metadata
 pub trait HasMetadata {
     /// A map, storing all metadata
-    fn metadata(&self) -> &SerdeAnyMap;
+    fn metadatas(&self) -> &SerdeAnyMap;
     /// A map, storing all metadata (mut)
-    fn metadata_mut(&mut self) -> &mut SerdeAnyMap;
+    fn metadatas_mut(&mut self) -> &mut SerdeAnyMap;
 
     /// Add a metadata to the metadata map
     #[inline]
@@ -83,7 +83,7 @@ pub trait HasMetadata {
     where
         M: SerdeAny,
     {
-        self.metadata_mut().insert(meta);
+        self.metadatas_mut().insert(meta);
     }
 
     /// Check for a metadata
@@ -92,7 +92,7 @@ pub trait HasMetadata {
     where
         M: SerdeAny,
     {
-        self.metadata().get::<M>().is_some()
+        self.metadatas().get::<M>().is_some()
     }
 }
 
@@ -307,13 +307,13 @@ where
 {
     /// Get all the metadata into an HashMap
     #[inline]
-    fn metadata(&self) -> &SerdeAnyMap {
+    fn metadatas(&self) -> &SerdeAnyMap {
         &self.metadata
     }
 
     /// Get all the metadata into an HashMap (mutable)
     #[inline]
-    fn metadata_mut(&mut self) -> &mut SerdeAnyMap {
+    fn metadatas_mut(&mut self) -> &mut SerdeAnyMap {
         &mut self.metadata
     }
 }

libafl/src/utils.rs

@@ -7,6 +7,11 @@ use xxhash_rust::xxh3::xxh3_64_with_seed;
 #[cfg(feature = "std")]
 use std::time::{SystemTime, UNIX_EPOCH};
 
+pub trait AsSlice<T> {
+    /// Convert to a slice
+    fn as_slice(&self) -> &[T];
+}
+
 pub type StdRand = RomuTrioRand;
 
 /// Ways to get random around here