SYS-OSS/FRET-LibAFL
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

finish minimizer scheduler

Browse Source
This commit is contained in:
Andrea Fioraldi 2021-02-26 22:22:10 +01:00
parent bb29e6dd72
commit fd83c10c1e
8 changed files with 119 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
fuzzers/libfuzzer_libpng/src/fuzzer.rs
View File

@ -5,7 +5,10 @@ use std::{env, path::PathBuf};
use libafl::{
bolts::{shmem::UnixShMem, tuples::tuple_list},
corpus::{Corpus, InMemoryCorpus, OnDiskCorpus, RandCorpusScheduler},
corpus::{
Corpus, InMemoryCorpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
RandCorpusScheduler,
},
events::setup_restarting_mgr,
executors::{inprocess::InProcessExecutor, Executor, ExitKind},
feedbacks::{CrashFeedback, MaxMapFeedback},
@ -101,7 +104,7 @@ fn fuzz(corpus_dirs: Vec<PathBuf>, objective_dir: PathBuf, broker_port: u16) ->
println!("We're a client, let's fuzz :)");
// Create a PNG dictionary if not existing
if state.metadata().get::<TokensMetadata>().is_none() {
if state.metadatas().get::<TokensMetadata>().is_none() {
state.add_metadata(TokensMetadata::new(vec![
vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header
"IHDR".as_bytes().to_vec(),
@ -115,8 +118,10 @@ fn fuzz(corpus_dirs: Vec<PathBuf>, objective_dir: PathBuf, broker_port: u16) ->
let mutator = HavocBytesMutator::default();
let stage = StdMutationalStage::new(mutator);
// A fuzzer with just one stage and a random policy to get testcasess from the corpus
let fuzzer = StdFuzzer::new(RandCorpusScheduler::new(), tuple_list!(stage));
// A fuzzer with just one stage and a minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerCorpusScheduler::new(RandCorpusScheduler::new());
//let scheduler = RandCorpusScheduler::new();
let fuzzer = StdFuzzer::new(scheduler, tuple_list!(stage));
// Create the executor for an in-process function with just one observer for edge coverage
let mut executor = InProcessExecutor::new(

88
libafl/src/corpus/minset.rs → libafl/src/corpus/minimizer.rs
View File

@ -1,15 +1,19 @@
use crate::{
bolts::serdeany::SerdeAny,
corpus::{Corpus, CorpusScheduler, Testcase},
feedbacks::MapIndexesMetadata,
inputs::{HasLen, Input},
state::{HasCorpus, HasMetadata},
state::{HasCorpus, HasMetadata, HasRand},
utils::{AsSlice, Rand},
Error,
};
use core::{iter::IntoIterator, marker::PhantomData};
use core::marker::PhantomData;
use hashbrown::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
pub const DEFAULT_SKIP_NOT_FAV_PROB: u64 = 95;
/// A testcase metadata saying if a testcase is favored
#[derive(Serialize, Deserialize)]
pub struct IsFavoredMetadata {}
@ -56,29 +60,29 @@ where
}
}
pub struct MinimizerCorpusScheduler<C, CS, F, I, IT, S>
pub struct MinimizerCorpusScheduler<C, CS, F, I, M, R, S>
where
CS: CorpusScheduler<I, S>,
F: FavFactor<I>,
I: Input,
IT: IntoIterator<Item = usize> + SerdeAny,
for<'a> &'a IT: IntoIterator<Item = usize>,
M: AsSlice<usize> + SerdeAny,
S: HasCorpus<C, I> + HasMetadata,
C: Corpus<I>,
{
base: CS,
phantom: PhantomData<(C, F, I, IT, S)>,
skip_not_fav_prob: u64,
phantom: PhantomData<(C, F, I, M, R, S)>,
}
impl<C, CS, F, I, IT, S> CorpusScheduler<I, S> for MinimizerCorpusScheduler<C, CS, F, I, IT, S>
impl<C, CS, F, I, M, R, S> CorpusScheduler<I, S> for MinimizerCorpusScheduler<C, CS, F, I, M, R, S>
where
CS: CorpusScheduler<I, S>,
F: FavFactor<I>,
I: Input,
IT: IntoIterator<Item = usize> + SerdeAny,
for<'a> &'a IT: IntoIterator<Item = usize>,
S: HasCorpus<C, I> + HasMetadata,
M: AsSlice<usize> + SerdeAny,
S: HasCorpus<C, I> + HasMetadata + HasRand<R>,
C: Corpus<I>,
R: Rand,
{
/// Add an entry to the corpus and return its index
fn on_add(&self, state: &mut S, idx: usize) -> Result<(), Error> {
@ -91,7 +95,7 @@ where
self.base.on_replace(state, idx, testcase)
}
/// Removes an entry from the corpus, returning it if it was present.
/// Removes an entry from the corpus, returning M if M was present.
fn on_remove(
&self,
state: &mut S,
@ -101,27 +105,38 @@ where
self.base.on_remove(state, idx, testcase)
}
// TODO: IntoIter
/// Gets the next entry
fn next(&self, state: &mut S) -> Result<usize, Error> {
self.cull(state)?;
self.base.next(state)
let mut idx = self.base.next(state)?;
while {
let has = !state
.corpus()
.get(idx)?
.borrow()
.has_metadata::<IsFavoredMetadata>();
has
} && state.rand_mut().below(100) < self.skip_not_fav_prob
{
idx = self.base.next(state)?;
}
Ok(idx)
}
}
impl<C, CS, F, I, IT, S> MinimizerCorpusScheduler<C, CS, F, I, IT, S>
impl<C, CS, F, I, M, R, S> MinimizerCorpusScheduler<C, CS, F, I, M, R, S>
where
CS: CorpusScheduler<I, S>,
F: FavFactor<I>,
I: Input,
IT: IntoIterator<Item = usize> + SerdeAny,
for<'a> &'a IT: IntoIterator<Item = usize>,
S: HasCorpus<C, I> + HasMetadata,
M: AsSlice<usize> + SerdeAny,
S: HasCorpus<C, I> + HasMetadata + HasRand<R>,
C: Corpus<I>,
R: Rand,
{
pub fn update_score(&self, state: &mut S, idx: usize) -> Result<(), Error> {
// Create a new top rated meta if not existing
if state.metadata().get::<TopRatedsMetadata>().is_none() {
if state.metadatas().get::<TopRatedsMetadata>().is_none() {
state.add_metadata(TopRatedsMetadata::new());
}
@ -129,31 +144,32 @@ where
{
let mut entry = state.corpus().get(idx)?.borrow_mut();
let factor = F::compute(&mut *entry)?;
for elem in entry.metadatas().get::<IT>().ok_or_else(|| {
let meta = entry.metadatas().get::<M>().ok_or_else(|| {
Error::KeyNotFound(format!(
"Metadata needed for MinimizerCorpusScheduler not found in testcase #{}",
idx
))
})? {
})?;
for elem in meta.as_slice() {
if let Some(old_idx) = state
.metadata()
.metadatas()
.get::<TopRatedsMetadata>()
.unwrap()
.map
.get(&elem)
.get(elem)
{
if factor > F::compute(&mut *state.corpus().get(*old_idx)?.borrow_mut())? {
continue;
}
}
new_favoreds.push((elem, idx));
new_favoreds.push((*elem, idx));
}
}
for pair in new_favoreds {
state
.metadata_mut()
.metadatas_mut()
.get_mut::<TopRatedsMetadata>()
.unwrap()
.map
@ -164,19 +180,20 @@ where
pub fn cull(&self, state: &mut S) -> Result<(), Error> {
let mut acc = HashSet::new();
let top_rated = state.metadata().get::<TopRatedsMetadata>().unwrap();
let top_rated = state.metadatas().get::<TopRatedsMetadata>().unwrap();
for key in top_rated.map.keys() {
if !acc.contains(key) {
let idx = top_rated.map.get(key).unwrap();
let mut entry = state.corpus().get(*idx)?.borrow_mut();
for elem in entry.metadatas().get::<IT>().ok_or_else(|| {
let meta = entry.metadatas().get::<M>().ok_or_else(|| {
Error::KeyNotFound(format!(
"Metadata needed for MinimizerCorpusScheduler not found in testcase #{}",
idx
))
})? {
acc.insert(elem);
})?;
for elem in meta.as_slice() {
acc.insert(*elem);
}
entry.add_metadata(IsFavoredMetadata {});
@ -189,7 +206,22 @@ where
pub fn new(base: CS) -> Self {
Self {
base: base,
skip_not_fav_prob: DEFAULT_SKIP_NOT_FAV_PROB,
phantom: PhantomData,
}
}
pub fn with_skip_prob(base: CS, skip_not_fav_prob: u64) -> Self {
Self {
base: base,
skip_not_fav_prob: skip_not_fav_prob,
phantom: PhantomData,
}
}
}
pub type LenTimeMinimizerCorpusScheduler<C, CS, I, M, R, S> =
MinimizerCorpusScheduler<C, CS, LenTimeMulFavFactor<I>, I, M, R, S>;
pub type IndexesLenTimeMinimizerCorpusScheduler<C, CS, I, R, S> =
MinimizerCorpusScheduler<C, CS, LenTimeMulFavFactor<I>, I, MapIndexesMetadata, R, S>;

8
libafl/src/corpus/mod.rs
View File

@ -14,7 +14,12 @@ pub use ondisk::OnDiskCorpus;
pub mod queue;
pub use queue::QueueCorpusScheduler;
pub mod minset;
pub mod minimizer;
pub use minimizer::{
FavFactor, IndexesLenTimeMinimizerCorpusScheduler, IsFavoredMetadata,
LenTimeMinimizerCorpusScheduler, LenTimeMulFavFactor, MinimizerCorpusScheduler,
TopRatedsMetadata,
};
use alloc::borrow::ToOwned;
use core::{cell::RefCell, marker::PhantomData};
@ -82,7 +87,6 @@ where
Ok(())
}
// TODO: IntoIter
/// Gets the next entry
fn next(&self, state: &mut S) -> Result<usize, Error>;
}

41
libafl/src/corpus/testcase.rs
View File

@ -6,8 +6,9 @@ use core::{convert::Into, default::Default, option::Option, time::Duration};
use serde::{Deserialize, Serialize};
use crate::{
bolts::serdeany::{SerdeAny, SerdeAnyMap},
bolts::serdeany::SerdeAnyMap,
inputs::{HasLen, Input},
state::HasMetadata,
Error,
};
@ -32,6 +33,23 @@ where
cached_len: Option<usize>,
}
impl<I> HasMetadata for Testcase<I>
where
I: Input,
{
/// Get all the metadatas into an HashMap
#[inline]
fn metadatas(&self) -> &SerdeAnyMap {
&self.metadatas
}
/// Get all the metadatas into an HashMap (mutable)
#[inline]
fn metadatas_mut(&mut self) -> &mut SerdeAnyMap {
&mut self.metadatas
}
}
/// Impl of a testcase
impl<I> Testcase<I>
where
@ -120,27 +138,6 @@ where
self.fitness = fitness;
}
/// Get all the metadatas into an HashMap
#[inline]
pub fn metadatas(&self) -> &SerdeAnyMap {
&self.metadatas
}
/// Get all the metadatas into an HashMap (mutable)
#[inline]
pub fn metadatas_mut(&mut self) -> &mut SerdeAnyMap {
&mut self.metadatas
}
/// Add a metadata
#[inline]
pub fn add_metadata<M>(&mut self, meta: M)
where
M: SerdeAny,
{
self.metadatas.insert(meta);
}
/// Get the execution time of the testcase
pub fn exec_time(&self) -> &Option<Duration> {
&self.exec_time

142
libafl/src/feedbacks/map.rs
View File

@ -13,15 +13,14 @@ use crate::{
feedbacks::Feedback,
inputs::Input,
observers::{MapObserver, Observer, ObserversTuple},
state::HasMetadata,
utils::AsSlice,
Error,
};
pub type MaxMapFeedback<T, O> = MapFeedback<T, MaxReducer<T>, O>;
pub type MinMapFeedback<T, O> = MapFeedback<T, MinReducer<T>, O>;
//pub type MaxMapTrackerFeedback<T, O> = MapFeedback<T, MaxReducer<T>, O>;
//pub type MinMapTrackerFeedback<T, O> = MapFeedback<T, MinReducer<T>, O>;
/// A Reducer function is used to aggregate values for the novelty search
pub trait Reducer<T>: Serialize + serde::de::DeserializeOwned + 'static
where
@ -74,122 +73,6 @@ where
}
}
/*
/// The most common AFL-like feedback type
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(bound = "T: serde::de::DeserializeOwned")]
pub struct MapFeedback<T, R, O>
where
T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned,
R: Reducer<T>,
O: MapObserver<T>,
{
/// Contains information about untouched entries
history_map: Vec<T>,
/// Name identifier of this instance
name: String,
/// Phantom Data of Reducer
phantom: PhantomData<(R, O)>,
}
impl<T, R, O, I> Feedback<I> for MapFeedback<T, R, O>
where
T: Integer
+ Default
+ Copy
+ 'static
+ serde::Serialize
+ serde::de::DeserializeOwned
+ core::fmt::Debug,
R: Reducer<T>,
O: MapObserver<T>,
I: Input,
{
fn is_interesting<OT: ObserversTuple>(
&mut self,
_input: &I,
observers: &OT,
_exit_kind: ExitKind,
) -> Result<u32, Error> {
let mut interesting = 0;
// TODO optimize
let observer = observers.match_name_type::<O>(&self.name).unwrap();
let size = observer.usable_count();
//println!("count: {:?}, map: {:?}, history: {:?}", size, observer.map(), &self.history_map);
for i in 0..size {
let history = self.history_map[i];
let item = observer.map()[i];
let reduced = R::reduce(history, item);
if history != reduced {
self.history_map[i] = reduced;
interesting += 1;
}
}
//println!("..interesting: {:?}, new_history: {:?}\n", interesting, &self.history_map);
//std::thread::sleep(std::time::Duration::from_millis(100));
Ok(interesting)
}
}
impl<T, R, O> Named for MapFeedback<T, R, O>
where
T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned,
R: Reducer<T>,
O: MapObserver<T>,
{
#[inline]
fn name(&self) -> &str {
self.name.as_str()
}
}
impl<T, R, O> MapFeedback<T, R, O>
where
T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned,
R: Reducer<T>,
O: MapObserver<T> + Observer,
{
/// Create new MapFeedback
pub fn new(name: &'static str, map_size: usize) -> Self {
Self {
history_map: vec![T::default(); map_size],
phantom: PhantomData,
name: name.to_string(),
}
}
/// Create new MapFeedback for the observer type.
/// Name should match that of the observer.
pub fn new_with_observer(name: &'static str, map_observer: &O) -> Self {
debug_assert_eq!(name, map_observer.name());
Self {
history_map: vec![T::default(); map_observer.map().len()],
phantom: PhantomData,
name: name.to_string(),
}
}
}
impl<T, R, O> MapFeedback<T, R, O>
where
T: Integer + Default + Copy + 'static + serde::Serialize + serde::de::DeserializeOwned,
R: Reducer<T>,
O: MapObserver<T>,
{
/// Create new MapFeedback using a map observer, and a map.
/// The map can be shared.
pub fn with_history_map(name: &'static str, history_map: Vec<T>) -> Self {
Self {
history_map: history_map,
name: name.to_string(),
phantom: PhantomData,
}
}
}
*/
/// A testcase metadata holding a list of indexes of a map
#[derive(Serialize, Deserialize)]
pub struct MapIndexesMetadata {
@ -198,12 +81,10 @@ pub struct MapIndexesMetadata {
crate::impl_serdeany!(MapIndexesMetadata);
impl IntoIterator for MapIndexesMetadata {
type Item = usize;
type IntoIter = alloc::vec::IntoIter<Self::Item>;
fn into_iter(self) -> Self::IntoIter {
self.list.into_iter()
impl AsSlice<usize> for MapIndexesMetadata {
/// Convert to a slice
fn as_slice(&self) -> &[usize] {
self.list.as_slice()
}
}
@ -221,15 +102,12 @@ pub struct MapNoveltiesMetadata {
crate::impl_serdeany!(MapNoveltiesMetadata);
impl IntoIterator for MapNoveltiesMetadata {
type Item = usize;
type IntoIter = alloc::vec::IntoIter<Self::Item>;
fn into_iter(self) -> Self::IntoIter {
self.list.into_iter()
impl AsSlice<usize> for MapNoveltiesMetadata {
/// Convert to a slice
fn as_slice(&self) -> &[usize] {
self.list.as_slice()
}
}
impl MapNoveltiesMetadata {
pub fn new(list: Vec<usize>) -> Self {
Self { list }

8
libafl/src/mutators/token_mutations.rs
View File

@ -37,7 +37,7 @@ where
{
let max_size = state.max_size();
let tokens_len = {
let meta = state.metadata().get::<TokensMetadata>();
let meta = state.metadatas().get::<TokensMetadata>();
if meta.is_none() {
return Ok(MutationResult::Skipped);
}
@ -51,7 +51,7 @@ where
let size = input.bytes().len();
let off = state.rand_mut().below((size + 1) as u64) as usize;
let meta = state.metadata().get::<TokensMetadata>().unwrap();
let meta = state.metadatas().get::<TokensMetadata>().unwrap();
let token = &meta.tokens[token_idx];
let mut len = token.len();
@ -83,7 +83,7 @@ where
}
let tokens_len = {
let meta = state.metadata().get::<TokensMetadata>();
let meta = state.metadatas().get::<TokensMetadata>();
if meta.is_none() {
return Ok(MutationResult::Skipped);
}
@ -96,7 +96,7 @@ where
let off = state.rand_mut().below(size as u64) as usize;
let meta = state.metadata().get::<TokensMetadata>().unwrap();
let meta = state.metadatas().get::<TokensMetadata>().unwrap();
let token = &meta.tokens[token_idx];
let mut len = token.len();
if off + len > size {

12
libafl/src/state/mod.rs
View File

@ -73,9 +73,9 @@ where
/// Trait for elements offering metadata
pub trait HasMetadata {
/// A map, storing all metadata
fn metadata(&self) -> &SerdeAnyMap;
fn metadatas(&self) -> &SerdeAnyMap;
/// A map, storing all metadata (mut)
fn metadata_mut(&mut self) -> &mut SerdeAnyMap;
fn metadatas_mut(&mut self) -> &mut SerdeAnyMap;
/// Add a metadata to the metadata map
#[inline]
@ -83,7 +83,7 @@ pub trait HasMetadata {
where
M: SerdeAny,
{
self.metadata_mut().insert(meta);
self.metadatas_mut().insert(meta);
}
/// Check for a metadata
@ -92,7 +92,7 @@ pub trait HasMetadata {
where
M: SerdeAny,
{
self.metadata().get::<M>().is_some()
self.metadatas().get::<M>().is_some()
}
}
@ -307,13 +307,13 @@ where
{
/// Get all the metadata into an HashMap
#[inline]
fn metadata(&self) -> &SerdeAnyMap {
fn metadatas(&self) -> &SerdeAnyMap {
&self.metadata
}
/// Get all the metadata into an HashMap (mutable)
#[inline]
fn metadata_mut(&mut self) -> &mut SerdeAnyMap {
fn metadatas_mut(&mut self) -> &mut SerdeAnyMap {
&mut self.metadata
}
}

5
libafl/src/utils.rs
View File

@ -7,6 +7,11 @@ use xxhash_rust::xxh3::xxh3_64_with_seed;
#[cfg(feature = "std")]
use std::time::{SystemTime, UNIX_EPOCH};
pub trait AsSlice<T> {
/// Convert to a slice
fn as_slice(&self) -> &[T];
}
pub type StdRand = RomuTrioRand;
/// Ways to get random around here