on_evaluation Scheduler method (#1106)

* add on evaluation hook in schedulers

* on_evaluation for WeightedScheduler

* fix PowerQueueScheduler

* fix fuzzers

* upd qemu

* tests

* upd
This commit is contained in:
Andrea Fioraldi 2023-02-28 11:33:26 +01:00 committed by GitHub
parent 59bf118a5a
commit dc800f0814
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 197 additions and 158 deletions

View File

@ -304,11 +304,12 @@ fn fuzz(
5, 5,
)?; )?;
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule(
&mut state, &mut state,
&edges_observer,
Some(PowerSchedule::EXPLORE), Some(PowerSchedule::EXPLORE),
)); ));

View File

@ -292,11 +292,12 @@ fn fuzz(
5, 5,
)?; )?;
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new( let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new(
&mut state, &mut state,
&edges_observer,
PowerSchedule::FAST, PowerSchedule::FAST,
)); ));

View File

@ -294,11 +294,12 @@ fn fuzz(
5, 5,
)?; )?;
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule(
&mut state, &mut state,
&edges_observer,
Some(PowerSchedule::EXPLORE), Some(PowerSchedule::EXPLORE),
)); ));

View File

@ -304,11 +304,12 @@ fn fuzz(
5, 5,
)?; )?;
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new( let scheduler = IndexesLenTimeMinimizerScheduler::new(PowerQueueScheduler::new(
&mut state, &mut state,
&edges_observer,
PowerSchedule::FAST, PowerSchedule::FAST,
)); ));

View File

@ -365,11 +365,12 @@ fn fuzz_binary(
5, 5,
)?; )?;
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule(
&mut state, &mut state,
&edges_observer,
Some(PowerSchedule::EXPLORE), Some(PowerSchedule::EXPLORE),
)); ));
@ -568,7 +569,7 @@ fn fuzz_text(
5, 5,
)?; )?;
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
let grimoire_mutator = StdScheduledMutator::with_max_stack_pow( let grimoire_mutator = StdScheduledMutator::with_max_stack_pow(
tuple_list!( tuple_list!(
@ -586,6 +587,7 @@ fn fuzz_text(
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule(
&mut state, &mut state,
&edges_observer,
Some(PowerSchedule::EXPLORE), Some(PowerSchedule::EXPLORE),
)); ));

View File

@ -142,13 +142,14 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations()));
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
let mut stages = tuple_list!(calibration, power); let mut stages = tuple_list!(calibration, power);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule(
&mut state, &mut state,
&edges_observer,
Some(PowerSchedule::FAST), Some(PowerSchedule::FAST),
)); ));

View File

@ -141,13 +141,14 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations()));
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
let mut stages = tuple_list!(calibration, power); let mut stages = tuple_list!(calibration, power);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule(
&mut state, &mut state,
&edges_observer,
Some(PowerSchedule::FAST), Some(PowerSchedule::FAST),
)); ));

View File

@ -107,13 +107,14 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations()));
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
let mut stages = tuple_list!(calibration, power); let mut stages = tuple_list!(calibration, power);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule( let scheduler = IndexesLenTimeMinimizerScheduler::new(StdWeightedScheduler::with_schedule(
&mut state, &mut state,
&edges_observer,
Some(PowerSchedule::FAST), Some(PowerSchedule::FAST),
)); ));

View File

@ -127,13 +127,16 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
// Setup a lain mutator with a mutational stage // Setup a lain mutator with a mutational stage
let mutator = LainMutator::new(); let mutator = LainMutator::new();
let power = StdPowerMutationalStage::new(mutator, &edges_observer); let power = StdPowerMutationalStage::new(mutator);
let mut stages = tuple_list!(calibration, power); let mut stages = tuple_list!(calibration, power);
// A minimization+queue policy to get testcasess from the corpus // A minimization+queue policy to get testcasess from the corpus
let scheduler = let scheduler = PacketLenMinimizerScheduler::new(PowerQueueScheduler::new(
PacketLenMinimizerScheduler::new(PowerQueueScheduler::new(&mut state, PowerSchedule::FAST)); &mut state,
&edges_observer,
PowerSchedule::FAST,
));
// A fuzzer with feedbacks and a corpus scheduler // A fuzzer with feedbacks and a corpus scheduler
let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective);

View File

@ -300,6 +300,18 @@ impl SchedulerTestcaseMetaData {
} }
} }
/// Create new [`struct@SchedulerTestcaseMetaData`] given `n_fuzz_entry`
#[must_use]
pub fn with_n_fuzz_entry(depth: u64, n_fuzz_entry: usize) -> Self {
Self {
bitmap_size: 0,
handicap: 0,
depth,
n_fuzz_entry,
cycle_and_time: (Duration::default(), 0),
}
}
/// Get the bitmap size /// Get the bitmap size
#[inline] #[inline]
#[must_use] #[must_use]

View File

@ -460,6 +460,9 @@ where
{ {
let exit_kind = self.execute_input(state, executor, manager, &input)?; let exit_kind = self.execute_input(state, executor, manager, &input)?;
let observers = executor.observers(); let observers = executor.observers();
self.scheduler.on_evaluation(state, &input, observers)?;
self.process_execution(state, manager, input, observers, &exit_kind, send_events) self.process_execution(state, manager, input, observers, &exit_kind, send_events)
} }
} }

View File

@ -125,13 +125,13 @@ where
CS::State: HasCorpus + HasMetadata + HasRand + Debug, CS::State: HasCorpus + HasMetadata + HasRand + Debug,
<CS::State as UsesInput>::Input: HasLen, <CS::State as UsesInput>::Input: HasLen,
{ {
fn on_add(&self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> { fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> {
self.update_accounting_score(state, idx)?; self.update_accounting_score(state, idx)?;
self.inner.on_add(state, idx) self.inner.on_add(state, idx)
} }
fn on_replace( fn on_replace(
&self, &mut self,
state: &mut Self::State, state: &mut Self::State,
idx: CorpusId, idx: CorpusId,
testcase: &Testcase<<Self::State as UsesInput>::Input>, testcase: &Testcase<<Self::State as UsesInput>::Input>,
@ -140,7 +140,7 @@ where
} }
fn on_remove( fn on_remove(
&self, &mut self,
state: &mut Self::State, state: &mut Self::State,
idx: CorpusId, idx: CorpusId,
testcase: &Option<Testcase<<Self::State as UsesInput>::Input>>, testcase: &Option<Testcase<<Self::State as UsesInput>::Input>>,
@ -148,7 +148,7 @@ where
self.inner.on_remove(state, idx, testcase) self.inner.on_remove(state, idx, testcase)
} }
fn next(&self, state: &mut Self::State) -> Result<CorpusId, Error> { fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {
if state if state
.metadata() .metadata()
.get::<TopAccountingMetadata>() .get::<TopAccountingMetadata>()
@ -158,7 +158,7 @@ where
} else { } else {
self.inner.cull(state)?; self.inner.cull(state)?;
} }
let mut idx = self.inner.base().next(state)?; let mut idx = self.inner.base_mut().next(state)?;
while { while {
let has = !state let has = !state
.corpus() .corpus()
@ -168,7 +168,7 @@ where
has has
} && state.rand_mut().below(100) < self.skip_non_favored_prob } && state.rand_mut().below(100) < self.skip_non_favored_prob
{ {
idx = self.inner.base().next(state)?; idx = self.inner.base_mut().next(state)?;
} }
Ok(idx) Ok(idx)
} }

View File

@ -82,14 +82,14 @@ where
CS::State: HasCorpus + HasMetadata + HasRand, CS::State: HasCorpus + HasMetadata + HasRand,
{ {
/// Add an entry to the corpus and return its index /// Add an entry to the corpus and return its index
fn on_add(&self, state: &mut CS::State, idx: CorpusId) -> Result<(), Error> { fn on_add(&mut self, state: &mut CS::State, idx: CorpusId) -> Result<(), Error> {
self.base.on_add(state, idx)?; self.base.on_add(state, idx)?;
self.update_score(state, idx) self.update_score(state, idx)
} }
/// Replaces the testcase at the given idx /// Replaces the testcase at the given idx
fn on_replace( fn on_replace(
&self, &mut self,
state: &mut CS::State, state: &mut CS::State,
idx: CorpusId, idx: CorpusId,
testcase: &Testcase<<CS::State as UsesInput>::Input>, testcase: &Testcase<<CS::State as UsesInput>::Input>,
@ -100,7 +100,7 @@ where
/// Removes an entry from the corpus, returning M if M was present. /// Removes an entry from the corpus, returning M if M was present.
fn on_remove( fn on_remove(
&self, &mut self,
state: &mut CS::State, state: &mut CS::State,
idx: CorpusId, idx: CorpusId,
testcase: &Option<Testcase<<CS::State as UsesInput>::Input>>, testcase: &Option<Testcase<<CS::State as UsesInput>::Input>>,
@ -163,7 +163,7 @@ where
} }
/// Gets the next entry /// Gets the next entry
fn next(&self, state: &mut CS::State) -> Result<CorpusId, Error> { fn next(&mut self, state: &mut CS::State) -> Result<CorpusId, Error> {
self.cull(state)?; self.cull(state)?;
let mut idx = self.base.next(state)?; let mut idx = self.base.next(state)?;
while { while {
@ -295,6 +295,11 @@ where
&self.base &self.base
} }
/// Get a reference to the base scheduler (mut)
pub fn base_mut(&mut self) -> &mut CS {
&mut self.base
}
/// Creates a new [`MinimizerScheduler`] that wraps a `base` [`Scheduler`] /// Creates a new [`MinimizerScheduler`] that wraps a `base` [`Scheduler`]
/// and has a default probability to skip non-faved [`Testcase`]s of [`DEFAULT_SKIP_NON_FAVORED_PROB`]. /// and has a default probability to skip non-faved [`Testcase`]s of [`DEFAULT_SKIP_NON_FAVORED_PROB`].
pub fn new(base: CS) -> Self { pub fn new(base: CS) -> Self {

View File

@ -33,6 +33,7 @@ use crate::{
bolts::rands::Rand, bolts::rands::Rand,
corpus::{Corpus, CorpusId, Testcase}, corpus::{Corpus, CorpusId, Testcase},
inputs::UsesInput, inputs::UsesInput,
observers::ObserversTuple,
random_corpus_id, random_corpus_id,
state::{HasCorpus, HasRand, UsesState}, state::{HasCorpus, HasRand, UsesState},
Error, Error,
@ -42,13 +43,13 @@ use crate::{
/// It has hooks to corpus add/replace/remove to allow complex scheduling algorithms to collect data. /// It has hooks to corpus add/replace/remove to allow complex scheduling algorithms to collect data.
pub trait Scheduler: UsesState { pub trait Scheduler: UsesState {
/// Added an entry to the corpus at the given index /// Added an entry to the corpus at the given index
fn on_add(&self, _state: &mut Self::State, _idx: CorpusId) -> Result<(), Error> { fn on_add(&mut self, _state: &mut Self::State, _idx: CorpusId) -> Result<(), Error> {
Ok(()) Ok(())
} }
/// Replaced the given testcase at the given idx /// Replaced the given testcase at the given idx
fn on_replace( fn on_replace(
&self, &mut self,
_state: &mut Self::State, _state: &mut Self::State,
_idx: CorpusId, _idx: CorpusId,
_prev: &Testcase<<Self::State as UsesInput>::Input>, _prev: &Testcase<<Self::State as UsesInput>::Input>,
@ -58,7 +59,7 @@ pub trait Scheduler: UsesState {
/// Removed the given entry from the corpus at the given index /// Removed the given entry from the corpus at the given index
fn on_remove( fn on_remove(
&self, &mut self,
_state: &mut Self::State, _state: &mut Self::State,
_idx: CorpusId, _idx: CorpusId,
_testcase: &Option<Testcase<<Self::State as UsesInput>::Input>>, _testcase: &Option<Testcase<<Self::State as UsesInput>::Input>>,
@ -66,8 +67,21 @@ pub trait Scheduler: UsesState {
Ok(()) Ok(())
} }
/// An input has been evaluated
fn on_evaluation<OT>(
&mut self,
_state: &mut Self::State,
_input: &<Self::State as UsesInput>::Input,
_observers: &OT,
) -> Result<(), Error>
where
OT: ObserversTuple<Self::State>,
{
Ok(())
}
/// Gets the next entry /// Gets the next entry
fn next(&self, state: &mut Self::State) -> Result<CorpusId, Error>; fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error>;
} }
/// Feed the fuzzer simply with a random testcase on request /// Feed the fuzzer simply with a random testcase on request
@ -88,7 +102,7 @@ where
S: HasCorpus + HasRand, S: HasCorpus + HasRand,
{ {
/// Gets the next entry at random /// Gets the next entry at random
fn next(&self, state: &mut Self::State) -> Result<CorpusId, Error> { fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {
if state.corpus().count() == 0 { if state.corpus().count() == 0 {
Err(Error::empty("No entries in corpus".to_owned())) Err(Error::empty("No entries in corpus".to_owned()))
} else { } else {

View File

@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize};
use crate::{ use crate::{
corpus::{Corpus, CorpusId, SchedulerTestcaseMetaData, Testcase}, corpus::{Corpus, CorpusId, SchedulerTestcaseMetaData, Testcase},
inputs::UsesInput, inputs::UsesInput,
observers::{MapObserver, ObserversTuple},
schedulers::Scheduler, schedulers::Scheduler,
state::{HasCorpus, HasMetadata, UsesState}, state::{HasCorpus, HasMetadata, UsesState},
Error, Error,
@ -163,24 +164,27 @@ pub enum PowerSchedule {
/// A corpus scheduler using power schedules /// A corpus scheduler using power schedules
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct PowerQueueScheduler<S> { pub struct PowerQueueScheduler<O, S> {
strat: PowerSchedule, strat: PowerSchedule,
phantom: PhantomData<S>, map_observer_name: String,
last_hash: usize,
phantom: PhantomData<(O, S)>,
} }
impl<S> UsesState for PowerQueueScheduler<S> impl<O, S> UsesState for PowerQueueScheduler<O, S>
where where
S: UsesInput, S: UsesInput,
{ {
type State = S; type State = S;
} }
impl<S> Scheduler for PowerQueueScheduler<S> impl<O, S> Scheduler for PowerQueueScheduler<O, S>
where where
S: HasCorpus + HasMetadata, S: HasCorpus + HasMetadata,
O: MapObserver,
{ {
/// Add an entry to the corpus and return its index /// Add an entry to the corpus and return its index
fn on_add(&self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> { fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> {
let current_idx = *state.corpus().current(); let current_idx = *state.corpus().current();
let mut depth = match current_idx { let mut depth = match current_idx {
@ -199,17 +203,15 @@ where
// Attach a `SchedulerTestcaseMetaData` to the queue entry. // Attach a `SchedulerTestcaseMetaData` to the queue entry.
depth += 1; depth += 1;
state state.corpus().get(idx)?.borrow_mut().add_metadata(
.corpus() SchedulerTestcaseMetaData::with_n_fuzz_entry(depth, self.last_hash),
.get(idx)? );
.borrow_mut()
.add_metadata(SchedulerTestcaseMetaData::new(depth));
Ok(()) Ok(())
} }
#[allow(clippy::cast_precision_loss)] #[allow(clippy::cast_precision_loss)]
fn on_replace( fn on_replace(
&self, &mut self,
state: &mut Self::State, state: &mut Self::State,
idx: CorpusId, idx: CorpusId,
prev: &Testcase<<Self::State as UsesInput>::Input>, prev: &Testcase<<Self::State as UsesInput>::Input>,
@ -251,7 +253,7 @@ where
#[allow(clippy::cast_precision_loss)] #[allow(clippy::cast_precision_loss)]
fn on_remove( fn on_remove(
&self, &mut self,
state: &mut Self::State, state: &mut Self::State,
_idx: CorpusId, _idx: CorpusId,
prev: &Option<Testcase<<Self::State as UsesInput>::Input>>, prev: &Option<Testcase<<Self::State as UsesInput>::Input>>,
@ -288,7 +290,36 @@ where
Ok(()) Ok(())
} }
fn next(&self, state: &mut Self::State) -> Result<CorpusId, Error> { fn on_evaluation<OT>(
&mut self,
state: &mut Self::State,
_input: &<Self::State as UsesInput>::Input,
observers: &OT,
) -> Result<(), Error>
where
OT: ObserversTuple<Self::State>,
{
let observer = observers
.match_name::<O>(&self.map_observer_name)
.ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))?;
let mut hash = observer.hash() as usize;
let psmeta = state
.metadata_mut()
.get_mut::<SchedulerMetadata>()
.ok_or_else(|| Error::key_not_found("SchedulerMetadata not found".to_string()))?;
hash %= psmeta.n_fuzz().len();
// Update the path frequency
psmeta.n_fuzz_mut()[hash] = psmeta.n_fuzz()[hash].saturating_add(1);
self.last_hash = hash;
Ok(())
}
fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {
if state.corpus().count() == 0 { if state.corpus().count() == 0 {
Err(Error::empty(String::from("No entries in corpus"))) Err(Error::empty(String::from("No entries in corpus")))
} else { } else {
@ -331,18 +362,21 @@ where
} }
} }
impl<S> PowerQueueScheduler<S> impl<O, S> PowerQueueScheduler<O, S>
where where
S: HasMetadata, S: HasMetadata,
O: MapObserver,
{ {
/// Create a new [`PowerQueueScheduler`] /// Create a new [`PowerQueueScheduler`]
#[must_use] #[must_use]
pub fn new(state: &mut S, strat: PowerSchedule) -> Self { pub fn new(state: &mut S, map_observer: &O, strat: PowerSchedule) -> Self {
if !state.has_metadata::<SchedulerMetadata>() { if !state.has_metadata::<SchedulerMetadata>() {
state.add_metadata::<SchedulerMetadata>(SchedulerMetadata::new(Some(strat))); state.add_metadata::<SchedulerMetadata>(SchedulerMetadata::new(Some(strat)));
} }
PowerQueueScheduler { PowerQueueScheduler {
strat, strat,
map_observer_name: map_observer.name().to_string(),
last_hash: 0,
phantom: PhantomData, phantom: PhantomData,
} }
} }

View File

@ -99,7 +99,7 @@ where
F: TestcaseScore<S>, F: TestcaseScore<S>,
S: HasCorpus + HasMetadata + HasRand, S: HasCorpus + HasMetadata + HasRand,
{ {
fn on_add(&self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> { fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> {
if state.metadata().get::<ProbabilityMetadata>().is_none() { if state.metadata().get::<ProbabilityMetadata>().is_none() {
state.add_metadata(ProbabilityMetadata::new()); state.add_metadata(ProbabilityMetadata::new());
} }
@ -108,7 +108,7 @@ where
/// Gets the next entry /// Gets the next entry
#[allow(clippy::cast_precision_loss)] #[allow(clippy::cast_precision_loss)]
fn next(&self, state: &mut Self::State) -> Result<CorpusId, Error> { fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {
if state.corpus().count() == 0 { if state.corpus().count() == 0 {
Err(Error::empty(String::from("No entries in corpus"))) Err(Error::empty(String::from("No entries in corpus")))
} else { } else {
@ -182,7 +182,7 @@ mod tests {
// the first 3 probabilities will be .69, .86, .44 // the first 3 probabilities will be .69, .86, .44
let rand = StdRand::with_seed(12); let rand = StdRand::with_seed(12);
let scheduler = UniformProbabilitySamplingScheduler::new(); let mut scheduler = UniformProbabilitySamplingScheduler::new();
let mut feedback = ConstFeedback::new(false); let mut feedback = ConstFeedback::new(false);
let mut objective = ConstFeedback::new(false); let mut objective = ConstFeedback::new(false);

View File

@ -29,7 +29,7 @@ where
S: HasCorpus, S: HasCorpus,
{ {
/// Gets the next entry in the queue /// Gets the next entry in the queue
fn next(&self, state: &mut Self::State) -> Result<CorpusId, Error> { fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {
if state.corpus().count() == 0 { if state.corpus().count() == 0 {
Err(Error::empty("No entries in corpus".to_owned())) Err(Error::empty("No entries in corpus".to_owned()))
} else { } else {
@ -79,7 +79,7 @@ mod tests {
#[test] #[test]
fn test_queuecorpus() { fn test_queuecorpus() {
let rand = StdRand::with_seed(4); let rand = StdRand::with_seed(4);
let scheduler = QueueScheduler::new(); let mut scheduler = QueueScheduler::new();
let mut q = let mut q =
OnDiskCorpus::<BytesInput>::new(PathBuf::from("target/.test/fancy/path")).unwrap(); OnDiskCorpus::<BytesInput>::new(PathBuf::from("target/.test/fancy/path")).unwrap();

View File

@ -93,7 +93,7 @@ where
S: HasCorpus + HasMetadata, S: HasCorpus + HasMetadata,
{ {
/// Gets the next entry in the queue /// Gets the next entry in the queue
fn next(&self, state: &mut Self::State) -> Result<CorpusId, Error> { fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {
if state.corpus().count() == 0 { if state.corpus().count() == 0 {
return Err(Error::empty("No entries in corpus".to_owned())); return Err(Error::empty("No entries in corpus".to_owned()));
} }

View File

@ -11,6 +11,7 @@ use crate::{
bolts::rands::Rand, bolts::rands::Rand,
corpus::{Corpus, CorpusId, SchedulerTestcaseMetaData, Testcase}, corpus::{Corpus, CorpusId, SchedulerTestcaseMetaData, Testcase},
inputs::UsesInput, inputs::UsesInput,
observers::{MapObserver, ObserversTuple},
random_corpus_id, random_corpus_id,
schedulers::{ schedulers::{
powersched::{PowerSchedule, SchedulerMetadata}, powersched::{PowerSchedule, SchedulerMetadata},
@ -88,25 +89,28 @@ crate::impl_serdeany!(WeightedScheduleMetadata);
/// A corpus scheduler using power schedules with weighted queue item selection algo. /// A corpus scheduler using power schedules with weighted queue item selection algo.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct WeightedScheduler<F, S> { pub struct WeightedScheduler<F, O, S> {
strat: Option<PowerSchedule>, strat: Option<PowerSchedule>,
phantom: PhantomData<(F, S)>, map_observer_name: String,
last_hash: usize,
phantom: PhantomData<(F, O, S)>,
} }
impl<F, S> WeightedScheduler<F, S> impl<F, O, S> WeightedScheduler<F, O, S>
where where
F: TestcaseScore<S>, F: TestcaseScore<S>,
O: MapObserver,
S: HasCorpus + HasMetadata + HasRand, S: HasCorpus + HasMetadata + HasRand,
{ {
/// Create a new [`WeightedScheduler`] without any power schedule /// Create a new [`WeightedScheduler`] without any power schedule
#[must_use] #[must_use]
pub fn new(state: &mut S) -> Self { pub fn new(state: &mut S, map_observer: &O) -> Self {
Self::with_schedule(state, None) Self::with_schedule(state, map_observer, None)
} }
/// Create a new [`WeightedScheduler`] /// Create a new [`WeightedScheduler`]
#[must_use] #[must_use]
pub fn with_schedule(state: &mut S, strat: Option<PowerSchedule>) -> Self { pub fn with_schedule(state: &mut S, map_observer: &O, strat: Option<PowerSchedule>) -> Self {
if !state.has_metadata::<SchedulerMetadata>() { if !state.has_metadata::<SchedulerMetadata>() {
state.add_metadata(SchedulerMetadata::new(strat)); state.add_metadata(SchedulerMetadata::new(strat));
} }
@ -116,6 +120,8 @@ where
} }
Self { Self {
strat, strat,
map_observer_name: map_observer.name().to_string(),
last_hash: 0,
phantom: PhantomData, phantom: PhantomData,
} }
} }
@ -216,20 +222,21 @@ where
} }
} }
impl<F, S> UsesState for WeightedScheduler<F, S> impl<F, O, S> UsesState for WeightedScheduler<F, O, S>
where where
S: UsesInput, S: UsesInput,
{ {
type State = S; type State = S;
} }
impl<F, S> Scheduler for WeightedScheduler<F, S> impl<F, O, S> Scheduler for WeightedScheduler<F, O, S>
where where
F: TestcaseScore<S>, F: TestcaseScore<S>,
O: MapObserver,
S: HasCorpus + HasMetadata + HasRand, S: HasCorpus + HasMetadata + HasRand,
{ {
/// Add an entry to the corpus and return its index /// Add an entry to the corpus and return its index
fn on_add(&self, state: &mut S, idx: CorpusId) -> Result<(), Error> { fn on_add(&mut self, state: &mut S, idx: CorpusId) -> Result<(), Error> {
let current_idx = *state.corpus().current(); let current_idx = *state.corpus().current();
let mut depth = match current_idx { let mut depth = match current_idx {
@ -248,11 +255,9 @@ where
// Attach a `SchedulerTestcaseMetaData` to the queue entry. // Attach a `SchedulerTestcaseMetaData` to the queue entry.
depth += 1; depth += 1;
state state.corpus().get(idx)?.borrow_mut().add_metadata(
.corpus() SchedulerTestcaseMetaData::with_n_fuzz_entry(depth, self.last_hash),
.get(idx)? );
.borrow_mut()
.add_metadata(SchedulerTestcaseMetaData::new(depth));
// Recreate the alias table // Recreate the alias table
self.create_alias_table(state)?; self.create_alias_table(state)?;
@ -260,7 +265,7 @@ where
} }
fn on_replace( fn on_replace(
&self, &mut self,
state: &mut S, state: &mut S,
idx: CorpusId, idx: CorpusId,
_testcase: &Testcase<S::Input>, _testcase: &Testcase<S::Input>,
@ -270,7 +275,7 @@ where
} }
fn on_remove( fn on_remove(
&self, &mut self,
state: &mut S, state: &mut S,
_idx: CorpusId, _idx: CorpusId,
_testcase: &Option<Testcase<S::Input>>, _testcase: &Option<Testcase<S::Input>>,
@ -280,8 +285,37 @@ where
Ok(()) Ok(())
} }
fn on_evaluation<OT>(
&mut self,
state: &mut Self::State,
_input: &<Self::State as UsesInput>::Input,
observers: &OT,
) -> Result<(), Error>
where
OT: ObserversTuple<Self::State>,
{
let observer = observers
.match_name::<O>(&self.map_observer_name)
.ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))?;
let mut hash = observer.hash() as usize;
let psmeta = state
.metadata_mut()
.get_mut::<SchedulerMetadata>()
.ok_or_else(|| Error::key_not_found("SchedulerMetadata not found".to_string()))?;
hash %= psmeta.n_fuzz().len();
// Update the path frequency
psmeta.n_fuzz_mut()[hash] = psmeta.n_fuzz()[hash].saturating_add(1);
self.last_hash = hash;
Ok(())
}
#[allow(clippy::similar_names, clippy::cast_precision_loss)] #[allow(clippy::similar_names, clippy::cast_precision_loss)]
fn next(&self, state: &mut S) -> Result<CorpusId, Error> { fn next(&mut self, state: &mut S) -> Result<CorpusId, Error> {
let corpus_counts = state.corpus().count(); let corpus_counts = state.corpus().count();
if corpus_counts == 0 { if corpus_counts == 0 {
Err(Error::empty(String::from("No entries in corpus"))) Err(Error::empty(String::from("No entries in corpus")))
@ -345,4 +379,4 @@ where
} }
/// The standard corpus weight, same as aflpp /// The standard corpus weight, same as aflpp
pub type StdWeightedScheduler<S> = WeightedScheduler<CorpusWeightTestcaseScore<S>, S>; pub type StdWeightedScheduler<O, S> = WeightedScheduler<CorpusWeightTestcaseScore<S>, O, S>;

View File

@ -1,50 +1,39 @@
//! The power schedules. This stage should be invoked after the calibration stage. //! The power schedules. This stage should be invoked after the calibration stage.
use alloc::string::{String, ToString};
use core::{fmt::Debug, marker::PhantomData}; use core::{fmt::Debug, marker::PhantomData};
use crate::{ use crate::{
bolts::tuples::MatchName, corpus::{Corpus, CorpusId},
corpus::{Corpus, CorpusId, SchedulerTestcaseMetaData},
executors::{Executor, HasObservers}, executors::{Executor, HasObservers},
fuzzer::Evaluator, fuzzer::Evaluator,
mutators::Mutator, mutators::Mutator,
observers::MapObserver, schedulers::{testcase_score::CorpusPowerTestcaseScore, TestcaseScore},
schedulers::{ stages::{mutational::MutatedTransform, MutationalStage, Stage},
powersched::SchedulerMetadata, testcase_score::CorpusPowerTestcaseScore, TestcaseScore,
},
stages::{
mutational::{MutatedTransform, MutatedTransformPost},
MutationalStage, Stage,
},
state::{HasClientPerfMonitor, HasCorpus, HasMetadata, HasRand, UsesState}, state::{HasClientPerfMonitor, HasCorpus, HasMetadata, HasRand, UsesState},
Error, Error,
}; };
/// The mutational stage using power schedules /// The mutational stage using power schedules
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct PowerMutationalStage<E, F, EM, I, M, O, Z> { pub struct PowerMutationalStage<E, F, EM, I, M, Z> {
map_observer_name: String,
mutator: M, mutator: M,
#[allow(clippy::type_complexity)] #[allow(clippy::type_complexity)]
phantom: PhantomData<(E, F, EM, I, O, Z)>, phantom: PhantomData<(E, F, EM, I, Z)>,
} }
impl<E, F, EM, I, M, O, Z> UsesState for PowerMutationalStage<E, F, EM, I, M, O, Z> impl<E, F, EM, I, M, Z> UsesState for PowerMutationalStage<E, F, EM, I, M, Z>
where where
E: UsesState, E: UsesState,
{ {
type State = E::State; type State = E::State;
} }
impl<E, F, EM, I, M, O, Z> MutationalStage<E, EM, I, M, Z> impl<E, F, EM, I, M, Z> MutationalStage<E, EM, I, M, Z> for PowerMutationalStage<E, F, EM, I, M, Z>
for PowerMutationalStage<E, F, EM, I, M, O, Z>
where where
E: Executor<EM, Z> + HasObservers, E: Executor<EM, Z> + HasObservers,
EM: UsesState<State = E::State>, EM: UsesState<State = E::State>,
F: TestcaseScore<E::State>, F: TestcaseScore<E::State>,
M: Mutator<I, E::State>, M: Mutator<I, E::State>,
O: MapObserver,
E::State: HasClientPerfMonitor + HasCorpus + HasMetadata + HasRand, E::State: HasClientPerfMonitor + HasCorpus + HasMetadata + HasRand,
Z: Evaluator<E, EM, State = E::State>, Z: Evaluator<E, EM, State = E::State>,
I: MutatedTransform<E::Input, E::State> + Clone, I: MutatedTransform<E::Input, E::State> + Clone,
@ -70,74 +59,14 @@ where
Ok(score) Ok(score)
} }
#[allow(clippy::cast_possible_wrap)]
fn perform_mutational(
&mut self,
fuzzer: &mut Z,
executor: &mut E,
state: &mut E::State,
manager: &mut EM,
corpus_idx: CorpusId,
) -> Result<(), Error> {
let num = self.iterations(state, corpus_idx)?;
let mut testcase = state.corpus().get(corpus_idx)?.borrow_mut();
let Ok(input) = I::try_transform_from(&mut testcase, state, corpus_idx) else { return Ok(()); };
drop(testcase);
for i in 0..num {
let mut input = input.clone();
self.mutator_mut().mutate(state, &mut input, i as i32)?;
let (untransformed, post) = input.try_transform_into(state)?;
let (_, corpus_idx) = fuzzer.evaluate_input(state, executor, manager, untransformed)?;
let observer = executor
.observers()
.match_name::<O>(&self.map_observer_name)
.ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))?;
let mut hash = observer.hash() as usize;
let psmeta = state
.metadata_mut()
.get_mut::<SchedulerMetadata>()
.ok_or_else(|| Error::key_not_found("SchedulerMetadata not found".to_string()))?;
hash %= psmeta.n_fuzz().len();
// Update the path frequency
psmeta.n_fuzz_mut()[hash] = psmeta.n_fuzz()[hash].saturating_add(1);
if let Some(idx) = corpus_idx {
state
.corpus()
.get(idx)?
.borrow_mut()
.metadata_mut()
.get_mut::<SchedulerTestcaseMetaData>()
.ok_or_else(|| {
Error::key_not_found("SchedulerTestcaseMetaData not found".to_string())
})?
.set_n_fuzz_entry(hash);
}
self.mutator_mut().post_exec(state, i as i32, corpus_idx)?;
post.post_exec(state, i as i32, corpus_idx)?;
}
Ok(())
}
} }
impl<E, F, EM, I, M, O, Z> Stage<E, EM, Z> for PowerMutationalStage<E, F, EM, I, M, O, Z> impl<E, F, EM, I, M, Z> Stage<E, EM, Z> for PowerMutationalStage<E, F, EM, I, M, Z>
where where
E: Executor<EM, Z> + HasObservers, E: Executor<EM, Z> + HasObservers,
EM: UsesState<State = E::State>, EM: UsesState<State = E::State>,
F: TestcaseScore<E::State>, F: TestcaseScore<E::State>,
M: Mutator<I, E::State>, M: Mutator<I, E::State>,
O: MapObserver,
E::State: HasClientPerfMonitor + HasCorpus + HasMetadata + HasRand, E::State: HasClientPerfMonitor + HasCorpus + HasMetadata + HasRand,
Z: Evaluator<E, EM, State = E::State>, Z: Evaluator<E, EM, State = E::State>,
I: MutatedTransform<E::Input, E::State> + Clone, I: MutatedTransform<E::Input, E::State> + Clone,
@ -157,36 +86,33 @@ where
} }
} }
impl<E, F, EM, M, O, Z> PowerMutationalStage<E, F, EM, E::Input, M, O, Z> impl<E, F, EM, M, Z> PowerMutationalStage<E, F, EM, E::Input, M, Z>
where where
E: Executor<EM, Z> + HasObservers, E: Executor<EM, Z> + HasObservers,
EM: UsesState<State = E::State>, EM: UsesState<State = E::State>,
F: TestcaseScore<E::State>, F: TestcaseScore<E::State>,
M: Mutator<E::Input, E::State>, M: Mutator<E::Input, E::State>,
O: MapObserver,
E::State: HasClientPerfMonitor + HasCorpus + HasMetadata + HasRand, E::State: HasClientPerfMonitor + HasCorpus + HasMetadata + HasRand,
Z: Evaluator<E, EM, State = E::State>, Z: Evaluator<E, EM, State = E::State>,
{ {
/// Creates a new [`PowerMutationalStage`] /// Creates a new [`PowerMutationalStage`]
pub fn new(mutator: M, map_observer_name: &O) -> Self { pub fn new(mutator: M) -> Self {
Self::transforming(mutator, map_observer_name) Self::transforming(mutator)
} }
} }
impl<E, F, EM, I, M, O, Z> PowerMutationalStage<E, F, EM, I, M, O, Z> impl<E, F, EM, I, M, Z> PowerMutationalStage<E, F, EM, I, M, Z>
where where
E: Executor<EM, Z> + HasObservers, E: Executor<EM, Z> + HasObservers,
EM: UsesState<State = E::State>, EM: UsesState<State = E::State>,
F: TestcaseScore<E::State>, F: TestcaseScore<E::State>,
M: Mutator<I, E::State>, M: Mutator<I, E::State>,
O: MapObserver,
E::State: HasClientPerfMonitor + HasCorpus + HasMetadata + HasRand, E::State: HasClientPerfMonitor + HasCorpus + HasMetadata + HasRand,
Z: Evaluator<E, EM, State = E::State>, Z: Evaluator<E, EM, State = E::State>,
{ {
/// Creates a new transforming [`PowerMutationalStage`] /// Creates a new transforming [`PowerMutationalStage`]
pub fn transforming(mutator: M, map_observer_name: &O) -> Self { pub fn transforming(mutator: M) -> Self {
Self { Self {
map_observer_name: map_observer_name.name().to_string(),
mutator, mutator,
phantom: PhantomData, phantom: PhantomData,
} }
@ -194,5 +120,5 @@ where
} }
/// The standard powerscheduling stage /// The standard powerscheduling stage
pub type StdPowerMutationalStage<E, EM, I, M, O, Z> = pub type StdPowerMutationalStage<E, EM, I, M, Z> =
PowerMutationalStage<E, CorpusPowerTestcaseScore<<E as UsesState>::State>, EM, I, M, O, Z>; PowerMutationalStage<E, CorpusPowerTestcaseScore<<E as UsesState>::State>, EM, I, M, Z>;

View File

@ -116,7 +116,7 @@ where
self.current_corpus_idx = Some(if let Some(corpus_idx) = self.current_corpus_idx { self.current_corpus_idx = Some(if let Some(corpus_idx) = self.current_corpus_idx {
corpus_idx corpus_idx
} else { } else {
fuzzer.scheduler().next(state)? fuzzer.scheduler_mut().next(state)?
}); });
self.testcases_to_do = self.iterations(state, self.current_corpus_idx.unwrap())?; self.testcases_to_do = self.iterations(state, self.current_corpus_idx.unwrap())?;

View File

@ -8,7 +8,7 @@ use which::which;
const QEMU_URL: &str = "https://github.com/AFLplusplus/qemu-libafl-bridge"; const QEMU_URL: &str = "https://github.com/AFLplusplus/qemu-libafl-bridge";
const QEMU_DIRNAME: &str = "qemu-libafl-bridge"; const QEMU_DIRNAME: &str = "qemu-libafl-bridge";
const QEMU_REVISION: &str = "f6a2e732e8e225ebb8d1a9399561af7330af31b3"; const QEMU_REVISION: &str = "0dc52ed6f3915f727aaec8648706760f278f0571";
fn build_dep_check(tools: &[&str]) { fn build_dep_check(tools: &[&str]) {
for tool in tools { for tool in tools {
@ -249,7 +249,6 @@ pub fn build(
.arg("--disable-vhost-vdpa") .arg("--disable-vhost-vdpa")
.arg("--disable-virglrenderer") .arg("--disable-virglrenderer")
.arg("--disable-virtfs") .arg("--disable-virtfs")
.arg("--disable-virtiofsd")
.arg("--disable-vmnet") .arg("--disable-vmnet")
.arg("--disable-vnc") .arg("--disable-vnc")
.arg("--disable-vnc-jpeg") .arg("--disable-vnc-jpeg")