From 04aecd97f6fe06828a1529a5c8bf3ed2e8687305 Mon Sep 17 00:00:00 2001 From: ToSeven <397341575@qq.com> Date: Wed, 6 Sep 2023 09:38:41 +0800 Subject: [PATCH] Add AFL-style metrics(pending,pend_fav, own_finds,imported) (#1351) * add the metrics(pending,own_finds,imported) * add the pend_fav metrics * push * Add the feature that AFLStats is computed and reported in AFLStatsStage * fix some cicd errors * AFLStats migrates to stage/stats.rs * fix the cicd error * fix some bugs and resolve the conflicts * fix some typos --------- Co-authored-by: toseven Co-authored-by: toka Co-authored-by: Dominik Maier --- libafl/src/fuzzer/mod.rs | 17 ++-- libafl/src/stages/mod.rs | 3 + libafl/src/stages/mutational.rs | 1 + libafl/src/stages/stats.rs | 162 ++++++++++++++++++++++++++++++++ libafl/src/state/mod.rs | 26 +++++ 5 files changed, 202 insertions(+), 7 deletions(-) create mode 100644 libafl/src/stages/stats.rs diff --git a/libafl/src/fuzzer/mod.rs b/libafl/src/fuzzer/mod.rs index 174c70d27c..a5df357a6e 100644 --- a/libafl/src/fuzzer/mod.rs +++ b/libafl/src/fuzzer/mod.rs @@ -24,8 +24,8 @@ use crate::{ stages::StagesTuple, start_timer, state::{ - HasClientPerfMonitor, HasCorpus, HasExecutions, HasLastReportTime, HasMetadata, - HasSolutions, UsesState, + HasClientPerfMonitor, HasCorpus, HasExecutions, HasImported, HasLastReportTime, + HasMetadata, HasSolutions, UsesState, }, Error, }; @@ -333,7 +333,8 @@ where F: Feedback, OF: Feedback, OT: ObserversTuple + Serialize + DeserializeOwned, - CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions + HasCorpus, + CS::State: + HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions + HasCorpus + HasImported, { /// Evaluate if a set of observation channels has an interesting state fn process_execution( @@ -415,6 +416,9 @@ where forward_id: None, }, )?; + } else { + // This testcase is from the other fuzzers. + *state.imported_mut() += 1; } Ok((res, Some(idx))) } @@ -450,7 +454,7 @@ where OT: ObserversTuple + Serialize + DeserializeOwned, F: Feedback, OF: Feedback, - CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions, + CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions + HasImported, { /// Process one input, adding to the respective corpora if needed and firing the right events #[inline] @@ -483,7 +487,7 @@ where F: Feedback, OF: Feedback, OT: ObserversTuple + Serialize + DeserializeOwned, - CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions, + CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions + HasImported, { /// Process one input, adding to the respective corpora if needed and firing the right events #[inline] @@ -591,6 +595,7 @@ where + HasMetadata + HasCorpus + HasTestcase + + HasImported + HasLastReportTime, ST: StagesTuple, { @@ -633,11 +638,9 @@ where { let mut testcase = state.testcase_mut(idx)?; let scheduled_count = testcase.scheduled_count(); - // increase scheduled count, this was fuzz_level in afl testcase.set_scheduled_count(scheduled_count + 1); } - Ok(idx) } } diff --git a/libafl/src/stages/mod.rs b/libafl/src/stages/mod.rs index 94dc54726c..1df95bdda9 100644 --- a/libafl/src/stages/mod.rs +++ b/libafl/src/stages/mod.rs @@ -27,6 +27,9 @@ pub use power::{PowerMutationalStage, StdPowerMutationalStage}; pub mod generalization; pub use generalization::GeneralizationStage; +pub mod stats; +pub use stats::AflStatsStage; + pub mod owned; pub use owned::StagesOwnedList; diff --git a/libafl/src/stages/mutational.rs b/libafl/src/stages/mutational.rs index 5cb590ed2a..8f3650ea2c 100644 --- a/libafl/src/stages/mutational.rs +++ b/libafl/src/stages/mutational.rs @@ -146,6 +146,7 @@ where post.post_exec(state, i as i32, corpus_idx)?; mark_feature_time!(state, PerfFeature::MutatePostExec); } + Ok(()) } } diff --git a/libafl/src/stages/stats.rs b/libafl/src/stages/stats.rs new file mode 100644 index 0000000000..5da960dd4a --- /dev/null +++ b/libafl/src/stages/stats.rs @@ -0,0 +1,162 @@ +//! Stage to compute/report AFL stats + +#[cfg(feature = "std")] +use alloc::string::ToString; +use core::{marker::PhantomData, time::Duration}; + +use libafl_bolts::current_time; +#[cfg(feature = "std")] +use serde_json::json; + +use crate::{ + corpus::{Corpus, CorpusId}, + events::EventFirer, + schedulers::minimizer::IsFavoredMetadata, + stages::Stage, + state::{HasCorpus, HasImported, HasMetadata, UsesState}, + Error, +}; +#[cfg(feature = "std")] +use crate::{events::Event, monitors::UserStats}; + +/// The [`AflStatsStage`] is a simple stage that computes and reports some stats. +#[derive(Debug, Clone)] +pub struct AflStatsStage +where + E: UsesState, + EM: EventFirer, + Z: UsesState, +{ + // the number of testcases that have been fuzzed + has_fuzzed_size: usize, + // the number of "favored" testcases + is_favored_size: usize, + // the number of testcases found by itself + own_finds_size: usize, + // the number of testcases imported by other fuzzers + imported_size: usize, + // the last time that we report all stats + last_report_time: Duration, + // the interval that we report all stats + stats_report_interval: Duration, + + phantom: PhantomData<(E, EM, Z)>, +} + +impl UsesState for AflStatsStage +where + E: UsesState, + EM: EventFirer, + Z: UsesState, +{ + type State = E::State; +} + +impl Stage for AflStatsStage +where + E: UsesState, + EM: EventFirer, + Z: UsesState, + E::State: HasImported + HasCorpus + HasMetadata, +{ + fn perform( + &mut self, + _fuzzer: &mut Z, + _executor: &mut E, + state: &mut E::State, + _manager: &mut EM, + corpus_idx: CorpusId, + ) -> Result<(), Error> { + // Report your stats every `STATS_REPORT_INTERVAL` + // compute pending, pending_favored, imported, own_finds + { + let testcase = state.corpus().get(corpus_idx)?.borrow(); + if testcase.scheduled_count() == 0 { + self.has_fuzzed_size += 1; + if testcase.has_metadata::() { + self.is_favored_size += 1; + } + } else { + return Ok(()); + } + } + + let corpus_size = state.corpus().count(); + let pending_size = corpus_size - self.has_fuzzed_size; + let pend_favored_size = corpus_size - self.is_favored_size; + self.imported_size = *state.imported(); + self.own_finds_size = corpus_size - self.imported_size; + + let cur = current_time(); + + if cur.checked_sub(self.last_report_time).unwrap_or_default() > self.stats_report_interval { + #[cfg(feature = "std")] + { + let json = json!({ + "pending":pending_size, + "pend_fav":pend_favored_size, + "own_finds":self.own_finds_size, + "imported":self.imported_size, + }); + _manager.fire( + state, + Event::UpdateUserStats { + name: "AflStats".to_string(), + value: UserStats::String(json.to_string()), + phantom: PhantomData, + }, + )?; + } + #[cfg(not(feature = "std"))] + log::info!( + "pending: {}, pend_favored: {}, own_finds: {}, imported: {}", + pending_size, + pend_favored_size, + self.own_finds_size, + self.imported_size + ); + self.last_report_time = cur; + } + + Ok(()) + } +} + +impl AflStatsStage +where + E: UsesState, + EM: EventFirer, + Z: UsesState, + E::State: HasImported + HasCorpus + HasMetadata, +{ + /// create a new instance of the [`AflStatsStage`] + #[must_use] + pub fn new(interval: Duration) -> Self { + Self { + stats_report_interval: interval, + ..Default::default() + } + } +} + +impl Default for AflStatsStage +where + E: UsesState, + EM: EventFirer, + Z: UsesState, + E::State: HasImported + HasCorpus + HasMetadata, +{ + /// the default instance of the [`AflStatsStage`] + #[must_use] + fn default() -> Self { + Self { + has_fuzzed_size: 0, + is_favored_size: 0, + own_finds_size: 0, + imported_size: 0, + last_report_time: current_time(), + stats_report_interval: Duration::from_secs(15), + phantom: PhantomData, + } + } +} diff --git a/libafl/src/state/mod.rs b/libafl/src/state/mod.rs index 6c31991d38..a78e932bad 100644 --- a/libafl/src/state/mod.rs +++ b/libafl/src/state/mod.rs @@ -210,6 +210,15 @@ pub trait HasExecutions { fn executions_mut(&mut self) -> &mut usize; } +/// Trait for some stats of AFL +pub trait HasImported { + ///the imported testcases counter + fn imported(&self) -> &usize; + + ///the imported testcases counter (mutable) + fn imported_mut(&mut self) -> &mut usize; +} + /// Trait for the starting time pub trait HasStartTime { /// The starting time @@ -244,6 +253,8 @@ pub struct StdState { executions: usize, /// At what time the fuzzing started start_time: Duration, + /// the number of new paths that imported from other fuzzers + imported: usize, /// The corpus corpus: C, // Solutions corpus @@ -407,6 +418,20 @@ impl HasExecutions for StdState { } } +impl HasImported for StdState { + /// Return the number of new paths that imported from other fuzzers + #[inline] + fn imported(&self) -> &usize { + &self.imported + } + + /// Return the number of new paths that imported from other fuzzers + #[inline] + fn imported_mut(&mut self) -> &mut usize { + &mut self.imported + } +} + impl HasLastReportTime for StdState { /// The last time we reported progress,if available/used. /// This information is used by fuzzer `maybe_report_progress`. @@ -812,6 +837,7 @@ where let mut state = Self { rand, executions: 0, + imported: 0, start_time: Duration::from_millis(0), metadata: SerdeAnyMap::default(), named_metadata: NamedSerdeAnyMap::default(),