Stability improve (#773)

* initial

* add

* fmt & fix

* dbg remove

* clp

* clp

* more

* clippy

* del

* fix

* remove unused

* fix

* doc

fuzzers/fuzzbench_qemu/Makefile.toml

@@ -97,4 +97,4 @@ script='''
 rm -f ./${FUZZER_NAME} libfuzzer_main.o
 make -C libpng-1.6.37 clean
 cargo clean
-'''
+'''

libafl/src/events/mod.rs

@@ -22,7 +22,8 @@ use crate::{
     inputs::Input,
     monitors::UserStats,
     observers::ObserversTuple,
-    state::{HasClientPerfMonitor, HasExecutions},
+    stages::calibrate::UnstableEntriesMetadata,
+    state::{HasClientPerfMonitor, HasExecutions, HasMetadata},
     Error,
 };
 
@@ -360,7 +361,7 @@ where
         monitor_timeout: Duration,
     ) -> Result<Duration, Error>
     where
-        S: HasExecutions + HasClientPerfMonitor,
+        S: HasExecutions + HasClientPerfMonitor + HasMetadata,
     {
         let executions = *state.executions();
         let cur = current_time();
@@ -377,13 +378,15 @@ where
                 },
             )?;
 
-            if let Some(x) = state.stability() {
+            /// Send the stability event to the broker
-                let stability = f64::from(*x);
+            if let Some(meta) = state.metadata().get::<UnstableEntriesMetadata>() {
+                let unstable_entries = meta.unstable_entries().len();
+                let map_len = meta.map_len();
                 self.fire(
                     state,
                     Event::UpdateUserStats {
                         name: "stability".to_string(),
-                        value: UserStats::Float(stability),
+                        value: UserStats::Ratio(unstable_entries as u64, map_len as u64),
                         phantom: PhantomData,
                     },
                 )?;

libafl/src/feedbacks/differential.rs

@@ -224,14 +224,6 @@ mod tests {
         fn introspection_monitor_mut(&mut self) -> &mut ClientPerfMonitor {
             unimplemented!()
         }
-
-        fn stability(&self) -> &Option<f32> {
-            unimplemented!()
-        }
-
-        fn stability_mut(&mut self) -> &mut Option<f32> {
-            unimplemented!()
-        }
     }
 
     fn test_diff(should_equal: bool) {

libafl/src/fuzzer/mod.rs

@@ -17,7 +17,7 @@ use crate::{
     schedulers::Scheduler,
     stages::StagesTuple,
     start_timer,
-    state::{HasClientPerfMonitor, HasCorpus, HasExecutions, HasSolutions},
+    state::{HasClientPerfMonitor, HasCorpus, HasExecutions, HasMetadata, HasSolutions},
     Error,
 };
 
@@ -150,7 +150,7 @@ pub trait Fuzzer<E, EM, I, S, ST>
 where
     I: Input,
     EM: ProgressReporter<I>,
-    S: HasExecutions + HasClientPerfMonitor,
+    S: HasExecutions + HasClientPerfMonitor + HasMetadata,
 {
     /// Fuzz for a single iteration.
     /// Returns the index of the last fuzzed corpus item.
@@ -519,7 +519,7 @@ where
     EM: EventManager<E, I, S, Self>,
     F: Feedback<I, S>,
     I: Input,
-    S: HasClientPerfMonitor + HasExecutions,
+    S: HasClientPerfMonitor + HasExecutions + HasMetadata,
     OF: Feedback<I, S>,
     ST: StagesTuple<E, EM, S, Self>,
 {

libafl/src/stages/calibrate.rs

@@ -1,8 +1,12 @@
 //! The calibration stage. The fuzzer measures the average exec time and the bitmap size.
 
-use alloc::string::{String, ToString};
+use alloc::{
+    string::{String, ToString},
+    vec::Vec,
+};
 use core::{fmt::Debug, marker::PhantomData, time::Duration};
 
+use hashbrown::HashSet;
 use num_traits::Bounded;
 use serde::{Deserialize, Serialize};
 
@@ -24,6 +28,39 @@ use crate::{
     Error,
 };
 
+crate::impl_serdeany!(UnstableEntriesMetadata);
+/// The metadata to keep unstable entries
+/// In libafl, the stability is the number of the unstable entries divided by the size of the map
+/// This is different from AFL++, which shows the number of the unstable entries divided by the number of filled entries.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct UnstableEntriesMetadata {
+    unstable_entries: HashSet<usize>,
+    map_len: usize,
+}
+
+impl UnstableEntriesMetadata {
+    #[must_use]
+    /// Create a new [`struct@UnstableEntriesMetadata`]
+    pub fn new(entries: HashSet<usize>, map_len: usize) -> Self {
+        Self {
+            unstable_entries: entries,
+            map_len,
+        }
+    }
+
+    /// Getter
+    #[must_use]
+    pub fn unstable_entries(&self) -> &HashSet<usize> {
+        &self.unstable_entries
+    }
+
+    /// Getter
+    #[must_use]
+    pub fn map_len(&self) -> usize {
+        self.map_len
+    }
+}
+
 /// The calibration stage will measure the average exec time and the target's stability for this input.
 #[derive(Clone, Debug)]
 pub struct CalibrationStage<I, O, OT, S>
@@ -39,8 +76,8 @@ where
     phantom: PhantomData<(I, O, OT, S)>,
 }
 
-const CAL_STAGE_START: usize = 4;
+const CAL_STAGE_START: usize = 4; // AFL++'s CAL_CYCLES_FAST + 1
-const CAL_STAGE_MAX: usize = 16;
+const CAL_STAGE_MAX: usize = 8; // AFL++'s CAL_CYCLES + 1
 
 impl<E, EM, I, O, OT, S, Z> Stage<E, EM, S, Z> for CalibrationStage<I, O, OT, S>
 where
@@ -109,7 +146,7 @@ where
         // run is found to be unstable, with CAL_STAGE_MAX total runs.
         let mut i = 1;
         let mut has_errors = false;
-        let mut unstable_entries: usize = 0;
+        let mut unstable_entries: Vec<usize> = vec![];
         let map_len: usize = map_first.len();
         while i < iter {
             let input = state
@@ -161,24 +198,40 @@ where
                 history_map.resize(map_len, O::Entry::default());
             }
 
-            for (first, (cur, history)) in
+            for (idx, (first, (cur, history))) in map_first
-                map_first.iter().zip(map.iter().zip(history_map.iter_mut()))
+                .iter()
+                .zip(map.iter().zip(history_map.iter_mut()))
+                .enumerate()
             {
                 if *first != *cur && *history != O::Entry::max_value() {
                     *history = O::Entry::max_value();
-                    unstable_entries += 1;
+                    unstable_entries.push(idx);
                 };
             }
 
+            if !unstable_entries.is_empty() && iter < CAL_STAGE_MAX {
+                iter += 2;
+            }
             i += 1;
         }
 
         #[allow(clippy::cast_precision_loss)]
-        if unstable_entries != 0 {
+        if !unstable_entries.is_empty() {
-            *state.stability_mut() = Some((map_len - unstable_entries) as f32 / (map_len as f32));
+            // If we see new stable entries executing this new corpus entries, then merge with the existing one
-
+            if state.has_metadata::<UnstableEntriesMetadata>() {
-            if iter < CAL_STAGE_MAX {
+                let existing = state
-                iter += 2;
+                    .metadata_mut()
+                    .get_mut::<UnstableEntriesMetadata>()
+                    .unwrap();
+                for item in unstable_entries {
+                    existing.unstable_entries.insert(item); // Insert newly found items
+                }
+                existing.map_len = map_len;
+            } else {
+                state.add_metadata::<UnstableEntriesMetadata>(UnstableEntriesMetadata::new(
+                    HashSet::from_iter(unstable_entries),
+                    map_len,
+                ));
             }
         };
 

libafl/src/stages/mod.rs

@@ -51,7 +51,7 @@ use crate::{
     inputs::Input,
     observers::ObserversTuple,
     schedulers::Scheduler,
-    state::{HasClientPerfMonitor, HasCorpus, HasExecutions, HasRand},
+    state::{HasClientPerfMonitor, HasCorpus, HasExecutions, HasMetadata, HasRand},
     Error, EvaluatorObservers, ExecutesInput, ExecutionProcessor, HasScheduler,
 };
 
@@ -179,7 +179,7 @@ where
     I: Input,
     OT: ObserversTuple<I, S>,
     PS: PushStage<CS, EM, I, OT, S, Z>,
-    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions,
+    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions + HasMetadata,
     Z: ExecutionProcessor<I, OT, S> + EvaluatorObservers<I, OT, S> + HasScheduler<CS, I, S>,
 {
     push_stage: PS,
@@ -193,7 +193,7 @@ where
     I: Input,
     OT: ObserversTuple<I, S>,
     PS: PushStage<CS, EM, I, OT, S, Z>,
-    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions,
+    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions + HasMetadata,
     Z: ExecutionProcessor<I, OT, S> + EvaluatorObservers<I, OT, S> + HasScheduler<CS, I, S>,
 {
     /// Create a new [`PushStageAdapter`], wrapping the given [`PushStage`]
@@ -215,7 +215,7 @@ where
     I: Input,
     OT: ObserversTuple<I, S>,
     PS: PushStage<CS, EM, I, OT, S, Z>,
-    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions,
+    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions + HasMetadata,
     Z: ExecutesInput<I, OT, S, Z>
         + ExecutionProcessor<I, OT, S>
         + EvaluatorObservers<I, OT, S>

libafl/src/stages/push/mod.rs

@@ -22,7 +22,7 @@ use crate::{
     inputs::Input,
     observers::ObserversTuple,
     schedulers::Scheduler,
-    state::{HasClientPerfMonitor, HasCorpus, HasExecutions, HasRand},
+    state::{HasClientPerfMonitor, HasCorpus, HasExecutions, HasMetadata, HasRand},
     Error, EvaluatorObservers, ExecutionProcessor, HasScheduler,
 };
 
@@ -186,7 +186,7 @@ where
     EM: EventFirer<I> + EventRestarter<S> + HasEventManagerId + ProgressReporter<I>,
     I: Input,
     OT: ObserversTuple<I, S>,
-    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions,
+    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions + HasMetadata,
     Z: ExecutionProcessor<I, OT, S> + EvaluatorObservers<I, OT, S> + HasScheduler<CS, I, S>,
 {
     /// Gets the [`PushStageHelper`]

libafl/src/stages/push/mutational.rs

@@ -18,7 +18,7 @@ use crate::{
     observers::ObserversTuple,
     schedulers::Scheduler,
     start_timer,
-    state::{HasClientPerfMonitor, HasCorpus, HasExecutions, HasRand},
+    state::{HasClientPerfMonitor, HasCorpus, HasExecutions, HasMetadata, HasRand},
     Error, EvaluatorObservers, ExecutionProcessor, HasScheduler,
 };
 
@@ -85,7 +85,7 @@ where
     I: Input,
     M: Mutator<I, S>,
     OT: ObserversTuple<I, S>,
-    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions,
+    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions + HasMetadata,
     Z: ExecutionProcessor<I, OT, S> + EvaluatorObservers<I, OT, S> + HasScheduler<CS, I, S>,
 {
     /// Creates a new default mutational stage
@@ -196,7 +196,7 @@ where
     I: Input,
     M: Mutator<I, S>,
     OT: ObserversTuple<I, S>,
-    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions,
+    S: HasClientPerfMonitor + HasCorpus<I> + HasRand + HasExecutions + HasMetadata,
     Z: ExecutionProcessor<I, OT, S> + EvaluatorObservers<I, OT, S> + HasScheduler<CS, I, S>,
 {
     type Item = Result<I, Error>;

libafl/src/state/mod.rs

@@ -77,12 +77,6 @@ pub trait HasClientPerfMonitor {
 
     /// Mutatable ref to [`ClientPerfMonitor`]
     fn introspection_monitor_mut(&mut self) -> &mut ClientPerfMonitor;
-
-    /// This node's stability
-    fn stability(&self) -> &Option<f32>;
-
-    /// This node's stability (mutable)
-    fn stability_mut(&mut self) -> &mut Option<f32>;
 }
 
 /// Trait for elements offering metadata
@@ -181,9 +175,6 @@ where
     named_metadata: NamedSerdeAnyMap,
     /// MaxSize testcase size for mutators that appreciate it
     max_size: usize,
-    /// The stability of the current fuzzing process
-    stability: Option<f32>,
-
     /// Performance statistics for this fuzzer
     #[cfg(feature = "introspection")]
     introspection_monitor: ClientPerfMonitor,
@@ -577,7 +568,6 @@ where
         let mut state = Self {
             rand,
             executions: 0,
-            stability: None,
             start_time: Duration::from_millis(0),
             metadata: SerdeAnyMap::default(),
             named_metadata: NamedSerdeAnyMap::default(),
@@ -609,18 +599,6 @@ where
     fn introspection_monitor_mut(&mut self) -> &mut ClientPerfMonitor {
         &mut self.introspection_monitor
     }
-
-    /// This node's stability
-    #[inline]
-    fn stability(&self) -> &Option<f32> {
-        &self.stability
-    }
-
-    /// This node's stability (mutable)
-    #[inline]
-    fn stability_mut(&mut self) -> &mut Option<f32> {
-        &mut self.stability
-    }
 }
 
 #[cfg(not(feature = "introspection"))]
@@ -638,18 +616,6 @@ where
     fn introspection_monitor_mut(&mut self) -> &mut ClientPerfMonitor {
         unimplemented!()
     }
-
-    /// This node's stability
-    #[inline]
-    fn stability(&self) -> &Option<f32> {
-        &self.stability
-    }
-
-    /// This node's stability (mutable)
-    #[inline]
-    fn stability_mut(&mut self) -> &mut Option<f32> {
-        &mut self.stability
-    }
 }
 
 #[cfg(feature = "python")]

libafl_targets/src/cmplog.rs

@@ -155,6 +155,7 @@ impl CmpMap for CmpLogMap {
 
 /// The global `CmpLog` map for the current `LibAFL` run.
 #[no_mangle]
+#[allow(clippy::large_stack_arrays)]
 pub static mut libafl_cmplog_map: CmpLogMap = CmpLogMap {
     headers: [CmpLogHeader {
         hits: 0,