diff --git a/MIGRATION.md b/MIGRATION.md index c01a48b788..af6972e069 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -16,6 +16,8 @@ - For the structs/traits that used to use `UsesState`, we bring back the generic for the state. - For `UsesState`, you can access to the input type through `HasCorpus` and `Corpus` traits - The `State` trait is now private in favour of individual and more specific traits +- Restrictions from certain schedulers and stages that required their inner observer to implement `MapObserver` have been lifted in favor of requiring `Hash` + - Related: removed `hash_simple` from `MapObserver` # 0.14.0 -> 0.14.1 - Removed `with_observers` from `Executor` trait. diff --git a/fuzzers/baby/baby_fuzzer_minimizing/src/main.rs b/fuzzers/baby/baby_fuzzer_minimizing/src/main.rs index 79524c2ecb..344206b466 100644 --- a/fuzzers/baby/baby_fuzzer_minimizing/src/main.rs +++ b/fuzzers/baby/baby_fuzzer_minimizing/src/main.rs @@ -39,7 +39,7 @@ pub fn main() -> Result<(), Error> { #[allow(static_mut_refs)] // only a problem in nightly let observer = unsafe { StdMapObserver::from_mut_ptr("signals", SIGNALS_PTR, SIGNALS.len()) }; - let factory = MapEqualityFactory::new(&observer); + let factory = ObserverEqualityFactory::new(&observer); // Feedback to rate the interestingness of an input let mut feedback = MaxMapFeedback::new(&observer); diff --git a/libafl/src/observers/map/const_map.rs b/libafl/src/observers/map/const_map.rs index ba2fe4ccce..10e2cee368 100644 --- a/libafl/src/observers/map/const_map.rs +++ b/libafl/src/observers/map/const_map.rs @@ -8,7 +8,6 @@ use core::{ ptr::NonNull, }; -use ahash::RandomState; use libafl_bolts::{ownedref::OwnedMutSizedSlice, HasLen, Named}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -111,11 +110,6 @@ where self.len() } - #[inline] - fn hash_simple(&self) -> u64 { - RandomState::with_seeds(0, 0, 0, 0).hash_one(self) - } - /// Reset the map #[inline] fn reset_map(&mut self) -> Result<(), Error> { diff --git a/libafl/src/observers/map/hitcount_map.rs b/libafl/src/observers/map/hitcount_map.rs index 1e369b0882..7eddb77723 100644 --- a/libafl/src/observers/map/hitcount_map.rs +++ b/libafl/src/observers/map/hitcount_map.rs @@ -221,11 +221,6 @@ where self.base.reset_map() } - #[inline] - fn hash_simple(&self) -> u64 { - self.base.hash_simple() - } - fn to_vec(&self) -> Vec { self.base.to_vec() } @@ -443,10 +438,6 @@ where self.base.reset_map() } - #[inline] - fn hash_simple(&self) -> u64 { - self.base.hash_simple() - } fn to_vec(&self) -> Vec { self.base.to_vec() } diff --git a/libafl/src/observers/map/mod.rs b/libafl/src/observers/map/mod.rs index 3d85a867bf..8ee79ac92c 100644 --- a/libafl/src/observers/map/mod.rs +++ b/libafl/src/observers/map/mod.rs @@ -7,7 +7,6 @@ use core::{ ops::{Deref, DerefMut}, }; -use ahash::RandomState; use libafl_bolts::{ownedref::OwnedMutSlice, AsSlice, AsSliceMut, HasLen, Named, Truncate}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -350,7 +349,7 @@ pub mod macros { /// /// TODO: enforce `iter() -> AssociatedTypeIter` when generic associated types stabilize pub trait MapObserver: - HasLen + Named + Serialize + DeserializeOwned + AsRef + AsMut + Hash + HasLen + Named + Serialize + DeserializeOwned + AsRef + AsMut // where // for<'it> &'it Self: IntoIterator { @@ -369,9 +368,6 @@ pub trait MapObserver: /// Count the set bytes in the map fn count_bytes(&self) -> u64; - /// Compute the hash of the map without needing to provide a hasher - fn hash_simple(&self) -> u64; - /// Get the initial value for `reset()` fn initial(&self) -> Self::Entry; @@ -526,11 +522,6 @@ where self.as_slice().len() } - #[inline] - fn hash_simple(&self) -> u64 { - RandomState::with_seeds(0, 0, 0, 0).hash_one(self) - } - #[inline] fn initial(&self) -> T { self.initial diff --git a/libafl/src/observers/map/multi_map.rs b/libafl/src/observers/map/multi_map.rs index ce7a04f786..bb002f9235 100644 --- a/libafl/src/observers/map/multi_map.rs +++ b/libafl/src/observers/map/multi_map.rs @@ -8,7 +8,6 @@ use core::{ slice::{Iter, IterMut}, }; -use ahash::RandomState; use libafl_bolts::{ ownedref::OwnedMutSlice, AsIter, AsIterMut, AsSlice, AsSliceMut, HasLen, Named, }; @@ -124,11 +123,6 @@ where res } - #[inline] - fn hash_simple(&self) -> u64 { - RandomState::with_seeds(0, 0, 0, 0).hash_one(self) - } - fn reset_map(&mut self) -> Result<(), Error> { let initial = self.initial(); for map in &mut self.maps { diff --git a/libafl/src/observers/map/owned_map.rs b/libafl/src/observers/map/owned_map.rs index 4ade2a948f..6689201098 100644 --- a/libafl/src/observers/map/owned_map.rs +++ b/libafl/src/observers/map/owned_map.rs @@ -7,7 +7,6 @@ use core::{ ops::{Deref, DerefMut}, }; -use ahash::RandomState; use libafl_bolts::{AsSlice, AsSliceMut, HasLen, Named}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -105,11 +104,6 @@ where self.as_slice().len() } - #[inline] - fn hash_simple(&self) -> u64 { - RandomState::with_seeds(0, 0, 0, 0).hash_one(self) - } - #[inline] fn initial(&self) -> T { self.initial diff --git a/libafl/src/observers/map/variable_map.rs b/libafl/src/observers/map/variable_map.rs index d98cca17dc..cfd3cf21eb 100644 --- a/libafl/src/observers/map/variable_map.rs +++ b/libafl/src/observers/map/variable_map.rs @@ -7,7 +7,6 @@ use core::{ ops::{Deref, DerefMut}, }; -use ahash::RandomState; use libafl_bolts::{ ownedref::{OwnedMutPtr, OwnedMutSlice}, AsSlice, AsSliceMut, HasLen, Named, @@ -113,11 +112,6 @@ where res } - #[inline] - fn hash_simple(&self) -> u64 { - RandomState::with_seeds(0, 0, 0, 0).hash_one(self) - } - /// Reset the map #[inline] fn reset_map(&mut self) -> Result<(), Error> { diff --git a/libafl/src/observers/value.rs b/libafl/src/observers/value.rs index 37d0a35960..8db0bc9a58 100644 --- a/libafl/src/observers/value.rs +++ b/libafl/src/observers/value.rs @@ -12,9 +12,8 @@ use ahash::RandomState; use libafl_bolts::{ownedref::OwnedRef, AsIter, AsIterMut, AsSlice, AsSliceMut, HasLen, Named}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use super::Observer; use crate::{ - observers::{MapObserver, ObserverWithHashField}, + observers::{MapObserver, Observer, ObserverWithHashField}, Error, }; @@ -289,6 +288,7 @@ impl AsMut for RefCellValueObserver<'_, T> { self } } + impl MapObserver for RefCellValueObserver<'_, A> where T: PartialEq + Copy + Hash + Default + DeserializeOwned + Serialize + Debug, @@ -308,12 +308,6 @@ where self.get_ref_mut()[idx] = val; } - /// Panics if the contained value is already mutably borrowed (calls - /// [`RefCell::borrow`]). - fn hash_simple(&self) -> u64 { - RandomState::with_seeds(0, 0, 0, 0).hash_one(self) - } - /// Panics if the contained value is already mutably borrowed (calls /// [`RefCell::borrow`]). fn usable_count(&self) -> usize { diff --git a/libafl/src/schedulers/mod.rs b/libafl/src/schedulers/mod.rs index 252da33b18..8d6add0f28 100644 --- a/libafl/src/schedulers/mod.rs +++ b/libafl/src/schedulers/mod.rs @@ -1,7 +1,7 @@ //! Schedule the access to the Corpus. use alloc::{borrow::ToOwned, string::ToString}; -use core::marker::PhantomData; +use core::{hash::Hash, marker::PhantomData}; pub mod testcase_score; pub use testcase_score::{LenTimeMulTestcaseScore, TestcaseScore}; @@ -28,6 +28,7 @@ pub use weighted::{StdWeightedScheduler, WeightedScheduler}; pub mod tuneable; use libafl_bolts::{ + generic_hash_std, rands::Rand, tuples::{Handle, MatchName, MatchNameRef}, }; @@ -35,7 +36,6 @@ pub use tuneable::*; use crate::{ corpus::{Corpus, CorpusId, HasTestcase, SchedulerTestcaseMetadata, Testcase}, - observers::MapObserver, random_corpus_id, state::{HasCorpus, HasRand}, Error, HasMetadata, @@ -107,17 +107,17 @@ pub fn on_evaluation_metadata_default( ) -> Result<(), Error> where CS: AflScheduler, - CS::MapObserverRef: AsRef, + CS::ObserverRef: AsRef, S: HasMetadata, - O: MapObserver, + O: Hash, OT: MatchName, { let observer = observers - .get(scheduler.map_observer_handle()) - .ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))? + .get(scheduler.observer_handle()) + .ok_or_else(|| Error::key_not_found("Observer not found".to_string()))? .as_ref(); - let mut hash = observer.hash_simple() as usize; + let mut hash = generic_hash_std(observer) as usize; let psmeta = state.metadata_mut::()?; @@ -153,8 +153,8 @@ where /// Defines the common metadata operations for the AFL-style schedulers pub trait AflScheduler { - /// The type of [`MapObserver`] that this scheduler will use as reference - type MapObserverRef; + /// The type of [`crate::observers::Observer`] that this scheduler will use as reference + type ObserverRef; /// Return the last hash fn last_hash(&self) -> usize; @@ -162,8 +162,8 @@ pub trait AflScheduler { /// Set the last hash fn set_last_hash(&mut self, value: usize); - /// Get the observer map observer name - fn map_observer_handle(&self) -> &Handle; + /// Get the observer handle + fn observer_handle(&self) -> &Handle; } /// Trait for Schedulers which track queue cycles diff --git a/libafl/src/schedulers/powersched.rs b/libafl/src/schedulers/powersched.rs index 91280b6b91..28a382954a 100644 --- a/libafl/src/schedulers/powersched.rs +++ b/libafl/src/schedulers/powersched.rs @@ -1,7 +1,7 @@ //! The queue corpus scheduler for power schedules. use alloc::vec::Vec; -use core::{marker::PhantomData, time::Duration}; +use core::{hash::Hash, marker::PhantomData, time::Duration}; use libafl_bolts::{ tuples::{Handle, Handled, MatchName}, @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use crate::{ corpus::{Corpus, CorpusId, HasTestcase, Testcase}, - observers::MapObserver, schedulers::{ on_add_metadata_default, on_evaluation_metadata_default, on_next_metadata_default, AflScheduler, HasQueueCycles, RemovableScheduler, Scheduler, @@ -276,7 +275,7 @@ pub enum BaseSchedule { pub struct PowerQueueScheduler { queue_cycles: u64, strat: PowerSchedule, - map_observer_handle: Handle, + observer_handle: Handle, last_hash: usize, phantom: PhantomData, } @@ -304,7 +303,7 @@ impl RemovableScheduler for PowerQueueScheduler { } impl AflScheduler for PowerQueueScheduler { - type MapObserverRef = C; + type ObserverRef = C; fn last_hash(&self) -> usize { self.last_hash @@ -314,8 +313,8 @@ impl AflScheduler for PowerQueueScheduler { self.last_hash = hash; } - fn map_observer_handle(&self) -> &Handle { - &self.map_observer_handle + fn observer_handle(&self) -> &Handle { + &self.observer_handle } } @@ -328,7 +327,7 @@ impl HasQueueCycles for PowerQueueScheduler { impl Scheduler for PowerQueueScheduler where S: HasCorpus + HasMetadata + HasTestcase, - O: MapObserver, + O: Hash, C: AsRef, { /// Called when a [`Testcase`] is added to the corpus @@ -383,12 +382,12 @@ where impl PowerQueueScheduler where - O: MapObserver, + O: Hash, C: AsRef + Named, { /// Create a new [`PowerQueueScheduler`] #[must_use] - pub fn new(state: &mut S, map_observer: &C, strat: PowerSchedule) -> Self + pub fn new(state: &mut S, observer: &C, strat: PowerSchedule) -> Self where S: HasMetadata, { @@ -398,7 +397,7 @@ where PowerQueueScheduler { queue_cycles: 0, strat, - map_observer_handle: map_observer.handle(), + observer_handle: observer.handle(), last_hash: 0, phantom: PhantomData, } diff --git a/libafl/src/schedulers/weighted.rs b/libafl/src/schedulers/weighted.rs index 38d14e9b60..c4a3720ae0 100644 --- a/libafl/src/schedulers/weighted.rs +++ b/libafl/src/schedulers/weighted.rs @@ -3,7 +3,7 @@ //! The queue corpus scheduler with weighted queue item selection [from AFL++](https://github.com/AFLplusplus/AFLplusplus/blob/1d4f1e48797c064ee71441ba555b29fc3f467983/src/afl-fuzz-queue.c#L32). //! This queue corpus scheduler needs calibration stage. -use core::marker::PhantomData; +use core::{hash::Hash, marker::PhantomData}; use hashbrown::HashMap; use libafl_bolts::{ @@ -13,14 +13,12 @@ use libafl_bolts::{ }; use serde::{Deserialize, Serialize}; -use super::powersched::PowerSchedule; use crate::{ corpus::{Corpus, CorpusId, HasTestcase, Testcase}, - observers::MapObserver, random_corpus_id, schedulers::{ on_add_metadata_default, on_evaluation_metadata_default, on_next_metadata_default, - powersched::{BaseSchedule, SchedulerMetadata}, + powersched::{BaseSchedule, PowerSchedule, SchedulerMetadata}, testcase_score::{CorpusWeightTestcaseScore, TestcaseScore}, AflScheduler, HasQueueCycles, RemovableScheduler, Scheduler, }, @@ -101,7 +99,7 @@ libafl_bolts::impl_serdeany!(WeightedScheduleMetadata); pub struct WeightedScheduler { table_invalidated: bool, strat: Option, - map_observer_handle: Handle, + observer_handle: Handle, last_hash: usize, queue_cycles: u64, phantom: PhantomData<(F, O)>, @@ -115,16 +113,16 @@ where { /// Create a new [`WeightedScheduler`] without any power schedule #[must_use] - pub fn new(state: &mut S, map_observer: &C) -> Self + pub fn new(state: &mut S, observer: &C) -> Self where S: HasMetadata, { - Self::with_schedule(state, map_observer, None) + Self::with_schedule(state, observer, None) } /// Create a new [`WeightedScheduler`] #[must_use] - pub fn with_schedule(state: &mut S, map_observer: &C, strat: Option) -> Self + pub fn with_schedule(state: &mut S, observer: &C, strat: Option) -> Self where S: HasMetadata, { @@ -133,7 +131,7 @@ where Self { strat, - map_observer_handle: map_observer.handle(), + observer_handle: observer.handle(), last_hash: 0, queue_cycles: 0, table_invalidated: true, @@ -284,7 +282,7 @@ impl RemovableScheduler for WeightedScheduler { } impl AflScheduler for WeightedScheduler { - type MapObserverRef = C; + type ObserverRef = C; fn last_hash(&self) -> usize { self.last_hash @@ -294,8 +292,8 @@ impl AflScheduler for WeightedScheduler { self.last_hash = hash; } - fn map_observer_handle(&self) -> &Handle { - &self.map_observer_handle + fn observer_handle(&self) -> &Handle { + &self.observer_handle } } @@ -309,7 +307,7 @@ impl Scheduler<::Input, S> for WeightedSchedule where C: AsRef + Named, F: TestcaseScore, - O: MapObserver, + O: Hash, S: HasCorpus + HasMetadata + HasRand + HasTestcase, { /// Called when a [`Testcase`] is added to the corpus diff --git a/libafl/src/stages/colorization.rs b/libafl/src/stages/colorization.rs index 6c06e10dce..bccc4a35e9 100644 --- a/libafl/src/stages/colorization.rs +++ b/libafl/src/stages/colorization.rs @@ -4,9 +4,10 @@ use alloc::{ collections::binary_heap::BinaryHeap, vec::Vec, }; -use core::{cmp::Ordering, fmt::Debug, marker::PhantomData, ops::Range}; +use core::{cmp::Ordering, fmt::Debug, hash::Hash, marker::PhantomData, ops::Range}; use libafl_bolts::{ + generic_hash_std, rands::Rand, tuples::{Handle, Handled}, Named, @@ -20,7 +21,7 @@ use crate::{ inputs::{HasMutatorBytes, HasMutatorResizableBytes}, mutators::mutations::buffer_copy, nonzero, - observers::{MapObserver, ObserversTuple}, + observers::ObserversTuple, stages::{RetryCountRestartHelper, Stage}, state::{HasCorpus, HasCurrentTestcase, HasRand}, Error, HasMetadata, HasNamedMetadata, @@ -81,7 +82,7 @@ where S: HasCorpus + HasMetadata + HasRand + HasNamedMetadata + HasCurrentCorpusId, E::Observers: ObserversTuple<::Input, S>, ::Input: HasMutatorResizableBytes + Clone, - O: MapObserver, + O: Hash, C: AsRef + Named, { #[inline] @@ -152,7 +153,7 @@ libafl_bolts::impl_serdeany!(TaintMetadata); impl ColorizationStage where EM: EventFirer<::Input, S>, - O: MapObserver, + O: Hash, C: AsRef + Named, E: HasObservers + Executor::Input, S, Z>, E::Observers: ObserversTuple<::Input, S>, @@ -317,7 +318,7 @@ where let observers = executor.observers(); let observer = observers[observer_handle].as_ref(); - let hash = observer.hash_simple() as usize; + let hash = generic_hash_std(observer) as usize; executor .observers_mut() diff --git a/libafl/src/stages/mod.rs b/libafl/src/stages/mod.rs index 8bf7cd4fb3..208b8f941b 100644 --- a/libafl/src/stages/mod.rs +++ b/libafl/src/stages/mod.rs @@ -37,7 +37,7 @@ use serde::{Deserialize, Serialize}; pub use sync::*; #[cfg(feature = "std")] pub use time_tracker::TimeTrackingStageWrapper; -pub use tmin::{MapEqualityFactory, MapEqualityFeedback, StdTMinMutationalStage}; +pub use tmin::{ObserverEqualityFactory, ObserverEqualityFeedback, StdTMinMutationalStage}; pub use tracing::{ShadowTracingStage, TracingStage}; pub use tuneable::*; use tuple_list::NonEmptyTuple; diff --git a/libafl/src/stages/tmin.rs b/libafl/src/stages/tmin.rs index 0303f37a25..259149dbf3 100644 --- a/libafl/src/stages/tmin.rs +++ b/libafl/src/stages/tmin.rs @@ -8,6 +8,7 @@ use core::{borrow::BorrowMut, fmt::Debug, hash::Hash, marker::PhantomData}; use ahash::RandomState; use libafl_bolts::{ + generic_hash_std, tuples::{Handle, Handled, MatchName, MatchNameRef}, HasLen, Named, }; @@ -25,7 +26,7 @@ use crate::{ inputs::Input, mark_feature_time, mutators::{MutationResult, Mutator}, - observers::{MapObserver, ObserversTuple}, + observers::ObserversTuple, schedulers::RemovableScheduler, stages::{ mutational::{MutatedTransform, MutatedTransformPost}, @@ -329,12 +330,12 @@ impl StdTMinMutationalStage { } } -/// A feedback which checks if the hash of the currently observed map is equal to the original hash +/// A feedback which checks if the hash of the current observed value is equal to the original hash /// provided #[derive(Clone, Debug)] -pub struct MapEqualityFeedback { +pub struct ObserverEqualityFeedback { name: Cow<'static, str>, - map_ref: Handle, + observer_handle: Handle, orig_hash: u64, #[cfg(feature = "track_hit_feedbacks")] // The previous run's result of `Self::is_interesting` @@ -342,25 +343,25 @@ pub struct MapEqualityFeedback { phantom: PhantomData<(M, S)>, } -impl Named for MapEqualityFeedback { +impl Named for ObserverEqualityFeedback { fn name(&self) -> &Cow<'static, str> { &self.name } } -impl HasObserverHandle for MapEqualityFeedback { +impl HasObserverHandle for ObserverEqualityFeedback { type Observer = C; fn observer_handle(&self) -> &Handle { - &self.map_ref + &self.observer_handle } } -impl StateInitializer for MapEqualityFeedback {} +impl StateInitializer for ObserverEqualityFeedback {} -impl Feedback for MapEqualityFeedback +impl Feedback for ObserverEqualityFeedback where - M: MapObserver, + M: Hash, C: AsRef, OT: MatchName, { @@ -375,7 +376,7 @@ where let obs = observers .get(self.observer_handle()) .expect("Should have been provided valid observer name."); - let res = obs.as_ref().hash_simple() == self.orig_hash; + let res = generic_hash_std(obs.as_ref()) == self.orig_hash; #[cfg(feature = "track_hit_feedbacks")] { self.last_result = Some(res); @@ -388,50 +389,51 @@ where } } -/// A feedback factory for ensuring that the maps for minimized inputs are the same +/// A feedback factory for ensuring that the values of the observers for minimized inputs are the same #[derive(Debug, Clone)] -pub struct MapEqualityFactory { - map_ref: Handle, +pub struct ObserverEqualityFactory { + observer_handle: Handle, phantom: PhantomData<(C, M, S)>, } -impl MapEqualityFactory +impl ObserverEqualityFactory where - M: MapObserver, + M: Hash, C: AsRef + Handled, { - /// Creates a new map equality feedback for the given observer + /// Creates a new observer equality feedback for the given observer pub fn new(obs: &C) -> Self { Self { - map_ref: obs.handle(), + observer_handle: obs.handle(), phantom: PhantomData, } } } -impl HasObserverHandle for MapEqualityFactory { +impl HasObserverHandle for ObserverEqualityFactory { type Observer = C; fn observer_handle(&self) -> &Handle { - &self.map_ref + &self.observer_handle } } -impl FeedbackFactory, OT> for MapEqualityFactory +impl FeedbackFactory, OT> + for ObserverEqualityFactory where - M: MapObserver, + M: Hash, C: AsRef + Handled, OT: ObserversTuple<::Input, S>, S: HasCorpus, { - fn create_feedback(&self, observers: &OT) -> MapEqualityFeedback { + fn create_feedback(&self, observers: &OT) -> ObserverEqualityFeedback { let obs = observers .get(self.observer_handle()) .expect("Should have been provided valid observer name."); - MapEqualityFeedback { - name: Cow::from("MapEq"), - map_ref: obs.handle(), - orig_hash: obs.as_ref().hash_simple(), + ObserverEqualityFeedback { + name: Cow::from("ObserverEq"), + observer_handle: obs.handle(), + orig_hash: generic_hash_std(obs.as_ref()), #[cfg(feature = "track_hit_feedbacks")] last_result: None, phantom: PhantomData, diff --git a/libafl_targets/src/sancov_8bit.rs b/libafl_targets/src/sancov_8bit.rs index ae1e916af8..e15f311a4c 100644 --- a/libafl_targets/src/sancov_8bit.rs +++ b/libafl_targets/src/sancov_8bit.rs @@ -83,7 +83,6 @@ mod observers { slice::{from_raw_parts, Iter, IterMut}, }; - use ahash::RandomState; use libafl::{ observers::{DifferentialObserver, MapObserver, Observer}, Error, @@ -230,11 +229,6 @@ mod observers { res } - #[inline] - fn hash_simple(&self) -> u64 { - RandomState::with_seeds(0, 0, 0, 0).hash_one(self) - } - fn reset_map(&mut self) -> Result<(), Error> { let initial = self.initial(); for map in unsafe { &mut *counter_maps_ptr_mut() } {