Remove MapObserver dependency from observer-dependent stages and schedulers in favour of generic hashing (#2851)

* Introdue SimpleHash separate from MapObserver

* Move to Hash for hashing

* Fix docs, remove even more restrictions

* fix libafl_targets

* fix fuzzer

* Remove broken and unnecessary derive

* Remove unnecessary trait restriction

* Remove unnecessary import

* Add changes to MIGRATION.md

* Remove more unnecessary imports
This commit is contained in:
Valentin Huber 2025-01-16 17:34:58 +01:00 committed by GitHub
parent 15aa498d5e
commit 93c5adde4d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 73 additions and 125 deletions

View File

@ -16,6 +16,8 @@
- For the structs/traits that used to use `UsesState`, we bring back the generic for the state.
- For `UsesState`, you can access to the input type through `HasCorpus` and `Corpus` traits
- The `State` trait is now private in favour of individual and more specific traits
- Restrictions from certain schedulers and stages that required their inner observer to implement `MapObserver` have been lifted in favor of requiring `Hash`
- Related: removed `hash_simple` from `MapObserver`
# 0.14.0 -> 0.14.1
- Removed `with_observers` from `Executor` trait.

View File

@ -39,7 +39,7 @@ pub fn main() -> Result<(), Error> {
#[allow(static_mut_refs)] // only a problem in nightly
let observer = unsafe { StdMapObserver::from_mut_ptr("signals", SIGNALS_PTR, SIGNALS.len()) };
let factory = MapEqualityFactory::new(&observer);
let factory = ObserverEqualityFactory::new(&observer);
// Feedback to rate the interestingness of an input
let mut feedback = MaxMapFeedback::new(&observer);

View File

@ -8,7 +8,6 @@ use core::{
ptr::NonNull,
};
use ahash::RandomState;
use libafl_bolts::{ownedref::OwnedMutSizedSlice, HasLen, Named};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
@ -111,11 +110,6 @@ where
self.len()
}
#[inline]
fn hash_simple(&self) -> u64 {
RandomState::with_seeds(0, 0, 0, 0).hash_one(self)
}
/// Reset the map
#[inline]
fn reset_map(&mut self) -> Result<(), Error> {

View File

@ -221,11 +221,6 @@ where
self.base.reset_map()
}
#[inline]
fn hash_simple(&self) -> u64 {
self.base.hash_simple()
}
fn to_vec(&self) -> Vec<u8> {
self.base.to_vec()
}
@ -443,10 +438,6 @@ where
self.base.reset_map()
}
#[inline]
fn hash_simple(&self) -> u64 {
self.base.hash_simple()
}
fn to_vec(&self) -> Vec<u8> {
self.base.to_vec()
}

View File

@ -7,7 +7,6 @@ use core::{
ops::{Deref, DerefMut},
};
use ahash::RandomState;
use libafl_bolts::{ownedref::OwnedMutSlice, AsSlice, AsSliceMut, HasLen, Named, Truncate};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
@ -350,7 +349,7 @@ pub mod macros {
///
/// TODO: enforce `iter() -> AssociatedTypeIter` when generic associated types stabilize
pub trait MapObserver:
HasLen + Named + Serialize + DeserializeOwned + AsRef<Self> + AsMut<Self> + Hash
HasLen + Named + Serialize + DeserializeOwned + AsRef<Self> + AsMut<Self>
// where
// for<'it> &'it Self: IntoIterator<Item = &'it Self::Entry>
{
@ -369,9 +368,6 @@ pub trait MapObserver:
/// Count the set bytes in the map
fn count_bytes(&self) -> u64;
/// Compute the hash of the map without needing to provide a hasher
fn hash_simple(&self) -> u64;
/// Get the initial value for `reset()`
fn initial(&self) -> Self::Entry;
@ -526,11 +522,6 @@ where
self.as_slice().len()
}
#[inline]
fn hash_simple(&self) -> u64 {
RandomState::with_seeds(0, 0, 0, 0).hash_one(self)
}
#[inline]
fn initial(&self) -> T {
self.initial

View File

@ -8,7 +8,6 @@ use core::{
slice::{Iter, IterMut},
};
use ahash::RandomState;
use libafl_bolts::{
ownedref::OwnedMutSlice, AsIter, AsIterMut, AsSlice, AsSliceMut, HasLen, Named,
};
@ -124,11 +123,6 @@ where
res
}
#[inline]
fn hash_simple(&self) -> u64 {
RandomState::with_seeds(0, 0, 0, 0).hash_one(self)
}
fn reset_map(&mut self) -> Result<(), Error> {
let initial = self.initial();
for map in &mut self.maps {

View File

@ -7,7 +7,6 @@ use core::{
ops::{Deref, DerefMut},
};
use ahash::RandomState;
use libafl_bolts::{AsSlice, AsSliceMut, HasLen, Named};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
@ -105,11 +104,6 @@ where
self.as_slice().len()
}
#[inline]
fn hash_simple(&self) -> u64 {
RandomState::with_seeds(0, 0, 0, 0).hash_one(self)
}
#[inline]
fn initial(&self) -> T {
self.initial

View File

@ -7,7 +7,6 @@ use core::{
ops::{Deref, DerefMut},
};
use ahash::RandomState;
use libafl_bolts::{
ownedref::{OwnedMutPtr, OwnedMutSlice},
AsSlice, AsSliceMut, HasLen, Named,
@ -113,11 +112,6 @@ where
res
}
#[inline]
fn hash_simple(&self) -> u64 {
RandomState::with_seeds(0, 0, 0, 0).hash_one(self)
}
/// Reset the map
#[inline]
fn reset_map(&mut self) -> Result<(), Error> {

View File

@ -12,9 +12,8 @@ use ahash::RandomState;
use libafl_bolts::{ownedref::OwnedRef, AsIter, AsIterMut, AsSlice, AsSliceMut, HasLen, Named};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use super::Observer;
use crate::{
observers::{MapObserver, ObserverWithHashField},
observers::{MapObserver, Observer, ObserverWithHashField},
Error,
};
@ -289,6 +288,7 @@ impl<T> AsMut<Self> for RefCellValueObserver<'_, T> {
self
}
}
impl<T, A> MapObserver for RefCellValueObserver<'_, A>
where
T: PartialEq + Copy + Hash + Default + DeserializeOwned + Serialize + Debug,
@ -308,12 +308,6 @@ where
self.get_ref_mut()[idx] = val;
}
/// Panics if the contained value is already mutably borrowed (calls
/// [`RefCell::borrow`]).
fn hash_simple(&self) -> u64 {
RandomState::with_seeds(0, 0, 0, 0).hash_one(self)
}
/// Panics if the contained value is already mutably borrowed (calls
/// [`RefCell::borrow`]).
fn usable_count(&self) -> usize {

View File

@ -1,7 +1,7 @@
//! Schedule the access to the Corpus.
use alloc::{borrow::ToOwned, string::ToString};
use core::marker::PhantomData;
use core::{hash::Hash, marker::PhantomData};
pub mod testcase_score;
pub use testcase_score::{LenTimeMulTestcaseScore, TestcaseScore};
@ -28,6 +28,7 @@ pub use weighted::{StdWeightedScheduler, WeightedScheduler};
pub mod tuneable;
use libafl_bolts::{
generic_hash_std,
rands::Rand,
tuples::{Handle, MatchName, MatchNameRef},
};
@ -35,7 +36,6 @@ pub use tuneable::*;
use crate::{
corpus::{Corpus, CorpusId, HasTestcase, SchedulerTestcaseMetadata, Testcase},
observers::MapObserver,
random_corpus_id,
state::{HasCorpus, HasRand},
Error, HasMetadata,
@ -107,17 +107,17 @@ pub fn on_evaluation_metadata_default<CS, O, OT, S>(
) -> Result<(), Error>
where
CS: AflScheduler,
CS::MapObserverRef: AsRef<O>,
CS::ObserverRef: AsRef<O>,
S: HasMetadata,
O: MapObserver,
O: Hash,
OT: MatchName,
{
let observer = observers
.get(scheduler.map_observer_handle())
.ok_or_else(|| Error::key_not_found("MapObserver not found".to_string()))?
.get(scheduler.observer_handle())
.ok_or_else(|| Error::key_not_found("Observer not found".to_string()))?
.as_ref();
let mut hash = observer.hash_simple() as usize;
let mut hash = generic_hash_std(observer) as usize;
let psmeta = state.metadata_mut::<SchedulerMetadata>()?;
@ -153,8 +153,8 @@ where
/// Defines the common metadata operations for the AFL-style schedulers
pub trait AflScheduler {
/// The type of [`MapObserver`] that this scheduler will use as reference
type MapObserverRef;
/// The type of [`crate::observers::Observer`] that this scheduler will use as reference
type ObserverRef;
/// Return the last hash
fn last_hash(&self) -> usize;
@ -162,8 +162,8 @@ pub trait AflScheduler {
/// Set the last hash
fn set_last_hash(&mut self, value: usize);
/// Get the observer map observer name
fn map_observer_handle(&self) -> &Handle<Self::MapObserverRef>;
/// Get the observer handle
fn observer_handle(&self) -> &Handle<Self::ObserverRef>;
}
/// Trait for Schedulers which track queue cycles

View File

@ -1,7 +1,7 @@
//! The queue corpus scheduler for power schedules.
use alloc::vec::Vec;
use core::{marker::PhantomData, time::Duration};
use core::{hash::Hash, marker::PhantomData, time::Duration};
use libafl_bolts::{
tuples::{Handle, Handled, MatchName},
@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize};
use crate::{
corpus::{Corpus, CorpusId, HasTestcase, Testcase},
observers::MapObserver,
schedulers::{
on_add_metadata_default, on_evaluation_metadata_default, on_next_metadata_default,
AflScheduler, HasQueueCycles, RemovableScheduler, Scheduler,
@ -276,7 +275,7 @@ pub enum BaseSchedule {
pub struct PowerQueueScheduler<C, O> {
queue_cycles: u64,
strat: PowerSchedule,
map_observer_handle: Handle<C>,
observer_handle: Handle<C>,
last_hash: usize,
phantom: PhantomData<O>,
}
@ -304,7 +303,7 @@ impl<C, I, O, S> RemovableScheduler<I, S> for PowerQueueScheduler<C, O> {
}
impl<C, O> AflScheduler for PowerQueueScheduler<C, O> {
type MapObserverRef = C;
type ObserverRef = C;
fn last_hash(&self) -> usize {
self.last_hash
@ -314,8 +313,8 @@ impl<C, O> AflScheduler for PowerQueueScheduler<C, O> {
self.last_hash = hash;
}
fn map_observer_handle(&self) -> &Handle<C> {
&self.map_observer_handle
fn observer_handle(&self) -> &Handle<C> {
&self.observer_handle
}
}
@ -328,7 +327,7 @@ impl<C, O> HasQueueCycles for PowerQueueScheduler<C, O> {
impl<C, I, O, S> Scheduler<I, S> for PowerQueueScheduler<C, O>
where
S: HasCorpus + HasMetadata + HasTestcase,
O: MapObserver,
O: Hash,
C: AsRef<O>,
{
/// Called when a [`Testcase`] is added to the corpus
@ -383,12 +382,12 @@ where
impl<C, O> PowerQueueScheduler<C, O>
where
O: MapObserver,
O: Hash,
C: AsRef<O> + Named,
{
/// Create a new [`PowerQueueScheduler`]
#[must_use]
pub fn new<S>(state: &mut S, map_observer: &C, strat: PowerSchedule) -> Self
pub fn new<S>(state: &mut S, observer: &C, strat: PowerSchedule) -> Self
where
S: HasMetadata,
{
@ -398,7 +397,7 @@ where
PowerQueueScheduler {
queue_cycles: 0,
strat,
map_observer_handle: map_observer.handle(),
observer_handle: observer.handle(),
last_hash: 0,
phantom: PhantomData,
}

View File

@ -3,7 +3,7 @@
//! The queue corpus scheduler with weighted queue item selection [from AFL++](https://github.com/AFLplusplus/AFLplusplus/blob/1d4f1e48797c064ee71441ba555b29fc3f467983/src/afl-fuzz-queue.c#L32).
//! This queue corpus scheduler needs calibration stage.
use core::marker::PhantomData;
use core::{hash::Hash, marker::PhantomData};
use hashbrown::HashMap;
use libafl_bolts::{
@ -13,14 +13,12 @@ use libafl_bolts::{
};
use serde::{Deserialize, Serialize};
use super::powersched::PowerSchedule;
use crate::{
corpus::{Corpus, CorpusId, HasTestcase, Testcase},
observers::MapObserver,
random_corpus_id,
schedulers::{
on_add_metadata_default, on_evaluation_metadata_default, on_next_metadata_default,
powersched::{BaseSchedule, SchedulerMetadata},
powersched::{BaseSchedule, PowerSchedule, SchedulerMetadata},
testcase_score::{CorpusWeightTestcaseScore, TestcaseScore},
AflScheduler, HasQueueCycles, RemovableScheduler, Scheduler,
},
@ -101,7 +99,7 @@ libafl_bolts::impl_serdeany!(WeightedScheduleMetadata);
pub struct WeightedScheduler<C, F, O> {
table_invalidated: bool,
strat: Option<PowerSchedule>,
map_observer_handle: Handle<C>,
observer_handle: Handle<C>,
last_hash: usize,
queue_cycles: u64,
phantom: PhantomData<(F, O)>,
@ -115,16 +113,16 @@ where
{
/// Create a new [`WeightedScheduler`] without any power schedule
#[must_use]
pub fn new<S>(state: &mut S, map_observer: &C) -> Self
pub fn new<S>(state: &mut S, observer: &C) -> Self
where
S: HasMetadata,
{
Self::with_schedule(state, map_observer, None)
Self::with_schedule(state, observer, None)
}
/// Create a new [`WeightedScheduler`]
#[must_use]
pub fn with_schedule<S>(state: &mut S, map_observer: &C, strat: Option<PowerSchedule>) -> Self
pub fn with_schedule<S>(state: &mut S, observer: &C, strat: Option<PowerSchedule>) -> Self
where
S: HasMetadata,
{
@ -133,7 +131,7 @@ where
Self {
strat,
map_observer_handle: map_observer.handle(),
observer_handle: observer.handle(),
last_hash: 0,
queue_cycles: 0,
table_invalidated: true,
@ -284,7 +282,7 @@ impl<C, F, I, O, S> RemovableScheduler<I, S> for WeightedScheduler<C, F, O> {
}
impl<C, F, O> AflScheduler for WeightedScheduler<C, F, O> {
type MapObserverRef = C;
type ObserverRef = C;
fn last_hash(&self) -> usize {
self.last_hash
@ -294,8 +292,8 @@ impl<C, F, O> AflScheduler for WeightedScheduler<C, F, O> {
self.last_hash = hash;
}
fn map_observer_handle(&self) -> &Handle<C> {
&self.map_observer_handle
fn observer_handle(&self) -> &Handle<C> {
&self.observer_handle
}
}
@ -309,7 +307,7 @@ impl<C, F, O, S> Scheduler<<S::Corpus as Corpus>::Input, S> for WeightedSchedule
where
C: AsRef<O> + Named,
F: TestcaseScore<S>,
O: MapObserver,
O: Hash,
S: HasCorpus + HasMetadata + HasRand + HasTestcase,
{
/// Called when a [`Testcase`] is added to the corpus

View File

@ -4,9 +4,10 @@ use alloc::{
collections::binary_heap::BinaryHeap,
vec::Vec,
};
use core::{cmp::Ordering, fmt::Debug, marker::PhantomData, ops::Range};
use core::{cmp::Ordering, fmt::Debug, hash::Hash, marker::PhantomData, ops::Range};
use libafl_bolts::{
generic_hash_std,
rands::Rand,
tuples::{Handle, Handled},
Named,
@ -20,7 +21,7 @@ use crate::{
inputs::{HasMutatorBytes, HasMutatorResizableBytes},
mutators::mutations::buffer_copy,
nonzero,
observers::{MapObserver, ObserversTuple},
observers::ObserversTuple,
stages::{RetryCountRestartHelper, Stage},
state::{HasCorpus, HasCurrentTestcase, HasRand},
Error, HasMetadata, HasNamedMetadata,
@ -81,7 +82,7 @@ where
S: HasCorpus + HasMetadata + HasRand + HasNamedMetadata + HasCurrentCorpusId,
E::Observers: ObserversTuple<<S::Corpus as Corpus>::Input, S>,
<S::Corpus as Corpus>::Input: HasMutatorResizableBytes + Clone,
O: MapObserver,
O: Hash,
C: AsRef<O> + Named,
{
#[inline]
@ -152,7 +153,7 @@ libafl_bolts::impl_serdeany!(TaintMetadata);
impl<C, E, EM, O, S, Z> ColorizationStage<C, E, EM, O, S, Z>
where
EM: EventFirer<<S::Corpus as Corpus>::Input, S>,
O: MapObserver,
O: Hash,
C: AsRef<O> + Named,
E: HasObservers + Executor<EM, <S::Corpus as Corpus>::Input, S, Z>,
E::Observers: ObserversTuple<<S::Corpus as Corpus>::Input, S>,
@ -317,7 +318,7 @@ where
let observers = executor.observers();
let observer = observers[observer_handle].as_ref();
let hash = observer.hash_simple() as usize;
let hash = generic_hash_std(observer) as usize;
executor
.observers_mut()

View File

@ -37,7 +37,7 @@ use serde::{Deserialize, Serialize};
pub use sync::*;
#[cfg(feature = "std")]
pub use time_tracker::TimeTrackingStageWrapper;
pub use tmin::{MapEqualityFactory, MapEqualityFeedback, StdTMinMutationalStage};
pub use tmin::{ObserverEqualityFactory, ObserverEqualityFeedback, StdTMinMutationalStage};
pub use tracing::{ShadowTracingStage, TracingStage};
pub use tuneable::*;
use tuple_list::NonEmptyTuple;

View File

@ -8,6 +8,7 @@ use core::{borrow::BorrowMut, fmt::Debug, hash::Hash, marker::PhantomData};
use ahash::RandomState;
use libafl_bolts::{
generic_hash_std,
tuples::{Handle, Handled, MatchName, MatchNameRef},
HasLen, Named,
};
@ -25,7 +26,7 @@ use crate::{
inputs::Input,
mark_feature_time,
mutators::{MutationResult, Mutator},
observers::{MapObserver, ObserversTuple},
observers::ObserversTuple,
schedulers::RemovableScheduler,
stages::{
mutational::{MutatedTransform, MutatedTransformPost},
@ -329,12 +330,12 @@ impl<E, EM, F, FF, M, S, Z> StdTMinMutationalStage<E, EM, F, FF, M, S, Z> {
}
}
/// A feedback which checks if the hash of the currently observed map is equal to the original hash
/// A feedback which checks if the hash of the current observed value is equal to the original hash
/// provided
#[derive(Clone, Debug)]
pub struct MapEqualityFeedback<C, M, S> {
pub struct ObserverEqualityFeedback<C, M, S> {
name: Cow<'static, str>,
map_ref: Handle<C>,
observer_handle: Handle<C>,
orig_hash: u64,
#[cfg(feature = "track_hit_feedbacks")]
// The previous run's result of `Self::is_interesting`
@ -342,25 +343,25 @@ pub struct MapEqualityFeedback<C, M, S> {
phantom: PhantomData<(M, S)>,
}
impl<C, M, S> Named for MapEqualityFeedback<C, M, S> {
impl<C, M, S> Named for ObserverEqualityFeedback<C, M, S> {
fn name(&self) -> &Cow<'static, str> {
&self.name
}
}
impl<C, M, S> HasObserverHandle for MapEqualityFeedback<C, M, S> {
impl<C, M, S> HasObserverHandle for ObserverEqualityFeedback<C, M, S> {
type Observer = C;
fn observer_handle(&self) -> &Handle<Self::Observer> {
&self.map_ref
&self.observer_handle
}
}
impl<C, M, S> StateInitializer<S> for MapEqualityFeedback<C, M, S> {}
impl<C, M, S> StateInitializer<S> for ObserverEqualityFeedback<C, M, S> {}
impl<C, EM, I, M, OT, S> Feedback<EM, I, OT, S> for MapEqualityFeedback<C, M, S>
impl<C, EM, I, M, OT, S> Feedback<EM, I, OT, S> for ObserverEqualityFeedback<C, M, S>
where
M: MapObserver,
M: Hash,
C: AsRef<M>,
OT: MatchName,
{
@ -375,7 +376,7 @@ where
let obs = observers
.get(self.observer_handle())
.expect("Should have been provided valid observer name.");
let res = obs.as_ref().hash_simple() == self.orig_hash;
let res = generic_hash_std(obs.as_ref()) == self.orig_hash;
#[cfg(feature = "track_hit_feedbacks")]
{
self.last_result = Some(res);
@ -388,50 +389,51 @@ where
}
}
/// A feedback factory for ensuring that the maps for minimized inputs are the same
/// A feedback factory for ensuring that the values of the observers for minimized inputs are the same
#[derive(Debug, Clone)]
pub struct MapEqualityFactory<C, M, S> {
map_ref: Handle<C>,
pub struct ObserverEqualityFactory<C, M, S> {
observer_handle: Handle<C>,
phantom: PhantomData<(C, M, S)>,
}
impl<C, M, S> MapEqualityFactory<C, M, S>
impl<C, M, S> ObserverEqualityFactory<C, M, S>
where
M: MapObserver,
M: Hash,
C: AsRef<M> + Handled,
{
/// Creates a new map equality feedback for the given observer
/// Creates a new observer equality feedback for the given observer
pub fn new(obs: &C) -> Self {
Self {
map_ref: obs.handle(),
observer_handle: obs.handle(),
phantom: PhantomData,
}
}
}
impl<C, M, S> HasObserverHandle for MapEqualityFactory<C, M, S> {
impl<C, M, S> HasObserverHandle for ObserverEqualityFactory<C, M, S> {
type Observer = C;
fn observer_handle(&self) -> &Handle<C> {
&self.map_ref
&self.observer_handle
}
}
impl<C, M, OT, S> FeedbackFactory<MapEqualityFeedback<C, M, S>, OT> for MapEqualityFactory<C, M, S>
impl<C, M, OT, S> FeedbackFactory<ObserverEqualityFeedback<C, M, S>, OT>
for ObserverEqualityFactory<C, M, S>
where
M: MapObserver,
M: Hash,
C: AsRef<M> + Handled,
OT: ObserversTuple<<S::Corpus as Corpus>::Input, S>,
S: HasCorpus,
{
fn create_feedback(&self, observers: &OT) -> MapEqualityFeedback<C, M, S> {
fn create_feedback(&self, observers: &OT) -> ObserverEqualityFeedback<C, M, S> {
let obs = observers
.get(self.observer_handle())
.expect("Should have been provided valid observer name.");
MapEqualityFeedback {
name: Cow::from("MapEq"),
map_ref: obs.handle(),
orig_hash: obs.as_ref().hash_simple(),
ObserverEqualityFeedback {
name: Cow::from("ObserverEq"),
observer_handle: obs.handle(),
orig_hash: generic_hash_std(obs.as_ref()),
#[cfg(feature = "track_hit_feedbacks")]
last_result: None,
phantom: PhantomData,

View File

@ -83,7 +83,6 @@ mod observers {
slice::{from_raw_parts, Iter, IterMut},
};
use ahash::RandomState;
use libafl::{
observers::{DifferentialObserver, MapObserver, Observer},
Error,
@ -230,11 +229,6 @@ mod observers {
res
}
#[inline]
fn hash_simple(&self) -> u64 {
RandomState::with_seeds(0, 0, 0, 0).hash_one(self)
}
fn reset_map(&mut self) -> Result<(), Error> {
let initial = self.initial();
for map in unsafe { &mut *counter_maps_ptr_mut() } {