diff --git a/libafl/Cargo.toml b/libafl/Cargo.toml index 6b26cff722..4c506badb0 100644 --- a/libafl/Cargo.toml +++ b/libafl/Cargo.toml @@ -53,9 +53,6 @@ gzip = ["libafl_bolts/gzip"] ## If set, will use the `fork()` syscall to spawn children, instead of launching a new command, if supported by the OS (has no effect on `Windows`). fork = ["libafl_bolts/derive"] -## Collected stats to decide if observers must be serialized or not (which should reduce mem use and increase speed) -adaptive_serialization = [] - ## If this feature is set, `LibAFL` targets (and the fuzzer) will crash on `SIGPIPE` on unix systems. handle_sigpipe = [] diff --git a/libafl/src/events/centralized.rs b/libafl/src/events/centralized.rs index f99496db02..b9ce3055ee 100644 --- a/libafl/src/events/centralized.rs +++ b/libafl/src/events/centralized.rs @@ -8,12 +8,8 @@ // 4. The "main broker", the gathers the stats from the fuzzer clients and broadcast the newly found testcases from the main evaluator. use alloc::{boxed::Box, string::String, vec::Vec}; -use core::fmt::Debug; -#[cfg(feature = "adaptive_serialization")] -use core::time::Duration; +use core::{fmt::Debug, time::Duration}; -#[cfg(feature = "adaptive_serialization")] -use libafl_bolts::tuples::{Handle, Handled}; #[cfg(feature = "llmp_compression")] use libafl_bolts::{ compress::GzipCompressor, @@ -22,6 +18,7 @@ use libafl_bolts::{ use libafl_bolts::{ llmp::{LlmpClient, LlmpClientDescription, Tag}, shmem::{NopShMemProvider, ShMemProvider}, + tuples::Handle, ClientId, }; use serde::{Deserialize, Serialize}; @@ -29,8 +26,6 @@ use serde::{Deserialize, Serialize}; use super::NopEventManager; #[cfg(feature = "llmp_compression")] use crate::events::llmp::COMPRESS_THRESHOLD; -#[cfg(feature = "adaptive_serialization")] -use crate::observers::TimeObserver; #[cfg(feature = "scalability_introspection")] use crate::state::HasScalabilityMonitor; use crate::{ @@ -42,7 +37,7 @@ use crate::{ executors::{Executor, HasObservers}, fuzzer::{EvaluatorObservers, ExecutionProcessor}, inputs::{Input, NopInput, UsesInput}, - observers::ObserversTuple, + observers::{ObserversTuple, TimeObserver}, state::{HasExecutions, HasLastReportTime, NopState, UsesState}, Error, HasMetadata, }; @@ -61,8 +56,7 @@ where client: LlmpClient, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor, - #[cfg(feature = "adaptive_serialization")] - time_ref: Handle, + time_ref: Option>, is_main: bool, } @@ -99,12 +93,12 @@ impl CentralizedEventManagerBuilder { Self { is_main } } - /// Creates a new [`CentralizedEventManager`]. - #[cfg(not(feature = "adaptive_serialization"))] + /// Creates a new `CentralizedEventManager` pub fn build_from_client( self, inner: EM, client: LlmpClient, + time_obs: Option>, ) -> Result, Error> where SP: ShMemProvider, @@ -115,28 +109,7 @@ impl CentralizedEventManagerBuilder { client, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - is_main: self.is_main, - }) - } - - /// Creates a new [`CentralizedEventManager`]. - #[cfg(feature = "adaptive_serialization")] - pub fn build_from_client( - self, - inner: EM, - client: LlmpClient, - time_obs: &TimeObserver, - ) -> Result, Error> - where - SP: ShMemProvider, - EM: UsesState, - { - Ok(CentralizedEventManager { - inner, - client, - #[cfg(feature = "llmp_compression")] - compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - time_ref: time_obs.handle(), + time_ref: time_obs, is_main: self.is_main, }) } @@ -145,12 +118,13 @@ impl CentralizedEventManagerBuilder { /// /// If the port is not yet bound, it will act as a broker; otherwise, it /// will act as a client. - #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] + #[cfg(feature = "std")] pub fn build_on_port( self, inner: EM, shmem_provider: SP, port: u16, + time_obs: Option>, ) -> Result, Error> where SP: ShMemProvider, @@ -162,45 +136,20 @@ impl CentralizedEventManagerBuilder { client, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - is_main: self.is_main, - }) - } - - /// Create a centralized event manager on a port - /// - /// If the port is not yet bound, it will act as a broker; otherwise, it - /// will act as a client. - #[cfg(all(feature = "std", feature = "adaptive_serialization"))] - pub fn build_on_port( - self, - inner: EM, - shmem_provider: SP, - port: u16, - time_obs: &TimeObserver, - ) -> Result, Error> - where - SP: ShMemProvider, - EM: UsesState, - { - let client = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; - Ok(CentralizedEventManager { - inner, - client, - #[cfg(feature = "llmp_compression")] - compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - time_ref: time_obs.handle(), + time_ref: time_obs, is_main: self.is_main, }) } /// If a client respawns, it may reuse the existing connection, previously /// stored by [`LlmpClient::to_env()`]. - #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] + #[cfg(feature = "std")] pub fn build_existing_client_from_env( self, inner: EM, shmem_provider: SP, env_name: &str, + time_obs: Option>, ) -> Result, Error> where EM: UsesState, @@ -211,41 +160,19 @@ impl CentralizedEventManagerBuilder { client: LlmpClient::on_existing_from_env(shmem_provider, env_name)?, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - is_main: self.is_main, - }) - } - - /// If a client respawns, it may reuse the existing connection, previously - /// stored by [`LlmpClient::to_env()`]. - #[cfg(all(feature = "std", feature = "adaptive_serialization"))] - pub fn build_existing_client_from_env( - self, - inner: EM, - shmem_provider: SP, - env_name: &str, - time_obs: &TimeObserver, - ) -> Result, Error> - where - EM: UsesState, - SP: ShMemProvider, - { - Ok(CentralizedEventManager { - inner, - client: LlmpClient::on_existing_from_env(shmem_provider, env_name)?, - #[cfg(feature = "llmp_compression")] - compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - time_ref: time_obs.handle(), + time_ref: time_obs, is_main: self.is_main, }) } /// Create an existing client from description - #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] + #[cfg(feature = "std")] pub fn existing_client_from_description( self, inner: EM, shmem_provider: SP, description: &LlmpClientDescription, + time_obs: Option>, ) -> Result, Error> where EM: UsesState, @@ -256,29 +183,7 @@ impl CentralizedEventManagerBuilder { client: LlmpClient::existing_client_from_description(shmem_provider, description)?, #[cfg(feature = "llmp_compression")] compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - is_main: self.is_main, - }) - } - - /// Create an existing client from description - #[cfg(all(feature = "std", feature = "adaptive_serialization"))] - pub fn existing_client_from_description( - self, - inner: EM, - shmem_provider: SP, - description: &LlmpClientDescription, - time_obs: &TimeObserver, - ) -> Result, Error> - where - EM: UsesState, - SP: ShMemProvider, - { - Ok(CentralizedEventManager { - inner, - client: LlmpClient::existing_client_from_description(shmem_provider, description)?, - #[cfg(feature = "llmp_compression")] - compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - time_ref: time_obs.handle(), + time_ref: time_obs, is_main: self.is_main, }) } @@ -291,7 +196,6 @@ where type State = EM::State; } -#[cfg(feature = "adaptive_serialization")] impl AdaptiveSerializer for CentralizedEventManager where EM: AdaptiveSerializer + UsesState, @@ -323,19 +227,11 @@ where self.inner.should_serialize_cnt_mut() } - fn time_ref(&self) -> &Handle { + fn time_ref(&self) -> &Option> { &self.time_ref } } -#[cfg(not(feature = "adaptive_serialization"))] -impl AdaptiveSerializer for CentralizedEventManager -where - EM: AdaptiveSerializer + UsesState, - SP: ShMemProvider, -{ -} - impl EventFirer for CentralizedEventManager where EM: AdaptiveSerializer + EventFirer + HasEventManagerId, @@ -399,15 +295,6 @@ where self.inner.log(state, severity_level, message) } - #[cfg(not(feature = "adaptive_serialization"))] - fn serialize_observers(&mut self, observers: &OT) -> Result>, Error> - where - OT: ObserversTuple + Serialize, - { - Ok(Some(postcard::to_allocvec(observers)?)) - } - - #[cfg(feature = "adaptive_serialization")] fn serialize_observers(&mut self, observers: &OT) -> Result>, Error> where OT: ObserversTuple + Serialize, diff --git a/libafl/src/events/launcher.rs b/libafl/src/events/launcher.rs index 5a22261ffb..9ecfc978ee 100644 --- a/libafl/src/events/launcher.rs +++ b/libafl/src/events/launcher.rs @@ -34,15 +34,6 @@ use libafl_bolts::llmp::LlmpBroker; use libafl_bolts::os::dup2; #[cfg(all(feature = "std", any(windows, not(feature = "fork"))))] use libafl_bolts::os::startable_self; -#[cfg(feature = "adaptive_serialization")] -use libafl_bolts::tuples::Handle; -#[cfg(all( - unix, - feature = "std", - feature = "fork", - feature = "adaptive_serialization" -))] -use libafl_bolts::tuples::Handled; #[cfg(all(unix, feature = "std", feature = "fork"))] use libafl_bolts::{ core_affinity::get_core_ids, @@ -51,13 +42,12 @@ use libafl_bolts::{ use libafl_bolts::{ core_affinity::{CoreId, Cores}, shmem::ShMemProvider, - tuples::tuple_list, + tuples::{tuple_list, Handle}, }; #[cfg(feature = "std")] use typed_builder::TypedBuilder; use super::hooks::EventManagerHooksTuple; -#[cfg(feature = "adaptive_serialization")] use crate::observers::TimeObserver; #[cfg(all(unix, feature = "std", feature = "fork"))] use crate::{ @@ -131,8 +121,9 @@ pub struct Launcher<'a, CF, EMH, MT, S, SP> { /// clusters. #[builder(default = None)] remote_broker_addr: Option, - #[cfg(feature = "adaptive_serialization")] - time_ref: Handle, + /// The time observer for addaptive serialization + #[builder(default = None)] + time_ref: Option>, /// If this launcher should spawn a new `broker` on `[Self::broker_port]` (default). /// The reason you may not want this is, if you already have a [`Launcher`] /// with a different configuration (for the same target) running on this machine. @@ -282,7 +273,6 @@ where .configuration(self.configuration) .serialize_state(self.serialize_state) .hooks(hooks); - #[cfg(feature = "adaptive_serialization")] let builder = builder.time_ref(self.time_ref.clone()); let (state, mgr) = builder.build().launch()?; @@ -308,7 +298,6 @@ where .serialize_state(self.serialize_state) .hooks(hooks); - #[cfg(feature = "adaptive_serialization")] let builder = builder.time_ref(self.time_ref.clone()); builder.build().launch()?; @@ -360,7 +349,6 @@ where .serialize_state(self.serialize_state) .hooks(hooks); - #[cfg(feature = "adaptive_serialization")] let builder = builder.time_ref(self.time_ref.clone()); let (state, mgr) = builder.build().launch()?; @@ -455,7 +443,6 @@ where .serialize_state(self.serialize_state) .hooks(hooks); - #[cfg(feature = "adaptive_serialization")] let builder = builder.time_ref(self.time_ref.clone()); builder.build().launch()?; @@ -506,8 +493,8 @@ pub struct CentralizedLauncher<'a, CF, IM, MF, MT, S, SP> { #[builder(default = 1338_u16)] centralized_broker_port: u16, /// The time observer by which to adaptively serialize - #[cfg(feature = "adaptive_serialization")] - time_obs: &'a TimeObserver, + #[builder(default = None)] + time_obs: Option>, /// The list of cores to run on cores: &'a Cores, /// A file name to write all client output to @@ -597,8 +584,7 @@ where .serialize_state(centralized_launcher.serialize_state) .hooks(tuple_list!()); - #[cfg(feature = "adaptive_serialization")] - let builder = builder.time_ref(centralized_launcher.time_obs.handle()); + let builder = builder.time_ref(centralized_launcher.time_obs.clone()); builder.build().launch() }; @@ -738,18 +724,11 @@ where let mut centralized_builder = CentralizedEventManager::builder(); centralized_builder = centralized_builder.is_main(true); - #[cfg(not(feature = "adaptive_serialization"))] let c_mgr = centralized_builder.build_on_port( mgr, self.shmem_provider.clone(), self.centralized_broker_port, - )?; - #[cfg(feature = "adaptive_serialization")] - let c_mgr = centralized_builder.build_on_port( - mgr, - self.shmem_provider.clone(), - self.centralized_broker_port, - self.time_obs, + self.time_obs.clone(), )?; self.main_run_client.take().unwrap()(state, c_mgr, *bind_to) @@ -760,18 +739,11 @@ where let centralized_builder = CentralizedEventManager::builder(); - #[cfg(not(feature = "adaptive_serialization"))] let c_mgr = centralized_builder.build_on_port( mgr, self.shmem_provider.clone(), self.centralized_broker_port, - )?; - #[cfg(feature = "adaptive_serialization")] - let c_mgr = centralized_builder.build_on_port( - mgr, - self.shmem_provider.clone(), - self.centralized_broker_port, - self.time_obs, + self.time_obs.clone(), )?; self.secondary_run_client.take().unwrap()(state, c_mgr, *bind_to) @@ -796,8 +768,7 @@ where .serialize_state(self.serialize_state) .hooks(tuple_list!()); - #[cfg(feature = "adaptive_serialization")] - let builder = builder.time_ref(self.time_obs.handle()); + let builder = builder.time_ref(self.time_obs.clone()); builder.build().launch()?; diff --git a/libafl/src/events/llmp/mgr.rs b/libafl/src/events/llmp/mgr.rs index 25c7472112..20a44552c6 100644 --- a/libafl/src/events/llmp/mgr.rs +++ b/libafl/src/events/llmp/mgr.rs @@ -8,8 +8,6 @@ use core::{marker::PhantomData, time::Duration}; #[cfg(feature = "std")] use std::net::TcpStream; -#[cfg(feature = "adaptive_serialization")] -use libafl_bolts::tuples::Handle; #[cfg(feature = "llmp_compression")] use libafl_bolts::{ compress::GzipCompressor, @@ -19,6 +17,7 @@ use libafl_bolts::{ current_time, llmp::{LlmpClient, LlmpClientDescription}, shmem::{NopShMemProvider, ShMemProvider}, + tuples::Handle, ClientId, }; #[cfg(feature = "std")] @@ -30,22 +29,18 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "llmp_compression")] use crate::events::llmp::COMPRESS_THRESHOLD; -#[cfg(feature = "adaptive_serialization")] -use crate::events::AdaptiveSerializer; -#[cfg(feature = "adaptive_serialization")] -use crate::observers::TimeObserver; use crate::{ events::{ hooks::EventManagerHooksTuple, llmp::{LLMP_TAG_EVENT_TO_BOTH, _LLMP_TAG_EVENT_TO_BROKER}, - CustomBufEventResult, CustomBufHandlerFn, Event, EventConfig, EventFirer, EventManager, - EventManagerId, EventProcessor, EventRestarter, HasCustomBufHandlers, HasEventManagerId, - ProgressReporter, + AdaptiveSerializer, CustomBufEventResult, CustomBufHandlerFn, Event, EventConfig, + EventFirer, EventManager, EventManagerId, EventProcessor, EventRestarter, + HasCustomBufHandlers, HasEventManagerId, ProgressReporter, }, executors::{Executor, HasObservers}, fuzzer::{Evaluator, EvaluatorObservers, ExecutionProcessor}, inputs::{NopInput, UsesInput}, - observers::ObserversTuple, + observers::{ObserversTuple, TimeObserver}, state::{HasExecutions, HasLastReportTime, NopState, State, UsesState}, Error, HasMetadata, }; @@ -74,16 +69,11 @@ where /// A node will not re-use the observer values sent over LLMP /// from nodes with other configurations. configuration: EventConfig, - #[cfg(feature = "adaptive_serialization")] serialization_time: Duration, - #[cfg(feature = "adaptive_serialization")] deserialization_time: Duration, - #[cfg(feature = "adaptive_serialization")] serializations_cnt: usize, - #[cfg(feature = "adaptive_serialization")] should_serialize_cnt: usize, - #[cfg(feature = "adaptive_serialization")] - pub(crate) time_ref: Handle, + pub(crate) time_ref: Option>, phantom: PhantomData, } @@ -149,12 +139,11 @@ impl LlmpEventManagerBuilder { } /// Create a manager from a raw LLMP client - #[cfg(feature = "adaptive_serialization")] pub fn build_from_client( self, llmp: LlmpClient, configuration: EventConfig, - time_ref: Handle, + time_ref: Option>, ) -> Result, Error> where SP: ShMemProvider, @@ -179,42 +168,17 @@ impl LlmpEventManagerBuilder { }) } - /// Create a manager from a raw LLMP client - #[cfg(not(feature = "adaptive_serialization"))] - pub fn build_from_client( - self, - llmp: LlmpClient, - configuration: EventConfig, - ) -> Result, Error> - where - SP: ShMemProvider, - S: State, - { - Ok(LlmpEventManager { - throttle: self.throttle, - last_sent: Duration::from_secs(0), - hooks: self.hooks, - always_interesting: self.always_interesting, - llmp, - #[cfg(feature = "llmp_compression")] - compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - configuration, - phantom: PhantomData, - custom_buf_handlers: vec![], - }) - } - /// Create an LLMP event manager on a port /// /// If the port is not yet bound, it will act as a broker; otherwise, it /// will act as a client. - #[cfg(all(feature = "std", feature = "adaptive_serialization"))] + #[cfg(feature = "std")] pub fn build_on_port( self, shmem_provider: SP, port: u16, configuration: EventConfig, - time_ref: Handle, + time_ref: Option>, ) -> Result, Error> where SP: ShMemProvider, @@ -240,45 +204,15 @@ impl LlmpEventManagerBuilder { }) } - /// Create an LLMP event manager on a port - /// - /// If the port is not yet bound, it will act as a broker; otherwise, it - /// will act as a client. - #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] - pub fn build_on_port( - self, - shmem_provider: SP, - port: u16, - configuration: EventConfig, - ) -> Result, Error> - where - SP: ShMemProvider, - S: State, - { - let llmp = LlmpClient::create_attach_to_tcp(shmem_provider, port)?; - Ok(LlmpEventManager { - throttle: self.throttle, - last_sent: Duration::from_secs(0), - hooks: self.hooks, - always_interesting: self.always_interesting, - llmp, - #[cfg(feature = "llmp_compression")] - compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - configuration, - phantom: PhantomData, - custom_buf_handlers: vec![], - }) - } - /// If a client respawns, it may reuse the existing connection, previously /// stored by [`LlmpClient::to_env()`]. - #[cfg(all(feature = "std", feature = "adaptive_serialization"))] + #[cfg(feature = "std")] pub fn build_existing_client_from_env( self, shmem_provider: SP, env_name: &str, configuration: EventConfig, - time_ref: Handle, + time_ref: Option>, ) -> Result, Error> where SP: ShMemProvider, @@ -304,42 +238,13 @@ impl LlmpEventManagerBuilder { }) } - /// If a client respawns, it may reuse the existing connection, previously - /// stored by [`LlmpClient::to_env()`]. - #[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] - pub fn build_existing_client_from_env( - self, - shmem_provider: SP, - env_name: &str, - configuration: EventConfig, - ) -> Result, Error> - where - SP: ShMemProvider, - S: State, - { - let llmp = LlmpClient::on_existing_from_env(shmem_provider, env_name)?; - Ok(LlmpEventManager { - throttle: self.throttle, - last_sent: Duration::from_secs(0), - hooks: self.hooks, - always_interesting: self.always_interesting, - llmp, - #[cfg(feature = "llmp_compression")] - compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - configuration, - phantom: PhantomData, - custom_buf_handlers: vec![], - }) - } - /// Create an existing client from description - #[cfg(feature = "adaptive_serialization")] pub fn build_existing_client_from_description( self, shmem_provider: SP, description: &LlmpClientDescription, configuration: EventConfig, - time_ref: Handle, + time_ref: Option>, ) -> Result, Error> where SP: ShMemProvider, @@ -364,36 +269,8 @@ impl LlmpEventManagerBuilder { custom_buf_handlers: vec![], }) } - - /// Create an existing client from description - #[cfg(not(feature = "adaptive_serialization"))] - pub fn build_existing_client_from_description( - self, - shmem_provider: SP, - description: &LlmpClientDescription, - configuration: EventConfig, - ) -> Result, Error> - where - SP: ShMemProvider, - S: State, - { - let llmp = LlmpClient::existing_client_from_description(shmem_provider, description)?; - Ok(LlmpEventManager { - throttle: self.throttle, - last_sent: Duration::from_secs(0), - hooks: self.hooks, - always_interesting: self.always_interesting, - llmp, - #[cfg(feature = "llmp_compression")] - compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD), - configuration, - phantom: PhantomData, - custom_buf_handlers: vec![], - }) - } } -#[cfg(feature = "adaptive_serialization")] impl AdaptiveSerializer for LlmpEventManager where SP: ShMemProvider, @@ -425,7 +302,7 @@ where &mut self.should_serialize_cnt } - fn time_ref(&self) -> &Handle { + fn time_ref(&self) -> &Option> { &self.time_ref } } @@ -556,11 +433,9 @@ where let res = if client_config.match_with(&self.configuration) && observers_buf.is_some() { - #[cfg(feature = "adaptive_serialization")] let start = current_time(); let observers: E::Observers = postcard::from_bytes(observers_buf.as_ref().unwrap())?; - #[cfg(feature = "adaptive_serialization")] { self.deserialization_time = current_time() - start; } @@ -671,15 +546,6 @@ where Ok(()) } - #[cfg(not(feature = "adaptive_serialization"))] - fn serialize_observers(&mut self, observers: &OT) -> Result>, Error> - where - OT: ObserversTuple + Serialize, - { - Ok(Some(postcard::to_allocvec(observers)?)) - } - - #[cfg(feature = "adaptive_serialization")] fn serialize_observers(&mut self, observers: &OT) -> Result>, Error> where OT: ObserversTuple + Serialize, diff --git a/libafl/src/events/llmp/restarting.rs b/libafl/src/events/llmp/restarting.rs index b4af6ca040..989ed7e470 100644 --- a/libafl/src/events/llmp/restarting.rs +++ b/libafl/src/events/llmp/restarting.rs @@ -22,9 +22,11 @@ use libafl_bolts::os::startable_self; use libafl_bolts::os::unix_signals::setup_signal_handler; #[cfg(all(feature = "std", feature = "fork", unix))] use libafl_bolts::os::{fork, ForkResult}; -#[cfg(feature = "adaptive_serialization")] -use libafl_bolts::tuples::{Handle, Handled}; -use libafl_bolts::{llmp::LlmpBroker, shmem::ShMemProvider, tuples::tuple_list}; +use libafl_bolts::{ + llmp::LlmpBroker, + shmem::ShMemProvider, + tuples::{tuple_list, Handle}, +}; #[cfg(feature = "std")] use libafl_bolts::{ llmp::LlmpConnection, os::CTRL_C_EXIT, shmem::StdShMemProvider, staterestore::StateRestorer, @@ -33,14 +35,10 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use typed_builder::TypedBuilder; -#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] -use crate::events::AdaptiveSerializer; -#[cfg(all(feature = "std", feature = "adaptive_serialization"))] +#[cfg(feature = "std")] use crate::events::AdaptiveSerializer; #[cfg(all(unix, feature = "std", not(miri)))] use crate::events::EVENTMGR_SIGHANDLER_STATE; -#[cfg(feature = "adaptive_serialization")] -use crate::observers::TimeObserver; use crate::{ events::{ hooks::EventManagerHooksTuple, Event, EventConfig, EventFirer, EventManager, @@ -51,7 +49,7 @@ use crate::{ fuzzer::{Evaluator, EvaluatorObservers, ExecutionProcessor}, inputs::UsesInput, monitors::Monitor, - observers::ObserversTuple, + observers::{ObserversTuple, TimeObserver}, state::{HasExecutions, HasLastReportTime, State, UsesState}, Error, HasMetadata, }; @@ -73,7 +71,7 @@ where save_state: LlmpShouldSaveState, } -#[cfg(all(feature = "std", feature = "adaptive_serialization"))] +#[cfg(feature = "std")] impl AdaptiveSerializer for LlmpRestartingEventManager where SP: ShMemProvider, @@ -105,19 +103,11 @@ where self.llmp_mgr.should_serialize_cnt_mut() } - fn time_ref(&self) -> &Handle { + fn time_ref(&self) -> &Option> { &self.llmp_mgr.time_ref } } -#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] -impl AdaptiveSerializer for LlmpRestartingEventManager -where - SP: ShMemProvider, - S: State, -{ -} - #[cfg(feature = "std")] impl UsesState for LlmpRestartingEventManager where @@ -330,7 +320,7 @@ pub enum ManagerKind { /// Sets up a restarting fuzzer, using the [`StdShMemProvider`], and standard features. /// The restarting mgr is a combination of restarter and runner, that can be used on systems with and without `fork` support. /// The restarter will spawn a new process each time the child crashes or timeouts. -#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))] +#[cfg(feature = "std")] #[allow(clippy::type_complexity)] pub fn setup_restarting_mgr_std( monitor: MT, @@ -345,7 +335,7 @@ pub fn setup_restarting_mgr_std( > where MT: Monitor + Clone, - S: State + HasExecutions, + S: State, { RestartingMgr::builder() .shmem_provider(StdShMemProvider::new()?) @@ -360,13 +350,14 @@ where /// Sets up a restarting fuzzer, using the [`StdShMemProvider`], and standard features. /// The restarting mgr is a combination of restarter and runner, that can be used on systems with and without `fork` support. /// The restarter will spawn a new process each time the child crashes or timeouts. -#[cfg(all(feature = "std", feature = "adaptive_serialization"))] +/// This one, additionally uses the timeobserver for the adaptive serialization +#[cfg(feature = "std")] #[allow(clippy::type_complexity)] -pub fn setup_restarting_mgr_std( +pub fn setup_restarting_mgr_std_adaptive( monitor: MT, broker_port: u16, configuration: EventConfig, - time_obs: &TimeObserver, + time_obs: Handle, ) -> Result< ( Option, @@ -376,7 +367,7 @@ pub fn setup_restarting_mgr_std( > where MT: Monitor + Clone, - S: State + HasExecutions, + S: State, { RestartingMgr::builder() .shmem_provider(StdShMemProvider::new()?) @@ -384,7 +375,7 @@ where .broker_port(broker_port) .configuration(configuration) .hooks(tuple_list!()) - .time_ref(time_obs.handle()) + .time_ref(Some(time_obs)) .build() .launch() } @@ -436,8 +427,8 @@ where serialize_state: LlmpShouldSaveState, /// The hooks passed to event manager: hooks: EMH, - #[cfg(feature = "adaptive_serialization")] - time_ref: Handle, + #[builder(default = None)] + time_ref: Option>, #[builder(setter(skip), default = PhantomData)] phantom_data: PhantomData<(EMH, S)>, } @@ -448,7 +439,7 @@ impl RestartingMgr where EMH: EventManagerHooksTuple + Copy + Clone, SP: ShMemProvider, - S: State + HasExecutions, + S: State, MT: Monitor + Clone, { /// Launch the broker and the clients and fuzz @@ -500,12 +491,6 @@ where return Err(Error::shutting_down()); } LlmpConnection::IsClient { client } => { - #[cfg(not(feature = "adaptive_serialization"))] - let mgr: LlmpEventManager = LlmpEventManager::builder() - .always_interesting(self.always_interesting) - .hooks(self.hooks) - .build_from_client(client, self.configuration)?; - #[cfg(feature = "adaptive_serialization")] let mgr: LlmpEventManager = LlmpEventManager::builder() .always_interesting(self.always_interesting) .hooks(self.hooks) @@ -532,16 +517,6 @@ where } ManagerKind::Client { cpu_core } => { // We are a client - #[cfg(not(feature = "adaptive_serialization"))] - let mgr = LlmpEventManager::builder() - .always_interesting(self.always_interesting) - .hooks(self.hooks) - .build_on_port( - self.shmem_provider.clone(), - self.broker_port, - self.configuration, - )?; - #[cfg(feature = "adaptive_serialization")] let mgr = LlmpEventManager::builder() .always_interesting(self.always_interesting) .hooks(self.hooks) @@ -671,15 +646,6 @@ where // If we're restarting, deserialize the old state. let (state, mut mgr) = if let Some((state_opt, mgr_description)) = staterestorer.restore()? { - #[cfg(not(feature = "adaptive_serialization"))] - let llmp_mgr = LlmpEventManager::builder() - .hooks(self.hooks) - .build_existing_client_from_description( - new_shmem_provider, - &mgr_description, - self.configuration, - )?; - #[cfg(feature = "adaptive_serialization")] let llmp_mgr = LlmpEventManager::builder() .hooks(self.hooks) .build_existing_client_from_description( @@ -699,15 +665,6 @@ where } else { log::info!("First run. Let's set it all up"); // Mgr to send and receive msgs from/to all other fuzzer instances - #[cfg(not(feature = "adaptive_serialization"))] - let mgr = LlmpEventManager::builder() - .hooks(self.hooks) - .build_existing_client_from_env( - new_shmem_provider, - _ENV_FUZZER_BROKER_CLIENT_INITIAL, - self.configuration, - )?; - #[cfg(feature = "adaptive_serialization")] let mgr = LlmpEventManager::builder() .hooks(self.hooks) .build_existing_client_from_env( @@ -748,14 +705,12 @@ where mod tests { use core::sync::atomic::{compiler_fence, Ordering}; - #[cfg(feature = "adaptive_serialization")] - use libafl_bolts::tuples::Handled; use libafl_bolts::{ llmp::{LlmpClient, LlmpSharedMap}, rands::StdRand, shmem::{ShMemProvider, StdShMemProvider}, staterestore::StateRestorer, - tuples::tuple_list, + tuples::{tuple_list, Handled}, ClientId, }; use serial_test::serial; @@ -782,7 +737,6 @@ mod tests { let rand = StdRand::with_seed(0); let time = TimeObserver::new("time"); - #[cfg(feature = "adaptive_serialization")] let time_ref = time.handle(); let mut corpus = InMemoryCorpus::::new(); @@ -811,13 +765,8 @@ mod tests { llmp_client.mark_safe_to_unmap(); } - #[cfg(not(feature = "adaptive_serialization"))] let mut llmp_mgr = LlmpEventManager::builder() - .build_from_client(llmp_client, "fuzzer".into()) - .unwrap(); - #[cfg(feature = "adaptive_serialization")] - let mut llmp_mgr = LlmpEventManager::builder() - .build_from_client(llmp_client, "fuzzer".into(), time_ref.clone()) + .build_from_client(llmp_client, "fuzzer".into(), Some(time_ref.clone())) .unwrap(); let scheduler = RandScheduler::new(); @@ -860,21 +809,12 @@ mod tests { assert!(sc_cpy.has_content()); let (mut state_clone, mgr_description) = staterestorer.restore().unwrap().unwrap(); - #[cfg(not(feature = "adaptive_serialization"))] let mut llmp_clone = LlmpEventManager::builder() .build_existing_client_from_description( shmem_provider, &mgr_description, "fuzzer".into(), - ) - .unwrap(); - #[cfg(feature = "adaptive_serialization")] - let mut llmp_clone = LlmpEventManager::builder() - .build_existing_client_from_description( - shmem_provider, - &mgr_description, - "fuzzer".into(), - time_ref, + Some(time_ref), ) .unwrap(); diff --git a/libafl/src/events/mod.rs b/libafl/src/events/mod.rs index 4fb32f0480..bc7cedd108 100644 --- a/libafl/src/events/mod.rs +++ b/libafl/src/events/mod.rs @@ -32,9 +32,11 @@ use ahash::RandomState; pub use launcher::*; #[cfg(all(unix, feature = "std"))] use libafl_bolts::os::unix_signals::{siginfo_t, ucontext_t, Handler, Signal, CTRL_C_EXIT}; -#[cfg(feature = "adaptive_serialization")] -use libafl_bolts::tuples::{Handle, MatchNameRef}; -use libafl_bolts::{current_time, ClientId}; +use libafl_bolts::{ + current_time, + tuples::{Handle, MatchNameRef}, + ClientId, +}; use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use uuid::Uuid; @@ -104,9 +106,9 @@ pub struct EventManagerId( #[cfg(feature = "introspection")] use crate::monitors::ClientPerfMonitor; -#[cfg(feature = "adaptive_serialization")] -use crate::observers::TimeObserver; -use crate::{inputs::UsesInput, stages::HasCurrentStage, state::UsesState}; +use crate::{ + inputs::UsesInput, observers::TimeObserver, stages::HasCurrentStage, state::UsesState, +}; /// The log event severity #[derive(Serialize, Deserialize, Debug, Clone, Copy)] @@ -839,11 +841,6 @@ where } /// Collected stats to decide if observers must be serialized or not -#[cfg(not(feature = "adaptive_serialization"))] -pub trait AdaptiveSerializer {} - -/// Collected stats to decide if observers must be serialized or not -#[cfg(feature = "adaptive_serialization")] pub trait AdaptiveSerializer { /// Expose the collected observers serialization time fn serialization_time(&self) -> Duration; @@ -864,7 +861,7 @@ pub trait AdaptiveSerializer { fn should_serialize_cnt_mut(&mut self) -> &mut usize; /// A [`Handle`] to the time observer to determine the `time_factor` - fn time_ref(&self) -> &Handle; + fn time_ref(&self) -> &Option>; /// Serialize the observer using the `time_factor` and `percentage_threshold`. /// These parameters are unique to each of the different types of `EventManager` @@ -878,35 +875,41 @@ pub trait AdaptiveSerializer { OT: ObserversTuple + Serialize, S: UsesInput, { - let exec_time = observers - .get(self.time_ref()) - .map(|o| o.last_runtime().unwrap_or(Duration::ZERO)) - .unwrap(); + match self.time_ref() { + Some(t) => { + let exec_time = observers + .get(t) + .map(|o| o.last_runtime().unwrap_or(Duration::ZERO)) + .unwrap(); - let mut must_ser = - (self.serialization_time() + self.deserialization_time()) * time_factor < exec_time; - if must_ser { - *self.should_serialize_cnt_mut() += 1; - } + let mut must_ser = (self.serialization_time() + self.deserialization_time()) + * time_factor + < exec_time; + if must_ser { + *self.should_serialize_cnt_mut() += 1; + } - if self.serializations_cnt() > 32 { - must_ser = (self.should_serialize_cnt() * 100 / self.serializations_cnt()) - > percentage_threshold; - } + if self.serializations_cnt() > 32 { + must_ser = (self.should_serialize_cnt() * 100 / self.serializations_cnt()) + > percentage_threshold; + } - if self.serialization_time() == Duration::ZERO - || must_ser - || self.serializations_cnt().trailing_zeros() >= 8 - { - let start = current_time(); - let ser = postcard::to_allocvec(observers)?; - *self.serialization_time_mut() = current_time() - start; + if self.serialization_time() == Duration::ZERO + || must_ser + || self.serializations_cnt().trailing_zeros() >= 8 + { + let start = current_time(); + let ser = postcard::to_allocvec(observers)?; + *self.serialization_time_mut() = current_time() - start; - *self.serializations_cnt_mut() += 1; - Ok(Some(ser)) - } else { - *self.serializations_cnt_mut() += 1; - Ok(None) + *self.serializations_cnt_mut() += 1; + Ok(Some(ser)) + } else { + *self.serializations_cnt_mut() += 1; + Ok(None) + } + } + None => Ok(None), } } } diff --git a/libafl_frida/src/asan/hook_funcs.rs b/libafl_frida/src/asan/hook_funcs.rs index de1293aa92..c5b1f2efee 100644 --- a/libafl_frida/src/asan/hook_funcs.rs +++ b/libafl_frida/src/asan/hook_funcs.rs @@ -11,9 +11,12 @@ use crate::{ errors::{AsanError, AsanErrors}, }, }; + +#[cfg(windows)] extern "system" { fn memcpy(dst: *mut c_void, src: *const c_void, size: usize) -> *mut c_void; } +#[cfg(windows)] extern "system" { fn memset(s: *mut c_void, c: i32, n: usize) -> *mut c_void; } diff --git a/libafl_sugar/Cargo.toml b/libafl_sugar/Cargo.toml index 8815dd2990..372bd89f9b 100644 --- a/libafl_sugar/Cargo.toml +++ b/libafl_sugar/Cargo.toml @@ -48,7 +48,7 @@ hexagon = ["libafl_qemu/hexagon"] pyo3-build-config = { version = "0.21", optional = true } [dependencies] -libafl = { path = "../libafl", version = "0.12.0", features = ["adaptive_serialization"] } +libafl = { path = "../libafl", version = "0.12.0" } libafl_bolts = { path = "../libafl_bolts", version = "0.12.0" } libafl_targets = { path = "../libafl_targets", version = "0.12.0" } diff --git a/libafl_sugar/src/forkserver.rs b/libafl_sugar/src/forkserver.rs index 6560fe153f..9ac5574670 100644 --- a/libafl_sugar/src/forkserver.rs +++ b/libafl_sugar/src/forkserver.rs @@ -295,7 +295,7 @@ impl<'a> ForkserverBytesCoverageSugar<'a> { .cores(self.cores) .broker_port(self.broker_port) .remote_broker_addr(self.remote_broker_addr) - .time_ref(time_ref); + .time_ref(Some(time_ref)); #[cfg(unix)] let launcher = launcher.stdout_file(Some("/dev/null")); match launcher.build().launch() { diff --git a/libafl_sugar/src/inmemory.rs b/libafl_sugar/src/inmemory.rs index 7c7f8b5d20..cc05399f18 100644 --- a/libafl_sugar/src/inmemory.rs +++ b/libafl_sugar/src/inmemory.rs @@ -351,7 +351,7 @@ where .cores(self.cores) .broker_port(self.broker_port) .remote_broker_addr(self.remote_broker_addr) - .time_ref(time_ref); + .time_ref(Some(time_ref)); #[cfg(unix)] let launcher = launcher.stdout_file(Some("/dev/null")); match launcher.build().launch() { diff --git a/libafl_sugar/src/qemu.rs b/libafl_sugar/src/qemu.rs index c3b6816028..aa1ca8839c 100644 --- a/libafl_sugar/src/qemu.rs +++ b/libafl_sugar/src/qemu.rs @@ -440,7 +440,7 @@ where .cores(self.cores) .broker_port(self.broker_port) .remote_broker_addr(self.remote_broker_addr) - .time_ref(time_ref); + .time_ref(Some(time_ref)); #[cfg(unix)] let launcher = launcher.stdout_file(Some("/dev/null")); diff --git a/libafl_targets/src/sancov_pcguard.rs b/libafl_targets/src/sancov_pcguard.rs index ef9c5fe192..9ebcb3a6c5 100644 --- a/libafl_targets/src/sancov_pcguard.rs +++ b/libafl_targets/src/sancov_pcguard.rs @@ -17,7 +17,9 @@ use libafl::executors::{hooks::ExecutorHook, HasObservers}; ))] use crate::coverage::EDGES_MAP; use crate::coverage::MAX_EDGES_FOUND; + #[cfg(feature = "sancov_ngram4")] +#[allow(unused)] use crate::EDGES_MAP_SIZE_IN_USE; #[cfg(feature = "pointer_maps")] use crate::{coverage::EDGES_MAP_PTR, EDGES_MAP_SIZE_MAX}; @@ -29,6 +31,7 @@ compile_error!( ); #[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))] +#[allow(unused)] use core::ops::ShlAssign; #[cfg(feature = "sancov_ngram4")]