SYS-OSS/FRET-LibAFL
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make adaptive serialization into default (#2296)

Browse Source 
* no adaptive

* add another api

* allow unused
This commit is contained in:
Dongjia "toka" Zhang 2024-06-09 00:10:30 +02:00 committed by GitHub
parent 454176427b
commit f4699ba385
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 113 additions and 443 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libafl/Cargo.toml
View File

@ -53,9 +53,6 @@ gzip = ["libafl_bolts/gzip"]
## If set, will use the `fork()` syscall to spawn children, instead of launching a new command, if supported by the OS (has no effect on `Windows`).
fork = ["libafl_bolts/derive"]
## Collected stats to decide if observers must be serialized or not (which should reduce mem use and increase speed)
adaptive_serialization = []
## If this feature is set, `LibAFL` targets (and the fuzzer) will crash on `SIGPIPE` on unix systems.
handle_sigpipe = []

147
libafl/src/events/centralized.rs
View File

@ -8,12 +8,8 @@
// 4. The "main broker", the gathers the stats from the fuzzer clients and broadcast the newly found testcases from the main evaluator.
use alloc::{boxed::Box, string::String, vec::Vec};
use core::fmt::Debug;
#[cfg(feature = "adaptive_serialization")]
use core::time::Duration;
use core::{fmt::Debug, time::Duration};
#[cfg(feature = "adaptive_serialization")]
use libafl_bolts::tuples::{Handle, Handled};
#[cfg(feature = "llmp_compression")]
use libafl_bolts::{
compress::GzipCompressor,
@ -22,6 +18,7 @@ use libafl_bolts::{
use libafl_bolts::{
llmp::{LlmpClient, LlmpClientDescription, Tag},
shmem::{NopShMemProvider, ShMemProvider},
tuples::Handle,
ClientId,
};
use serde::{Deserialize, Serialize};
@ -29,8 +26,6 @@ use serde::{Deserialize, Serialize};
use super::NopEventManager;
#[cfg(feature = "llmp_compression")]
use crate::events::llmp::COMPRESS_THRESHOLD;
#[cfg(feature = "adaptive_serialization")]
use crate::observers::TimeObserver;
#[cfg(feature = "scalability_introspection")]
use crate::state::HasScalabilityMonitor;
use crate::{
@ -42,7 +37,7 @@ use crate::{
executors::{Executor, HasObservers},
fuzzer::{EvaluatorObservers, ExecutionProcessor},
inputs::{Input, NopInput, UsesInput},
observers::ObserversTuple,
observers::{ObserversTuple, TimeObserver},
state::{HasExecutions, HasLastReportTime, NopState, UsesState},
Error, HasMetadata,
};
@ -61,8 +56,7 @@ where
client: LlmpClient<SP>,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor,
#[cfg(feature = "adaptive_serialization")]
time_ref: Handle<TimeObserver>,
time_ref: Option<Handle<TimeObserver>>,
is_main: bool,
}
@ -99,12 +93,12 @@ impl CentralizedEventManagerBuilder {
Self { is_main }
}
/// Creates a new [`CentralizedEventManager`].
#[cfg(not(feature = "adaptive_serialization"))]
/// Creates a new `CentralizedEventManager`
pub fn build_from_client<EM, SP>(
self,
inner: EM,
client: LlmpClient<SP>,
time_obs: Option<Handle<TimeObserver>>,
) -> Result<CentralizedEventManager<EM, SP>, Error>
where
SP: ShMemProvider,
@ -115,28 +109,7 @@ impl CentralizedEventManagerBuilder {
client,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
is_main: self.is_main,
})
}
/// Creates a new [`CentralizedEventManager`].
#[cfg(feature = "adaptive_serialization")]
pub fn build_from_client<EM, SP>(
self,
inner: EM,
client: LlmpClient<SP>,
time_obs: &TimeObserver,
) -> Result<CentralizedEventManager<EM, SP>, Error>
where
SP: ShMemProvider,
EM: UsesState,
{
Ok(CentralizedEventManager {
inner,
client,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
time_ref: time_obs.handle(),
time_ref: time_obs,
is_main: self.is_main,
})
}
@ -145,12 +118,13 @@ impl CentralizedEventManagerBuilder {
///
/// If the port is not yet bound, it will act as a broker; otherwise, it
/// will act as a client.
#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))]
#[cfg(feature = "std")]
pub fn build_on_port<EM, SP>(
self,
inner: EM,
shmem_provider: SP,
port: u16,
time_obs: Option<Handle<TimeObserver>>,
) -> Result<CentralizedEventManager<EM, SP>, Error>
where
SP: ShMemProvider,
@ -162,45 +136,20 @@ impl CentralizedEventManagerBuilder {
client,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
is_main: self.is_main,
})
}
/// Create a centralized event manager on a port
///
/// If the port is not yet bound, it will act as a broker; otherwise, it
/// will act as a client.
#[cfg(all(feature = "std", feature = "adaptive_serialization"))]
pub fn build_on_port<EM, SP>(
self,
inner: EM,
shmem_provider: SP,
port: u16,
time_obs: &TimeObserver,
) -> Result<CentralizedEventManager<EM, SP>, Error>
where
SP: ShMemProvider,
EM: UsesState,
{
let client = LlmpClient::create_attach_to_tcp(shmem_provider, port)?;
Ok(CentralizedEventManager {
inner,
client,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
time_ref: time_obs.handle(),
time_ref: time_obs,
is_main: self.is_main,
})
}
/// If a client respawns, it may reuse the existing connection, previously
/// stored by [`LlmpClient::to_env()`].
#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))]
#[cfg(feature = "std")]
pub fn build_existing_client_from_env<EM, SP>(
self,
inner: EM,
shmem_provider: SP,
env_name: &str,
time_obs: Option<Handle<TimeObserver>>,
) -> Result<CentralizedEventManager<EM, SP>, Error>
where
EM: UsesState,
@ -211,41 +160,19 @@ impl CentralizedEventManagerBuilder {
client: LlmpClient::on_existing_from_env(shmem_provider, env_name)?,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
is_main: self.is_main,
})
}
/// If a client respawns, it may reuse the existing connection, previously
/// stored by [`LlmpClient::to_env()`].
#[cfg(all(feature = "std", feature = "adaptive_serialization"))]
pub fn build_existing_client_from_env<EM, SP>(
self,
inner: EM,
shmem_provider: SP,
env_name: &str,
time_obs: &TimeObserver,
) -> Result<CentralizedEventManager<EM, SP>, Error>
where
EM: UsesState,
SP: ShMemProvider,
{
Ok(CentralizedEventManager {
inner,
client: LlmpClient::on_existing_from_env(shmem_provider, env_name)?,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
time_ref: time_obs.handle(),
time_ref: time_obs,
is_main: self.is_main,
})
}
/// Create an existing client from description
#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))]
#[cfg(feature = "std")]
pub fn existing_client_from_description<EM, SP>(
self,
inner: EM,
shmem_provider: SP,
description: &LlmpClientDescription,
time_obs: Option<Handle<TimeObserver>>,
) -> Result<CentralizedEventManager<EM, SP>, Error>
where
EM: UsesState,
@ -256,29 +183,7 @@ impl CentralizedEventManagerBuilder {
client: LlmpClient::existing_client_from_description(shmem_provider, description)?,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
is_main: self.is_main,
})
}
/// Create an existing client from description
#[cfg(all(feature = "std", feature = "adaptive_serialization"))]
pub fn existing_client_from_description<EM, SP>(
self,
inner: EM,
shmem_provider: SP,
description: &LlmpClientDescription,
time_obs: &TimeObserver,
) -> Result<CentralizedEventManager<EM, SP>, Error>
where
EM: UsesState,
SP: ShMemProvider,
{
Ok(CentralizedEventManager {
inner,
client: LlmpClient::existing_client_from_description(shmem_provider, description)?,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
time_ref: time_obs.handle(),
time_ref: time_obs,
is_main: self.is_main,
})
}
@ -291,7 +196,6 @@ where
type State = EM::State;
}
#[cfg(feature = "adaptive_serialization")]
impl<EM, SP> AdaptiveSerializer for CentralizedEventManager<EM, SP>
where
EM: AdaptiveSerializer + UsesState,
@ -323,19 +227,11 @@ where
self.inner.should_serialize_cnt_mut()
}
fn time_ref(&self) -> &Handle<TimeObserver> {
fn time_ref(&self) -> &Option<Handle<TimeObserver>> {
&self.time_ref
}
}
#[cfg(not(feature = "adaptive_serialization"))]
impl<EM, SP> AdaptiveSerializer for CentralizedEventManager<EM, SP>
where
EM: AdaptiveSerializer + UsesState,
SP: ShMemProvider,
{
}
impl<EM, SP> EventFirer for CentralizedEventManager<EM, SP>
where
EM: AdaptiveSerializer + EventFirer + HasEventManagerId,
@ -399,15 +295,6 @@ where
self.inner.log(state, severity_level, message)
}
#[cfg(not(feature = "adaptive_serialization"))]
fn serialize_observers<OT>(&mut self, observers: &OT) -> Result<Option<Vec<u8>>, Error>
where
OT: ObserversTuple<Self::State> + Serialize,
{
Ok(Some(postcard::to_allocvec(observers)?))
}
#[cfg(feature = "adaptive_serialization")]
fn serialize_observers<OT>(&mut self, observers: &OT) -> Result<Option<Vec<u8>>, Error>
where
OT: ObserversTuple<Self::State> + Serialize,

49
libafl/src/events/launcher.rs
View File

@ -34,15 +34,6 @@ use libafl_bolts::llmp::LlmpBroker;
use libafl_bolts::os::dup2;
#[cfg(all(feature = "std", any(windows, not(feature = "fork"))))]
use libafl_bolts::os::startable_self;
#[cfg(feature = "adaptive_serialization")]
use libafl_bolts::tuples::Handle;
#[cfg(all(
unix,
feature = "std",
feature = "fork",
feature = "adaptive_serialization"
))]
use libafl_bolts::tuples::Handled;
#[cfg(all(unix, feature = "std", feature = "fork"))]
use libafl_bolts::{
core_affinity::get_core_ids,
@ -51,13 +42,12 @@ use libafl_bolts::{
use libafl_bolts::{
core_affinity::{CoreId, Cores},
shmem::ShMemProvider,
tuples::tuple_list,
tuples::{tuple_list, Handle},
};
#[cfg(feature = "std")]
use typed_builder::TypedBuilder;
use super::hooks::EventManagerHooksTuple;
#[cfg(feature = "adaptive_serialization")]
use crate::observers::TimeObserver;
#[cfg(all(unix, feature = "std", feature = "fork"))]
use crate::{
@ -131,8 +121,9 @@ pub struct Launcher<'a, CF, EMH, MT, S, SP> {
/// clusters.
#[builder(default = None)]
remote_broker_addr: Option<SocketAddr>,
#[cfg(feature = "adaptive_serialization")]
time_ref: Handle<TimeObserver>,
/// The time observer for addaptive serialization
#[builder(default = None)]
time_ref: Option<Handle<TimeObserver>>,
/// If this launcher should spawn a new `broker` on `[Self::broker_port]` (default).
/// The reason you may not want this is, if you already have a [`Launcher`]
/// with a different configuration (for the same target) running on this machine.
@ -282,7 +273,6 @@ where
.configuration(self.configuration)
.serialize_state(self.serialize_state)
.hooks(hooks);
#[cfg(feature = "adaptive_serialization")]
let builder = builder.time_ref(self.time_ref.clone());
let (state, mgr) = builder.build().launch()?;
@ -308,7 +298,6 @@ where
.serialize_state(self.serialize_state)
.hooks(hooks);
#[cfg(feature = "adaptive_serialization")]
let builder = builder.time_ref(self.time_ref.clone());
builder.build().launch()?;
@ -360,7 +349,6 @@ where
.serialize_state(self.serialize_state)
.hooks(hooks);
#[cfg(feature = "adaptive_serialization")]
let builder = builder.time_ref(self.time_ref.clone());
let (state, mgr) = builder.build().launch()?;
@ -455,7 +443,6 @@ where
.serialize_state(self.serialize_state)
.hooks(hooks);
#[cfg(feature = "adaptive_serialization")]
let builder = builder.time_ref(self.time_ref.clone());
builder.build().launch()?;
@ -506,8 +493,8 @@ pub struct CentralizedLauncher<'a, CF, IM, MF, MT, S, SP> {
#[builder(default = 1338_u16)]
centralized_broker_port: u16,
/// The time observer by which to adaptively serialize
#[cfg(feature = "adaptive_serialization")]
time_obs: &'a TimeObserver,
#[builder(default = None)]
time_obs: Option<Handle<TimeObserver>>,
/// The list of cores to run on
cores: &'a Cores,
/// A file name to write all client output to
@ -597,8 +584,7 @@ where
.serialize_state(centralized_launcher.serialize_state)
.hooks(tuple_list!());
#[cfg(feature = "adaptive_serialization")]
let builder = builder.time_ref(centralized_launcher.time_obs.handle());
let builder = builder.time_ref(centralized_launcher.time_obs.clone());
builder.build().launch()
};
@ -738,18 +724,11 @@ where
let mut centralized_builder = CentralizedEventManager::builder();
centralized_builder = centralized_builder.is_main(true);
#[cfg(not(feature = "adaptive_serialization"))]
let c_mgr = centralized_builder.build_on_port(
mgr,
self.shmem_provider.clone(),
self.centralized_broker_port,
)?;
#[cfg(feature = "adaptive_serialization")]
let c_mgr = centralized_builder.build_on_port(
mgr,
self.shmem_provider.clone(),
self.centralized_broker_port,
self.time_obs,
self.time_obs.clone(),
)?;
self.main_run_client.take().unwrap()(state, c_mgr, *bind_to)
@ -760,18 +739,11 @@ where
let centralized_builder = CentralizedEventManager::builder();
#[cfg(not(feature = "adaptive_serialization"))]
let c_mgr = centralized_builder.build_on_port(
mgr,
self.shmem_provider.clone(),
self.centralized_broker_port,
)?;
#[cfg(feature = "adaptive_serialization")]
let c_mgr = centralized_builder.build_on_port(
mgr,
self.shmem_provider.clone(),
self.centralized_broker_port,
self.time_obs,
self.time_obs.clone(),
)?;
self.secondary_run_client.take().unwrap()(state, c_mgr, *bind_to)
@ -796,8 +768,7 @@ where
.serialize_state(self.serialize_state)
.hooks(tuple_list!());
#[cfg(feature = "adaptive_serialization")]
let builder = builder.time_ref(self.time_obs.handle());
let builder = builder.time_ref(self.time_obs.clone());
builder.build().launch()?;

160
libafl/src/events/llmp/mgr.rs
View File

@ -8,8 +8,6 @@ use core::{marker::PhantomData, time::Duration};
#[cfg(feature = "std")]
use std::net::TcpStream;
#[cfg(feature = "adaptive_serialization")]
use libafl_bolts::tuples::Handle;
#[cfg(feature = "llmp_compression")]
use libafl_bolts::{
compress::GzipCompressor,
@ -19,6 +17,7 @@ use libafl_bolts::{
current_time,
llmp::{LlmpClient, LlmpClientDescription},
shmem::{NopShMemProvider, ShMemProvider},
tuples::Handle,
ClientId,
};
#[cfg(feature = "std")]
@ -30,22 +29,18 @@ use serde::{Deserialize, Serialize};
#[cfg(feature = "llmp_compression")]
use crate::events::llmp::COMPRESS_THRESHOLD;
#[cfg(feature = "adaptive_serialization")]
use crate::events::AdaptiveSerializer;
#[cfg(feature = "adaptive_serialization")]
use crate::observers::TimeObserver;
use crate::{
events::{
hooks::EventManagerHooksTuple,
llmp::{LLMP_TAG_EVENT_TO_BOTH, _LLMP_TAG_EVENT_TO_BROKER},
CustomBufEventResult, CustomBufHandlerFn, Event, EventConfig, EventFirer, EventManager,
EventManagerId, EventProcessor, EventRestarter, HasCustomBufHandlers, HasEventManagerId,
ProgressReporter,
AdaptiveSerializer, CustomBufEventResult, CustomBufHandlerFn, Event, EventConfig,
EventFirer, EventManager, EventManagerId, EventProcessor, EventRestarter,
HasCustomBufHandlers, HasEventManagerId, ProgressReporter,
},
executors::{Executor, HasObservers},
fuzzer::{Evaluator, EvaluatorObservers, ExecutionProcessor},
inputs::{NopInput, UsesInput},
observers::ObserversTuple,
observers::{ObserversTuple, TimeObserver},
state::{HasExecutions, HasLastReportTime, NopState, State, UsesState},
Error, HasMetadata,
};
@ -74,16 +69,11 @@ where
/// A node will not re-use the observer values sent over LLMP
/// from nodes with other configurations.
configuration: EventConfig,
#[cfg(feature = "adaptive_serialization")]
serialization_time: Duration,
#[cfg(feature = "adaptive_serialization")]
deserialization_time: Duration,
#[cfg(feature = "adaptive_serialization")]
serializations_cnt: usize,
#[cfg(feature = "adaptive_serialization")]
should_serialize_cnt: usize,
#[cfg(feature = "adaptive_serialization")]
pub(crate) time_ref: Handle<TimeObserver>,
pub(crate) time_ref: Option<Handle<TimeObserver>>,
phantom: PhantomData<S>,
}
@ -149,12 +139,11 @@ impl<EMH> LlmpEventManagerBuilder<EMH> {
}
/// Create a manager from a raw LLMP client
#[cfg(feature = "adaptive_serialization")]
pub fn build_from_client<S, SP>(
self,
llmp: LlmpClient<SP>,
configuration: EventConfig,
time_ref: Handle<TimeObserver>,
time_ref: Option<Handle<TimeObserver>>,
) -> Result<LlmpEventManager<EMH, S, SP>, Error>
where
SP: ShMemProvider,
@ -179,42 +168,17 @@ impl<EMH> LlmpEventManagerBuilder<EMH> {
})
}
/// Create a manager from a raw LLMP client
#[cfg(not(feature = "adaptive_serialization"))]
pub fn build_from_client<S, SP>(
self,
llmp: LlmpClient<SP>,
configuration: EventConfig,
) -> Result<LlmpEventManager<EMH, S, SP>, Error>
where
SP: ShMemProvider,
S: State,
{
Ok(LlmpEventManager {
throttle: self.throttle,
last_sent: Duration::from_secs(0),
hooks: self.hooks,
always_interesting: self.always_interesting,
llmp,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
configuration,
phantom: PhantomData,
custom_buf_handlers: vec![],
})
}
/// Create an LLMP event manager on a port
///
/// If the port is not yet bound, it will act as a broker; otherwise, it
/// will act as a client.
#[cfg(all(feature = "std", feature = "adaptive_serialization"))]
#[cfg(feature = "std")]
pub fn build_on_port<S, SP>(
self,
shmem_provider: SP,
port: u16,
configuration: EventConfig,
time_ref: Handle<TimeObserver>,
time_ref: Option<Handle<TimeObserver>>,
) -> Result<LlmpEventManager<EMH, S, SP>, Error>
where
SP: ShMemProvider,
@ -240,45 +204,15 @@ impl<EMH> LlmpEventManagerBuilder<EMH> {
})
}
/// Create an LLMP event manager on a port
///
/// If the port is not yet bound, it will act as a broker; otherwise, it
/// will act as a client.
#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))]
pub fn build_on_port<S, SP>(
self,
shmem_provider: SP,
port: u16,
configuration: EventConfig,
) -> Result<LlmpEventManager<EMH, S, SP>, Error>
where
SP: ShMemProvider,
S: State,
{
let llmp = LlmpClient::create_attach_to_tcp(shmem_provider, port)?;
Ok(LlmpEventManager {
throttle: self.throttle,
last_sent: Duration::from_secs(0),
hooks: self.hooks,
always_interesting: self.always_interesting,
llmp,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
configuration,
phantom: PhantomData,
custom_buf_handlers: vec![],
})
}
/// If a client respawns, it may reuse the existing connection, previously
/// stored by [`LlmpClient::to_env()`].
#[cfg(all(feature = "std", feature = "adaptive_serialization"))]
#[cfg(feature = "std")]
pub fn build_existing_client_from_env<S, SP>(
self,
shmem_provider: SP,
env_name: &str,
configuration: EventConfig,
time_ref: Handle<TimeObserver>,
time_ref: Option<Handle<TimeObserver>>,
) -> Result<LlmpEventManager<EMH, S, SP>, Error>
where
SP: ShMemProvider,
@ -304,42 +238,13 @@ impl<EMH> LlmpEventManagerBuilder<EMH> {
})
}
/// If a client respawns, it may reuse the existing connection, previously
/// stored by [`LlmpClient::to_env()`].
#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))]
pub fn build_existing_client_from_env<S, SP>(
self,
shmem_provider: SP,
env_name: &str,
configuration: EventConfig,
) -> Result<LlmpEventManager<EMH, S, SP>, Error>
where
SP: ShMemProvider,
S: State,
{
let llmp = LlmpClient::on_existing_from_env(shmem_provider, env_name)?;
Ok(LlmpEventManager {
throttle: self.throttle,
last_sent: Duration::from_secs(0),
hooks: self.hooks,
always_interesting: self.always_interesting,
llmp,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
configuration,
phantom: PhantomData,
custom_buf_handlers: vec![],
})
}
/// Create an existing client from description
#[cfg(feature = "adaptive_serialization")]
pub fn build_existing_client_from_description<S, SP>(
self,
shmem_provider: SP,
description: &LlmpClientDescription,
configuration: EventConfig,
time_ref: Handle<TimeObserver>,
time_ref: Option<Handle<TimeObserver>>,
) -> Result<LlmpEventManager<EMH, S, SP>, Error>
where
SP: ShMemProvider,
@ -364,36 +269,8 @@ impl<EMH> LlmpEventManagerBuilder<EMH> {
custom_buf_handlers: vec![],
})
}
/// Create an existing client from description
#[cfg(not(feature = "adaptive_serialization"))]
pub fn build_existing_client_from_description<S, SP>(
self,
shmem_provider: SP,
description: &LlmpClientDescription,
configuration: EventConfig,
) -> Result<LlmpEventManager<EMH, S, SP>, Error>
where
SP: ShMemProvider,
S: State,
{
let llmp = LlmpClient::existing_client_from_description(shmem_provider, description)?;
Ok(LlmpEventManager {
throttle: self.throttle,
last_sent: Duration::from_secs(0),
hooks: self.hooks,
always_interesting: self.always_interesting,
llmp,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor::with_threshold(COMPRESS_THRESHOLD),
configuration,
phantom: PhantomData,
custom_buf_handlers: vec![],
})
}
}
#[cfg(feature = "adaptive_serialization")]
impl<EMH, S, SP> AdaptiveSerializer for LlmpEventManager<EMH, S, SP>
where
SP: ShMemProvider,
@ -425,7 +302,7 @@ where
&mut self.should_serialize_cnt
}
fn time_ref(&self) -> &Handle<TimeObserver> {
fn time_ref(&self) -> &Option<Handle<TimeObserver>> {
&self.time_ref
}
}
@ -556,11 +433,9 @@ where
let res = if client_config.match_with(&self.configuration)
&& observers_buf.is_some()
{
#[cfg(feature = "adaptive_serialization")]
let start = current_time();
let observers: E::Observers =
postcard::from_bytes(observers_buf.as_ref().unwrap())?;
#[cfg(feature = "adaptive_serialization")]
{
self.deserialization_time = current_time() - start;
}
@ -671,15 +546,6 @@ where
Ok(())
}
#[cfg(not(feature = "adaptive_serialization"))]
fn serialize_observers<OT>(&mut self, observers: &OT) -> Result<Option<Vec<u8>>, Error>
where
OT: ObserversTuple<Self::State> + Serialize,
{
Ok(Some(postcard::to_allocvec(observers)?))
}
#[cfg(feature = "adaptive_serialization")]
fn serialize_observers<OT>(&mut self, observers: &OT) -> Result<Option<Vec<u8>>, Error>
where
OT: ObserversTuple<Self::State> + Serialize,

106
libafl/src/events/llmp/restarting.rs
View File

@ -22,9 +22,11 @@ use libafl_bolts::os::startable_self;
use libafl_bolts::os::unix_signals::setup_signal_handler;
#[cfg(all(feature = "std", feature = "fork", unix))]
use libafl_bolts::os::{fork, ForkResult};
#[cfg(feature = "adaptive_serialization")]
use libafl_bolts::tuples::{Handle, Handled};
use libafl_bolts::{llmp::LlmpBroker, shmem::ShMemProvider, tuples::tuple_list};
use libafl_bolts::{
llmp::LlmpBroker,
shmem::ShMemProvider,
tuples::{tuple_list, Handle},
};
#[cfg(feature = "std")]
use libafl_bolts::{
llmp::LlmpConnection, os::CTRL_C_EXIT, shmem::StdShMemProvider, staterestore::StateRestorer,
@ -33,14 +35,10 @@ use serde::{Deserialize, Serialize};
#[cfg(feature = "std")]
use typed_builder::TypedBuilder;
#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))]
use crate::events::AdaptiveSerializer;
#[cfg(all(feature = "std", feature = "adaptive_serialization"))]
#[cfg(feature = "std")]
use crate::events::AdaptiveSerializer;
#[cfg(all(unix, feature = "std", not(miri)))]
use crate::events::EVENTMGR_SIGHANDLER_STATE;
#[cfg(feature = "adaptive_serialization")]
use crate::observers::TimeObserver;
use crate::{
events::{
hooks::EventManagerHooksTuple, Event, EventConfig, EventFirer, EventManager,
@ -51,7 +49,7 @@ use crate::{
fuzzer::{Evaluator, EvaluatorObservers, ExecutionProcessor},
inputs::UsesInput,
monitors::Monitor,
observers::ObserversTuple,
observers::{ObserversTuple, TimeObserver},
state::{HasExecutions, HasLastReportTime, State, UsesState},
Error, HasMetadata,
};
@ -73,7 +71,7 @@ where
save_state: LlmpShouldSaveState,
}
#[cfg(all(feature = "std", feature = "adaptive_serialization"))]
#[cfg(feature = "std")]
impl<EMH, S, SP> AdaptiveSerializer for LlmpRestartingEventManager<EMH, S, SP>
where
SP: ShMemProvider,
@ -105,19 +103,11 @@ where
self.llmp_mgr.should_serialize_cnt_mut()
}
fn time_ref(&self) -> &Handle<TimeObserver> {
fn time_ref(&self) -> &Option<Handle<TimeObserver>> {
&self.llmp_mgr.time_ref
}
}
#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))]
impl<EMH, S, SP> AdaptiveSerializer for LlmpRestartingEventManager<EMH, S, SP>
where
SP: ShMemProvider,
S: State,
{
}
#[cfg(feature = "std")]
impl<EMH, S, SP> UsesState for LlmpRestartingEventManager<EMH, S, SP>
where
@ -330,7 +320,7 @@ pub enum ManagerKind {
/// Sets up a restarting fuzzer, using the [`StdShMemProvider`], and standard features.
/// The restarting mgr is a combination of restarter and runner, that can be used on systems with and without `fork` support.
/// The restarter will spawn a new process each time the child crashes or timeouts.
#[cfg(all(feature = "std", not(feature = "adaptive_serialization")))]
#[cfg(feature = "std")]
#[allow(clippy::type_complexity)]
pub fn setup_restarting_mgr_std<MT, S>(
monitor: MT,
@ -345,7 +335,7 @@ pub fn setup_restarting_mgr_std<MT, S>(
>
where
MT: Monitor + Clone,
S: State + HasExecutions,
S: State,
{
RestartingMgr::builder()
.shmem_provider(StdShMemProvider::new()?)
@ -360,13 +350,14 @@ where
/// Sets up a restarting fuzzer, using the [`StdShMemProvider`], and standard features.
/// The restarting mgr is a combination of restarter and runner, that can be used on systems with and without `fork` support.
/// The restarter will spawn a new process each time the child crashes or timeouts.
#[cfg(all(feature = "std", feature = "adaptive_serialization"))]
/// This one, additionally uses the timeobserver for the adaptive serialization
#[cfg(feature = "std")]
#[allow(clippy::type_complexity)]
pub fn setup_restarting_mgr_std<MT, S>(
pub fn setup_restarting_mgr_std_adaptive<MT, S>(
monitor: MT,
broker_port: u16,
configuration: EventConfig,
time_obs: &TimeObserver,
time_obs: Handle<TimeObserver>,
) -> Result<
(
Option<S>,
@ -376,7 +367,7 @@ pub fn setup_restarting_mgr_std<MT, S>(
>
where
MT: Monitor + Clone,
S: State + HasExecutions,
S: State,
{
RestartingMgr::builder()
.shmem_provider(StdShMemProvider::new()?)
@ -384,7 +375,7 @@ where
.broker_port(broker_port)
.configuration(configuration)
.hooks(tuple_list!())
.time_ref(time_obs.handle())
.time_ref(Some(time_obs))
.build()
.launch()
}
@ -436,8 +427,8 @@ where
serialize_state: LlmpShouldSaveState,
/// The hooks passed to event manager:
hooks: EMH,
#[cfg(feature = "adaptive_serialization")]
time_ref: Handle<TimeObserver>,
#[builder(default = None)]
time_ref: Option<Handle<TimeObserver>>,
#[builder(setter(skip), default = PhantomData)]
phantom_data: PhantomData<(EMH, S)>,
}
@ -448,7 +439,7 @@ impl<EMH, MT, S, SP> RestartingMgr<EMH, MT, S, SP>
where
EMH: EventManagerHooksTuple<S> + Copy + Clone,
SP: ShMemProvider,
S: State + HasExecutions,
S: State,
MT: Monitor + Clone,
{
/// Launch the broker and the clients and fuzz
@ -500,12 +491,6 @@ where
return Err(Error::shutting_down());
}
LlmpConnection::IsClient { client } => {
#[cfg(not(feature = "adaptive_serialization"))]
let mgr: LlmpEventManager<EMH, S, SP> = LlmpEventManager::builder()
.always_interesting(self.always_interesting)
.hooks(self.hooks)
.build_from_client(client, self.configuration)?;
#[cfg(feature = "adaptive_serialization")]
let mgr: LlmpEventManager<EMH, S, SP> = LlmpEventManager::builder()
.always_interesting(self.always_interesting)
.hooks(self.hooks)
@ -532,16 +517,6 @@ where
}
ManagerKind::Client { cpu_core } => {
// We are a client
#[cfg(not(feature = "adaptive_serialization"))]
let mgr = LlmpEventManager::builder()
.always_interesting(self.always_interesting)
.hooks(self.hooks)
.build_on_port(
self.shmem_provider.clone(),
self.broker_port,
self.configuration,
)?;
#[cfg(feature = "adaptive_serialization")]
let mgr = LlmpEventManager::builder()
.always_interesting(self.always_interesting)
.hooks(self.hooks)
@ -671,15 +646,6 @@ where
// If we're restarting, deserialize the old state.
let (state, mut mgr) =
if let Some((state_opt, mgr_description)) = staterestorer.restore()? {
#[cfg(not(feature = "adaptive_serialization"))]
let llmp_mgr = LlmpEventManager::builder()
.hooks(self.hooks)
.build_existing_client_from_description(
new_shmem_provider,
&mgr_description,
self.configuration,
)?;
#[cfg(feature = "adaptive_serialization")]
let llmp_mgr = LlmpEventManager::builder()
.hooks(self.hooks)
.build_existing_client_from_description(
@ -699,15 +665,6 @@ where
} else {
log::info!("First run. Let's set it all up");
// Mgr to send and receive msgs from/to all other fuzzer instances
#[cfg(not(feature = "adaptive_serialization"))]
let mgr = LlmpEventManager::builder()
.hooks(self.hooks)
.build_existing_client_from_env(
new_shmem_provider,
_ENV_FUZZER_BROKER_CLIENT_INITIAL,
self.configuration,
)?;
#[cfg(feature = "adaptive_serialization")]
let mgr = LlmpEventManager::builder()
.hooks(self.hooks)
.build_existing_client_from_env(
@ -748,14 +705,12 @@ where
mod tests {
use core::sync::atomic::{compiler_fence, Ordering};
#[cfg(feature = "adaptive_serialization")]
use libafl_bolts::tuples::Handled;
use libafl_bolts::{
llmp::{LlmpClient, LlmpSharedMap},
rands::StdRand,
shmem::{ShMemProvider, StdShMemProvider},
staterestore::StateRestorer,
tuples::tuple_list,
tuples::{tuple_list, Handled},
ClientId,
};
use serial_test::serial;
@ -782,7 +737,6 @@ mod tests {
let rand = StdRand::with_seed(0);
let time = TimeObserver::new("time");
#[cfg(feature = "adaptive_serialization")]
let time_ref = time.handle();
let mut corpus = InMemoryCorpus::<BytesInput>::new();
@ -811,13 +765,8 @@ mod tests {
llmp_client.mark_safe_to_unmap();
}
#[cfg(not(feature = "adaptive_serialization"))]
let mut llmp_mgr = LlmpEventManager::builder()
.build_from_client(llmp_client, "fuzzer".into())
.unwrap();
#[cfg(feature = "adaptive_serialization")]
let mut llmp_mgr = LlmpEventManager::builder()
.build_from_client(llmp_client, "fuzzer".into(), time_ref.clone())
.build_from_client(llmp_client, "fuzzer".into(), Some(time_ref.clone()))
.unwrap();
let scheduler = RandScheduler::new();
@ -860,21 +809,12 @@ mod tests {
assert!(sc_cpy.has_content());
let (mut state_clone, mgr_description) = staterestorer.restore().unwrap().unwrap();
#[cfg(not(feature = "adaptive_serialization"))]
let mut llmp_clone = LlmpEventManager::builder()
.build_existing_client_from_description(
shmem_provider,
&mgr_description,
"fuzzer".into(),
)
.unwrap();
#[cfg(feature = "adaptive_serialization")]
let mut llmp_clone = LlmpEventManager::builder()
.build_existing_client_from_description(
shmem_provider,
&mgr_description,
"fuzzer".into(),
time_ref,
Some(time_ref),
)
.unwrap();

77
libafl/src/events/mod.rs
View File

@ -32,9 +32,11 @@ use ahash::RandomState;
pub use launcher::*;
#[cfg(all(unix, feature = "std"))]
use libafl_bolts::os::unix_signals::{siginfo_t, ucontext_t, Handler, Signal, CTRL_C_EXIT};
#[cfg(feature = "adaptive_serialization")]
use libafl_bolts::tuples::{Handle, MatchNameRef};
use libafl_bolts::{current_time, ClientId};
use libafl_bolts::{
current_time,
tuples::{Handle, MatchNameRef},
ClientId,
};
use serde::{Deserialize, Serialize};
#[cfg(feature = "std")]
use uuid::Uuid;
@ -104,9 +106,9 @@ pub struct EventManagerId(
#[cfg(feature = "introspection")]
use crate::monitors::ClientPerfMonitor;
#[cfg(feature = "adaptive_serialization")]
use crate::observers::TimeObserver;
use crate::{inputs::UsesInput, stages::HasCurrentStage, state::UsesState};
use crate::{
inputs::UsesInput, observers::TimeObserver, stages::HasCurrentStage, state::UsesState,
};
/// The log event severity
#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
@ -839,11 +841,6 @@ where
}
/// Collected stats to decide if observers must be serialized or not
#[cfg(not(feature = "adaptive_serialization"))]
pub trait AdaptiveSerializer {}
/// Collected stats to decide if observers must be serialized or not
#[cfg(feature = "adaptive_serialization")]
pub trait AdaptiveSerializer {
/// Expose the collected observers serialization time
fn serialization_time(&self) -> Duration;
@ -864,7 +861,7 @@ pub trait AdaptiveSerializer {
fn should_serialize_cnt_mut(&mut self) -> &mut usize;
/// A [`Handle`] to the time observer to determine the `time_factor`
fn time_ref(&self) -> &Handle<TimeObserver>;
fn time_ref(&self) -> &Option<Handle<TimeObserver>>;
/// Serialize the observer using the `time_factor` and `percentage_threshold`.
/// These parameters are unique to each of the different types of `EventManager`
@ -878,35 +875,41 @@ pub trait AdaptiveSerializer {
OT: ObserversTuple<S> + Serialize,
S: UsesInput,
{
let exec_time = observers
.get(self.time_ref())
.map(|o| o.last_runtime().unwrap_or(Duration::ZERO))
.unwrap();
match self.time_ref() {
Some(t) => {
let exec_time = observers
.get(t)
.map(|o| o.last_runtime().unwrap_or(Duration::ZERO))
.unwrap();
let mut must_ser =
(self.serialization_time() + self.deserialization_time()) * time_factor < exec_time;
if must_ser {
*self.should_serialize_cnt_mut() += 1;
}
let mut must_ser = (self.serialization_time() + self.deserialization_time())
* time_factor
< exec_time;
if must_ser {
*self.should_serialize_cnt_mut() += 1;
}
if self.serializations_cnt() > 32 {
must_ser = (self.should_serialize_cnt() * 100 / self.serializations_cnt())
> percentage_threshold;
}
if self.serializations_cnt() > 32 {
must_ser = (self.should_serialize_cnt() * 100 / self.serializations_cnt())
> percentage_threshold;
}
if self.serialization_time() == Duration::ZERO
|| must_ser
|| self.serializations_cnt().trailing_zeros() >= 8
{
let start = current_time();
let ser = postcard::to_allocvec(observers)?;
*self.serialization_time_mut() = current_time() - start;
if self.serialization_time() == Duration::ZERO
|| must_ser
|| self.serializations_cnt().trailing_zeros() >= 8
{
let start = current_time();
let ser = postcard::to_allocvec(observers)?;
*self.serialization_time_mut() = current_time() - start;
*self.serializations_cnt_mut() += 1;
Ok(Some(ser))
} else {
*self.serializations_cnt_mut() += 1;
Ok(None)
*self.serializations_cnt_mut() += 1;
Ok(Some(ser))
} else {
*self.serializations_cnt_mut() += 1;
Ok(None)
}
}
None => Ok(None),
}
}
}

3
libafl_frida/src/asan/hook_funcs.rs
View File

@ -11,9 +11,12 @@ use crate::{
errors::{AsanError, AsanErrors},
},
};
#[cfg(windows)]
extern "system" {
fn memcpy(dst: *mut c_void, src: *const c_void, size: usize) -> *mut c_void;
}
#[cfg(windows)]
extern "system" {
fn memset(s: *mut c_void, c: i32, n: usize) -> *mut c_void;
}

2
libafl_sugar/Cargo.toml
View File

@ -48,7 +48,7 @@ hexagon = ["libafl_qemu/hexagon"]
pyo3-build-config = { version = "0.21", optional = true }
[dependencies]
libafl = { path = "../libafl", version = "0.12.0", features = ["adaptive_serialization"] }
libafl = { path = "../libafl", version = "0.12.0" }
libafl_bolts = { path = "../libafl_bolts", version = "0.12.0" }
libafl_targets = { path = "../libafl_targets", version = "0.12.0" }

2
libafl_sugar/src/forkserver.rs
View File

@ -295,7 +295,7 @@ impl<'a> ForkserverBytesCoverageSugar<'a> {
.cores(self.cores)
.broker_port(self.broker_port)
.remote_broker_addr(self.remote_broker_addr)
.time_ref(time_ref);
.time_ref(Some(time_ref));
#[cfg(unix)]
let launcher = launcher.stdout_file(Some("/dev/null"));
match launcher.build().launch() {

2
libafl_sugar/src/inmemory.rs
View File

@ -351,7 +351,7 @@ where
.cores(self.cores)
.broker_port(self.broker_port)
.remote_broker_addr(self.remote_broker_addr)
.time_ref(time_ref);
.time_ref(Some(time_ref));
#[cfg(unix)]
let launcher = launcher.stdout_file(Some("/dev/null"));
match launcher.build().launch() {

2
libafl_sugar/src/qemu.rs
View File

@ -440,7 +440,7 @@ where
.cores(self.cores)
.broker_port(self.broker_port)
.remote_broker_addr(self.remote_broker_addr)
.time_ref(time_ref);
.time_ref(Some(time_ref));
#[cfg(unix)]
let launcher = launcher.stdout_file(Some("/dev/null"));

3
libafl_targets/src/sancov_pcguard.rs
View File

@ -17,7 +17,9 @@ use libafl::executors::{hooks::ExecutorHook, HasObservers};
))]
use crate::coverage::EDGES_MAP;
use crate::coverage::MAX_EDGES_FOUND;
#[cfg(feature = "sancov_ngram4")]
#[allow(unused)]
use crate::EDGES_MAP_SIZE_IN_USE;
#[cfg(feature = "pointer_maps")]
use crate::{coverage::EDGES_MAP_PTR, EDGES_MAP_SIZE_MAX};
@ -29,6 +31,7 @@ compile_error!(
);
#[cfg(any(feature = "sancov_ngram4", feature = "sancov_ngram8"))]
#[allow(unused)]
use core::ops::ShlAssign;
#[cfg(feature = "sancov_ngram4")]