Refactor configurations with EventConfig (#277)

* switch to EventConfig

* adapt fuzzers

* fix tests

* fix sugar

* clippy

* ahash
This commit is contained in:
Andrea Fioraldi 2021-08-26 13:25:03 +02:00 committed by GitHub
parent 7d249dd306
commit 6ae36ce658
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 134 additions and 77 deletions

View File

@ -16,6 +16,7 @@ use libafl::{
ondisk::OnDiskMetadataFormat, Corpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
QueueCorpusScheduler,
},
events::EventConfig,
executors::{
inprocess::InProcessExecutor, timeout::TimeoutExecutor, Executor, ExitKind, HasObservers,
ShadowExecutor,
@ -442,7 +443,7 @@ unsafe fn fuzz(
};
Launcher::builder()
.configuration(configuration)
.configuration(EventConfig::from_name(&configuration))
.shmem_provider(shmem_provider)
.stats(stats)
.run_client(&mut run_client)

View File

@ -18,6 +18,7 @@ use libafl::{
Corpus, InMemoryCorpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
QueueCorpusScheduler,
},
events::EventConfig,
executors::{inprocess::InProcessExecutor, ExitKind, TimeoutExecutor},
feedback_or,
feedbacks::{CrashFeedback, MapFeedbackState, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
@ -231,7 +232,7 @@ pub fn libafl_main() {
Launcher::builder()
.shmem_provider(shmem_provider)
.configuration("launcher default".into())
.configuration(EventConfig::from_name("default"))
.stats(stats)
.run_client(&mut run_client)
.cores(&cores)

View File

@ -10,7 +10,7 @@ use libafl::{
tuples::{tuple_list, Merge},
},
corpus::{Corpus, InMemoryCorpus, OnDiskCorpus, RandCorpusScheduler},
events::setup_restarting_mgr_std,
events::{setup_restarting_mgr_std, EventConfig},
executors::{inprocess::InProcessExecutor, ExitKind},
feedback_or,
feedbacks::{CrashFeedback, MapFeedbackState, MaxMapFeedback},
@ -60,7 +60,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
// The restarting state will spawn the same process again as child, then restarted it each time it crashes.
let (state, mut restarting_mgr) =
match setup_restarting_mgr_std(stats, broker_port, "default".into()) {
match setup_restarting_mgr_std(stats, broker_port, EventConfig::from_name("default")) {
Ok(tuple) => tuple,
Err(Error::ShuttingDown) => {
println!("\nFinished fuzzing. Good bye.");

View File

@ -14,7 +14,7 @@ use libafl::{
Corpus, InMemoryCorpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
PowerQueueCorpusScheduler,
},
events::{setup_restarting_mgr_std, EventRestarter},
events::{setup_restarting_mgr_std, EventConfig, EventRestarter},
executors::{inprocess::InProcessExecutor, ExitKind, TimeoutExecutor},
feedback_or, feedback_or_fast,
feedbacks::{CrashFeedback, MapFeedbackState, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
@ -61,7 +61,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
// The restarting state will spawn the same process again as child, then restarted it each time it crashes.
let (state, mut restarting_mgr) =
match setup_restarting_mgr_std(stats, broker_port, "default".into()) {
match setup_restarting_mgr_std(stats, broker_port, EventConfig::AlwaysUnique) {
Ok(res) => res,
Err(err) => match err {
Error::ShuttingDown => {

View File

@ -20,6 +20,7 @@ use libafl::{
Corpus, InMemoryCorpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
QueueCorpusScheduler,
},
events::EventConfig,
executors::{inprocess::InProcessExecutor, ExitKind, TimeoutExecutor},
feedback_or, feedback_or_fast,
feedbacks::{CrashFeedback, MapFeedbackState, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
@ -166,7 +167,7 @@ pub fn libafl_main() {
match Launcher::builder()
.shmem_provider(shmem_provider)
.configuration("launcher default".into())
.configuration(EventConfig::from_name("default"))
.stats(stats)
.run_client(&mut run_client)
.cores(&cores)

View File

@ -6,7 +6,7 @@ use std::{env, path::PathBuf};
use libafl::{
bolts::{current_nanos, rands::StdRand, tuples::tuple_list},
corpus::{Corpus, InMemoryCorpus, OnDiskCorpus, RandCorpusScheduler},
events::{setup_restarting_mgr_std, EventRestarter},
events::{setup_restarting_mgr_std, EventConfig, EventRestarter},
executors::{inprocess::InProcessExecutor, ExitKind},
feedbacks::{MapFeedbackState, MaxMapFeedback, ReachabilityFeedback},
fuzzer::{Fuzzer, StdFuzzer},
@ -52,7 +52,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
// The restarting state will spawn the same process again as child, then restarted it each time it crashes.
let (state, mut restarting_mgr) =
match setup_restarting_mgr_std(stats, broker_port, "default".into()) {
match setup_restarting_mgr_std(stats, broker_port, EventConfig::AlwaysUnique) {
Ok(res) => res,
Err(err) => match err {
Error::ShuttingDown => {

View File

@ -9,7 +9,7 @@ use libafl::{
Corpus, InMemoryCorpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
QueueCorpusScheduler,
},
events::setup_restarting_mgr_std,
events::{setup_restarting_mgr_std, EventConfig},
executors::{inprocess::InProcessExecutor, ExitKind, ShadowExecutor},
feedback_or,
feedbacks::{CrashFeedback, MapFeedbackState, MaxMapFeedback, TimeFeedback},
@ -53,7 +53,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
// The restarting state will spawn the same process again as child, then restarted it each time it crashes.
let (state, mut restarting_mgr) =
match setup_restarting_mgr_std(stats, broker_port, "default".into()) {
match setup_restarting_mgr_std(stats, broker_port, EventConfig::from_name("default")) {
Ok(res) => res,
Err(err) => match err {
Error::ShuttingDown => {

View File

@ -14,7 +14,7 @@ use libafl::{
Corpus, InMemoryCorpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
QueueCorpusScheduler,
},
events::setup_restarting_mgr_std,
events::{setup_restarting_mgr_std, EventConfig},
executors::{
command::CommandConfigurator, inprocess::InProcessExecutor, ExitKind, ShadowExecutor,
},
@ -88,7 +88,7 @@ fn fuzz(
// The restarting state will spawn the same process again as child, then restarted it each time it crashes.
let (state, mut restarting_mgr) =
match setup_restarting_mgr_std(stats, broker_port, "default".into()) {
match setup_restarting_mgr_std(stats, broker_port, EventConfig::from_name("default")) {
Ok(res) => res,
Err(err) => match err {
Error::ShuttingDown => {

View File

@ -4,7 +4,7 @@ use serde::de::DeserializeOwned;
#[cfg(feature = "std")]
use crate::{
bolts::shmem::ShMemProvider,
events::{LlmpRestartingEventManager, ManagerKind, RestartingMgr},
events::{EventConfig, LlmpRestartingEventManager, ManagerKind, RestartingMgr},
inputs::Input,
observers::ObserversTuple,
stats::Stats,
@ -54,7 +54,7 @@ where
/// The stats instance to use
stats: ST,
/// The configuration
configuration: String,
configuration: EventConfig,
/// The 'main' function to run for each client forked. This probably shouldn't return
run_client: LauncherClientFnRef<'a, I, OT, S, SP>,
/// The broker port to use (or to attach to, in case [`Self::spawn_broker`] is `false`)
@ -129,7 +129,7 @@ where
.kind(ManagerKind::Client {
cpu_core: Some(*bind_to),
})
.configuration(self.configuration.clone())
.configuration(self.configuration)
.build()
.launch()?;
@ -151,7 +151,7 @@ where
.broker_port(self.broker_port)
.kind(ManagerKind::Broker)
.remote_broker_addr(self.remote_broker_addr)
.configuration(self.configuration.clone())
.configuration(self.configuration)
.build()
.launch()?;
@ -196,7 +196,7 @@ where
id: core_conf.parse()?,
}),
})
.configuration(self.configuration.clone())
.configuration(self.configuration)
.build()
.launch()?;
@ -252,7 +252,7 @@ where
.broker_port(self.broker_port)
.kind(ManagerKind::Broker)
.remote_broker_addr(self.remote_broker_addr)
.configuration(self.configuration.clone())
.configuration(self.configuration)
.build()
.launch()?;

View File

@ -1,6 +1,6 @@
//! LLMP-backed event manager for scalable multi-processed fuzzing
use alloc::string::{String, ToString};
use alloc::string::ToString;
use core::{marker::PhantomData, time::Duration};
#[cfg(feature = "std")]
@ -25,8 +25,8 @@ use crate::{
shmem::ShMemProvider,
},
events::{
BrokerEventResult, Event, EventFirer, EventManager, EventManagerId, EventProcessor,
EventRestarter, HasEventManagerId,
BrokerEventResult, Event, EventConfig, EventFirer, EventManager, EventManagerId,
EventProcessor, EventRestarter, HasEventManagerId,
},
executors::{Executor, HasObservers},
fuzzer::{EvaluatorObservers, ExecutionProcessor},
@ -257,7 +257,7 @@ where
llmp: llmp::LlmpClient<SP>,
#[cfg(feature = "llmp_compression")]
compressor: GzipCompressor,
configuration: String,
configuration: EventConfig,
phantom: PhantomData<(I, OT, S)>,
}
@ -280,7 +280,7 @@ where
SP: ShMemProvider + 'static,
{
/// Create a manager from a raw llmp client
pub fn new(llmp: llmp::LlmpClient<SP>, configuration: String) -> Result<Self, Error> {
pub fn new(llmp: llmp::LlmpClient<SP>, configuration: EventConfig) -> Result<Self, Error> {
Ok(Self {
llmp,
#[cfg(feature = "llmp_compression")]
@ -297,7 +297,7 @@ where
pub fn new_on_port(
shmem_provider: SP,
port: u16,
configuration: String,
configuration: EventConfig,
) -> Result<Self, Error> {
Ok(Self {
llmp: llmp::LlmpClient::create_attach_to_tcp(shmem_provider, port)?,
@ -313,7 +313,7 @@ where
pub fn existing_client_from_env(
shmem_provider: SP,
env_name: &str,
configuration: String,
configuration: EventConfig,
) -> Result<Self, Error> {
Ok(Self {
llmp: LlmpClient::on_existing_from_env(shmem_provider, env_name)?,
@ -333,7 +333,7 @@ where
pub fn existing_client_from_description(
shmem_provider: SP,
description: &LlmpClientDescription,
configuration: String,
configuration: EventConfig,
) -> Result<Self, Error> {
Ok(Self {
llmp: llmp::LlmpClient::existing_client_from_description(shmem_provider, description)?,
@ -377,12 +377,14 @@ where
} => {
#[cfg(feature = "std")]
println!(
"Received new Testcase from {} ({})",
"Received new Testcase from {} ({:?})",
_client_id, client_config
);
let _res = if client_config == self.configuration {
let observers: OT = postcard::from_bytes(&observers_buf)?;
let _res = if client_config.match_with(&self.configuration)
&& observers_buf.is_some()
{
let observers: OT = postcard::from_bytes(observers_buf.as_ref().unwrap())?;
fuzzer.process_execution(state, self, input, &observers, &exit_kind, false)?
} else {
fuzzer.evaluate_input_with_observers(state, executor, self, input, false)?
@ -435,8 +437,8 @@ where
Ok(())
}
fn configuration(&self) -> &str {
&self.configuration
fn configuration(&self) -> EventConfig {
self.configuration
}
}
@ -550,7 +552,7 @@ where
self.llmp_mgr.fire(state, event)
}
fn configuration(&self) -> &str {
fn configuration(&self) -> EventConfig {
self.llmp_mgr.configuration()
}
}
@ -674,7 +676,7 @@ pub enum ManagerKind {
pub fn setup_restarting_mgr_std<I, OT, S, ST>(
stats: ST,
broker_port: u16,
configuration: String,
configuration: EventConfig,
) -> Result<
(
Option<S>,
@ -717,7 +719,7 @@ where
/// manager.
shmem_provider: SP,
/// The configuration
configuration: String,
configuration: EventConfig,
/// The stats to use
#[builder(default = None)]
stats: Option<ST>,
@ -783,10 +785,8 @@ where
return Err(Error::ShuttingDown);
}
LlmpConnection::IsClient { client } => {
let mgr = LlmpEventManager::<I, OT, S, SP>::new(
client,
self.configuration.clone(),
)?;
let mgr =
LlmpEventManager::<I, OT, S, SP>::new(client, self.configuration)?;
(mgr, None)
}
}
@ -807,7 +807,7 @@ where
let mgr = LlmpEventManager::<I, OT, S, SP>::new_on_port(
self.shmem_provider.clone(),
self.broker_port,
self.configuration.clone(),
self.configuration,
)?;
(mgr, cpu_core)
@ -894,7 +894,7 @@ where
LlmpEventManager::existing_client_from_description(
new_shmem_provider,
&mgr_description,
self.configuration.clone(),
self.configuration,
)?,
staterestorer,
),
@ -905,7 +905,7 @@ where
let mgr = LlmpEventManager::<I, OT, S, SP>::existing_client_from_env(
new_shmem_provider,
_ENV_FUZZER_BROKER_CLIENT_INITIAL,
self.configuration.clone(),
self.configuration,
)?;
(None, LlmpRestartingEventManager::new(mgr, staterestorer))
@ -974,8 +974,7 @@ mod tests {
}
let mut llmp_mgr =
LlmpEventManager::<BytesInput, (), _, _>::new(llmp_client, "fuzzer".to_string())
.unwrap();
LlmpEventManager::<BytesInput, (), _, _>::new(llmp_client, "fuzzer".into()).unwrap();
let scheduler = RandCorpusScheduler::new();
@ -1017,7 +1016,7 @@ mod tests {
let mut llmp_clone = LlmpEventManager::existing_client_from_description(
shmem_provider,
&mgr_description,
"fuzzer".to_string(),
"fuzzer".into(),
)
.unwrap();

View File

@ -5,8 +5,9 @@ pub use simple::*;
pub mod llmp;
pub use llmp::*;
use ahash::AHasher;
use alloc::{string::String, vec::Vec};
use core::{fmt, marker::PhantomData, time::Duration};
use core::{fmt, hash::Hasher, marker::PhantomData, time::Duration};
use serde::{Deserialize, Serialize};
use crate::{
@ -65,6 +66,49 @@ pub enum BrokerEventResult {
Forward,
}
/// Distinguish a fuzzer by its config
#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
pub enum EventConfig {
AlwaysUnique,
FromName { name_hash: u64 },
}
impl EventConfig {
#[must_use]
pub fn from_name(name: &str) -> Self {
let mut hasher = AHasher::new_with_keys(0, 0);
hasher.write(name.as_bytes());
EventConfig::FromName {
name_hash: hasher.finish(),
}
}
#[must_use]
pub fn match_with(&self, other: &EventConfig) -> bool {
match self {
EventConfig::AlwaysUnique => false,
EventConfig::FromName { name_hash: a } => match other {
EventConfig::AlwaysUnique => false,
EventConfig::FromName { name_hash: b } => (a == b),
},
}
}
}
impl From<&str> for EventConfig {
#[must_use]
fn from(name: &str) -> Self {
Self::from_name(name)
}
}
impl From<String> for EventConfig {
#[must_use]
fn from(name: String) -> Self {
Self::from_name(&name)
}
}
/*
/// A custom event, for own messages, with own handler.
pub trait CustomEvent<I>: SerdeAny
@ -94,13 +138,13 @@ where
/// The input for the new testcase
input: I,
/// The state of the observers when this testcase was found
observers_buf: Vec<u8>,
observers_buf: Option<Vec<u8>>,
/// The exit kind
exit_kind: ExitKind,
/// The new corpus size of this client
corpus_size: usize,
/// The client config for this observers/testcase combination
client_config: String,
client_config: EventConfig,
/// The time of generation of the event
time: Duration,
/// The executions of this client
@ -228,8 +272,8 @@ where
}
/// Get the configuration
fn configuration(&self) -> &str {
"<default>"
fn configuration(&self) -> EventConfig {
EventConfig::AlwaysUnique
}
}
@ -319,7 +363,7 @@ mod tests {
current_time,
tuples::{tuple_list, Named},
},
events::Event,
events::{Event, EventConfig},
executors::ExitKind,
inputs::bytes::BytesInput,
observers::StdMapObserver,
@ -336,10 +380,10 @@ mod tests {
let i = BytesInput::new(vec![0]);
let e = Event::NewTestcase {
input: i,
observers_buf,
observers_buf: Some(observers_buf),
exit_kind: ExitKind::Ok,
corpus_size: 123,
client_config: "conf".into(),
client_config: EventConfig::AlwaysUnique,
time: current_time(),
executions: 0,
};
@ -358,7 +402,7 @@ mod tests {
executions: _,
} => {
let o: tuple_list_type!(StdMapObserver::<u32>) =
postcard::from_bytes(&observers_buf).unwrap();
postcard::from_bytes(observers_buf.as_ref().unwrap()).unwrap();
assert_eq!("test", o.0.name());
}
_ => panic!("mistmatch"),

View File

@ -3,7 +3,7 @@
use crate::{
bolts::current_time,
corpus::{Corpus, CorpusScheduler, Testcase},
events::{Event, EventFirer, EventManager},
events::{Event, EventConfig, EventFirer, EventManager},
executors::{Executor, ExitKind, HasObservers},
feedbacks::Feedback,
inputs::Input,
@ -374,7 +374,13 @@ where
self.scheduler_mut().on_add(state, idx)?;
if send_events {
let observers_buf = manager.serialize_observers(observers)?;
// TODO set None for fast targets
let observers_buf = match manager.configuration() {
EventConfig::AlwaysUnique => None,
EventConfig::FromName { .. } => {
Some(manager.serialize_observers(observers)?)
}
};
manager.fire(
state,
Event::NewTestcase {
@ -382,7 +388,7 @@ where
observers_buf,
exit_kind: exit_kind.clone(),
corpus_size: state.corpus().count(),
client_config: manager.configuration().to_string(),
client_config: manager.configuration(),
time: current_time(),
executions: *state.executions(),
},
@ -494,7 +500,10 @@ where
let idx = state.corpus_mut().add(testcase)?;
self.scheduler_mut().on_add(state, idx)?;
let observers_buf = manager.serialize_observers(observers)?;
let observers_buf = match manager.configuration() {
EventConfig::AlwaysUnique => None,
EventConfig::FromName { .. } => Some(manager.serialize_observers(observers)?),
};
manager.fire(
state,
Event::NewTestcase {
@ -502,7 +511,7 @@ where
observers_buf,
exit_kind,
corpus_size: state.corpus().count(),
client_config: manager.configuration().to_string(),
client_config: manager.configuration(),
time: current_time(),
executions: *state.executions(),
},

View File

@ -357,13 +357,14 @@ where
/// loads inputs from a directory
/// If `forced` is `true`, the value will be loaded,
/// even if it's not considered to be `interesting`.
fn load_from_directory<E, EM, Z>(
pub fn load_from_directory<E, EM, Z>(
&mut self,
fuzzer: &mut Z,
executor: &mut E,
manager: &mut EM,
in_dir: &Path,
forced: bool,
loader: &dyn Fn(&mut Z, &mut Self, &Path) -> Result<I, Error>,
) -> Result<(), Error>
where
Z: Evaluator<E, EM, I, Self>,
@ -381,7 +382,7 @@ where
if attr.is_file() && attr.len() > 0 {
println!("Loading file {:?} ...", &path);
let input = I::from_file(&path)?;
let input = loader(fuzzer, self, &path)?;
if forced {
let _ = fuzzer.add_input(self, executor, manager, input)?;
} else {
@ -391,7 +392,7 @@ where
}
}
} else if attr.is_dir() {
self.load_from_directory(fuzzer, executor, manager, &path, forced)?;
self.load_from_directory(fuzzer, executor, manager, &path, forced, loader)?;
}
}
@ -413,7 +414,9 @@ where
EM: EventFirer<I, Self>,
{
for in_dir in in_dirs {
self.load_from_directory(fuzzer, executor, manager, in_dir, forced)?;
self.load_from_directory(fuzzer, executor, manager, in_dir, forced, &|_, _, path| {
I::from_file(&path)
})?;
}
manager.fire(
self,

View File

@ -103,14 +103,11 @@ fn main() {
let cxxflags: Vec<&str> = cxxflags.trim().split_whitespace().collect();
let mut ldflags: Vec<&str> = ldflags.trim().split_whitespace().collect();
match env::var("CARGO_CFG_TARGET_VENDOR").unwrap().as_str() {
if env::var("CARGO_CFG_TARGET_VENDOR").unwrap().as_str() == "apple" {
// Needed on macos.
// Explanation at https://github.com/banach-space/llvm-tutor/blob/787b09ed31ff7f0e7bdd42ae20547d27e2991512/lib/CMakeLists.txt#L59
"apple" => {
ldflags.push("-undefined");
ldflags.push("dynamic_lookup");
}
_ => (),
};
let _ = Command::new(llvm_bindir.join("clang++"))

View File

@ -14,6 +14,7 @@ use libafl::{
CachedOnDiskCorpus, Corpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
QueueCorpusScheduler,
},
events::EventConfig,
executors::{inprocess::InProcessExecutor, ExitKind, TimeoutExecutor},
feedback_or, feedback_or_fast,
feedbacks::{CrashFeedback, MapFeedbackState, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
@ -74,10 +75,10 @@ where
{
#[allow(clippy::too_many_lines, clippy::similar_names)]
pub fn run(&mut self) {
let conf = self
.configuration
.take()
.unwrap_or_else(|| "default".into());
let conf = match self.configuration.as_ref() {
Some(name) => EventConfig::from_name(name),
None => EventConfig::AlwaysUnique,
};
let timeout = Duration::from_secs(self.timeout.unwrap_or(DEFAULT_TIMEOUT_SECS));

View File

@ -14,6 +14,7 @@ use libafl::{
CachedOnDiskCorpus, Corpus, IndexesLenTimeMinimizerCorpusScheduler, OnDiskCorpus,
QueueCorpusScheduler,
},
events::EventConfig,
executors::{ExitKind, TimeoutExecutor},
feedback_or, feedback_or_fast,
feedbacks::{CrashFeedback, MapFeedbackState, MaxMapFeedback, TimeFeedback, TimeoutFeedback},
@ -87,10 +88,10 @@ where
H: FnMut(&[u8]),
{
pub fn run(&mut self) {
let conf = self
.configuration
.take()
.unwrap_or_else(|| "default".into());
let conf = match self.configuration.as_ref() {
Some(name) => EventConfig::from_name(name),
None => EventConfig::AlwaysUnique,
};
let timeout = Duration::from_secs(self.timeout.unwrap_or(DEFAULT_TIMEOUT_SECS));