From c893c053b13165b5d723f9f8a48f6139fce66177 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Wed, 9 Dec 2020 22:30:31 +0100 Subject: [PATCH 1/2] moved to romu_trio rand --- afl/benches/rand_speeds.rs | 14 ++++- afl/src/engines/mod.rs | 3 +- afl/src/events/mod.rs | 4 +- afl/src/feedbacks/mod.rs | 1 - afl/src/utils.rs | 123 ++++++++++++++++++++++++++++--------- 5 files changed, 107 insertions(+), 38 deletions(-) diff --git a/afl/benches/rand_speeds.rs b/afl/benches/rand_speeds.rs index 087738659f..495d7018e0 100644 --- a/afl/benches/rand_speeds.rs +++ b/afl/benches/rand_speeds.rs @@ -1,14 +1,22 @@ //! Compare the speed of rand implementations -use afl::utils::{Rand, XorShift64Rand, Xoshiro256StarRand}; +use afl::utils::{ + Lehmer64Rand, Rand, RomuDuoJrRand, RomuTrioRand, XorShift64Rand, Xoshiro256StarRand, +}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; fn criterion_benchmark(c: &mut Criterion) { - let mut xorshift = XorShift64Rand::new(0); - let mut xoshiro = Xoshiro256StarRand::new(0); + let mut xorshift = XorShift64Rand::new(1); + let mut xoshiro = Xoshiro256StarRand::new(1); + let mut romu = RomuDuoJrRand::new(1); + let mut lehmer = Lehmer64Rand::new(1); + let mut romu_trio = RomuTrioRand::new(1); c.bench_function("xorshift", |b| b.iter(|| black_box(xorshift.next()))); c.bench_function("xoshiro", |b| b.iter(|| black_box(xoshiro.next()))); + c.bench_function("romu", |b| b.iter(|| black_box(romu.next()))); + c.bench_function("romu_trio", |b| b.iter(|| black_box(romu_trio.next()))); + c.bench_function("lehmer", |b| b.iter(|| black_box(lehmer.next()))); } criterion_group!(benches, criterion_benchmark); diff --git a/afl/src/engines/mod.rs b/afl/src/engines/mod.rs index 426fac6c0b..2f75445217 100644 --- a/afl/src/engines/mod.rs +++ b/afl/src/engines/mod.rs @@ -301,7 +301,8 @@ where let cur = current_milliseconds(); if cur - last > 60 * 100 { last = cur; - manager.fire(Event::update_stats(state.executions(), state.executions_over_seconds()), + manager.fire( + Event::update_stats(state.executions(), state.executions_over_seconds()), state, corpus, )?; // TODO self.new_execs}); diff --git a/afl/src/events/mod.rs b/afl/src/events/mod.rs index 25a0106f7a..cadbe0f1c9 100644 --- a/afl/src/events/mod.rs +++ b/afl/src/events/mod.rs @@ -33,9 +33,7 @@ pub enum BrokerEventResult { Forward, } -pub trait ShowStats { - -} +pub trait ShowStats {} /* diff --git a/afl/src/feedbacks/mod.rs b/afl/src/feedbacks/mod.rs index 9117333c1f..39f2d505dd 100644 --- a/afl/src/feedbacks/mod.rs +++ b/afl/src/feedbacks/mod.rs @@ -300,4 +300,3 @@ where } } */ - diff --git a/afl/src/utils.rs b/afl/src/utils.rs index 21ea1d3d4d..c23aaf81f6 100644 --- a/afl/src/utils.rs +++ b/afl/src/utils.rs @@ -9,14 +9,16 @@ use xxhash_rust::xxh3::xxh3_64_with_seed; #[cfg(feature = "std")] use std::time::{SystemTime, UNIX_EPOCH}; -pub type StdRand = Xoshiro256StarRand; +pub type StdRand = RomuTrioRand; /// Ways to get random around here pub trait Rand: Debug { // Sets the seed of this Rand fn set_seed(&mut self, seed: u64); + // Gets the next 64 bit value fn next(&mut self) -> u64; + // Gets a value below the given 64 bit val (inclusive) fn below(&mut self, upper_bound_excl: u64) -> u64 { if upper_bound_excl <= 1 { @@ -73,6 +75,15 @@ where const HASH_CONST: u64 = 0xa5b35705; +#[cfg(feature = "std")] +/// Gets current nanoseconds since UNIX_EPOCH +pub fn current_nanos() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 +} + /// XXH3 Based, hopefully speedy, rnd implementation /// #[derive(Copy, Clone, Debug, Default)] @@ -91,6 +102,7 @@ impl Rand for Xoshiro256StarRand { self.seeded = true; } + #[inline] fn next(&mut self) -> u64 { let ret: u64 = self.rand_seed[0] .wrapping_add(self.rand_seed[3]) @@ -125,19 +137,11 @@ impl Xoshiro256StarRand { ret } - pub fn to_rc_refcell(self) -> Rc> { - self.into() - } - /// Creates a rand instance, pre-seeded with the current time in nanoseconds. /// Needs stdlib timer #[cfg(feature = "std")] pub fn preseeded() -> Self { - let seed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() as u64; - Self::new(seed) + Self::new(current_nanos()) } } @@ -155,6 +159,7 @@ impl Rand for XorShift64Rand { self.seeded = true; } + #[inline] fn next(&mut self) -> u64 { let mut x = self.rand_seed; x ^= x << 13; @@ -179,19 +184,11 @@ impl XorShift64Rand { ret } - pub fn to_rc_refcell(self) -> Rc> { - self.into() - } - /// Creates a rand instance, pre-seeded with the current time in nanoseconds. /// Needs stdlib timer #[cfg(feature = "std")] pub fn preseeded() -> Self { - let seed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() as u64; - Self::new(seed) + Self::new(current_nanos()) } } @@ -209,6 +206,7 @@ impl Rand for Lehmer64Rand { self.seeded = true; } + #[inline] fn next(&mut self) -> u64 { self.rand_seed *= 0xda942042e4dd58b5; return (self.rand_seed >> 64) as u64; @@ -222,26 +220,91 @@ impl Into>> for Lehmer64Rand { } impl Lehmer64Rand { - /// Creates a new Xoshiro rand with the given seed + /// Creates a new Lehmer rand with the given seed pub fn new(seed: u64) -> Self { let mut ret: Self = Default::default(); - ret.set_seed(seed); // TODO: Proper random seed? + ret.set_seed(seed); ret } - pub fn to_rc_refcell(self) -> Rc> { - self.into() - } - /// Creates a rand instance, pre-seeded with the current time in nanoseconds. /// Needs stdlib timer #[cfg(feature = "std")] pub fn preseeded() -> Self { - let seed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() as u64; - Self::new(seed) + Self::new(current_nanos()) + } +} + +/// Extremely quick rand implementation +/// see https://arxiv.org/pdf/2002.11331.pdf +#[derive(Copy, Clone, Debug, Default)] +pub struct RomuTrioRand { + x_state: u64, + y_state: u64, + z_state: u64, +} + +impl RomuTrioRand { + pub fn new(seed: u64) -> Self { + let mut rand = Self::default(); + rand.set_seed(seed); + rand + } + + /// Creates a rand instance, pre-seeded with the current time in nanoseconds. + /// Needs stdlib timer + #[cfg(feature = "std")] + pub fn preseeded() -> Self { + Self::new(current_nanos()) + } +} + +impl Rand for RomuTrioRand { + fn set_seed(&mut self, seed: u64) { + self.x_state = seed ^ 0x12345; + self.y_state = seed ^ 0x6789A; + self.z_state = seed ^ 0xBCDEF; + } + + #[inline] + fn next(&mut self) -> u64 { + let xp = self.x_state; + let yp = self.y_state; + let zp = self.z_state; + self.x_state = 15241094284759029579u64.wrapping_mul(zp); + self.y_state = yp.wrapping_sub(xp).rotate_left(12); + self.z_state = zp.wrapping_sub(yp).rotate_left(44); + xp + } +} + +/// see https://arxiv.org/pdf/2002.11331.pdf +#[derive(Copy, Clone, Debug, Default)] +pub struct RomuDuoJrRand { + x_state: u64, + y_state: u64, +} + +impl RomuDuoJrRand { + pub fn new(seed: u64) -> Self { + let mut rand = Self::default(); + rand.set_seed(seed); + rand + } +} + +impl Rand for RomuDuoJrRand { + fn set_seed(&mut self, seed: u64) { + self.x_state = seed ^ 0x12345; + self.y_state = seed ^ 0x6789A; + } + + #[inline] + fn next(&mut self) -> u64 { + let xp = self.x_state; + self.x_state = 15241094284759029579u64.wrapping_mul(self.y_state); + self.y_state = self.y_state.wrapping_sub(xp).rotate_left(27); + xp } } From 620657a8eaca2808931cf61cfc068a7379d917be Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Wed, 9 Dec 2020 23:18:16 +0100 Subject: [PATCH 2/2] started event manager --- afl/Cargo.toml | 2 +- afl/src/events/llmp.rs | 5 + afl/src/events/llmp_manager.rs | 89 -------------- afl/src/events/mod.rs | 185 ++++++++++++++++++++++++++--- afl/src/events/shmem_translated.rs | 6 +- afl/src/utils.rs | 7 ++ 6 files changed, 184 insertions(+), 110 deletions(-) delete mode 100644 afl/src/events/llmp_manager.rs diff --git a/afl/Cargo.toml b/afl/Cargo.toml index fcac574c2a..15287084c6 100644 --- a/afl/Cargo.toml +++ b/afl/Cargo.toml @@ -33,4 +33,4 @@ num = "*" xxhash-rust = { version = "0.8.0", features = ["xxh3"] } # xxh3 hashing for rust serde = { version = "1.0", default-features = false, features = ["alloc"] } # serialization lib erased-serde = "0.3.12" -postcard = "0.5.1" # no_std compatible serde serialization fromat \ No newline at end of file +postcard = { version = "0.5.1", features = ["alloc"] } # no_std compatible serde serialization fromat \ No newline at end of file diff --git a/afl/src/events/llmp.rs b/afl/src/events/llmp.rs index 0f72928f15..fce9887cce 100644 --- a/afl/src/events/llmp.rs +++ b/afl/src/events/llmp.rs @@ -865,6 +865,11 @@ impl LlmpBroker { } })) } + + /// Broadcasts the given buf to all lients + fn send_buf(&mut self, tag: u32, buf: &[u8]) -> Result<(), AflError> { + self.llmp_out.send_buf(tag, buf) + } } /// `n` clients connect to a broker. They share an outgoing map with the broker, diff --git a/afl/src/events/llmp_manager.rs b/afl/src/events/llmp_manager.rs deleted file mode 100644 index 97e3e3f65e..0000000000 --- a/afl/src/events/llmp_manager.rs +++ /dev/null @@ -1,89 +0,0 @@ -use core::marker::PhantomData; -use core::ptr; -use std::{ffi::c_void, io::Read, io::Write, net::TcpListener}; - -use crate::{ - corpus::Corpus, engines::State, executors::Executor, inputs::Input, utils::Rand, AflError, -}; - -use super::{ - llmp_translated::{LlmpBroker, LlmpClient, LlmpClientloopFn, LlmpMsgHookFn}, - Event, EventManager, -}; - -/// Eventmanager for multi-processed application -#[cfg(feature = "std")] -pub struct LLMPEventManager -where - C: Corpus, - I: Input, - E: Executor, - R: Rand, - //CE: CustomEvent, -{ - // TODO... - phantom: PhantomData<(C, E, I, R)>, - is_broker: bool, -} - -#[cfg(feature = "std")] -impl EventManager for LLMPEventManager -where - C: Corpus, - E: Executor, - I: Input, - R: Rand, - //CE: CustomEvent, -{ - fn enabled(&self) -> bool { - true - } - - fn fire(&mut self, _event: Event) -> Result<(), AflError> { - //self.events.push(event); - - // TODO: Serde serialize, llmp send - - Ok(()) - } - - fn process(&mut self, _state: &mut State, _corpus: &mut C) -> Result { - // TODO: iterators - /* - let mut handled = vec![]; - for x in self.events.iter() { - handled.push(x.handle_in_broker(state, corpus)?); - } - handled - .iter() - .zip(self.events.iter()) - .map(|(x, event)| match x { - BrokerEventResult::Forward => event.handle_in_client(state, corpus), - // Ignore broker-only events - BrokerEventResult::Handled => Ok(()), - }) - .for_each(drop); - let count = self.events.len(); - dbg!("Handled {} events", count); - self.events.clear(); - - let num = self.events.len(); - for event in &self.events {} - - self.events.clear(); - */ - - Ok(0) - } -} - -#[cfg(feature = "std")] -impl LLMPEventManager -where - C: Corpus, - I: Input, - E: Executor, - R: Rand, -{ - -} diff --git a/afl/src/events/mod.rs b/afl/src/events/mod.rs index cadbe0f1c9..9d9c3a352e 100644 --- a/afl/src/events/mod.rs +++ b/afl/src/events/mod.rs @@ -1,19 +1,16 @@ -//#[cfg(feature = "std")] -//pub mod llmp; +#[cfg(feature = "std")] +mod llmp; +#[cfg(feature = "std")] +pub mod shmem_translated; use alloc::string::String; use core::marker::PhantomData; use serde::{Deserialize, Serialize}; -//#[cfg(feature = "std")] -//pub mod llmp_translated; // TODO: Abstract away. //#[cfg(feature = "std")] //pub mod shmem_translated; -//#[cfg(feature = "std")] -//pub use crate::events::llmp::LLMPEventManager; - #[cfg(feature = "std")] use std::io::Write; @@ -190,10 +187,6 @@ where I: Input, R: Rand, { - /// Check if this EventaManager support a given Event type - /// To compare events, use Event::name().as_ptr() - fn enabled(&self) -> bool; - /// Fire an Event fn fire<'a>( &mut self, @@ -328,10 +321,6 @@ where W: Write, //CE: CustomEvent, { - fn enabled(&self) -> bool { - true - } - fn fire<'a>( &mut self, event: Event<'a, C, E, I, R>, @@ -371,3 +360,169 @@ where } } } + +/// Eventmanager for multi-processed application +#[cfg(feature = "std")] +pub struct LlmpBrokerEventManager +where + C: Corpus, + I: Input, + E: Executor, + R: Rand, + //CE: CustomEvent, +{ + llmp_broker: llmp::LlmpBroker, + phantom: PhantomData<(C, E, I, R)>, +} + +/// Eventmanager for multi-processed application +#[cfg(feature = "std")] +pub struct LlmpClientEventManager +where + C: Corpus, + I: Input, + E: Executor, + R: Rand, + //CE: CustomEvent, +{ + llmp_client: llmp::LlmpClient, + phantom: PhantomData<(C, E, I, R)>, +} + +#[cfg(feature = "std")] +impl EventManager for LlmpBrokerEventManager +where + C: Corpus, + E: Executor, + I: Input, + R: Rand, + //CE: CustomEvent, +{ + /// Fire an Event + fn fire<'a>( + &mut self, + event: Event<'a, C, E, I, R>, + state: &mut State, + corpus: &mut C, + ) -> Result<(), AflError> { + // TODO let serialized = postcard::to_vec(&event)?; + // self.llmp_broker.send_buf(&serialized)?; + Ok(()) + } + + fn process(&mut self, _state: &mut State, _corpus: &mut C) -> Result { + // TODO: iterators + /* + let mut handled = vec![]; + for x in self.events.iter() { + handled.push(x.handle_in_broker(state, corpus)?); + } + handled + .iter() + .zip(self.events.iter()) + .map(|(x, event)| match x { + BrokerEventResult::Forward => event.handle_in_client(state, corpus), + // Ignore broker-only events + BrokerEventResult::Handled => Ok(()), + }) + .for_each(drop); + let count = self.events.len(); + dbg!("Handled {} events", count); + self.events.clear(); + + let num = self.events.len(); + for event in &self.events {} + + self.events.clear(); + */ + + Ok(0) + } + + fn on_recv(&self, _state: &mut State, _corpus: &mut C) -> Result<(), AflError> { + // TODO: Better way to move out of testcase, or get ref + //Ok(corpus.add(self.testcase.take().unwrap())) + Ok(()) + } + + fn handle_in_broker( + &self, + event: &Event, + /*broker: &dyn EventManager,*/ _state: &mut State, + _corpus: &mut C, + ) -> Result { + match event { + Event::LoadInitial { + sender_id: _, + phantom: _, + } => Ok(BrokerEventResult::Handled), + Event::NewTestcase { + sender_id: _, + input: _, + observers: _, + } => Ok(BrokerEventResult::Forward), + Event::UpdateStats { + sender_id: _, + executions: _, + execs_over_sec: _, + phantom: _, + } => { + // TODO + Ok(BrokerEventResult::Handled) + } + Event::Crash { + sender_id: _, + input: _, + phantom: _, + } => Ok(BrokerEventResult::Handled), + Event::Timeout { + sender_id: _, + input: _, + phantom: _, + } => { + // TODO + Ok(BrokerEventResult::Handled) + } + Event::Log { + sender_id, + severity_level, + message, + phantom: _, + } => { + //TODO: broker.log() + #[cfg(feature = "std")] + println!("{}[{}]: {}", sender_id, severity_level, message); + Ok(BrokerEventResult::Handled) + }, + Event::None { + phantom: _, + } => Ok(BrokerEventResult::Handled) + //Event::Custom {sender_id, custom_event} => custom_event.handle_in_broker(state, corpus), + //_ => Ok(BrokerEventResult::Forward), + } + } + + fn handle_in_client( + &self, + event: Event, + /*client: &dyn EventManager,*/ _state: &mut State, + corpus: &mut C, + ) -> Result<(), AflError> { + match event { + Event::NewTestcase { + sender_id: _, + input: _, + observers: _, + } => { + // here u should match sender_id, if equal to the current one do not re-execute + // we need to pass engine to process() too, TODO + #[cfg(feature = "std")] + println!("PLACEHOLDER: received NewTestcase"); + Ok(()) + } + _ => Err(AflError::Unknown( + "Received illegal message that message should not have arrived.".into(), + )), + } + } +} diff --git a/afl/src/events/shmem_translated.rs b/afl/src/events/shmem_translated.rs index 1506c805d9..60939f5b09 100644 --- a/afl/src/events/shmem_translated.rs +++ b/afl/src/events/shmem_translated.rs @@ -180,11 +180,7 @@ unsafe fn afl_shmem_init(shm: *mut AflShmem, map_size: usize) -> *mut c_uchar { } /// Uses a shmap id string to open a shared map -unsafe fn afl_shmem_by_str( - shm: *mut AflShmem, - shm_str: &CStr, - map_size: usize, -) -> *mut c_uchar { +unsafe fn afl_shmem_by_str(shm: *mut AflShmem, shm_str: &CStr, map_size: usize) -> *mut c_uchar { if shm.is_null() || shm_str.to_bytes().len() == 0 || map_size == 0 { return 0 as *mut c_uchar; } diff --git a/afl/src/utils.rs b/afl/src/utils.rs index c23aaf81f6..71618de124 100644 --- a/afl/src/utils.rs +++ b/afl/src/utils.rs @@ -291,6 +291,13 @@ impl RomuDuoJrRand { rand.set_seed(seed); rand } + + /// Creates a rand instance, pre-seeded with the current time in nanoseconds. + /// Needs stdlib timer + #[cfg(feature = "std")] + pub fn preseeded() -> Self { + Self::new(current_nanos()) + } } impl Rand for RomuDuoJrRand {