SYS-OSS/FRET-LibAFL
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move Input loading and dumping APIs from Testcase to Corpus (#1201)

Browse Source 
* Less allocatiosn for filenames

* clippy for wasm fuzzer

* Reworked filename and rename APIs

* python, clippy

* fmt

* More cleanup, fixed metadata location

* clippy

* fix fuzzbench_text / cached len, invert parameters (state first)

* clippy

* oops

* Caching for paths

* simplified, fixed

* no_std

* cached_len

* Nider API for input getting
This commit is contained in:
Dominik Maier 2023-04-18 12:14:49 +02:00 committed by GitHub
parent fd68c8a81f
commit 96e24d1c8b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 389 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
fuzzers/baby_fuzzer_wasm/src/lib.rs
View File

@ -23,15 +23,16 @@ use crate::utils::set_panic_hook;
// defined for internal use by libafl
#[no_mangle]
#[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)]
pub extern "C" fn external_current_millis() -> u64 {
let window: Window = web_sys::window().expect("should be in browser to run this demo");
let performance: Performance = window
.performance()
.expect("should be in browser to run this demo");
let now = performance.now() as u64;
now
performance.now() as u64
}
#[allow(clippy::missing_panics_doc)]
#[wasm_bindgen]
pub fn fuzz() {
set_panic_hook();
@ -39,7 +40,7 @@ pub fn fuzz() {
let mut signals = [0u8; 64];
let signals_ptr = signals.as_mut_ptr();
let signals_set = |i: usize| unsafe {
*signals_ptr.offset(i as isize) += 1;
*signals_ptr.add(i) += 1;
};
// The closure that we want to fuzz

1
fuzzers/fuzzbench_text/Cargo.toml
View File

@ -28,6 +28,7 @@ clap = { version = "4.0", features = ["default"] }
nix = "0.26"
mimalloc = { version = "*", default-features = false }
content_inspector = "0.2.4"
#log = "0.4"
[lib]
name = "fuzzbench"

22
fuzzers/fuzzbench_text/src/lib.rs
View File

@ -288,6 +288,12 @@ fn fuzz_binary(
// This way, we are able to continue fuzzing afterwards.
let mut shmem_provider = StdShMemProvider::new()?;
/*
// For debugging
let mut mgr = SimpleEventManager::new(monitor);
let mut state = None;
*/
let (state, mut mgr) = match SimpleRestartingEventManager::launch(monitor, &mut shmem_provider)
{
// The restarting state will spawn the same process again as child, then restarted it each time it crashes.
@ -435,8 +441,8 @@ fn fuzz_binary(
if state.must_load_initial_inputs() {
state
.load_initial_inputs(&mut fuzzer, &mut executor, &mut mgr, &[seed_dir.clone()])
.unwrap_or_else(|_| {
println!("Failed to load initial corpus at {:?}", &seed_dir);
.unwrap_or_else(|e| {
println!("Failed to load initial corpus at {seed_dir:?} - {e:?}");
process::exit(0);
});
println!("We imported {} inputs from disk.", state.corpus().count());
@ -491,6 +497,14 @@ fn fuzz_text(
// This way, we are able to continue fuzzing afterwards.
let mut shmem_provider = StdShMemProvider::new()?;
/*
// For debugging
log::set_max_level(log::LevelFilter::Trace);
SimpleStderrLogger::set_logger()?;
let mut mgr = SimpleEventManager::new(monitor);
let mut state = None;
*/
let (state, mut mgr) = match SimpleRestartingEventManager::launch(monitor, &mut shmem_provider)
{
// The restarting state will spawn the same process again as child, then restarted it each time it crashes.
@ -654,8 +668,8 @@ fn fuzz_text(
if state.must_load_initial_inputs() {
state
.load_initial_inputs(&mut fuzzer, &mut executor, &mut mgr, &[seed_dir.clone()])
.unwrap_or_else(|_| {
println!("Failed to load initial corpus at {:?}", &seed_dir);
.unwrap_or_else(|e| {
println!("Failed to load initial corpus at {seed_dir:?} {e:?}");
process::exit(0);
});
println!("We imported {} inputs from disk.", state.corpus().count());

2
fuzzers/tutorial/src/metadata.rs
View File

@ -25,7 +25,7 @@ impl<S> TestcaseScore<S> for PacketLenTestcaseScore
where
S: HasCorpus<Input = PacketData> + HasMetadata,
{
fn compute(entry: &mut Testcase<PacketData>, _state: &S) -> Result<f64, Error> {
fn compute(_state: &S, entry: &mut Testcase<PacketData>) -> Result<f64, Error> {
Ok(entry
.metadata_map()
.get::<PacketLenMetadata>()

7
libafl/.fancyfile.metadata Normal file
View File

@ -0,0 +1,7 @@
{
"metadata": {
"map": {}
},
"exec_time": null,
"executions": 0
}

12
libafl/src/corpus/cached.rs
View File

@ -73,7 +73,7 @@ where
fn get(&self, idx: CorpusId) -> Result<&RefCell<Testcase<I>>, Error> {
let testcase = { self.inner.get(idx)? };
if testcase.borrow().input().is_none() {
let _: &I = testcase.borrow_mut().load_input()?;
self.load_input_into(&mut testcase.borrow_mut())?;
let mut borrowed_num = 0;
while self.cached_indexes.borrow().len() >= self.cache_max_len {
let removed = self.cached_indexes.borrow_mut().pop_front().unwrap();
@ -128,6 +128,16 @@ where
fn nth(&self, nth: usize) -> CorpusId {
self.inner.nth(nth)
}
#[inline]
fn load_input_into(&self, testcase: &mut Testcase<Self::Input>) -> Result<(), Error> {
self.inner.load_input_into(testcase)
}
#[inline]
fn store_input_from(&self, testcase: &Testcase<Self::Input>) -> Result<(), Error> {
self.inner.store_input_from(testcase)
}
}
impl<I> HasTestcase for CachedOnDiskCorpus<I>

11
libafl/src/corpus/inmemory.rs
View File

@ -380,6 +380,17 @@ where
fn nth(&self, nth: usize) -> CorpusId {
self.storage.keys[nth]
}
#[inline]
fn load_input_into(&self, _: &mut Testcase<Self::Input>) -> Result<(), Error> {
// Inputs never get evicted, nothing to load here.
Ok(())
}
#[inline]
fn store_input_from(&self, _: &Testcase<Self::Input>) -> Result<(), Error> {
Ok(())
}
}
impl<I> HasTestcase for InMemoryCorpus<I>

165
libafl/src/corpus/inmemory_ondisk.rs
View File

@ -3,6 +3,7 @@
//! For a lower memory footprint, consider using [`crate::corpus::CachedOnDiskCorpus`]
//! which only stores a certain number of [`Testcase`]s and removes additional ones in a FIFO manner.
use alloc::string::String;
use core::{cell::RefCell, time::Duration};
#[cfg(feature = "std")]
use std::{fs, fs::File, io::Write};
@ -72,7 +73,9 @@ where
#[inline]
fn add(&mut self, testcase: Testcase<I>) -> Result<CorpusId, Error> {
let idx = self.inner.add(testcase)?;
self.save_testcase(&mut self.get(idx).unwrap().borrow_mut(), idx)?;
let testcase = &mut self.get(idx).unwrap().borrow_mut();
self.save_testcase(testcase, idx)?;
*testcase.input_mut() = None;
Ok(idx)
}
@ -81,7 +84,9 @@ where
fn replace(&mut self, idx: CorpusId, testcase: Testcase<I>) -> Result<Testcase<I>, Error> {
let entry = self.inner.replace(idx, testcase)?;
self.remove_testcase(&entry)?;
self.save_testcase(&mut self.get(idx).unwrap().borrow_mut(), idx)?;
let testcase = &mut self.get(idx).unwrap().borrow_mut();
self.save_testcase(testcase, idx)?;
*testcase.input_mut() = None;
Ok(entry)
}
@ -135,6 +140,28 @@ where
fn nth(&self, nth: usize) -> CorpusId {
self.inner.nth(nth)
}
fn load_input_into(&self, testcase: &mut Testcase<Self::Input>) -> Result<(), Error> {
if testcase.input_mut().is_none() {
let Some(file_path) = testcase.file_path().as_ref() else {
return Err(Error::illegal_argument("No file path set for testcase. Could not load inputs."));
};
let input = I::from_file(file_path)?;
testcase.set_input(input);
}
Ok(())
}
fn store_input_from(&self, testcase: &Testcase<Self::Input>) -> Result<(), Error> {
// Store the input to disk
let Some(file_path) = testcase.file_path() else {
return Err(Error::illegal_argument("No file path set for testcase. Could not store input to disk."));
};
let Some(input) = testcase.input() else {
return Err(Error::illegal_argument("No input available for testcase. Could not store anything."));
};
input.to_file(file_path)
}
}
impl<I> HasTestcase for InMemoryOnDiskCorpus<I>
@ -212,47 +239,109 @@ where
})
}
/// Sets the filename for a [`Testcase`].
/// If an error gets returned from the corpus (i.e., file exists), we'll have to retry with a different filename.
#[inline]
pub fn rename_testcase(
&self,
testcase: &mut Testcase<I>,
filename: String,
) -> Result<(), Error> {
if testcase.filename().is_some() {
// We are renaming!
let old_filename = testcase.filename_mut().take().unwrap();
let new_filename = filename;
// Do operations below when new filename is specified
if old_filename == new_filename {
*testcase.filename_mut() = Some(old_filename);
return Ok(());
}
let new_lock_filename = format!(".{new_filename}.lafl_lock");
// Try to create lock file for new testcases
if OpenOptions::new()
.create(true)
.write(true)
.open(self.dir_path.join(new_lock_filename))
.is_err()
{
*testcase.filename_mut() = Some(old_filename);
return Err(Error::illegal_state(
"unable to create lock file for new testcase",
));
}
let new_file_path = self.dir_path.join(&new_filename);
fs::rename(testcase.file_path().as_ref().unwrap(), &new_file_path)?;
let new_metadata_path = {
if let Some(old_metadata_path) = testcase.metadata_path() {
// We have metadata. Let's rename it.
let new_metadata_path = self.dir_path.join(format!(".{new_filename}.metadata"));
fs::rename(old_metadata_path, &new_metadata_path)?;
Some(new_metadata_path)
} else {
None
}
};
*testcase.metadata_path_mut() = new_metadata_path;
*testcase.filename_mut() = Some(new_filename);
*testcase.file_path_mut() = Some(new_file_path);
Ok(())
} else {
Err(Error::illegal_argument(
"Cannot rename testcase without name!",
))
}
}
fn save_testcase(&self, testcase: &mut Testcase<I>, idx: CorpusId) -> Result<(), Error> {
if testcase.filename().is_none() {
let file_name_orig = testcase.filename_mut().take().unwrap_or_else(|| {
// TODO walk entry metadata to ask for pieces of filename (e.g. :havoc in AFL)
let file_orig = testcase.input().as_ref().unwrap().generate_name(idx.0);
let mut file = file_orig.clone();
testcase.input().as_ref().unwrap().generate_name(idx.0)
});
if testcase.file_path().is_some() {
// We already have a valid path, no need to do calculate anything
*testcase.filename_mut() = Some(file_name_orig);
} else {
// New testcase, we need to save it.
let mut file_name = file_name_orig.clone();
let mut ctr = 2;
let filename = loop {
let lockfile = format!(".{file}.lafl_lock");
let (file_name, lockfile_path) = loop {
let lockfile_name = format!(".{file_name}.lafl_lock");
let lockfile_path = self.dir_path.join(lockfile_name);
if OpenOptions::new()
.write(true)
.create_new(true)
.open(self.dir_path.join(lockfile))
.open(&lockfile_path)
.is_ok()
{
break file;
break (file_name, lockfile_path);
}
file = format!("{file_orig}-{ctr}");
file_name = format!("{file_name_orig}-{ctr}");
ctr += 1;
};
let file_path = self.dir_path.join(filename.clone());
let filename_str = file_path.to_str().expect("Invalid Path");
testcase.set_filename(filename_str.into())?;
*testcase.file_path_mut() = Some(self.dir_path.join(&file_name));
*testcase.filename_mut() = Some(file_name);
fs::remove_file(lockfile_path)?;
}
let lock_file_path = self.dir_path.join(format!(".{filename}.lafl_lock"));
fs::remove_file(lock_file_path)?;
};
if self.meta_format.is_some() {
let mut filename = PathBuf::from(testcase.filename().as_ref().unwrap());
filename.set_file_name(format!(
".{}.metadata",
filename.file_name().unwrap().to_string_lossy()
));
let mut tmpfile_name = PathBuf::from(&filename);
tmpfile_name.set_file_name(format!(
".{}.tmp",
tmpfile_name.file_name().unwrap().to_string_lossy()
));
let metafile_name = format!(".{}.metadata", testcase.filename().as_ref().unwrap());
let metafile_path = self.dir_path.join(&metafile_name);
let mut tmpfile_path = metafile_path.clone();
tmpfile_path.set_file_name(format!(".{metafile_name}.tmp",));
let ondisk_meta = OnDiskMetadata {
metadata: testcase.metadata_map(),
@ -260,7 +349,7 @@ where
executions: testcase.executions(),
};
let mut tmpfile = File::create(&tmpfile_name)?;
let mut tmpfile = File::create(&tmpfile_path)?;
let serialized = match self.meta_format.as_ref().unwrap() {
OnDiskMetadataFormat::Postcard => postcard::to_allocvec(&ondisk_meta)?,
@ -272,25 +361,23 @@ where
.unwrap(),
};
tmpfile.write_all(&serialized)?;
fs::rename(&tmpfile_name, &filename)?;
fs::rename(&tmpfile_path, &metafile_path)?;
*testcase.metadata_path_mut() = Some(metafile_path);
}
testcase
.store_input()
.expect("Could not save testcase to disk");
self.store_input_from(testcase)?;
Ok(())
}
fn remove_testcase(&self, testcase: &Testcase<I>) -> Result<(), Error> {
if let Some(filename) = testcase.filename() {
fs::remove_file(filename)?;
}
fs::remove_file(self.dir_path.join(filename))?;
if self.meta_format.is_some() {
let mut filename = PathBuf::from(testcase.filename().as_ref().unwrap());
filename.set_file_name(format!(
".{}.metadata",
filename.file_name().unwrap().to_string_lossy()
));
fs::remove_file(filename)?;
fs::remove_file(self.dir_path.join(format!(".{filename}.metadata")))?;
}
// also try to remove the corresponding `.lafl_lock` file if it still exists
// (even though it shouldn't exist anymore, at this point in time)
let _ = fs::remove_file(self.dir_path.join(format!(".{filename}.lafl_lock")));
}
Ok(())
}

2
libafl/src/corpus/minimizer.rs
View File

@ -115,7 +115,7 @@ where
while let Some(idx) = cur_id {
let (weight, input) = {
let mut testcase = state.corpus().get(idx)?.borrow_mut();
let weight = TS::compute(&mut *testcase, state)?
let weight = TS::compute(state, &mut *testcase)?
.to_u64()
.expect("Weight must be computable.");
let input = testcase

28
libafl/src/corpus/mod.rs
View File

@ -71,7 +71,7 @@ macro_rules! random_corpus_id {
}};
}
/// Corpus with all current testcases
/// Corpus with all current [`Testcase`]s, or solutions
pub trait Corpus: UsesInput + Serialize + for<'de> Deserialize<'de> {
/// Returns the number of elements
fn count(&self) -> usize;
@ -84,7 +84,7 @@ pub trait Corpus: UsesInput + Serialize + for<'de> Deserialize<'de> {
/// Add an entry to the corpus and return its index
fn add(&mut self, testcase: Testcase<Self::Input>) -> Result<CorpusId, Error>;
/// Replaces the testcase at the given idx, returning the existing.
/// Replaces the [`Testcase`] at the given idx, returning the existing.
fn replace(
&mut self,
idx: CorpusId,
@ -130,9 +130,23 @@ pub trait Corpus: UsesInput + Serialize + for<'de> Deserialize<'de> {
.nth(nth)
.expect("Failed to get the {nth} CorpusId")
}
/// Method to load the input for this [`Testcase`] from persistent storage,
/// if necessary, and if was not already loaded (`== Some(input)`).
/// After this call, `testcase.input()` must always return `Some(input)`.
fn load_input_into(&self, testcase: &mut Testcase<Self::Input>) -> Result<(), Error>;
/// Method to store the input of this `Testcase` to persistent storage, if necessary.
fn store_input_from(&self, testcase: &Testcase<Self::Input>) -> Result<(), Error>;
/// Loads the `Input` for a given [`CorpusId`] from the [`Corpus`], and returns the clone.
fn cloned_input_for_id(&self, idx: CorpusId) -> Result<Self::Input, Error> {
let mut testcase = self.get(idx)?.borrow_mut();
Ok(testcase.load_input(self)?.clone())
}
}
/// `Iterator` over the ids of a `Corpus`
/// [`Iterator`] over the ids of a [`Corpus`]
#[derive(Debug)]
pub struct CorpusIdIterator<'a, C>
where
@ -368,6 +382,14 @@ pub mod pybind {
unwrap_me!(self.wrapper, c, { c.last() })
}
fn load_input_into(&self, testcase: &mut Testcase<BytesInput>) -> Result<(), Error> {
unwrap_me!(self.wrapper, c, { c.load_input_into(testcase) })
}
fn store_input_from(&self, testcase: &Testcase<BytesInput>) -> Result<(), Error> {
unwrap_me!(self.wrapper, c, { c.store_input_from(testcase) })
}
/*fn ids<'a>(&'a self) -> CorpusIdIterator<'a, Self> {
CorpusIdIterator {
corpus: self,

10
libafl/src/corpus/ondisk.rs
View File

@ -138,6 +138,16 @@ where
fn nth(&self, nth: usize) -> CorpusId {
self.inner.nth(nth)
}
#[inline]
fn load_input_into(&self, testcase: &mut Testcase<Self::Input>) -> Result<(), Error> {
self.inner.load_input_into(testcase)
}
#[inline]
fn store_input_from(&self, testcase: &Testcase<Self::Input>) -> Result<(), Error> {
self.inner.store_input_from(testcase)
}
}
impl<I> HasTestcase for OnDiskCorpus<I>

152
libafl/src/corpus/testcase.rs
View File

@ -9,10 +9,11 @@ use core::{
time::Duration,
};
#[cfg(feature = "std")]
use std::fs;
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use super::Corpus;
use crate::{
bolts::{serdeany::SerdeAnyMap, HasLen},
corpus::CorpusId,
@ -43,12 +44,18 @@ pub struct Testcase<I>
where
I: Input,
{
/// The input of this testcase
/// The [`Input`] of this [`Testcase`], or `None`, if it is not currently in memory
input: Option<I>,
/// Filename, if this testcase is backed by a file in the filesystem
/// The filename for this [`Testcase`]
filename: Option<String>,
/// Map of metadata associated with this testcase
/// Complete path to the [`Input`] on disk, if this [`Testcase`] is backed by a file in the filesystem
#[cfg(feature = "std")]
file_path: Option<PathBuf>,
/// Map of metadata associated with this [`Testcase`]
metadata: SerdeAnyMap,
/// Complete path to the metadata [`SerdeAnyMap`] on disk, if this [`Testcase`] is backed by a file in the filesystem
#[cfg(feature = "std")]
metadata_path: Option<PathBuf>,
/// Time needed to execute the input
exec_time: Option<Duration>,
/// Cached len of the input, if any
@ -83,36 +90,13 @@ impl<I> Testcase<I>
where
I: Input,
{
/// Returns this testcase with a loaded input
pub fn load_input(&mut self) -> Result<&I, Error> {
if self.input.is_none() {
self.input = Some(I::from_file(self.filename.as_ref().unwrap())?);
}
/// Returns this [`Testcase`] with a loaded `Input`]
pub fn load_input<C: Corpus<Input = I>>(&mut self, corpus: &C) -> Result<&I, Error> {
corpus.load_input_into(self)?;
Ok(self.input.as_ref().unwrap())
}
/// Store the input to disk if possible
pub fn store_input(&mut self) -> Result<bool, Error> {
match self.filename() {
Some(fname) => {
let saved = match self.input() {
None => false,
Some(i) => {
i.to_file(fname)?;
true
}
};
if saved {
// remove the input from memory
*self.input_mut() = None;
}
Ok(saved)
}
None => Ok(false),
}
}
/// Get the input, if any
/// Get the input, if available any
#[inline]
pub fn input(&self) -> &Option<I> {
&self.input
@ -144,55 +128,32 @@ where
&mut self.filename
}
/// Set the filename
/// Get the filename path, if any
#[inline]
#[cfg(feature = "std")]
pub fn set_filename(&mut self, filename: String) -> Result<(), Error> {
use std::fs::OpenOptions;
if self.filename.is_some() {
let f = self.filename.clone().unwrap();
let old_filename = f.as_str();
let new_filename = filename.as_str();
// Do operations below when new filename is specified
if old_filename.eq(new_filename) {
return Ok(());
}
let new_lock_filename = format!(".{new_filename}.lafl_lock");
// Try to create lock file for new testcases
if OpenOptions::new()
.create(true)
.write(true)
.open(&new_lock_filename)
.is_err()
{
return Err(Error::illegal_state(
"unable to create lock file for new testcase",
));
}
fs::rename(old_filename, new_filename)?;
let old_metadata_filename = format!(".{old_filename}.metadata");
let new_metadata_filename = format!(".{new_filename}.metadata");
fs::rename(old_metadata_filename, new_metadata_filename)?;
fs::remove_file(&new_lock_filename)?;
}
self.filename = Some(filename);
Ok(())
pub fn file_path(&self) -> &Option<PathBuf> {
&self.file_path
}
/// Get the filename path, if any (mutable)
#[inline]
#[cfg(feature = "no_std")]
pub fn set_filename(&mut self, filename: String) -> Result<(), Error> {
self.filename = Some(filename);
Ok(())
#[cfg(feature = "std")]
pub fn file_path_mut(&mut self) -> &mut Option<PathBuf> {
&mut self.file_path
}
/// Get the metadata path, if any
#[inline]
#[cfg(feature = "std")]
pub fn metadata_path(&self) -> &Option<PathBuf> {
&self.metadata_path
}
/// Get the metadata path, if any (mutable)
#[inline]
#[cfg(feature = "std")]
pub fn metadata_path_mut(&mut self) -> &mut Option<PathBuf> {
&mut self.metadata_path
}
/// Get the execution time of the testcase
@ -313,6 +274,10 @@ where
scheduled_count: 0,
executions: 0,
parent_id: None,
#[cfg(feature = "std")]
file_path: None,
#[cfg(feature = "std")]
metadata_path: None,
}
}
}
@ -322,25 +287,30 @@ impl<I> Testcase<I>
where
I: Input + HasLen,
{
/// Get the cached len
/// Get the cached `len`. Will `Error::EmptyOptional` if `len` is not yet cached.
#[inline]
pub fn cached_len(&mut self) -> Result<usize, Error> {
Ok(match &self.input {
pub fn cached_len(&mut self) -> Option<usize> {
self.cached_len
}
/// Get the `len` or calculate it, if not yet calculated.
#[allow(clippy::len_without_is_empty)]
pub fn load_len<C: Corpus<Input = I>>(&mut self, corpus: &C) -> Result<usize, Error> {
match &self.input {
Some(i) => {
let l = i.len();
self.cached_len = Some(l);
l
Ok(l)
}
None => {
if let Some(l) = self.cached_len {
l
Ok(l)
} else {
let l = self.load_input()?.len();
self.cached_len = Some(l);
l
corpus.load_input_into(self)?;
self.load_len(corpus)
}
}
}
})
}
}
@ -471,11 +441,7 @@ pub mod pybind {
use pyo3::{prelude::*, types::PyDict};
use super::{HasMetadata, Testcase};
use crate::{
bolts::ownedref::OwnedMutPtr,
inputs::{BytesInput, HasBytesVec},
pybind::PythonMetadata,
};
use crate::{bolts::ownedref::OwnedMutPtr, inputs::BytesInput, pybind::PythonMetadata};
/// `PythonTestcase` with fixed generics
pub type PythonTestcase = Testcase<BytesInput>;
@ -514,14 +480,6 @@ pub mod pybind {
}
}
fn load_input(&mut self) -> &[u8] {
self.inner
.as_mut()
.load_input()
.expect("Failed to load input")
.bytes()
}
#[getter]
fn exec_time_ms(&self) -> Option<u128> {
self.inner.as_ref().exec_time().map(|t| t.as_millis())

12
libafl/src/feedbacks/nautilus.rs
View File

@ -9,14 +9,14 @@ use serde_json;
use crate::{
bolts::tuples::Named,
corpus::Testcase,
corpus::{Corpus, Testcase},
events::EventFirer,
executors::ExitKind,
feedbacks::Feedback,
generators::NautilusContext,
inputs::{NautilusInput, UsesInput},
observers::ObserversTuple,
state::{HasClientPerfMonitor, HasMetadata},
state::{HasClientPerfMonitor, HasCorpus, HasMetadata},
Error,
};
@ -82,7 +82,10 @@ impl<'a, S> Named for NautilusFeedback<'a, S> {
impl<'a, S> Feedback<S> for NautilusFeedback<'a, S>
where
S: HasMetadata + HasClientPerfMonitor + UsesInput<Input = NautilusInput>,
S: HasMetadata
+ HasClientPerfMonitor
+ UsesInput<Input = NautilusInput>
+ HasCorpus<Input = NautilusInput>,
{
#[allow(clippy::wrong_self_convention)]
fn is_interesting<EM, OT>(
@ -109,7 +112,8 @@ where
where
OT: ObserversTuple<S>,
{
let input = testcase.load_input()?.clone();
state.corpus().load_input_into(testcase)?;
let input = testcase.input().as_ref().unwrap().clone();
let meta = state
.metadata_map_mut()
.get_mut::<NautilusChunksMetadata>()

37
libafl/src/mutators/encoded_mutations.rs
View File

@ -332,13 +332,11 @@ where
}
}
let other_size = state
.corpus()
.get(idx)?
.borrow_mut()
.load_input()?
.codes()
.len();
let other_size = {
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
other_testcase.load_input(state.corpus())?.codes().len()
};
if other_size < 2 {
return Ok(MutationResult::Skipped);
}
@ -348,9 +346,6 @@ where
let to = state.rand_mut().below(size as u64) as usize;
let mut len = 1 + state.rand_mut().below((other_size - from) as u64) as usize;
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
let other = other_testcase.load_input()?;
if size + len > max_size {
if max_size > size {
len = max_size - size;
@ -359,6 +354,10 @@ where
}
}
let other_testcase = state.corpus().get(idx)?.borrow_mut();
// no need to `load_input` again - we did that above already.
let other = other_testcase.input().as_ref().unwrap();
input.codes_mut().resize(size + len, 0);
unsafe {
buffer_self_copy(input.codes_mut(), to, to + len, size - to);
@ -410,13 +409,12 @@ where
}
}
let other_size = state
.corpus()
.get(idx)?
.borrow_mut()
.load_input()?
.codes()
.len();
let other_size = {
// new scope to make the borrow checker happy
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
other_testcase.load_input(state.corpus())?.codes().len()
};
if other_size < 2 {
return Ok(MutationResult::Skipped);
}
@ -425,8 +423,9 @@ where
let len = state.rand_mut().below(min(other_size - from, size) as u64) as usize;
let to = state.rand_mut().below((size - len) as u64) as usize;
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
let other = other_testcase.load_input()?;
let other_testcase = state.corpus().get(idx)?.borrow_mut();
// no need to load the input again, it'll already be present at this point.
let other = other_testcase.input().as_ref().unwrap();
unsafe {
buffer_copy(input.codes_mut(), other.codes(), from, to, len);

5
libafl/src/mutators/gramatron.rs
View File

@ -117,11 +117,10 @@ where
let rand_num = state.rand_mut().next() as usize;
let mut other_testcase = state.testcase_mut(idx)?;
other_testcase.load_input()?; // Preload the input
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
if !other_testcase.has_metadata::<GramatronIdxMapMetadata>() {
let meta = GramatronIdxMapMetadata::new(other_testcase.input().as_ref().unwrap());
let meta = GramatronIdxMapMetadata::new(other_testcase.load_input(state.corpus())?);
other_testcase.add_metadata(meta);
}
let meta = other_testcase

47
libafl/src/mutators/mutations.rs
View File

@ -1114,13 +1114,11 @@ where
}
}
let other_size = state
.corpus()
.get(idx)?
.borrow_mut()
.load_input()?
.bytes()
.len();
let other_size = {
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
other_testcase.load_input(state.corpus())?.bytes().len()
};
if other_size < 2 {
return Ok(MutationResult::Skipped);
}
@ -1128,9 +1126,6 @@ where
let range = rand_range(state, other_size, min(other_size, max_size - size));
let target = state.rand_mut().below(size as u64) as usize;
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
let other = other_testcase.load_input()?;
input.bytes_mut().resize(size + range.len(), 0);
unsafe {
buffer_self_copy(
@ -1139,6 +1134,13 @@ where
target + range.len(),
size - target,
);
}
let other_testcase = state.corpus().get(idx)?.borrow_mut();
// No need to load the input again, it'll still be cached.
let other = other_testcase.input().as_ref().unwrap();
unsafe {
buffer_copy(
input.bytes_mut(),
other.bytes(),
@ -1193,13 +1195,11 @@ where
}
}
let other_size = state
.corpus()
.get(idx)?
.borrow_mut()
.load_input()?
.bytes()
.len();
let other_size = {
let mut testcase = state.corpus().get(idx)?.borrow_mut();
testcase.load_input(state.corpus())?.bytes().len()
};
if other_size < 2 {
return Ok(MutationResult::Skipped);
}
@ -1207,8 +1207,9 @@ where
let target = state.rand_mut().below(size as u64) as usize;
let range = rand_range(state, other_size, min(other_size, size - target));
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
let other = other_testcase.load_input()?;
let other_testcase = state.corpus().get(idx)?.borrow_mut();
// No need to load the input again, it'll still be cached.
let other = other_testcase.input().as_ref().unwrap();
unsafe {
buffer_copy(
@ -1279,7 +1280,7 @@ where
let (first_diff, last_diff) = {
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
let other = other_testcase.load_input()?;
let other = other_testcase.load_input(state.corpus())?;
let mut counter: u32 = 0;
loop {
@ -1297,8 +1298,10 @@ where
let split_at = state.rand_mut().between(first_diff, last_diff) as usize;
let mut other_testcase = state.corpus().get(idx)?.borrow_mut();
let other = other_testcase.load_input()?;
let other_testcase = state.corpus().get(idx)?.borrow_mut();
// Input will already be loaded.
let other = other_testcase.input().as_ref().unwrap();
input
.bytes_mut()
.splice(split_at.., other.bytes()[split_at..].iter().copied());

10
libafl/src/mutators/scheduled.rs
View File

@ -440,10 +440,7 @@ mod tests {
.add(Testcase::new(vec![b'd', b'e', b'f'].into()))
.unwrap();
let testcase = corpus
.get(corpus.first().unwrap())
.expect("Corpus did not contain entries");
let mut input = testcase.borrow_mut().load_input().unwrap().clone();
let mut input = corpus.cloned_input_for_id(corpus.first().unwrap()).unwrap();
let mut feedback = ConstFeedback::new(false);
let mut objective = ConstFeedback::new(false);
@ -481,10 +478,7 @@ mod tests {
.add(Testcase::new(vec![b'd', b'e', b'f'].into()))
.unwrap();
let testcase = corpus
.get(corpus.first().unwrap())
.expect("Corpus did not contain entries");
let mut input = testcase.borrow_mut().load_input().unwrap().clone();
let mut input = corpus.cloned_input_for_id(corpus.first().unwrap()).unwrap();
let input_prior = input.clone();
let mut feedback = ConstFeedback::new(false);

2
libafl/src/schedulers/ecofuzz.rs
View File

@ -369,7 +369,7 @@ where
{
/// Compute the `weight` used in weighted corpus entry selection algo
#[allow(clippy::cast_precision_loss, clippy::cast_lossless)]
fn compute(entry: &mut Testcase<S::Input>, state: &S) -> Result<f64, Error> {
fn compute(state: &S, entry: &mut Testcase<S::Input>) -> Result<f64, Error> {
// subtract # initial inputs to the corpus count
let mut energy = 0;
let mut average_cost = (state.corpus().count() / state.executions()) as u64;

6
libafl/src/schedulers/minimizer.rs
View File

@ -116,7 +116,7 @@ where
let mut map = HashMap::new();
for i in state.corpus().ids() {
let mut old = state.corpus().get(i)?.borrow_mut();
let factor = F::compute(&mut *old, state)?;
let factor = F::compute(state, &mut *old)?;
if let Some(old_map) = old.metadata_map_mut().get_mut::<M>() {
let mut e_iter = entries.iter();
let mut map_iter = old_map.as_slice().iter(); // ASSERTION: guaranteed to be in order?
@ -256,7 +256,7 @@ where
let mut new_favoreds = vec![];
{
let mut entry = state.corpus().get(idx)?.borrow_mut();
let factor = F::compute(&mut *entry, state)?;
let factor = F::compute(state, &mut *entry)?;
let meta = entry.metadata_map_mut().get_mut::<M>().ok_or_else(|| {
Error::key_not_found(format!(
"Metadata needed for MinimizerScheduler not found in testcase #{idx}"
@ -270,7 +270,7 @@ where
continue;
}
let mut old = state.corpus().get(*old_idx)?.borrow_mut();
if factor > F::compute(&mut *old, state)? {
if factor > F::compute(state, &mut *old)? {
continue;
}

4
libafl/src/schedulers/probabilistic_sampling.rs
View File

@ -70,7 +70,7 @@ where
#[allow(clippy::cast_precision_loss)]
#[allow(clippy::unused_self)]
pub fn store_probability(&self, state: &mut S, idx: CorpusId) -> Result<(), Error> {
let factor = F::compute(&mut *state.corpus().get(idx)?.borrow_mut(), state)?;
let factor = F::compute(state, &mut *state.corpus().get(idx)?.borrow_mut())?;
if factor == 0.0 {
return Err(Error::illegal_state(
"Infinity probability calculated for probabilistic sampling scheduler",
@ -186,7 +186,7 @@ mod tests {
where
S: HasMetadata + HasCorpus,
{
fn compute(_: &mut Testcase<S::Input>, _state: &S) -> Result<f64, Error> {
fn compute(_state: &S, _: &mut Testcase<S::Input>) -> Result<f64, Error> {
Ok(FACTOR)
}
}

7
libafl/src/schedulers/queue.rs
View File

@ -107,10 +107,7 @@ mod tests {
let mut q =
OnDiskCorpus::<BytesInput>::new(PathBuf::from("target/.test/fancy/path")).unwrap();
let t = Testcase::with_filename(
BytesInput::new(vec![0_u8; 4]),
"target/.test/fancy/path/fancyfile".into(),
);
let t = Testcase::with_filename(BytesInput::new(vec![0_u8; 4]), "fancyfile".into());
q.add(t).unwrap();
let objective_q =
@ -133,7 +130,7 @@ mod tests {
.unwrap()
.clone();
assert_eq!(filename, "target/.test/fancy/path/fancyfile");
assert_eq!(filename, "fancyfile");
fs::remove_dir_all("target/.test/fancy").unwrap();
}

11
libafl/src/schedulers/testcase_score.rs
View File

@ -20,7 +20,7 @@ where
S: HasMetadata + HasCorpus,
{
/// Computes the favor factor of a [`Testcase`]. Lower is better.
fn compute(entry: &mut Testcase<S::Input>, state: &S) -> Result<f64, Error>;
fn compute(state: &S, entry: &mut Testcase<S::Input>) -> Result<f64, Error>;
}
/// Multiply the testcase size with the execution time.
@ -36,9 +36,10 @@ where
S::Input: HasLen,
{
#[allow(clippy::cast_precision_loss, clippy::cast_lossless)]
fn compute(entry: &mut Testcase<S::Input>, _state: &S) -> Result<f64, Error> {
fn compute(state: &S, entry: &mut Testcase<S::Input>) -> Result<f64, Error> {
// TODO maybe enforce entry.exec_time().is_some()
Ok(entry.exec_time().map_or(1, |d| d.as_millis()) as f64 * entry.cached_len()? as f64)
Ok(entry.exec_time().map_or(1, |d| d.as_millis()) as f64
* entry.load_len(state.corpus())? as f64)
}
}
@ -65,7 +66,7 @@ where
clippy::cast_sign_loss,
clippy::cast_lossless
)]
fn compute(entry: &mut Testcase<S::Input>, state: &S) -> Result<f64, Error> {
fn compute(state: &S, entry: &mut Testcase<S::Input>) -> Result<f64, Error> {
let psmeta = state.metadata::<SchedulerMetadata>()?;
let fuzz_mu = if let Some(strat) = psmeta.strat() {
@ -273,7 +274,7 @@ where
{
/// Compute the `weight` used in weighted corpus entry selection algo
#[allow(clippy::cast_precision_loss, clippy::cast_lossless)]
fn compute(entry: &mut Testcase<S::Input>, state: &S) -> Result<f64, Error> {
fn compute(state: &S, entry: &mut Testcase<S::Input>) -> Result<f64, Error> {
let mut weight = 1.0;
let psmeta = state.metadata::<SchedulerMetadata>()?;

2
libafl/src/schedulers/weighted.rs
View File

@ -154,7 +154,7 @@ where
for i in state.corpus().ids() {
let mut testcase = state.corpus().get(i)?.borrow_mut();
let weight = F::compute(&mut *testcase, state)?;
let weight = F::compute(state, &mut *testcase)?;
weights.insert(i, weight);
sum += weight;
}

14
libafl/src/stages/calibrate.rs
View File

@ -115,12 +115,7 @@ where
let mut iter = self.stage_max;
let input = state
.corpus()
.get(corpus_idx)?
.borrow_mut()
.load_input()?
.clone();
let input = state.corpus().cloned_input_for_id(corpus_idx)?;
// Run once to get the initial calibration map
executor.observers_mut().pre_exec_all(state, &input)?;
@ -158,12 +153,7 @@ where
let mut has_errors = false;
while i < iter {
let input = state
.corpus()
.get(corpus_idx)?
.borrow_mut()
.load_input()?
.clone();
let input = state.corpus().cloned_input_for_id(corpus_idx)?;
executor.observers_mut().pre_exec_all(state, &input)?;
start = current_time();

8
libafl/src/stages/colorization.rs
View File

@ -155,13 +155,7 @@ where
corpus_idx: CorpusId,
name: &str,
) -> Result<E::Input, Error> {
let mut input = state
.corpus()
.get(corpus_idx)?
.borrow_mut()
.load_input()
.unwrap()
.clone();
let mut input = state.corpus().cloned_input_for_id(corpus_idx)?;
// The backup of the input
let backup = input.clone();
// This is the buffer we'll randomly mutate during type_replace

26
libafl/src/stages/dump.rs
View File

@ -1,6 +1,6 @@
//! The [`DumpToDiskStage`] is a stage that dumps the corpus and the solutions to disk to e.g. allow AFL to sync
use alloc::vec::Vec;
use alloc::{string::String, vec::Vec};
use core::{clone::Clone, marker::PhantomData};
use std::{fs, fs::File, io::Write, path::PathBuf};
@ -68,10 +68,16 @@ where
while let Some(i) = corpus_idx {
let mut testcase = state.corpus().get(i)?.borrow_mut();
let input = testcase.load_input()?;
let bytes = (self.to_bytes)(input);
state.corpus().load_input_into(&mut testcase)?;
let bytes = (self.to_bytes)(testcase.input().as_ref().unwrap());
let fname = self.corpus_dir.join(format!("id_{i}"));
let fname = self.corpus_dir.join(format!(
"id_{i}_{}",
testcase
.filename()
.as_ref()
.map_or_else(|| "unnamed", String::as_str)
));
let mut f = File::create(fname)?;
drop(f.write_all(&bytes));
@ -80,10 +86,16 @@ where
while let Some(i) = solutions_idx {
let mut testcase = state.solutions().get(i)?.borrow_mut();
let input = testcase.load_input()?;
let bytes = (self.to_bytes)(input);
state.solutions().load_input_into(&mut testcase)?;
let bytes = (self.to_bytes)(testcase.input().as_ref().unwrap());
let fname = self.solutions_dir.join(format!("id_{i}"));
let fname = self.solutions_dir.join(format!(
"id_{i}_{}",
testcase
.filename()
.as_ref()
.map_or_else(|| "unnamed", String::as_str)
));
let mut f = File::create(fname)?;
drop(f.write_all(&bytes));

7
libafl/src/stages/generalization.rs
View File

@ -79,10 +79,13 @@ where
) -> Result<(), Error> {
let (mut payload, original, novelties) = {
start_timer!(state);
state.corpus().get(corpus_idx)?.borrow_mut().load_input()?;
{
let corpus = state.corpus();
let mut testcase = corpus.get(corpus_idx)?.borrow_mut();
corpus.load_input_into(&mut testcase)?;
}
mark_feature_time!(state, PerfFeature::GetInputFromCorpus);
let mut entry = state.corpus().get(corpus_idx)?.borrow_mut();
let input = entry.input_mut().as_mut().unwrap();
let payload: Vec<_> = input.bytes().iter().map(|&x| Some(x)).collect();

6
libafl/src/stages/mutational.rs
View File

@ -64,16 +64,18 @@ where
impl<I, S> MutatedTransform<I, S> for I
where
I: Input + Clone,
S: HasCorpus<Input = I>,
{
type Post = ();
#[inline]
fn try_transform_from(
base: &mut Testcase<I>,
_state: &S,
state: &S,
_corpus_idx: CorpusId,
) -> Result<Self, Error> {
Ok(base.load_input()?.clone())
state.corpus().load_input_into(base)?;
Ok(base.input().as_ref().unwrap().clone())
}
#[inline]

2
libafl/src/stages/power.rs
View File

@ -55,7 +55,7 @@ where
fn iterations(&self, state: &mut E::State, corpus_idx: CorpusId) -> Result<u64, Error> {
// Update handicap
let mut testcase = state.corpus().get(corpus_idx)?.borrow_mut();
let score = F::compute(&mut *testcase, state)? as u64;
let score = F::compute(state, &mut *testcase)? as u64;
Ok(score)
}

17
libafl/src/stages/push/mutational.rs
View File

@ -137,14 +137,15 @@ where
}
start_timer!(state);
let mut input = state
.corpus()
.get(self.current_corpus_idx.unwrap())
.unwrap()
.borrow_mut()
.load_input()
.unwrap()
.clone();
let input = state
.corpus_mut()
.cloned_input_for_id(self.current_corpus_idx.unwrap());
let mut input = match input {
Err(e) => return Some(Err(e)),
Ok(input) => input,
};
mark_feature_time!(state, PerfFeature::GetInputFromCorpus);
start_timer!(state);

2
libafl/src/stages/sync.rs
View File

@ -268,7 +268,7 @@ where
last_id.map_or_else(|| state.corpus().first(), |id| state.corpus().next(id));
while let Some(id) = cur_id {
let input = state.testcase_mut(id)?.load_input()?.clone();
let input = state.corpus().cloned_input_for_id(id)?;
self.client.fire(
state,

7
libafl/src/stages/tmin.rs
View File

@ -72,12 +72,7 @@ where
let num = self.iterations(state, base_corpus_idx)?;
start_timer!(state);
let mut base = state
.corpus()
.get(base_corpus_idx)?
.borrow_mut()
.load_input()?
.clone();
let mut base = state.corpus().cloned_input_for_id(base_corpus_idx)?;
let mut hasher = RandomState::with_seeds(0, 0, 0, 0).build_hasher();
base.hash(&mut hasher);
let base_hash = hasher.finish();

23
libafl/src/stages/tracing.rs
View File

@ -51,12 +51,8 @@ where
corpus_idx: CorpusId,
) -> Result<(), Error> {
start_timer!(state);
let input = state
.corpus()
.get(corpus_idx)?
.borrow_mut()
.load_input()?
.clone();
let input = state.corpus().cloned_input_for_id(corpus_idx)?;
mark_feature_time!(state, PerfFeature::GetInputFromCorpus);
start_timer!(state);
@ -142,12 +138,7 @@ where
) -> Result<(), Error> {
// First run with the un-mutated input
let unmutated_input = state
.corpus()
.get(corpus_idx)?
.borrow_mut()
.load_input()?
.clone();
let unmutated_input = state.corpus().cloned_input_for_id(corpus_idx)?;
if let Some(name) = &self.cmplog_observer_name {
if let Some(ob) = self
@ -277,12 +268,8 @@ where
corpus_idx: CorpusId,
) -> Result<(), Error> {
start_timer!(state);
let input = state
.corpus()
.get(corpus_idx)?
.borrow_mut()
.load_input()?
.clone();
let input = state.corpus().cloned_input_for_id(corpus_idx)?;
mark_feature_time!(state, PerfFeature::GetInputFromCorpus);
start_timer!(state);