diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3d857f0359..ae73618122 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -46,5 +46,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Test - run: cargo doc \ No newline at end of file + - name: Build Docs + run: cargo doc + - name: Test Docs + run: cargo test --doc \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 8973fa815d..efb844584e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,5 @@ members = [ #example fuzzers "fuzzers/libfuzzer_libpng", -] \ No newline at end of file + "fuzzers/libfuzzer_libmozjpeg", +] diff --git a/fuzzers/libfuzzer_libmozjpeg/.gitignore b/fuzzers/libfuzzer_libmozjpeg/.gitignore new file mode 100644 index 0000000000..335ec9573d --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/.gitignore @@ -0,0 +1 @@ +*.tar.gz diff --git a/fuzzers/libfuzzer_libmozjpeg/Cargo.toml b/fuzzers/libfuzzer_libmozjpeg/Cargo.toml new file mode 100644 index 0000000000..ede0d941fc --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "libfuzzer_libmozjpeg" +version = "0.1.0" +authors = ["Marcin Kozlowski "] +edition = "2018" +build = "build.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["std"] +std = [] + +#[profile.release] +#lto = true +#codegen-units = 1 +#opt-level = 3 +#debug = true + +[build-dependencies] +cc = { version = "1.0", features = ["parallel"] } +num_cpus = "1.0" + +[dependencies] +libafl = { path = "../../libafl/" } + +[[example]] +name = "libfuzzer_libmozjpeg" +path = "./src/fuzzer.rs" +test = false +bench = false diff --git a/fuzzers/libfuzzer_libmozjpeg/README.md b/fuzzers/libfuzzer_libmozjpeg/README.md new file mode 100644 index 0000000000..931138306c --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/README.md @@ -0,0 +1,28 @@ +# Libfuzzer for libmozjpeg + +This folder contains an example fuzzer for libmozjpeg, using LLMP for fast multi-process fuzzing and crash detection. +It has been tested on Linux. + +## Build + +To build this example, run `cargo build --example libfuzzer_libmozjpeg --release`. +This will call (the build.rs)[./builld.rs], which in turn downloads a libmozjpeg archive from the web. +Then, it will link (the fuzzer)[./src/fuzzer.rs] against (the c++ harness)[./harness.cc] and the instrumented `libmozjpeg`. +Afterwards, the fuzzer will be ready to run, from `../../target/examples/libfuzzer_libmozjpeg`. + +## Run + +The first time you run the binary, the broker will open a tcp port (currently on port `1337`), waiting for fuzzer clients to connect. This port is local and only used for the initial handshake. All further communication happens via shared map, to be independent of the kernel. + +Each following execution will run a fuzzer client. +As this example uses in-process fuzzing, we added a Restarting Event Manager (`setup_restarting_mgr`). +This means each client will start itself again to listen for crashes and timeouts. +By restarting the actual fuzzer, it can recover from these exit conditions. + +For convenience, you may just run `./test.sh` in this folder or: + +broker.sh - starts the broker +start.sh - starts as many clients as there are cores +stop.sh - stop everything + + diff --git a/fuzzers/libfuzzer_libmozjpeg/broker.sh b/fuzzers/libfuzzer_libmozjpeg/broker.sh new file mode 100755 index 0000000000..bbe5b82ae6 --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/broker.sh @@ -0,0 +1,3 @@ +#!/bin/bash +taskset -c 0 ./.libfuzzer_test.elf + diff --git a/fuzzers/libfuzzer_libmozjpeg/build.rs b/fuzzers/libfuzzer_libmozjpeg/build.rs new file mode 100644 index 0000000000..ee0062f3e3 --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/build.rs @@ -0,0 +1,112 @@ +// build.rs + +use std::env; +use std::path::Path; +use std::process::Command; + +const LIBMOZJPEG_URL: &str = "https://github.com/mozilla/mozjpeg/archive/v4.0.3.tar.gz"; + +fn main() { + let out_dir = env::var_os("OUT_DIR").unwrap(); + let cwd = env::current_dir().unwrap().to_string_lossy().to_string(); + let out_dir = out_dir.to_string_lossy().to_string(); + let out_dir_path = Path::new(&out_dir); + + println!("cargo:rerun-if-changed=./runtime/rt.c",); + println!("cargo:rerun-if-changed=harness.cc"); + + let libmozjpeg = format!("{}/mozjpeg-4.0.3", &out_dir); + let libmozjpeg_path = Path::new(&libmozjpeg); + let libmozjpeg_tar = format!("{}/v4.0.3.tar.gz", &cwd); + + // Enforce clang for its -fsanitize-coverage support. + std::env::set_var("CC", "clang"); + std::env::set_var("CXX", "clang++"); + + if !libmozjpeg_path.is_dir() { + if !Path::new(&libmozjpeg_tar).is_file() { + println!("cargo:warning=Libmozjpeg not found, downloading..."); + // Download libmozjpeg + Command::new("wget") + .arg("-c") + .arg(LIBMOZJPEG_URL) + .arg("-O") + .arg(&libmozjpeg_tar) + .status() + .unwrap(); + } + Command::new("tar") + .current_dir(&out_dir_path) + .arg("-xvf") + .arg(&libmozjpeg_tar) + .status() + .unwrap(); + Command::new(format!("{}/cmake", &libmozjpeg)) + .current_dir(&out_dir_path) + .args(&[ + "-G\"Unix Makefiles\"", + "--disable-shared", + &libmozjpeg, + "CC=clang", + "CFLAGS=-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard", + "LDFLAGS=-g -fPIE -fsanitize-coverage=trace-pc-guard", + ]) + .env("CC", "clang") + .env("CXX", "clang++") + .env( + "CFLAGS", + "-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard", + ) + .env( + "CXXFLAGS", + "-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard", + ) + .env("LDFLAGS", "-g -fPIE -fsanitize-coverage=trace-pc-guard"); + Command::new("make") + .current_dir(&libmozjpeg_path) + //.arg(&format!("-j{}", num_cpus::get())) + .args(&[ + "CC=clang", + "CXX=clang++", + "CFLAGS=-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard", + "LDFLAGS=-g -fPIE -fsanitize-coverage=trace-pc-guard", + "CXXFLAGS=-D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard", + ]) + .env("CC", "clang") + .env("CXX", "clang++") + .env( + "CFLAGS", + "-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard", + ) + .env( + "CXXFLAGS", + "-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard", + ) + .env("LDFLAGS", "-g -fPIE -fsanitize-coverage=trace-pc-guard") + .status() + .unwrap(); + } + + cc::Build::new() + .file("../libfuzzer_runtime/rt.c") + .compile("libfuzzer-sys"); + + cc::Build::new() + .include(&libmozjpeg_path) + .flag("-fsanitize-coverage=trace-pc-guard") + .file("./harness.cc") + .compile("libfuzzer-harness"); + + println!("cargo:rustc-link-search=native={}", &out_dir); + println!("cargo:rustc-link-search=native={}/", &libmozjpeg); + println!("cargo:rustc-link-lib=static=jpeg"); + + //Deps for libmozjpeg: -pthread -lz -lm + println!("cargo:rustc-link-lib=dylib=m"); + println!("cargo:rustc-link-lib=dylib=z"); + + //For the C++ harness + println!("cargo:rustc-link-lib=static=stdc++"); + + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/fuzzers/libfuzzer_libmozjpeg/corpus/blank.jpg b/fuzzers/libfuzzer_libmozjpeg/corpus/blank.jpg new file mode 100644 index 0000000000..1cda9a53dc Binary files /dev/null and b/fuzzers/libfuzzer_libmozjpeg/corpus/blank.jpg differ diff --git a/fuzzers/libfuzzer_libmozjpeg/harness.cc b/fuzzers/libfuzzer_libmozjpeg/harness.cc new file mode 100644 index 0000000000..6e43bb662f --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/harness.cc @@ -0,0 +1,92 @@ +#include +#include "jpeglib.h" +#include +#include + + +struct my_error_mgr { + struct jpeg_error_mgr pub; /* "public" fields */ + + jmp_buf setjmp_buffer; /* for return to caller */ +}; + +typedef struct my_error_mgr *my_error_ptr; + +/* + * Here's the routine that will replace the standard error_exit method: + */ + + METHODDEF(void) +my_error_exit(j_common_ptr cinfo) +{ + /* cinfo->err really points to a my_error_mgr struct, so coerce pointer */ + my_error_ptr myerr = (my_error_ptr)cinfo->err; + + /* Always display the message. */ + /* We could postpone this until after returning, if we chose. */ + (*cinfo->err->output_message) (cinfo); + + /* Return control to the setjmp point */ + longjmp(myerr->setjmp_buffer, 1); +} + + + + +int do_read_JPEG_file(struct jpeg_decompress_struct *cinfo, const uint8_t *input, size_t len) +{ + struct my_error_mgr jerr; + /* More stuff */ + JSAMPARRAY buffer; /* Output row buffer */ + int row_stride; /* physical row width in output buffer */ + /* Step 1: allocate and initialize JPEG decompression object */ + /* We set up the normal JPEG error routines, then override error_exit. */ + cinfo->err = jpeg_std_error(&jerr.pub); + jerr.pub.error_exit = my_error_exit; + /* Establish the setjmp return context for my_error_exit to use. */ + if (setjmp(jerr.setjmp_buffer)) { + jpeg_destroy_decompress(cinfo); + return 0; + } + /* Now we can initialize the JPEG decompression object. */ + jpeg_create_decompress(cinfo); + /* Step 2: specify data source (eg, a file) */ + jpeg_mem_src(cinfo,input, len ); + /* Step 3: read file parameters with jpeg_read_header() */ + (void)jpeg_read_header(cinfo, TRUE); + /* Step 4: set parameters for decompression */ + /* In this example, we don't need to change any of the defaults set by + * jpeg_read_header(), so we do nothing here. + */ + /* Step 5: Start decompressor */ + (void)jpeg_start_decompress(cinfo); + /* JSAMPLEs per row in output buffer */ + row_stride = cinfo->output_width * cinfo->output_components; + /* Make a one-row-high sample array that will go away when done with image */ + buffer = (*cinfo->mem->alloc_sarray) + ((j_common_ptr)cinfo, JPOOL_IMAGE, row_stride, 1); + /* Step 6: while (scan lines remain to be read) */ + /* jpeg_read_scanlines(...); */ + while (cinfo->output_scanline < cinfo->output_height) { + (void)jpeg_read_scanlines(cinfo, buffer, 1); + /* Assume put_scanline_someplace wants a pointer and sample count. */ + //put_scanline_someplace(buffer[0], row_stride); + } + /* Step 7: Finish decompression */ + (void)jpeg_finish_decompress(cinfo); + /* Step 8: Release JPEG decompression object */ + //jpeg_destroy_decompress(cinfo); + return 1; +} + + + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + + + struct jpeg_decompress_struct cinfo; + do_read_JPEG_file(&cinfo,data,size); + return 0; + +} + diff --git a/fuzzers/libfuzzer_libmozjpeg/jpeg.tkns b/fuzzers/libfuzzer_libmozjpeg/jpeg.tkns new file mode 100644 index 0000000000..f6215d224d --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/jpeg.tkns @@ -0,0 +1,22 @@ +# +# AFL dictionary for JPEG images +# ------------------------------ +# +# Created by Michal Zalewski +# + +header_jfif="JFIF\x00" +header_jfxx="JFXX\x00" + +section_ffc0="\xff\xc0" +section_ffc2="\xff\xc2" +section_ffc4="\xff\xc4" +section_ffd0="\xff\xd0" +section_ffd8="\xff\xd8" +section_ffd9="\xff\xd9" +section_ffda="\xff\xda" +section_ffdb="\xff\xdb" +section_ffdd="\xff\xdd" +section_ffe0="\xff\xe0" +section_ffe1="\xff\xe1" +section_fffe="\xff\xfe" \ No newline at end of file diff --git a/fuzzers/libfuzzer_libmozjpeg/src/fuzzer.rs b/fuzzers/libfuzzer_libmozjpeg/src/fuzzer.rs new file mode 100644 index 0000000000..58ec07983a --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/src/fuzzer.rs @@ -0,0 +1,152 @@ +//! A libfuzzer-like fuzzer with llmp-multithreading support and restarts +//! The example harness is built for libmozjpeg. + +use std::{env, path::PathBuf}; + +use libafl::{ + bolts::{shmem::UnixShMem, tuples::tuple_list}, + corpus::{Corpus, InMemoryCorpus, OnDiskCorpus, RandCorpusScheduler}, + events::setup_restarting_mgr, + executors::{inprocess::InProcessExecutor, Executor, ExitKind}, + feedbacks::{CrashFeedback, MaxMapFeedback}, + fuzzer::{Fuzzer, HasCorpusScheduler, StdFuzzer}, + inputs::Input, + mutators::scheduled::HavocBytesMutator, + mutators::token_mutations::Tokens, + observers::StdMapObserver, + stages::mutational::StdMutationalStage, + state::{HasCorpus, HasMetadata, State}, + stats::SimpleStats, + utils::{current_nanos, StdRand}, + Error, +}; + +/// We will interact with a C++ target, so use external c functionality +extern "C" { + /// int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) + fn LLVMFuzzerTestOneInput(data: *const u8, size: usize) -> i32; + + // afl_libfuzzer_init calls LLVMFUzzerInitialize() + fn afl_libfuzzer_init() -> i32; + + static __lafl_edges_map: *mut u8; + static __lafl_cmp_map: *mut u8; + static __lafl_max_edges_size: u32; +} + +/// The wrapped harness function, calling out to the LLVM-style harness +fn harness(_executor: &E, buf: &[u8]) -> ExitKind +where + E: Executor, + I: Input, +{ + // println!("{:?}", buf); + unsafe { + LLVMFuzzerTestOneInput(buf.as_ptr(), buf.len()); + } + ExitKind::Ok +} + +/// The main fn, usually parsing parameters, and starting the fuzzer +pub fn main() { + // Registry the metadata types used in this fuzzer + // Needed only on no_std + //RegistryBuilder::register::(); + + println!( + "Workdir: {:?}", + env::current_dir().unwrap().to_string_lossy().to_string() + ); + fuzz( + vec![PathBuf::from("./corpus")], + PathBuf::from("./crashes"), + 1337, + ) + .expect("An error occurred while fuzzing"); +} + +/// The actual fuzzer +fn fuzz(corpus_dirs: Vec, objective_dir: PathBuf, broker_port: u16) -> Result<(), Error> { + // 'While the stats are state, they are usually used in the broker - which is likely never restarted + let stats = SimpleStats::new(|s| println!("{}", s)); + + // The restarting state will spawn the same process again as child, then restarted it each time it crashes. + let (state, mut restarting_mgr) = + setup_restarting_mgr::<_, _, UnixShMem, _>(stats, broker_port) + .expect("Failed to setup the restarter".into()); + + // Create an observation channel using the coverage map + let edges_observer = + StdMapObserver::new_from_ptr("edges", unsafe { __lafl_edges_map }, unsafe { + __lafl_max_edges_size as usize + }); + + // If not restarting, create a State from scratch + let mut state = state.unwrap_or_else(|| { + State::new( + // RNG + StdRand::with_seed(current_nanos()), + // Corpus that will be evolved, we keep it in memory for performance + InMemoryCorpus::new(), + // Feedbacks to rate the interestingness of an input + tuple_list!(MaxMapFeedback::new_with_observer(&edges_observer)), + // Corpus in which we store solutions (crashes in this example), + // on disk so the user can get them after stopping the fuzzer + OnDiskCorpus::new(objective_dir).unwrap(), + // Feedbacks to recognize an input as solution + tuple_list!(CrashFeedback::new()), + ) + }); + + println!("We're a client, let's fuzz :)"); + + // Add the JPEG tokens if not existing + if state.metadata().get::().is_none() { + state.add_metadata(Tokens::from_tokens_file("./jpeg.tkns")?); + } + + // Setup a basic mutator with a mutational stage + let mutator = HavocBytesMutator::default(); + let stage = StdMutationalStage::new(mutator); + + // A fuzzer with just one stage and a random policy to get testcasess from the corpus + let fuzzer = StdFuzzer::new(RandCorpusScheduler::new(), tuple_list!(stage)); + + // Create the executor for an in-process function with just one observer for edge coverage + let mut executor = InProcessExecutor::new( + "in-process(edges)", + harness, + tuple_list!(edges_observer), + &mut state, + &mut restarting_mgr, + ); + + // The actual target run starts here. + // Call LLVMFUzzerInitialize() if present. + unsafe { + if afl_libfuzzer_init() == -1 { + println!("Warning: LLVMFuzzerInitialize failed with -1") + } + } + + // In case the corpus is empty (on first run), reset + if state.corpus().count() < 1 { + state + .load_initial_inputs( + &mut executor, + &mut restarting_mgr, + fuzzer.scheduler(), + &corpus_dirs, + ) + .expect(&format!( + "Failed to load initial corpus at {:?}", + &corpus_dirs + )); + println!("We imported {} inputs from disk.", state.corpus().count()); + } + + fuzzer.fuzz_loop(&mut state, &mut executor, &mut restarting_mgr)?; + + // Never reached + Ok(()) +} diff --git a/fuzzers/libfuzzer_libmozjpeg/start.sh b/fuzzers/libfuzzer_libmozjpeg/start.sh new file mode 100755 index 0000000000..9e0b74d5de --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/start.sh @@ -0,0 +1,9 @@ +#!/bin/bash +cores=$(grep -c ^processor /proc/cpuinfo) +for (( c=1;c<$cores;c++)) +do + echo $c + taskset -c $c ./.libfuzzer_test.elf 2>/dev/null & + sleep 0.1 +done + diff --git a/fuzzers/libfuzzer_libmozjpeg/stop.sh b/fuzzers/libfuzzer_libmozjpeg/stop.sh new file mode 100755 index 0000000000..a97094121b --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/stop.sh @@ -0,0 +1,2 @@ +#!/bin/bash +killall -9 .libfuzzer_test.elf diff --git a/fuzzers/libfuzzer_libmozjpeg/test.sh b/fuzzers/libfuzzer_libmozjpeg/test.sh new file mode 100755 index 0000000000..55df5cee43 --- /dev/null +++ b/fuzzers/libfuzzer_libmozjpeg/test.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +mkdir -p ./crashes + +cargo build --example libfuzzer_libmozjpeg --release || exit 1 +cp ../../target/release/examples/libfuzzer_libmozjpeg ./.libfuzzer_test.elf + +# The broker +RUST_BACKTRACE=full taskset -c 0 ./.libfuzzer_test.elf & +# Give the broker time to spawn +sleep 2 +echo "Spawning client" +# The 1st fuzzer client, pin to cpu 0x1 +RUST_BACKTRACE=full taskset -c 1 ./.libfuzzer_test.elf 2>/dev/null + +killall .libfuzzer_test.elf +rm -rf ./.libfuzzer_test.elf diff --git a/fuzzers/libfuzzer_libpng/src/fuzzer.rs b/fuzzers/libfuzzer_libpng/src/fuzzer.rs index 4adf8acbbd..62845e7799 100644 --- a/fuzzers/libfuzzer_libpng/src/fuzzer.rs +++ b/fuzzers/libfuzzer_libpng/src/fuzzer.rs @@ -14,8 +14,7 @@ use libafl::{ feedbacks::{CrashFeedback, MaxMapFeedback}, fuzzer::{Fuzzer, HasCorpusScheduler, StdFuzzer}, inputs::Input, - mutators::scheduled::HavocBytesMutator, - mutators::token_mutations::TokensMetadata, + mutators::{scheduled::HavocBytesMutator, token_mutations::Tokens}, observers::{HitcountsMapObserver, StdMapObserver}, stages::mutational::StdMutationalStage, state::{HasCorpus, HasMetadata, State}, @@ -54,7 +53,7 @@ where pub fn main() { // Registry the metadata types used in this fuzzer // Needed only on no_std - //RegistryBuilder::register::(); + //RegistryBuilder::register::(); println!( "Workdir: {:?}", @@ -109,8 +108,8 @@ fn fuzz(corpus_dirs: Vec, objective_dir: PathBuf, broker_port: u16) -> println!("We're a client, let's fuzz :)"); // Create a PNG dictionary if not existing - if state.metadatas().get::().is_none() { - state.add_metadata(TokensMetadata::new(vec![ + if state.metadata().get::().is_none() { + state.add_metadata(Tokens::new(vec![ vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header "IHDR".as_bytes().to_vec(), "IDAT".as_bytes().to_vec(), diff --git a/libafl/src/corpus/minimizer.rs b/libafl/src/corpus/minimizer.rs index 98b3ab9495..42c37d9d84 100644 --- a/libafl/src/corpus/minimizer.rs +++ b/libafl/src/corpus/minimizer.rs @@ -136,7 +136,7 @@ where { pub fn update_score(&self, state: &mut S, idx: usize) -> Result<(), Error> { // Create a new top rated meta if not existing - if state.metadatas().get::().is_none() { + if state.metadata().get::().is_none() { state.add_metadata(TopRatedsMetadata::new()); } @@ -144,7 +144,7 @@ where { let mut entry = state.corpus().get(idx)?.borrow_mut(); let factor = F::compute(&mut *entry)?; - let meta = entry.metadatas().get::().ok_or_else(|| { + let meta = entry.metadata().get::().ok_or_else(|| { Error::KeyNotFound(format!( "Metadata needed for MinimizerCorpusScheduler not found in testcase #{}", idx @@ -152,7 +152,7 @@ where })?; for elem in meta.as_slice() { if let Some(old_idx) = state - .metadatas() + .metadata() .get::() .unwrap() .map @@ -169,7 +169,7 @@ where for pair in new_favoreds { state - .metadatas_mut() + .metadata_mut() .get_mut::() .unwrap() .map @@ -179,17 +179,17 @@ where } pub fn cull(&self, state: &mut S) -> Result<(), Error> { - if state.metadatas().get::().is_none() { + if state.metadata().get::().is_none() { return Ok(()); } let mut acc = HashSet::new(); - let top_rated = state.metadatas().get::().unwrap(); + let top_rated = state.metadata().get::().unwrap(); for key in top_rated.map.keys() { if !acc.contains(key) { let idx = top_rated.map.get(key).unwrap(); let mut entry = state.corpus().get(*idx)?.borrow_mut(); - let meta = entry.metadatas().get::().ok_or_else(|| { + let meta = entry.metadata().get::().ok_or_else(|| { Error::KeyNotFound(format!( "Metadata needed for MinimizerCorpusScheduler not found in testcase #{}", idx diff --git a/libafl/src/corpus/ondisk.rs b/libafl/src/corpus/ondisk.rs index b87301ff5e..f597d45bdd 100644 --- a/libafl/src/corpus/ondisk.rs +++ b/libafl/src/corpus/ondisk.rs @@ -37,7 +37,7 @@ where fn add(&mut self, mut testcase: Testcase) -> Result { match testcase.filename() { None => { - // TODO walk entry metadatas to ask for pices of filename (e.g. :havoc in AFL) + // TODO walk entry metadata to ask for pices of filename (e.g. :havoc in AFL) let filename = self.dir_path.join(format!("id_{}", &self.entries.len())); let filename_str = filename.to_str().expect("Invalid Path"); testcase.set_filename(filename_str.into()); diff --git a/libafl/src/corpus/testcase.rs b/libafl/src/corpus/testcase.rs index 0da3c6505a..625e005ca2 100644 --- a/libafl/src/corpus/testcase.rs +++ b/libafl/src/corpus/testcase.rs @@ -25,8 +25,8 @@ where filename: Option, /// Accumulated fitness from all the feedbacks fitness: u32, - /// Map of metadatas associated with this testcase - metadatas: SerdeAnyMap, + /// Map of metadata associated with this testcase + metadata: SerdeAnyMap, /// Time needed to execute the input exec_time: Option, /// Cached len of the input, if any @@ -37,16 +37,16 @@ impl HasMetadata for Testcase where I: Input, { - /// Get all the metadatas into an HashMap + /// Get all the metadata into an HashMap #[inline] - fn metadatas(&self) -> &SerdeAnyMap { - &self.metadatas + fn metadata(&self) -> &SerdeAnyMap { + &self.metadata } - /// Get all the metadatas into an HashMap (mutable) + /// Get all the metadata into an HashMap (mutable) #[inline] - fn metadatas_mut(&mut self) -> &mut SerdeAnyMap { - &mut self.metadatas + fn metadata_mut(&mut self) -> &mut SerdeAnyMap { + &mut self.metadata } } @@ -158,7 +158,7 @@ where input: Some(input.into()), filename: None, fitness: 0, - metadatas: SerdeAnyMap::new(), + metadata: SerdeAnyMap::new(), exec_time: None, cached_len: None, } @@ -171,7 +171,7 @@ where input: Some(input), filename: Some(filename), fitness: 0, - metadatas: SerdeAnyMap::new(), + metadata: SerdeAnyMap::new(), exec_time: None, cached_len: None, } @@ -184,7 +184,7 @@ where input: Some(input.into()), filename: None, fitness: fitness, - metadatas: SerdeAnyMap::new(), + metadata: SerdeAnyMap::new(), exec_time: None, cached_len: None, } @@ -196,7 +196,7 @@ where input: None, filename: None, fitness: 0, - metadatas: SerdeAnyMap::new(), + metadata: SerdeAnyMap::new(), exec_time: None, cached_len: None, } diff --git a/libafl/src/events/mod.rs b/libafl/src/events/mod.rs index 280c1ac8bf..01cb94038d 100644 --- a/libafl/src/events/mod.rs +++ b/libafl/src/events/mod.rs @@ -73,7 +73,7 @@ where I: Input, { // TODO use an ID to keep track of the original index in the sender Corpus - // The sender can then use it to send Testcase metadatas with CustomEvent + // The sender can then use it to send Testcase metadata with CustomEvent /// A fuzzer found a new testcase. Rejoice! NewTestcase { /// The input for the new testcase diff --git a/libafl/src/mutators/mutations.rs b/libafl/src/mutators/mutations.rs index 09c884ac22..a4ac24c46d 100644 --- a/libafl/src/mutators/mutations.rs +++ b/libafl/src/mutators/mutations.rs @@ -11,12 +11,6 @@ use crate::{ use alloc::{borrow::ToOwned, vec::Vec}; use core::cmp::{max, min}; -#[cfg(feature = "std")] -use std::{ - fs::File, - io::{BufRead, BufReader}, -}; - /// The result of a mutation. /// If the mutation got skipped, the target /// will not be executed with the returned input. @@ -810,75 +804,8 @@ pub fn str_decode(item: &str) -> Result, Error> { return Ok(token); } -/// Adds a token to a dictionary, checking it is not a duplicate -pub fn add_token(tokens: &mut Vec>, token: &Vec) -> u32 { - if tokens.contains(token) { - return 0; - } - tokens.push(token.to_vec()); - return 1; -} - -/// Read a dictionary file and return the number of entries read -#[cfg(feature = "std")] -pub fn read_tokens_file(f: &str, tokens: &mut Vec>) -> Result { - let mut entries = 0; - - println!("Loading tokens file {:?} ...", &f); - - let file = File::open(&f)?; // panic if not found - let reader = BufReader::new(file); - - for line in reader.lines() { - let line = line.unwrap(); - let line = line.trim_start().trim_end(); - - // we are only interested in '"..."', not prefixed 'foo = ' - let start = line.chars().nth(0); - if line.len() == 0 || start == Some('#') { - continue; - } - let pos_quote = match line.find("\"") { - Some(x) => x, - _ => return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)), - }; - if line.chars().nth(line.len() - 1) != Some('"') { - return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)); - } - - // extract item - let item = match line.get(pos_quote + 1..line.len() - 1) { - Some(x) => x, - _ => return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)), - }; - if item.len() == 0 { - continue; - } - - // decode - let token: Vec = match str_decode(item) { - Ok(val) => val, - Err(_) => { - return Err(Error::IllegalArgument( - "Illegal line (hex decoding): ".to_owned() + line, - )) - } - }; - - // add - entries += add_token(tokens, &token); - } - - Ok(entries) -} - #[cfg(test)] mod tests { - #[cfg(feature = "std")] - use std::fs; - - #[cfg(feature = "std")] - use crate::mutators::read_tokens_file; use super::*; use crate::{ @@ -889,26 +816,6 @@ mod tests { utils::StdRand, }; - #[cfg(feature = "std")] - #[test] - fn test_read_tokens() { - let _ = fs::remove_file("test.tkns"); - let data = r###" -# comment -token1@123="AAA" -token1="A\x41A" -"A\AA" -token2="B" - "###; - fs::write("test.tkns", data).expect("Unable to write test.tkns"); - let mut v: Vec> = Vec::new(); - let res = read_tokens_file(&"test.tkns".to_string(), &mut v).unwrap(); - #[cfg(feature = "std")] - println!("Token file entries: {:?}", res); - assert_eq!(res, 2); - let _ = fs::remove_file("test.tkns"); - } - #[test] fn test_mutators() { let mut inputs = vec![ diff --git a/libafl/src/mutators/token_mutations.rs b/libafl/src/mutators/token_mutations.rs index 6702afeb80..9a43602bfb 100644 --- a/libafl/src/mutators/token_mutations.rs +++ b/libafl/src/mutators/token_mutations.rs @@ -1,6 +1,13 @@ //! Tokens are what afl calls extras or dictionaries. //! They may be inserted as part of mutations during fuzzing. +#[cfg(feature = "std")] +use std::{ + fs::File, + io::{BufRead, BufReader}, + path::Path, +}; + use crate::{ inputs::{HasBytesVec, Input}, mutators::*, @@ -16,15 +23,101 @@ use mutations::buffer_copy; /// A state metadata holding a list of tokens #[derive(Serialize, Deserialize)] -pub struct TokensMetadata { - tokens: Vec>, +pub struct Tokens { + token_vec: Vec>, } -crate::impl_serdeany!(TokensMetadata); +crate::impl_serdeany!(Tokens); -impl TokensMetadata { - pub fn new(tokens: Vec>) -> Self { - Self { tokens: tokens } +/// The metadata used for token mutators +impl Tokens { + /// Creates a new tokens metadata (old-skool afl name: `dictornary`) + pub fn new(token_vec: Vec>) -> Self { + Self { token_vec } + } + + /// Creates a new instance from a file + #[cfg(feature = "std")] + pub fn from_tokens_file

(file: P) -> Result + where + P: AsRef, + { + let mut ret = Self::new(vec![]); + ret.add_tokens_from_file(file)?; + Ok(ret) + } + + /// Adds a token to a dictionary, checking it is not a duplicate + /// Returns `false` if the token was already present and did not get added. + pub fn add_token(&mut self, token: &Vec) -> bool { + if self.token_vec.contains(token) { + return false; + } + self.token_vec.push(token.to_vec()); + return true; + } + + /// Reads a tokens file, returning the count of new entries read + #[cfg(feature = "std")] + pub fn add_tokens_from_file

(&mut self, file: P) -> Result + where + P: AsRef, + { + let mut entries = 0; + + // println!("Loading tokens file {:?} ...", file); + + let file = File::open(file)?; // panic if not found + let reader = BufReader::new(file); + + for line in reader.lines() { + let line = line.unwrap(); + let line = line.trim_start().trim_end(); + + // we are only interested in '"..."', not prefixed 'foo = ' + let start = line.chars().nth(0); + if line.len() == 0 || start == Some('#') { + continue; + } + let pos_quote = match line.find("\"") { + Some(x) => x, + _ => return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)), + }; + if line.chars().nth(line.len() - 1) != Some('"') { + return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)); + } + + // extract item + let item = match line.get(pos_quote + 1..line.len() - 1) { + Some(x) => x, + _ => return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)), + }; + if item.len() == 0 { + continue; + } + + // decode + let token: Vec = match str_decode(item) { + Ok(val) => val, + Err(_) => { + return Err(Error::IllegalArgument( + "Illegal line (hex decoding): ".to_owned() + line, + )) + } + }; + + // add + if self.add_token(&token) { + entries += 1; + } + } + + Ok(entries) + } + + /// Gets the tokens stored in this db + pub fn tokens(&self) -> &[Vec] { + return &self.token_vec; } } @@ -37,22 +130,22 @@ where { let max_size = state.max_size(); let tokens_len = { - let meta = state.metadatas().get::(); + let meta = state.metadata().get::(); if meta.is_none() { return Ok(MutationResult::Skipped); } - if meta.unwrap().tokens.len() == 0 { + if meta.unwrap().tokens().len() == 0 { return Ok(MutationResult::Skipped); } - meta.unwrap().tokens.len() + meta.unwrap().tokens().len() }; let token_idx = state.rand_mut().below(tokens_len as u64) as usize; let size = input.bytes().len(); let off = state.rand_mut().below((size + 1) as u64) as usize; - let meta = state.metadatas().get::().unwrap(); - let token = &meta.tokens[token_idx]; + let meta = state.metadata().get::().unwrap(); + let token = &meta.tokens()[token_idx]; let mut len = token.len(); if size + len > max_size { @@ -83,21 +176,21 @@ where } let tokens_len = { - let meta = state.metadatas().get::(); + let meta = state.metadata().get::(); if meta.is_none() { return Ok(MutationResult::Skipped); } - if meta.unwrap().tokens.len() == 0 { + if meta.unwrap().tokens().len() == 0 { return Ok(MutationResult::Skipped); } - meta.unwrap().tokens.len() + meta.unwrap().tokens().len() }; let token_idx = state.rand_mut().below(tokens_len as u64) as usize; let off = state.rand_mut().below(size as u64) as usize; - let meta = state.metadatas().get::().unwrap(); - let token = &meta.tokens[token_idx]; + let meta = state.metadata().get::().unwrap(); + let token = &meta.tokens()[token_idx]; let mut len = token.len(); if off + len > size { len = size - off; @@ -107,3 +200,30 @@ where Ok(MutationResult::Mutated) } + +#[cfg(test)] +mod tests { + #[cfg(feature = "std")] + use std::fs; + + use super::Tokens; + + #[cfg(feature = "std")] + #[test] + fn test_read_tokens() { + let _ = fs::remove_file("test.tkns"); + let data = r###" +# comment +token1@123="AAA" +token1="A\x41A" +"A\AA" +token2="B" + "###; + fs::write("test.tkns", data).expect("Unable to write test.tkns"); + let tokens = Tokens::from_tokens_file(&"test.tkns").unwrap(); + #[cfg(feature = "std")] + println!("Token file entries: {:?}", tokens.tokens()); + assert_eq!(tokens.tokens().len(), 2); + let _ = fs::remove_file("test.tkns"); + } +} diff --git a/libafl/src/state/mod.rs b/libafl/src/state/mod.rs index b9ce39b076..ef6a467eae 100644 --- a/libafl/src/state/mod.rs +++ b/libafl/src/state/mod.rs @@ -73,9 +73,9 @@ where /// Trait for elements offering metadata pub trait HasMetadata { /// A map, storing all metadata - fn metadatas(&self) -> &SerdeAnyMap; + fn metadata(&self) -> &SerdeAnyMap; /// A map, storing all metadata (mut) - fn metadatas_mut(&mut self) -> &mut SerdeAnyMap; + fn metadata_mut(&mut self) -> &mut SerdeAnyMap; /// Add a metadata to the metadata map #[inline] @@ -83,7 +83,7 @@ pub trait HasMetadata { where M: SerdeAny, { - self.metadatas_mut().insert(meta); + self.metadata_mut().insert(meta); } /// Check for a metadata @@ -92,7 +92,7 @@ pub trait HasMetadata { where M: SerdeAny, { - self.metadatas().get::().is_some() + self.metadata().get::().is_some() } } @@ -317,13 +317,13 @@ where { /// Get all the metadata into an HashMap #[inline] - fn metadatas(&self) -> &SerdeAnyMap { + fn metadata(&self) -> &SerdeAnyMap { &self.metadata } /// Get all the metadata into an HashMap (mutable) #[inline] - fn metadatas_mut(&mut self) -> &mut SerdeAnyMap { + fn metadata_mut(&mut self) -> &mut SerdeAnyMap { &mut self.metadata } }