Libmozjpeg example added (#15)

* WIP Harness for libmozjpeg

* Taskset removal (wrong invocation, without -c)

* Clean up
Fixed taskset in test.sh

* Docs

* Formatting

* Formatting

* Formatting

* Formatting

* jpeg example now uses a tokens file

* fixed testcases

* fixing build

* fixed more bugs

* metadatas->metadata

* token files

* added doctest test

Co-authored-by: Dominik Maier <domenukk@gmail.com>

.github/workflows/build_and_test.yml

@@ -46,5 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Test
+    - name: Build Docs
       run: cargo doc
+    - name: Test Docs
+      run: cargo test --doc

Cargo.toml

@@ -11,4 +11,5 @@ members = [
 
     #example fuzzers
     "fuzzers/libfuzzer_libpng",
+    "fuzzers/libfuzzer_libmozjpeg",
 ]

fuzzers/libfuzzer_libmozjpeg/.gitignore

@@ -0,0 +1 @@
+*.tar.gz

fuzzers/libfuzzer_libmozjpeg/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "libfuzzer_libmozjpeg"
+version = "0.1.0"
+authors = ["Marcin Kozlowski <marcinguy@gmail.com>"]
+edition = "2018"
+build = "build.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+default = ["std"]
+std = []
+
+#[profile.release]
+#lto = true
+#codegen-units = 1
+#opt-level = 3
+#debug = true
+
+[build-dependencies]
+cc = { version = "1.0", features = ["parallel"] }
+num_cpus = "1.0"
+
+[dependencies]
+libafl = { path = "../../libafl/" }
+
+[[example]]
+name = "libfuzzer_libmozjpeg"
+path = "./src/fuzzer.rs"
+test = false
+bench = false

fuzzers/libfuzzer_libmozjpeg/README.md

@@ -0,0 +1,28 @@
+# Libfuzzer for libmozjpeg
+
+This folder contains an example fuzzer for libmozjpeg, using LLMP for fast multi-process fuzzing and crash detection.
+It has been tested on Linux.
+
+## Build
+
+To build this example, run `cargo build --example libfuzzer_libmozjpeg --release`.
+This will call (the build.rs)[./builld.rs], which in turn downloads a libmozjpeg archive from the web.
+Then, it will link (the fuzzer)[./src/fuzzer.rs] against (the c++ harness)[./harness.cc] and the instrumented `libmozjpeg`.
+Afterwards, the fuzzer will be ready to run, from `../../target/examples/libfuzzer_libmozjpeg`.
+
+## Run
+
+The first time you run the binary, the broker will open a tcp port (currently on port `1337`), waiting for fuzzer clients to connect. This port is local and only used for the initial handshake. All further communication happens via shared map, to be independent of the kernel.
+
+Each following execution will run a fuzzer client.
+As this example uses in-process fuzzing, we added a Restarting Event Manager (`setup_restarting_mgr`).
+This means each client will start itself again to listen for crashes and timeouts.
+By restarting the actual fuzzer, it can recover from these exit conditions.
+
+For convenience, you may just run `./test.sh` in this folder or:
+
+broker.sh - starts the broker
+start.sh - starts as many clients as there are cores
+stop.sh - stop everything
+
+

fuzzers/libfuzzer_libmozjpeg/broker.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+taskset -c 0 ./.libfuzzer_test.elf
+

fuzzers/libfuzzer_libmozjpeg/build.rs

@@ -0,0 +1,112 @@
+// build.rs
+
+use std::env;
+use std::path::Path;
+use std::process::Command;
+
+const LIBMOZJPEG_URL: &str = "https://github.com/mozilla/mozjpeg/archive/v4.0.3.tar.gz";
+
+fn main() {
+    let out_dir = env::var_os("OUT_DIR").unwrap();
+    let cwd = env::current_dir().unwrap().to_string_lossy().to_string();
+    let out_dir = out_dir.to_string_lossy().to_string();
+    let out_dir_path = Path::new(&out_dir);
+
+    println!("cargo:rerun-if-changed=./runtime/rt.c",);
+    println!("cargo:rerun-if-changed=harness.cc");
+
+    let libmozjpeg = format!("{}/mozjpeg-4.0.3", &out_dir);
+    let libmozjpeg_path = Path::new(&libmozjpeg);
+    let libmozjpeg_tar = format!("{}/v4.0.3.tar.gz", &cwd);
+
+    // Enforce clang for its -fsanitize-coverage support.
+    std::env::set_var("CC", "clang");
+    std::env::set_var("CXX", "clang++");
+
+    if !libmozjpeg_path.is_dir() {
+        if !Path::new(&libmozjpeg_tar).is_file() {
+            println!("cargo:warning=Libmozjpeg not found, downloading...");
+            // Download libmozjpeg
+            Command::new("wget")
+                .arg("-c")
+                .arg(LIBMOZJPEG_URL)
+                .arg("-O")
+                .arg(&libmozjpeg_tar)
+                .status()
+                .unwrap();
+        }
+        Command::new("tar")
+            .current_dir(&out_dir_path)
+            .arg("-xvf")
+            .arg(&libmozjpeg_tar)
+            .status()
+            .unwrap();
+        Command::new(format!("{}/cmake", &libmozjpeg))
+            .current_dir(&out_dir_path)
+            .args(&[
+                "-G\"Unix Makefiles\"",
+                "--disable-shared",
+                &libmozjpeg,
+                "CC=clang",
+                "CFLAGS=-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard",
+                "LDFLAGS=-g -fPIE -fsanitize-coverage=trace-pc-guard",
+            ])
+            .env("CC", "clang")
+            .env("CXX", "clang++")
+            .env(
+                "CFLAGS",
+                "-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard",
+            )
+            .env(
+                "CXXFLAGS",
+                "-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard",
+            )
+            .env("LDFLAGS", "-g -fPIE -fsanitize-coverage=trace-pc-guard");
+        Command::new("make")
+            .current_dir(&libmozjpeg_path)
+            //.arg(&format!("-j{}", num_cpus::get()))
+            .args(&[
+                "CC=clang",
+                "CXX=clang++",
+                "CFLAGS=-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard",
+                "LDFLAGS=-g -fPIE -fsanitize-coverage=trace-pc-guard",
+                "CXXFLAGS=-D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard",
+            ])
+            .env("CC", "clang")
+            .env("CXX", "clang++")
+            .env(
+                "CFLAGS",
+                "-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard",
+            )
+            .env(
+                "CXXFLAGS",
+                "-O3 -g -D_DEFAULT_SOURCE -fPIE -fsanitize-coverage=trace-pc-guard",
+            )
+            .env("LDFLAGS", "-g -fPIE -fsanitize-coverage=trace-pc-guard")
+            .status()
+            .unwrap();
+    }
+
+    cc::Build::new()
+        .file("../libfuzzer_runtime/rt.c")
+        .compile("libfuzzer-sys");
+
+    cc::Build::new()
+        .include(&libmozjpeg_path)
+        .flag("-fsanitize-coverage=trace-pc-guard")
+        .file("./harness.cc")
+        .compile("libfuzzer-harness");
+
+    println!("cargo:rustc-link-search=native={}", &out_dir);
+    println!("cargo:rustc-link-search=native={}/", &libmozjpeg);
+    println!("cargo:rustc-link-lib=static=jpeg");
+
+    //Deps for libmozjpeg: -pthread -lz -lm
+    println!("cargo:rustc-link-lib=dylib=m");
+    println!("cargo:rustc-link-lib=dylib=z");
+
+    //For the C++ harness
+    println!("cargo:rustc-link-lib=static=stdc++");
+
+    println!("cargo:rerun-if-changed=build.rs");
+}

fuzzers/libfuzzer_libmozjpeg/harness.cc

@@ -0,0 +1,92 @@
+#include <stdio.h>
+#include "jpeglib.h"
+#include <setjmp.h>
+#include <stdint.h>
+
+
+struct my_error_mgr {
+	struct jpeg_error_mgr pub;    /* "public" fields */
+
+	jmp_buf setjmp_buffer;        /* for return to caller */
+};
+
+typedef struct my_error_mgr *my_error_ptr;
+
+/*
+ * Here's the routine that will replace the standard error_exit method:
+ */
+
+	METHODDEF(void)
+my_error_exit(j_common_ptr cinfo)
+{
+	/* cinfo->err really points to a my_error_mgr struct, so coerce pointer */
+	my_error_ptr myerr = (my_error_ptr)cinfo->err;
+
+	/* Always display the message. */
+	/* We could postpone this until after returning, if we chose. */
+	(*cinfo->err->output_message) (cinfo);
+
+	/* Return control to the setjmp point */
+	longjmp(myerr->setjmp_buffer, 1);
+}
+
+
+
+
+int do_read_JPEG_file(struct jpeg_decompress_struct *cinfo, const uint8_t *input, size_t len)
+{
+	struct my_error_mgr jerr;
+	/* More stuff */
+	JSAMPARRAY buffer;            /* Output row buffer */
+	int row_stride;               /* physical row width in output buffer */
+	/* Step 1: allocate and initialize JPEG decompression object */
+	/* We set up the normal JPEG error routines, then override error_exit. */
+	cinfo->err = jpeg_std_error(&jerr.pub);
+	jerr.pub.error_exit = my_error_exit;
+	/* Establish the setjmp return context for my_error_exit to use. */
+	if (setjmp(jerr.setjmp_buffer)) {
+		jpeg_destroy_decompress(cinfo);
+		return 0;
+	}
+	/* Now we can initialize the JPEG decompression object. */
+	jpeg_create_decompress(cinfo);
+	/* Step 2: specify data source (eg, a file) */
+	jpeg_mem_src(cinfo,input, len );
+	/* Step 3: read file parameters with jpeg_read_header() */
+	(void)jpeg_read_header(cinfo, TRUE);
+	/* Step 4: set parameters for decompression */
+	/* In this example, we don't need to change any of the defaults set by
+	 * jpeg_read_header(), so we do nothing here.
+	 */
+	/* Step 5: Start decompressor */
+	(void)jpeg_start_decompress(cinfo);
+	/* JSAMPLEs per row in output buffer */
+	row_stride = cinfo->output_width * cinfo->output_components;
+	/* Make a one-row-high sample array that will go away when done with image */
+	buffer = (*cinfo->mem->alloc_sarray)
+		((j_common_ptr)cinfo, JPOOL_IMAGE, row_stride, 1);
+	/* Step 6: while (scan lines remain to be read) */
+	/*           jpeg_read_scanlines(...); */
+	while (cinfo->output_scanline < cinfo->output_height) {
+		(void)jpeg_read_scanlines(cinfo, buffer, 1);
+		/* Assume put_scanline_someplace wants a pointer and sample count. */
+		//put_scanline_someplace(buffer[0], row_stride);
+	}
+	/* Step 7: Finish decompression */
+	(void)jpeg_finish_decompress(cinfo);
+	/* Step 8: Release JPEG decompression object */
+	//jpeg_destroy_decompress(cinfo);
+	return 1;
+}
+
+
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+
+
+	struct jpeg_decompress_struct cinfo;
+	do_read_JPEG_file(&cinfo,data,size);  
+	return 0;
+
+}
+

fuzzers/libfuzzer_libmozjpeg/jpeg.tkns

@@ -0,0 +1,22 @@
+#
+# AFL dictionary for JPEG images
+# ------------------------------
+#
+# Created by Michal Zalewski
+#
+
+header_jfif="JFIF\x00"
+header_jfxx="JFXX\x00"
+
+section_ffc0="\xff\xc0"
+section_ffc2="\xff\xc2"
+section_ffc4="\xff\xc4"
+section_ffd0="\xff\xd0"
+section_ffd8="\xff\xd8"
+section_ffd9="\xff\xd9"
+section_ffda="\xff\xda"
+section_ffdb="\xff\xdb"
+section_ffdd="\xff\xdd"
+section_ffe0="\xff\xe0"
+section_ffe1="\xff\xe1"
+section_fffe="\xff\xfe"

fuzzers/libfuzzer_libmozjpeg/src/fuzzer.rs

@@ -0,0 +1,152 @@
+//! A libfuzzer-like fuzzer with llmp-multithreading support and restarts
+//! The example harness is built for libmozjpeg.
+
+use std::{env, path::PathBuf};
+
+use libafl::{
+    bolts::{shmem::UnixShMem, tuples::tuple_list},
+    corpus::{Corpus, InMemoryCorpus, OnDiskCorpus, RandCorpusScheduler},
+    events::setup_restarting_mgr,
+    executors::{inprocess::InProcessExecutor, Executor, ExitKind},
+    feedbacks::{CrashFeedback, MaxMapFeedback},
+    fuzzer::{Fuzzer, HasCorpusScheduler, StdFuzzer},
+    inputs::Input,
+    mutators::scheduled::HavocBytesMutator,
+    mutators::token_mutations::Tokens,
+    observers::StdMapObserver,
+    stages::mutational::StdMutationalStage,
+    state::{HasCorpus, HasMetadata, State},
+    stats::SimpleStats,
+    utils::{current_nanos, StdRand},
+    Error,
+};
+
+/// We will interact with a C++ target, so use external c functionality
+extern "C" {
+    /// int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+    fn LLVMFuzzerTestOneInput(data: *const u8, size: usize) -> i32;
+
+    // afl_libfuzzer_init calls LLVMFUzzerInitialize()
+    fn afl_libfuzzer_init() -> i32;
+
+    static __lafl_edges_map: *mut u8;
+    static __lafl_cmp_map: *mut u8;
+    static __lafl_max_edges_size: u32;
+}
+
+/// The wrapped harness function, calling out to the LLVM-style harness
+fn harness<E, I>(_executor: &E, buf: &[u8]) -> ExitKind
+where
+    E: Executor<I>,
+    I: Input,
+{
+    // println!("{:?}", buf);
+    unsafe {
+        LLVMFuzzerTestOneInput(buf.as_ptr(), buf.len());
+    }
+    ExitKind::Ok
+}
+
+/// The main fn, usually parsing parameters, and starting the fuzzer
+pub fn main() {
+    // Registry the metadata types used in this fuzzer
+    // Needed only on no_std
+    //RegistryBuilder::register::<Tokens>();
+
+    println!(
+        "Workdir: {:?}",
+        env::current_dir().unwrap().to_string_lossy().to_string()
+    );
+    fuzz(
+        vec![PathBuf::from("./corpus")],
+        PathBuf::from("./crashes"),
+        1337,
+    )
+    .expect("An error occurred while fuzzing");
+}
+
+/// The actual fuzzer
+fn fuzz(corpus_dirs: Vec<PathBuf>, objective_dir: PathBuf, broker_port: u16) -> Result<(), Error> {
+    // 'While the stats are state, they are usually used in the broker - which is likely never restarted
+    let stats = SimpleStats::new(|s| println!("{}", s));
+
+    // The restarting state will spawn the same process again as child, then restarted it each time it crashes.
+    let (state, mut restarting_mgr) =
+        setup_restarting_mgr::<_, _, UnixShMem, _>(stats, broker_port)
+            .expect("Failed to setup the restarter".into());
+
+    // Create an observation channel using the coverage map
+    let edges_observer =
+        StdMapObserver::new_from_ptr("edges", unsafe { __lafl_edges_map }, unsafe {
+            __lafl_max_edges_size as usize
+        });
+
+    // If not restarting, create a State from scratch
+    let mut state = state.unwrap_or_else(|| {
+        State::new(
+            // RNG
+            StdRand::with_seed(current_nanos()),
+            // Corpus that will be evolved, we keep it in memory for performance
+            InMemoryCorpus::new(),
+            // Feedbacks to rate the interestingness of an input
+            tuple_list!(MaxMapFeedback::new_with_observer(&edges_observer)),
+            // Corpus in which we store solutions (crashes in this example),
+            // on disk so the user can get them after stopping the fuzzer
+            OnDiskCorpus::new(objective_dir).unwrap(),
+            // Feedbacks to recognize an input as solution
+            tuple_list!(CrashFeedback::new()),
+        )
+    });
+
+    println!("We're a client, let's fuzz :)");
+
+    // Add the JPEG tokens if not existing
+    if state.metadata().get::<Tokens>().is_none() {
+        state.add_metadata(Tokens::from_tokens_file("./jpeg.tkns")?);
+    }
+
+    // Setup a basic mutator with a mutational stage
+    let mutator = HavocBytesMutator::default();
+    let stage = StdMutationalStage::new(mutator);
+
+    // A fuzzer with just one stage and a random policy to get testcasess from the corpus
+    let fuzzer = StdFuzzer::new(RandCorpusScheduler::new(), tuple_list!(stage));
+
+    // Create the executor for an in-process function with just one observer for edge coverage
+    let mut executor = InProcessExecutor::new(
+        "in-process(edges)",
+        harness,
+        tuple_list!(edges_observer),
+        &mut state,
+        &mut restarting_mgr,
+    );
+
+    // The actual target run starts here.
+    // Call LLVMFUzzerInitialize() if present.
+    unsafe {
+        if afl_libfuzzer_init() == -1 {
+            println!("Warning: LLVMFuzzerInitialize failed with -1")
+        }
+    }
+
+    // In case the corpus is empty (on first run), reset
+    if state.corpus().count() < 1 {
+        state
+            .load_initial_inputs(
+                &mut executor,
+                &mut restarting_mgr,
+                fuzzer.scheduler(),
+                &corpus_dirs,
+            )
+            .expect(&format!(
+                "Failed to load initial corpus at {:?}",
+                &corpus_dirs
+            ));
+        println!("We imported {} inputs from disk.", state.corpus().count());
+    }
+
+    fuzzer.fuzz_loop(&mut state, &mut executor, &mut restarting_mgr)?;
+
+    // Never reached
+    Ok(())
+}

fuzzers/libfuzzer_libmozjpeg/start.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+cores=$(grep -c ^processor /proc/cpuinfo)
+for (( c=1;c<$cores;c++))
+do
+    echo $c
+    taskset -c $c ./.libfuzzer_test.elf  2>/dev/null &
+    sleep 0.1
+done
+

fuzzers/libfuzzer_libmozjpeg/stop.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+killall -9 .libfuzzer_test.elf

fuzzers/libfuzzer_libmozjpeg/test.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+mkdir -p ./crashes
+
+cargo build --example libfuzzer_libmozjpeg --release || exit 1
+cp ../../target/release/examples/libfuzzer_libmozjpeg ./.libfuzzer_test.elf
+
+# The broker
+RUST_BACKTRACE=full taskset -c 0 ./.libfuzzer_test.elf &
+# Give the broker time to spawn
+sleep 2
+echo "Spawning client"
+# The 1st fuzzer client, pin to cpu 0x1
+RUST_BACKTRACE=full taskset -c 1 ./.libfuzzer_test.elf 2>/dev/null
+
+killall .libfuzzer_test.elf
+rm -rf ./.libfuzzer_test.elf

fuzzers/libfuzzer_libpng/src/fuzzer.rs

@@ -14,8 +14,7 @@ use libafl::{
     feedbacks::{CrashFeedback, MaxMapFeedback},
     fuzzer::{Fuzzer, HasCorpusScheduler, StdFuzzer},
     inputs::Input,
-    mutators::scheduled::HavocBytesMutator,
-    mutators::token_mutations::TokensMetadata,
+    mutators::{scheduled::HavocBytesMutator, token_mutations::Tokens},
     observers::{HitcountsMapObserver, StdMapObserver},
     stages::mutational::StdMutationalStage,
     state::{HasCorpus, HasMetadata, State},
@@ -54,7 +53,7 @@ where
 pub fn main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<TokensMetadata>();
+    //RegistryBuilder::register::<Tokens>();
 
     println!(
         "Workdir: {:?}",
@@ -109,8 +108,8 @@ fn fuzz(corpus_dirs: Vec<PathBuf>, objective_dir: PathBuf, broker_port: u16) ->
     println!("We're a client, let's fuzz :)");
 
     // Create a PNG dictionary if not existing
-    if state.metadatas().get::<TokensMetadata>().is_none() {
-        state.add_metadata(TokensMetadata::new(vec![
+    if state.metadata().get::<Tokens>().is_none() {
+        state.add_metadata(Tokens::new(vec![
             vec![137, 80, 78, 71, 13, 10, 26, 10], // PNG header
             "IHDR".as_bytes().to_vec(),
             "IDAT".as_bytes().to_vec(),

libafl/src/corpus/minimizer.rs

@@ -136,7 +136,7 @@ where
 {
     pub fn update_score(&self, state: &mut S, idx: usize) -> Result<(), Error> {
         // Create a new top rated meta if not existing
-        if state.metadatas().get::<TopRatedsMetadata>().is_none() {
+        if state.metadata().get::<TopRatedsMetadata>().is_none() {
             state.add_metadata(TopRatedsMetadata::new());
         }
 
@@ -144,7 +144,7 @@ where
         {
             let mut entry = state.corpus().get(idx)?.borrow_mut();
             let factor = F::compute(&mut *entry)?;
-            let meta = entry.metadatas().get::<M>().ok_or_else(|| {
+            let meta = entry.metadata().get::<M>().ok_or_else(|| {
                 Error::KeyNotFound(format!(
                     "Metadata needed for MinimizerCorpusScheduler not found in testcase #{}",
                     idx
@@ -152,7 +152,7 @@ where
             })?;
             for elem in meta.as_slice() {
                 if let Some(old_idx) = state
-                    .metadatas()
+                    .metadata()
                     .get::<TopRatedsMetadata>()
                     .unwrap()
                     .map
@@ -169,7 +169,7 @@ where
 
         for pair in new_favoreds {
             state
-                .metadatas_mut()
+                .metadata_mut()
                 .get_mut::<TopRatedsMetadata>()
                 .unwrap()
                 .map
@@ -179,17 +179,17 @@ where
     }
 
     pub fn cull(&self, state: &mut S) -> Result<(), Error> {
-        if state.metadatas().get::<TopRatedsMetadata>().is_none() {
+        if state.metadata().get::<TopRatedsMetadata>().is_none() {
             return Ok(());
         }
         let mut acc = HashSet::new();
-        let top_rated = state.metadatas().get::<TopRatedsMetadata>().unwrap();
+        let top_rated = state.metadata().get::<TopRatedsMetadata>().unwrap();
 
         for key in top_rated.map.keys() {
             if !acc.contains(key) {
                 let idx = top_rated.map.get(key).unwrap();
                 let mut entry = state.corpus().get(*idx)?.borrow_mut();
-                let meta = entry.metadatas().get::<M>().ok_or_else(|| {
+                let meta = entry.metadata().get::<M>().ok_or_else(|| {
                     Error::KeyNotFound(format!(
                         "Metadata needed for MinimizerCorpusScheduler not found in testcase #{}",
                         idx

libafl/src/corpus/ondisk.rs

@@ -37,7 +37,7 @@ where
     fn add(&mut self, mut testcase: Testcase<I>) -> Result<usize, Error> {
         match testcase.filename() {
             None => {
-                // TODO walk entry metadatas to ask for pices of filename (e.g. :havoc in AFL)
+                // TODO walk entry metadata to ask for pices of filename (e.g. :havoc in AFL)
                 let filename = self.dir_path.join(format!("id_{}", &self.entries.len()));
                 let filename_str = filename.to_str().expect("Invalid Path");
                 testcase.set_filename(filename_str.into());

libafl/src/corpus/testcase.rs

@@ -25,8 +25,8 @@ where
     filename: Option<String>,
     /// Accumulated fitness from all the feedbacks
     fitness: u32,
-    /// Map of metadatas associated with this testcase
-    metadatas: SerdeAnyMap,
+    /// Map of metadata associated with this testcase
+    metadata: SerdeAnyMap,
     /// Time needed to execute the input
     exec_time: Option<Duration>,
     /// Cached len of the input, if any
@@ -37,16 +37,16 @@ impl<I> HasMetadata for Testcase<I>
 where
     I: Input,
 {
-    /// Get all the metadatas into an HashMap
+    /// Get all the metadata into an HashMap
     #[inline]
-    fn metadatas(&self) -> &SerdeAnyMap {
-        &self.metadatas
+    fn metadata(&self) -> &SerdeAnyMap {
+        &self.metadata
     }
 
-    /// Get all the metadatas into an HashMap (mutable)
+    /// Get all the metadata into an HashMap (mutable)
     #[inline]
-    fn metadatas_mut(&mut self) -> &mut SerdeAnyMap {
-        &mut self.metadatas
+    fn metadata_mut(&mut self) -> &mut SerdeAnyMap {
+        &mut self.metadata
     }
 }
 
@@ -158,7 +158,7 @@ where
             input: Some(input.into()),
             filename: None,
             fitness: 0,
-            metadatas: SerdeAnyMap::new(),
+            metadata: SerdeAnyMap::new(),
             exec_time: None,
             cached_len: None,
         }
@@ -171,7 +171,7 @@ where
             input: Some(input),
             filename: Some(filename),
             fitness: 0,
-            metadatas: SerdeAnyMap::new(),
+            metadata: SerdeAnyMap::new(),
             exec_time: None,
             cached_len: None,
         }
@@ -184,7 +184,7 @@ where
             input: Some(input.into()),
             filename: None,
             fitness: fitness,
-            metadatas: SerdeAnyMap::new(),
+            metadata: SerdeAnyMap::new(),
             exec_time: None,
             cached_len: None,
         }
@@ -196,7 +196,7 @@ where
             input: None,
             filename: None,
             fitness: 0,
-            metadatas: SerdeAnyMap::new(),
+            metadata: SerdeAnyMap::new(),
             exec_time: None,
             cached_len: None,
         }

libafl/src/events/mod.rs

@@ -73,7 +73,7 @@ where
     I: Input,
 {
     // TODO use an ID to keep track of the original index in the sender Corpus
-    // The sender can then use it to send Testcase metadatas with CustomEvent
+    // The sender can then use it to send Testcase metadata with CustomEvent
     /// A fuzzer found a new testcase. Rejoice!
     NewTestcase {
         /// The input for the new testcase

libafl/src/mutators/mutations.rs

@@ -11,12 +11,6 @@ use crate::{
 use alloc::{borrow::ToOwned, vec::Vec};
 use core::cmp::{max, min};
 
-#[cfg(feature = "std")]
-use std::{
-    fs::File,
-    io::{BufRead, BufReader},
-};
-
 /// The result of a mutation.
 /// If the mutation got skipped, the target
 /// will not be executed with the returned input.
@@ -810,75 +804,8 @@ pub fn str_decode(item: &str) -> Result<Vec<u8>, Error> {
     return Ok(token);
 }
 
-/// Adds a token to a dictionary, checking it is not a duplicate
-pub fn add_token(tokens: &mut Vec<Vec<u8>>, token: &Vec<u8>) -> u32 {
-    if tokens.contains(token) {
-        return 0;
-    }
-    tokens.push(token.to_vec());
-    return 1;
-}
-
-/// Read a dictionary file and return the number of entries read
-#[cfg(feature = "std")]
-pub fn read_tokens_file(f: &str, tokens: &mut Vec<Vec<u8>>) -> Result<u32, Error> {
-    let mut entries = 0;
-
-    println!("Loading tokens file {:?} ...", &f);
-
-    let file = File::open(&f)?; // panic if not found
-    let reader = BufReader::new(file);
-
-    for line in reader.lines() {
-        let line = line.unwrap();
-        let line = line.trim_start().trim_end();
-
-        // we are only interested in '"..."', not prefixed 'foo = '
-        let start = line.chars().nth(0);
-        if line.len() == 0 || start == Some('#') {
-            continue;
-        }
-        let pos_quote = match line.find("\"") {
-            Some(x) => x,
-            _ => return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)),
-        };
-        if line.chars().nth(line.len() - 1) != Some('"') {
-            return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line));
-        }
-
-        // extract item
-        let item = match line.get(pos_quote + 1..line.len() - 1) {
-            Some(x) => x,
-            _ => return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)),
-        };
-        if item.len() == 0 {
-            continue;
-        }
-
-        // decode
-        let token: Vec<u8> = match str_decode(item) {
-            Ok(val) => val,
-            Err(_) => {
-                return Err(Error::IllegalArgument(
-                    "Illegal line (hex decoding): ".to_owned() + line,
-                ))
-            }
-        };
-
-        // add
-        entries += add_token(tokens, &token);
-    }
-
-    Ok(entries)
-}
-
 #[cfg(test)]
 mod tests {
-    #[cfg(feature = "std")]
-    use std::fs;
-
-    #[cfg(feature = "std")]
-    use crate::mutators::read_tokens_file;
 
     use super::*;
     use crate::{
@@ -889,26 +816,6 @@ mod tests {
         utils::StdRand,
     };
 
-    #[cfg(feature = "std")]
-    #[test]
-    fn test_read_tokens() {
-        let _ = fs::remove_file("test.tkns");
-        let data = r###"
-# comment
-token1@123="AAA"
-token1="A\x41A"
-"A\AA"
-token2="B"
-        "###;
-        fs::write("test.tkns", data).expect("Unable to write test.tkns");
-        let mut v: Vec<Vec<u8>> = Vec::new();
-        let res = read_tokens_file(&"test.tkns".to_string(), &mut v).unwrap();
-        #[cfg(feature = "std")]
-        println!("Token file entries: {:?}", res);
-        assert_eq!(res, 2);
-        let _ = fs::remove_file("test.tkns");
-    }
-
     #[test]
     fn test_mutators() {
         let mut inputs = vec![

libafl/src/mutators/token_mutations.rs

@@ -1,6 +1,13 @@
 //! Tokens are what afl calls extras or dictionaries.
 //! They may be inserted as part of mutations during fuzzing.
 
+#[cfg(feature = "std")]
+use std::{
+    fs::File,
+    io::{BufRead, BufReader},
+    path::Path,
+};
+
 use crate::{
     inputs::{HasBytesVec, Input},
     mutators::*,
@@ -16,15 +23,101 @@ use mutations::buffer_copy;
 
 /// A state metadata holding a list of tokens
 #[derive(Serialize, Deserialize)]
-pub struct TokensMetadata {
-    tokens: Vec<Vec<u8>>,
+pub struct Tokens {
+    token_vec: Vec<Vec<u8>>,
 }
 
-crate::impl_serdeany!(TokensMetadata);
+crate::impl_serdeany!(Tokens);
 
-impl TokensMetadata {
-    pub fn new(tokens: Vec<Vec<u8>>) -> Self {
-        Self { tokens: tokens }
+/// The metadata used for token mutators
+impl Tokens {
+    /// Creates a new tokens metadata (old-skool afl name: `dictornary`)
+    pub fn new(token_vec: Vec<Vec<u8>>) -> Self {
+        Self { token_vec }
+    }
+
+    /// Creates a new instance from a file
+    #[cfg(feature = "std")]
+    pub fn from_tokens_file<P>(file: P) -> Result<Self, Error>
+    where
+        P: AsRef<Path>,
+    {
+        let mut ret = Self::new(vec![]);
+        ret.add_tokens_from_file(file)?;
+        Ok(ret)
+    }
+
+    /// Adds a token to a dictionary, checking it is not a duplicate
+    /// Returns `false` if the token was already present and did not get added.
+    pub fn add_token(&mut self, token: &Vec<u8>) -> bool {
+        if self.token_vec.contains(token) {
+            return false;
+        }
+        self.token_vec.push(token.to_vec());
+        return true;
+    }
+
+    /// Reads a tokens file, returning the count of new entries read
+    #[cfg(feature = "std")]
+    pub fn add_tokens_from_file<P>(&mut self, file: P) -> Result<u32, Error>
+    where
+        P: AsRef<Path>,
+    {
+        let mut entries = 0;
+
+        // println!("Loading tokens file {:?} ...", file);
+
+        let file = File::open(file)?; // panic if not found
+        let reader = BufReader::new(file);
+
+        for line in reader.lines() {
+            let line = line.unwrap();
+            let line = line.trim_start().trim_end();
+
+            // we are only interested in '"..."', not prefixed 'foo = '
+            let start = line.chars().nth(0);
+            if line.len() == 0 || start == Some('#') {
+                continue;
+            }
+            let pos_quote = match line.find("\"") {
+                Some(x) => x,
+                _ => return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)),
+            };
+            if line.chars().nth(line.len() - 1) != Some('"') {
+                return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line));
+            }
+
+            // extract item
+            let item = match line.get(pos_quote + 1..line.len() - 1) {
+                Some(x) => x,
+                _ => return Err(Error::IllegalArgument("Illegal line: ".to_owned() + line)),
+            };
+            if item.len() == 0 {
+                continue;
+            }
+
+            // decode
+            let token: Vec<u8> = match str_decode(item) {
+                Ok(val) => val,
+                Err(_) => {
+                    return Err(Error::IllegalArgument(
+                        "Illegal line (hex decoding): ".to_owned() + line,
+                    ))
+                }
+            };
+
+            // add
+            if self.add_token(&token) {
+                entries += 1;
+            }
+        }
+
+        Ok(entries)
+    }
+
+    /// Gets the tokens stored in this db
+    pub fn tokens(&self) -> &[Vec<u8>] {
+        return &self.token_vec;
     }
 }
 
@@ -37,22 +130,22 @@ where
 {
     let max_size = state.max_size();
     let tokens_len = {
-        let meta = state.metadatas().get::<TokensMetadata>();
+        let meta = state.metadata().get::<Tokens>();
         if meta.is_none() {
             return Ok(MutationResult::Skipped);
         }
-        if meta.unwrap().tokens.len() == 0 {
+        if meta.unwrap().tokens().len() == 0 {
             return Ok(MutationResult::Skipped);
         }
-        meta.unwrap().tokens.len()
+        meta.unwrap().tokens().len()
     };
     let token_idx = state.rand_mut().below(tokens_len as u64) as usize;
 
     let size = input.bytes().len();
     let off = state.rand_mut().below((size + 1) as u64) as usize;
 
-    let meta = state.metadatas().get::<TokensMetadata>().unwrap();
-    let token = &meta.tokens[token_idx];
+    let meta = state.metadata().get::<Tokens>().unwrap();
+    let token = &meta.tokens()[token_idx];
     let mut len = token.len();
 
     if size + len > max_size {
@@ -83,21 +176,21 @@ where
     }
 
     let tokens_len = {
-        let meta = state.metadatas().get::<TokensMetadata>();
+        let meta = state.metadata().get::<Tokens>();
         if meta.is_none() {
             return Ok(MutationResult::Skipped);
         }
-        if meta.unwrap().tokens.len() == 0 {
+        if meta.unwrap().tokens().len() == 0 {
             return Ok(MutationResult::Skipped);
         }
-        meta.unwrap().tokens.len()
+        meta.unwrap().tokens().len()
     };
     let token_idx = state.rand_mut().below(tokens_len as u64) as usize;
 
     let off = state.rand_mut().below(size as u64) as usize;
 
-    let meta = state.metadatas().get::<TokensMetadata>().unwrap();
-    let token = &meta.tokens[token_idx];
+    let meta = state.metadata().get::<Tokens>().unwrap();
+    let token = &meta.tokens()[token_idx];
     let mut len = token.len();
     if off + len > size {
         len = size - off;
@@ -107,3 +200,30 @@ where
 
     Ok(MutationResult::Mutated)
 }
+
+#[cfg(test)]
+mod tests {
+    #[cfg(feature = "std")]
+    use std::fs;
+
+    use super::Tokens;
+
+    #[cfg(feature = "std")]
+    #[test]
+    fn test_read_tokens() {
+        let _ = fs::remove_file("test.tkns");
+        let data = r###"
+# comment
+token1@123="AAA"
+token1="A\x41A"
+"A\AA"
+token2="B"
+        "###;
+        fs::write("test.tkns", data).expect("Unable to write test.tkns");
+        let tokens = Tokens::from_tokens_file(&"test.tkns").unwrap();
+        #[cfg(feature = "std")]
+        println!("Token file entries: {:?}", tokens.tokens());
+        assert_eq!(tokens.tokens().len(), 2);
+        let _ = fs::remove_file("test.tkns");
+    }
+}

libafl/src/state/mod.rs

@@ -73,9 +73,9 @@ where
 /// Trait for elements offering metadata
 pub trait HasMetadata {
     /// A map, storing all metadata
-    fn metadatas(&self) -> &SerdeAnyMap;
+    fn metadata(&self) -> &SerdeAnyMap;
     /// A map, storing all metadata (mut)
-    fn metadatas_mut(&mut self) -> &mut SerdeAnyMap;
+    fn metadata_mut(&mut self) -> &mut SerdeAnyMap;
 
     /// Add a metadata to the metadata map
     #[inline]
@@ -83,7 +83,7 @@ pub trait HasMetadata {
     where
         M: SerdeAny,
     {
-        self.metadatas_mut().insert(meta);
+        self.metadata_mut().insert(meta);
     }
 
     /// Check for a metadata
@@ -92,7 +92,7 @@ pub trait HasMetadata {
     where
         M: SerdeAny,
     {
-        self.metadatas().get::<M>().is_some()
+        self.metadata().get::<M>().is_some()
     }
 }
 
@@ -317,13 +317,13 @@ where
 {
     /// Get all the metadata into an HashMap
     #[inline]
-    fn metadatas(&self) -> &SerdeAnyMap {
+    fn metadata(&self) -> &SerdeAnyMap {
         &self.metadata
     }
 
     /// Get all the metadata into an HashMap (mutable)
     #[inline]
-    fn metadatas_mut(&mut self) -> &mut SerdeAnyMap {
+    fn metadata_mut(&mut self) -> &mut SerdeAnyMap {
         &mut self.metadata
     }
 }