OnDiskCorpus: Write metadata by default, metadata gzip compression (#995)

* Write metadata by default

* fix fuzzers

* Cleanup, gzip feature

* Fix casing for ondisk corpus

* fix fmt, clippy

* clippy

* clippy for gdiplus fuzzer

* fmt

fuzzers/forkserver_libafl_cc/src/main.rs

@@ -115,7 +115,7 @@ pub fn main() {
         // New maximization map feedback linked to the edges observer and the feedback state
         MaxMapFeedback::new_tracking(&edges_observer, true, false),
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not

fuzzers/forkserver_simple/src/main.rs

@@ -115,7 +115,7 @@ pub fn main() {
         // New maximization map feedback linked to the edges observer and the feedback state
         MaxMapFeedback::new_tracking(&edges_observer, true, false),
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not

fuzzers/frida_gdiplus/src/fuzzer.rs

@@ -1,10 +1,10 @@
-//! A libfuzzer-like fuzzer with llmp-multithreading support and restarts
+//! A `libfuzzer`-like fuzzer with `llmp`-multithreading support and restarts
-//! The example harness is built for gdiplus.
+//! The example harness is built for `gdiplus`.
-//! NOTE: This file is 1-to-1 copy of the ../../frida_libpng/fuzzer.rs, which
+//! NOTE: This file is 1-to-1 copy of `../../frida_libpng/fuzzer.rs`, which
 //! is platform independent. Hence, this file contains code for other platforms
-//! but it's only meaningful for Windows because of the gdiplus target. If you
+//! but it's only meaningful for Windows because of the `gdiplus` target. If you
-//! going to make it compilable only for windows, don't foret to modify the
+//! going to make it compilable only for Windows, don't forget to modify the
-//! scripts/test_all_fuzzers.sh to opt-out this fuzzer from that test.
+//! `scripts/test_all_fuzzers.sh` to opt-out this fuzzer from that test.
 
 use mimalloc::MiMalloc;
 #[global_allocator]
@@ -23,7 +23,7 @@ use libafl::{
         tuples::{tuple_list, Merge},
         AsSlice,
     },
-    corpus::{ondisk::OnDiskMetadataFormat, CachedOnDiskCorpus, Corpus, OnDiskCorpus},
+    corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus},
     events::{llmp::LlmpRestartingEventManager, EventConfig},
     executors::{inprocess::InProcessExecutor, ExitKind, ShadowExecutor},
     feedback_and_fast, feedback_or, feedback_or_fast,
@@ -60,16 +60,16 @@ pub fn main() {
     let options = parse_args();
 
     unsafe {
-        match fuzz(options) {
+        match fuzz(&options) {
             Ok(()) | Err(Error::ShuttingDown) => println!("\nFinished fuzzing. Good bye."),
-            Err(e) => panic!("Error during fuzzing: {:?}", e),
+            Err(e) => panic!("Error during fuzzing: {e:?}"),
         }
     }
 }
 
 /// The actual fuzzer
 #[allow(clippy::too_many_lines, clippy::too_many_arguments)]
-unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
+unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> {
     // 'While the stats are state, they are usually used in the broker - which is likely never restarted
     let monitor = MultiMonitor::new(|s| println!("{s}"));
 
@@ -102,10 +102,10 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
 
                 #[cfg(unix)]
                 let mut frida_helper =
-                    FridaInstrumentationHelper::new(&gum, &options, tuple_list!(coverage, asan));
+                    FridaInstrumentationHelper::new(&gum, options, tuple_list!(coverage, asan));
                 #[cfg(windows)]
                 let mut frida_helper =
-                    FridaInstrumentationHelper::new(&gum, &options, tuple_list!(coverage));
+                    FridaInstrumentationHelper::new(&gum, options, tuple_list!(coverage));
 
                 // Create an observation channel using the coverage map
                 let edges_observer = HitcountsMapObserver::new(StdMapObserver::from_mut_ptr(
@@ -123,7 +123,7 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
                     // New maximization map feedback linked to the edges observer and the feedback state
                     MaxMapFeedback::new_tracking(&edges_observer, true, false),
                     // Time feedback, this one does not need a feedback state
-                    TimeFeedback::new_with_observer(&time_observer)
+                    TimeFeedback::with_observer(&time_observer)
                 );
 
                 // Feedbacks to recognize an input as solution
@@ -146,11 +146,7 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
                         CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(),
                         // Corpus in which we store solutions (crashes in this example),
                         // on disk so the user can get them after stopping the fuzzer
-                        OnDiskCorpus::new_save_meta(
+                        OnDiskCorpus::new(&options.output).unwrap(),
-                            options.output.to_path_buf(),
-                            Some(OnDiskMetadataFormat::JsonPretty),
-                        )
-                        .unwrap(),
                         &mut feedback,
                         &mut objective,
                     )
@@ -225,7 +221,7 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
                 let cmplog = CmpLogRuntime::new();
 
                 let mut frida_helper =
-                    FridaInstrumentationHelper::new(&gum, &options, tuple_list!(coverage, cmplog));
+                    FridaInstrumentationHelper::new(&gum, options, tuple_list!(coverage, cmplog));
 
                 // Create an observation channel using the coverage map
                 let edges_observer = HitcountsMapObserver::new(StdMapObserver::from_mut_ptr(
@@ -243,7 +239,7 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
                     // New maximization map feedback linked to the edges observer and the feedback state
                     MaxMapFeedback::new_tracking(&edges_observer, true, false),
                     // Time feedback, this one does not need a feedback state
-                    TimeFeedback::new_with_observer(&time_observer)
+                    TimeFeedback::with_observer(&time_observer)
                 );
 
                 #[cfg(unix)]
@@ -261,14 +257,11 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
                         // RNG
                         StdRand::with_seed(current_nanos()),
                         // Corpus that will be evolved, we keep it in memory for performance
-                        CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(),
+                        CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64)
+                            .unwrap(),
                         // Corpus in which we store solutions (crashes in this example),
                         // on disk so the user can get them after stopping the fuzzer
-                        OnDiskCorpus::new_save_meta(
+                        OnDiskCorpus::new(&options.output).unwrap(),
-                            options.output.to_path_buf(),
-                            Some(OnDiskMetadataFormat::JsonPretty),
-                        )
-                        .unwrap(),
                         &mut feedback,
                         &mut objective,
                     )
@@ -358,7 +351,7 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
                 let coverage = CoverageRuntime::new();
 
                 let mut frida_helper =
-                    FridaInstrumentationHelper::new(&gum, &options, tuple_list!(coverage));
+                    FridaInstrumentationHelper::new(&gum, options, tuple_list!(coverage));
 
                 // Create an observation channel using the coverage map
                 let edges_observer = HitcountsMapObserver::new(StdMapObserver::from_mut_ptr(
@@ -376,7 +369,7 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
                     // New maximization map feedback linked to the edges observer and the feedback state
                     MaxMapFeedback::new_tracking(&edges_observer, true, false),
                     // Time feedback, this one does not need a feedback state
-                    TimeFeedback::new_with_observer(&time_observer)
+                    TimeFeedback::with_observer(&time_observer)
                 );
 
                 #[cfg(unix)]
@@ -394,14 +387,11 @@ unsafe fn fuzz(options: FuzzerOptions) -> Result<(), Error> {
                         // RNG
                         StdRand::with_seed(current_nanos()),
                         // Corpus that will be evolved, we keep it in memory for performance
-                        CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(),
+                        CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64)
+                            .unwrap(),
                         // Corpus in which we store solutions (crashes in this example),
                         // on disk so the user can get them after stopping the fuzzer
-                        OnDiskCorpus::new_save_meta(
+                        OnDiskCorpus::new(&options.output).unwrap(),
-                            options.output.to_path_buf(),
-                            Some(OnDiskMetadataFormat::JsonPretty),
-                        )
-                        .unwrap(),
                         &mut feedback,
                         &mut objective,
                     )

fuzzers/frida_libpng/src/fuzzer.rs

@@ -117,7 +117,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> {
                     // New maximization map feedback linked to the edges observer and the feedback state
                     MaxMapFeedback::new_tracking(&edges_observer, true, false),
                     // Time feedback, this one does not need a feedback state
-                    TimeFeedback::new_with_observer(&time_observer)
+                    TimeFeedback::with_observer(&time_observer)
                 );
 
                 // Feedbacks to recognize an input as solution
@@ -137,14 +137,11 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> {
                         // RNG
                         StdRand::with_seed(current_nanos()),
                         // Corpus that will be evolved, we keep it in memory for performance
-                        CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(),
+                        CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64)
+                            .unwrap(),
                         // Corpus in which we store solutions (crashes in this example),
                         // on disk so the user can get them after stopping the fuzzer
-                        OnDiskCorpus::new_save_meta(
+                        OnDiskCorpus::new(options.output.clone()).unwrap(),
-                            options.output.clone(),
-                            Some(OnDiskMetadataFormat::JsonPretty),
-                        )
-                        .unwrap(),
                         &mut feedback,
                         &mut objective,
                     )
@@ -237,7 +234,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> {
                     // New maximization map feedback linked to the edges observer and the feedback state
                     MaxMapFeedback::new_tracking(&edges_observer, true, false),
                     // Time feedback, this one does not need a feedback state
-                    TimeFeedback::new_with_observer(&time_observer)
+                    TimeFeedback::with_observer(&time_observer)
                 );
 
                 #[cfg(unix)]
@@ -255,14 +252,11 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> {
                         // RNG
                         StdRand::with_seed(current_nanos()),
                         // Corpus that will be evolved, we keep it in memory for performance
-                        CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(),
+                        CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64)
+                            .unwrap(),
                         // Corpus in which we store solutions (crashes in this example),
                         // on disk so the user can get them after stopping the fuzzer
-                        OnDiskCorpus::new_save_meta(
+                        OnDiskCorpus::new(options.output.clone()).unwrap(),
-                            options.output.clone(),
-                            Some(OnDiskMetadataFormat::JsonPretty),
-                        )
-                        .unwrap(),
                         &mut feedback,
                         &mut objective,
                     )
@@ -370,7 +364,7 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> {
                     // New maximization map feedback linked to the edges observer and the feedback state
                     MaxMapFeedback::new_tracking(&edges_observer, true, false),
                     // Time feedback, this one does not need a feedback state
-                    TimeFeedback::new_with_observer(&time_observer)
+                    TimeFeedback::with_observer(&time_observer)
                 );
 
                 #[cfg(unix)]
@@ -388,14 +382,11 @@ unsafe fn fuzz(options: &FuzzerOptions) -> Result<(), Error> {
                         // RNG
                         StdRand::with_seed(current_nanos()),
                         // Corpus that will be evolved, we keep it in memory for performance
-                        CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(),
+                        CachedOnDiskCorpus::no_meta(PathBuf::from("./corpus_discovered"), 64)
+                            .unwrap(),
                         // Corpus in which we store solutions (crashes in this example),
                         // on disk so the user can get them after stopping the fuzzer
-                        OnDiskCorpus::new_save_meta(
+                        OnDiskCorpus::new(options.output.clone()).unwrap(),
-                            options.output.clone(),
-                            Some(OnDiskMetadataFormat::JsonPretty),
-                        )
-                        .unwrap(),
                         &mut feedback,
                         &mut objective,
                     )

fuzzers/fuzzbench/src/lib.rs

@@ -259,7 +259,7 @@ fn fuzz(
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not

fuzzers/fuzzbench_fork_qemu/src/fuzzer.rs

@@ -259,7 +259,7 @@ fn fuzz(
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not

fuzzers/fuzzbench_forkserver/src/main.rs

@@ -259,7 +259,7 @@ fn fuzz(
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not

fuzzers/fuzzbench_qemu/src/fuzzer.rs

@@ -271,7 +271,7 @@ fn fuzz(
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not

fuzzers/fuzzbench_text/src/lib.rs

@@ -321,7 +321,7 @@ fn fuzz_binary(
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
     // A feedback to choose if an input is a solution or not
     let mut objective = CrashFeedback::new();
@@ -522,7 +522,7 @@ fn fuzz_text(
     let mut feedback = feedback_or!(
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not

fuzzers/libafl_atheris/src/lib.rs

@@ -231,7 +231,7 @@ pub fn LLVMFuzzerRunDriver(
             // New maximization map feedback linked to the edges observer and the feedback state
             MaxMapFeedback::new_tracking(&edges_observer, true, false),
             // Time feedback, this one does not need a feedback state
-            TimeFeedback::new_with_observer(&time_observer)
+            TimeFeedback::with_observer(&time_observer)
         );
 
         // A feedback to choose if an input is a solution or not

fuzzers/libfuzzer_libpng/src/lib.rs

@@ -100,7 +100,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not

fuzzers/libfuzzer_libpng_accounting/src/lib.rs

@@ -146,7 +146,7 @@ pub fn libafl_main() {
             // New maximization map feedback linked to the edges observer and the feedback state
             MaxMapFeedback::new_tracking(&edges_observer, true, false),
             // Time feedback, this one does not need a feedback state
-            TimeFeedback::new_with_observer(&time_observer)
+            TimeFeedback::with_observer(&time_observer)
         );
 
         // A feedback to choose if an input is a solution or not

fuzzers/libfuzzer_libpng_cmin/src/lib.rs

@@ -99,7 +99,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not
@@ -186,7 +186,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
     // Call LLVMFUzzerInitialize() if present.
     let args: Vec<String> = env::args().collect();
     if libfuzzer_initialize(&args) == -1 {
-        println!("Warning: LLVMFuzzerInitialize failed with -1")
+        println!("Warning: LLVMFuzzerInitialize failed with -1");
     }
 
     // In case the corpus is empty (on first run), reset

fuzzers/libfuzzer_libpng_ctx/src/lib.rs

@@ -140,7 +140,7 @@ pub fn libafl_main() {
             // New maximization map feedback linked to the edges observer and the feedback state
             MaxMapFeedback::new_tracking(&edges_observer, true, false),
             // Time feedback, this one does not need a feedback state
-            TimeFeedback::new_with_observer(&time_observer)
+            TimeFeedback::with_observer(&time_observer)
         );
 
         // A feedback to choose if an input is a solution or not
@@ -213,7 +213,7 @@ pub fn libafl_main() {
         // Call LLVMFUzzerInitialize() if present.
         let args: Vec<String> = env::args().collect();
         if libfuzzer_initialize(&args) == -1 {
-            println!("Warning: LLVMFuzzerInitialize failed with -1")
+            println!("Warning: LLVMFuzzerInitialize failed with -1");
         }
 
         // In case the corpus is empty (on first run), reset

fuzzers/libfuzzer_libpng_launcher/src/lib.rs

@@ -145,7 +145,7 @@ pub fn libafl_main() {
             // New maximization map feedback linked to the edges observer and the feedback state
             MaxMapFeedback::new_tracking(&edges_observer, true, false),
             // Time feedback, this one does not need a feedback state
-            TimeFeedback::new_with_observer(&time_observer)
+            TimeFeedback::with_observer(&time_observer)
         );
 
         // A feedback to choose if an input is a solution or not
@@ -218,7 +218,7 @@ pub fn libafl_main() {
         // Call LLVMFUzzerInitialize() if present.
         let args: Vec<String> = env::args().collect();
         if libfuzzer_initialize(&args) == -1 {
-            println!("Warning: LLVMFuzzerInitialize failed with -1")
+            println!("Warning: LLVMFuzzerInitialize failed with -1");
         }
 
         // In case the corpus is empty (on first run), reset

fuzzers/libfuzzer_reachability/src/lib.rs

@@ -132,7 +132,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
     // Call LLVMFUzzerInitialize() if present.
     let args: Vec<String> = env::args().collect();
     if libfuzzer_initialize(&args) == -1 {
-        println!("Warning: LLVMFuzzerInitialize failed with -1")
+        println!("Warning: LLVMFuzzerInitialize failed with -1");
     }
 
     // In case the corpus is empty (on first run), reset

fuzzers/libfuzzer_stb_image/src/main.rs

@@ -81,7 +81,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
         // New maximization map feedback linked to the edges observer and the feedback state
         MaxMapFeedback::new_tracking(&edges_observer, true, false),
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not
@@ -138,7 +138,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
     // Call LLVMFUzzerInitialize() if present.
     let args: Vec<String> = env::args().collect();
     if libfuzzer_initialize(&args) == -1 {
-        println!("Warning: LLVMFuzzerInitialize failed with -1")
+        println!("Warning: LLVMFuzzerInitialize failed with -1");
     }
 
     // In case the corpus is empty (on first run), reset

fuzzers/libfuzzer_stb_image_concolic/fuzzer/build.rs

@@ -11,12 +11,12 @@ use which::which;
 
 fn build_dep_check(tools: &[&str]) {
     for tool in tools {
-        println!("Checking for build tool {}...", tool);
+        println!("Checking for build tool {tool}...");
 
         if let Ok(path) = which(tool) {
             println!("Found build tool {}", path.to_str().unwrap());
         } else {
-            println!("ERROR: missing build tool {}", tool);
+            println!("ERROR: missing build tool {tool}");
             exit(1);
         };
     }
@@ -67,11 +67,11 @@ fn main() {
         .expect("Failed to build runtime");
 
     std::fs::copy(
-        &runtime_dir
+        runtime_dir
             .join("target")
             .join("release")
             .join("libSymRuntime.so"),
-        &runtime_dir.join("libSymRuntime.so"),
+        runtime_dir.join("libSymRuntime.so"),
     )
     .unwrap();
 

fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs

@@ -118,7 +118,7 @@ fn fuzz(
         // New maximization map feedback linked to the edges observer and the feedback state
         MaxMapFeedback::new_tracking(&edges_observer, true, false),
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not
@@ -175,19 +175,14 @@ fn fuzz(
     // Call LLVMFUzzerInitialize() if present.
     let args: Vec<String> = env::args().collect();
     if libfuzzer_initialize(&args) == -1 {
-        println!("Warning: LLVMFuzzerInitialize failed with -1")
+        println!("Warning: LLVMFuzzerInitialize failed with -1");
     }
 
     // In case the corpus is empty (on first run), reset
     if state.corpus().count() < 1 {
         state
-            .load_initial_inputs(
+            .load_initial_inputs(&mut fuzzer, &mut executor, &mut restarting_mgr, corpus_dirs)
-                &mut fuzzer,
+            .unwrap_or_else(|_| panic!("Failed to load initial corpus at {corpus_dirs:?}"));
-                &mut executor,
-                &mut restarting_mgr,
-                &corpus_dirs,
-            )
-            .unwrap_or_else(|_| panic!("Failed to load initial corpus at {:?}", &corpus_dirs));
         println!("We imported {} inputs from disk.", state.corpus().count());
     }
 

fuzzers/libfuzzer_stb_image_sugar/src/main.rs

@@ -32,7 +32,7 @@ fn fuzz(input_dirs: &[PathBuf], output_dir: PathBuf, cores: &Cores, broker_port:
     // Call LLVMFUzzerInitialize() if present.
     let args: Vec<String> = env::args().collect();
     if libfuzzer_initialize(&args) == -1 {
-        println!("Warning: LLVMFuzzerInitialize failed with -1")
+        println!("Warning: LLVMFuzzerInitialize failed with -1");
     }
 
     InMemoryBytesCoverageSugar::builder()

fuzzers/libfuzzer_windows_asan/src/lib.rs

@@ -76,7 +76,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer)
+        TimeFeedback::with_observer(&time_observer)
     );
 
     // A feedback to choose if an input is a solution or not
@@ -150,7 +150,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
     // Call LLVMFUzzerInitialize() if present.
     let args: Vec<String> = env::args().collect();
     if libfuzzer_initialize(&args) == -1 {
-        println!("Warning: LLVMFuzzerInitialize failed with -1")
+        println!("Warning: LLVMFuzzerInitialize failed with -1");
     }
 
     // In case the corpus is empty (on first run), reset

fuzzers/qemu_arm_launcher/src/fuzzer.rs

@@ -125,7 +125,7 @@ pub fn fuzz() {
             // New maximization map feedback linked to the edges observer and the feedback state
             MaxMapFeedback::new_tracking(&edges_observer, true, false),
             // Time feedback, this one does not need a feedback state
-            TimeFeedback::new_with_observer(&time_observer)
+            TimeFeedback::with_observer(&time_observer)
         );
 
         // A feedback to choose if an input is a solution or not

fuzzers/qemu_launcher/src/fuzzer.rs

@@ -129,7 +129,7 @@ pub fn fuzz() {
             // New maximization map feedback linked to the edges observer and the feedback state
             MaxMapFeedback::new_tracking(&edges_observer, true, false),
             // Time feedback, this one does not need a feedback state
-            TimeFeedback::new_with_observer(&time_observer)
+            TimeFeedback::with_observer(&time_observer)
         );
 
         // A feedback to choose if an input is a solution or not

fuzzers/qemu_systemmode/src/fuzzer.rs

@@ -157,7 +157,7 @@ pub fn fuzz() {
             // New maximization map feedback linked to the edges observer and the feedback state
             MaxMapFeedback::new_tracking(&edges_observer, true, true),
             // Time feedback, this one does not need a feedback state
-            TimeFeedback::new_with_observer(&time_observer)
+            TimeFeedback::with_observer(&time_observer)
         );
 
         // A feedback to choose if an input is a solution or not

fuzzers/tinyinst_simple/src/main.rs

@@ -27,7 +27,7 @@ fn main() {
     let args = vec![".\\test\\test.exe".to_string(), "@@".to_string()];
 
     let observer = unsafe { ListObserver::new("cov", &mut COVERAGE) };
-    let mut feedback = ListFeedback::new_with_observer(&observer);
+    let mut feedback = ListFeedback::with_observer(&observer);
 
     let input = BytesInput::new(b"bad".to_vec());
     let rand = StdRand::new();

fuzzers/tutorial/src/lib.rs

@@ -96,7 +96,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
         // New maximization map feedback linked to the edges observer and the feedback state
         map_feedback,
         // Time feedback, this one does not need a feedback state
-        TimeFeedback::new_with_observer(&time_observer),
+        TimeFeedback::with_observer(&time_observer),
         PacketLenFeedback::new()
     );
 
@@ -154,7 +154,7 @@ fn fuzz(corpus_dirs: &[PathBuf], objective_dir: PathBuf, broker_port: u16) -> Re
     // Call LLVMFUzzerInitialize() if present.
     let args: Vec<String> = env::args().collect();
     if libfuzzer_initialize(&args) == -1 {
-        println!("Warning: LLVMFuzzerInitialize failed with -1")
+        println!("Warning: LLVMFuzzerInitialize failed with -1");
     }
 
     // In case the corpus is empty (on first run), reset

libafl/Cargo.toml

@@ -12,7 +12,7 @@ edition = "2021"
 categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"]
 
 [features]
-default = ["std", "derive", "llmp_compression", "rand_trait", "fork", "prelude"]
+default = ["std", "derive", "llmp_compression", "rand_trait", "fork", "prelude", "gzip"]
 std = ["serde_json", "serde_json/std", "hostname", "nix", "serde/std", "bincode", "wait-timeout", "regex", "byteorder", "once_cell", "uuid", "tui_monitor", "ctor", "backtrace", "uds"] # print, env, launcher ... support
 derive = ["libafl_derive"] # provide derive(SerdeAny) macro.
 fork = [] # uses the fork() syscall to spawn children, instead of launching a new command, if supported by the OS (has no effect on Windows, no_std).
@@ -23,22 +23,24 @@ python = ["pyo3", "concat-idents"]
 prelude = [] # Expose libafl::prelude for access without additional using directives
 tui_monitor = ["tui", "crossterm"] # enable TuiMonitor with crossterm
 prometheus_monitor = ["std", "async-std", "prometheus-client", "tide", "futures"]
-cli = ["clap"]  # expose bolts::cli
+cli = ["clap"]  # expose bolts::cli for easy commandline parsing
-qemu_cli = ["cli"]
+qemu_cli = ["cli"] # Commandline flags for qemu-based fuzzers
-frida_cli = ["cli"]
+frida_cli = ["cli"] # Commandline flags for frida-based fuzzers
 afl_exec_sec = [] # calculate exec/sec like AFL
-errors_backtrace = ["backtrace"]
+errors_backtrace = ["backtrace"] # Create backtraces at Error creation
-cmin = ["z3"] # corpus minimisation
+cmin = ["z3"] # for corpus minimisation
-corpus_btreemap = []
+corpus_btreemap = [] # Switches from HashMap to BTreeMap for CorpusId
+gzip = ["miniz_oxide"] # Enables gzip compression in certain parts of the lib
 
 # features hiding dependencies licensed under GPL
 gpl = []
 # features hiding dependencies licensed under AGPL
 agpl = ["gpl", "nautilus"]
 nautilus = ["grammartec", "std", "serde_json/std"]
+
 # LLMP features
 llmp_bind_public = [] # If set, llmp will bind to 0.0.0.0, allowing cross-device communication. Binds to localhost by default.
-llmp_compression = ["miniz_oxide"] # llmp compression using GZip
+llmp_compression = ["gzip"] # llmp compression using GZip
 llmp_debug = [] # Enables debug output for LLMP
 llmp_small_maps = [] # reduces initial map size for llmp
 

libafl/src/corpus/cached.rs

@@ -2,7 +2,7 @@
 
 use alloc::collections::vec_deque::VecDeque;
 use core::cell::RefCell;
-use std::path::PathBuf;
+use std::path::Path;
 
 use serde::{Deserialize, Serialize};
 
@@ -15,7 +15,7 @@ use crate::{
     Error,
 };
 
-/// A corpus that keep in memory a maximun number of testcases. The eviction policy is FIFO.
+/// A corpus that keeps a maximum number of [`Testcase`]s in memory. The eviction policy is FIFO.
 #[cfg(feature = "std")]
 #[derive(Default, Serialize, Deserialize, Clone, Debug)]
 #[serde(bound = "I: serde::de::DeserializeOwned")]
@@ -133,32 +133,58 @@ where
     I: Input,
 {
     /// Creates the [`CachedOnDiskCorpus`].
-    pub fn new(dir_path: PathBuf, cache_max_len: usize) -> Result<Self, Error> {
+    ///
-        if cache_max_len == 0 {
+    /// This corpus stores (and reads) all testcases to/from disk
-            return Err(Error::illegal_argument(
+    ///
-                "The max cache len in CachedOnDiskCorpus cannot be 0",
+    /// By default, it stores metadata for each [`Testcase`] as prettified json.
-            ));
+    /// Metadata will be written to a file named `.<testcase>.metadata`
-        }
+    /// the metadata may include objective reason, specific information for a fuzz job, and more.
-        Ok(Self {
+    ///
-            inner: OnDiskCorpus::new(dir_path)?,
+    /// If you don't want metadata, use [`CachedOnDiskCorpus::no_meta`].
-            cached_indexes: RefCell::new(VecDeque::new()),
+    /// to pick a different metadata format, use [`CachedOnDiskCorpus::with_meta_format`].
-            cache_max_len,
+    ///
-        })
+    /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`.
+    pub fn new<P>(dir_path: P, cache_max_len: usize) -> Result<Self, Error>
+    where
+        P: AsRef<Path>,
+    {
+        Self::_new(OnDiskCorpus::new(dir_path)?, cache_max_len)
     }
 
-    /// Creates the [`CachedOnDiskCorpus`] specifying the type of `Metadata` to be saved to disk.
+    /// Creates an [`CachedOnDiskCorpus`] that does not store [`Testcase`] metadata to disk.
-    pub fn new_save_meta(
+    pub fn no_meta<P>(dir_path: P, cache_max_len: usize) -> Result<Self, Error>
-        dir_path: PathBuf,
+    where
-        meta_format: Option<OnDiskMetadataFormat>,
+        P: AsRef<Path>,
+    {
+        Self::_new(OnDiskCorpus::no_meta(dir_path)?, cache_max_len)
+    }
+
+    /// Creates the [`CachedOnDiskCorpus`] specifying the format in which `Metadata` will be saved to disk.
+    ///
+    /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`.
+    pub fn with_meta_format<P>(
+        dir_path: P,
         cache_max_len: usize,
-    ) -> Result<Self, Error> {
+        meta_format: OnDiskMetadataFormat,
+    ) -> Result<Self, Error>
+    where
+        P: AsRef<Path>,
+    {
+        Self::_new(
+            OnDiskCorpus::with_meta_format(dir_path, meta_format)?,
+            cache_max_len,
+        )
+    }
+
+    /// Internal constructor `fn`
+    fn _new(on_disk_corpus: OnDiskCorpus<I>, cache_max_len: usize) -> Result<Self, Error> {
         if cache_max_len == 0 {
             return Err(Error::illegal_argument(
                 "The max cache len in CachedOnDiskCorpus cannot be 0",
             ));
         }
         Ok(Self {
-            inner: OnDiskCorpus::new_save_meta(dir_path, meta_format)?,
+            inner: on_disk_corpus,
             cached_indexes: RefCell::new(VecDeque::new()),
             cache_max_len,
         })

libafl/src/corpus/ondisk.rs

@@ -1,4 +1,7 @@
-//! The ondisk corpus stores unused testcases to disk.
+//! The ondisk corpus stores [`Testcase`]s to disk.
+//! Additionally, all of them are kept in memory.
+//! For a lower memory footprint, consider using [`crate::corpus::CachedOnDiskCorpus`]
+//! which only stores a certain number of testcases and removes additional ones in a FIFO manner.
 
 use core::{cell::RefCell, time::Duration};
 #[cfg(feature = "std")]
@@ -10,6 +13,8 @@ use std::{
 
 use serde::{Deserialize, Serialize};
 
+#[cfg(feature = "gzip")]
+use crate::bolts::compress::GzipCompressor;
 use crate::{
     bolts::serdeany::SerdeAnyMap,
     corpus::{Corpus, CorpusId, InMemoryCorpus, Testcase},
@@ -28,9 +33,12 @@ pub enum OnDiskMetadataFormat {
     Json,
     /// JSON formatted for readability
     JsonPretty,
+    #[cfg(feature = "gzip")]
+    /// The same as [`OnDiskMetadataFormat::JsonPretty`], but compressed
+    JsonGzip,
 }
 
-/// A corpus able to store testcases to disk, and load them from disk, when they are being used.
+/// The [`Testcase`] metadata that'll be stored to disk
 #[cfg(feature = "std")]
 #[derive(Debug, Serialize)]
 pub struct OnDiskMetadata<'a> {
@@ -39,7 +47,9 @@ pub struct OnDiskMetadata<'a> {
     executions: &'a usize,
 }
 
-/// A corpus able to store testcases to disk, and load them from disk, when they are being used.
+/// A corpus able to store [`Testcase`]s to disk, and load them from disk, when they are being used.
+///
+/// Metadata is written to a `.<filename>.metadata` file in the same folder by default.
 #[cfg(feature = "std")]
 #[derive(Default, Serialize, Deserialize, Clone, Debug)]
 #[serde(bound = "I: serde::de::DeserializeOwned")]
@@ -142,33 +152,54 @@ impl<I> OnDiskCorpus<I>
 where
     I: Input,
 {
-    /// Creates the [`OnDiskCorpus`].
+    /// Creates an [`OnDiskCorpus`].
+    ///
+    /// This corpus stores all testcases to disk, and keeps all of them in memory, as well.
+    ///
+    /// By default, it stores metadata for each [`Testcase`] as prettified json.
+    /// Metadata will be written to a file named `.<testcase>.metadata`
+    /// The metadata may include objective reason, specific information for a fuzz job, and more.
+    ///
+    /// If you don't want metadata, use [`OnDiskCorpus::no_meta`].
+    /// To pick a different metadata format, use [`OnDiskCorpus::with_meta_format`].
+    ///
     /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`.
     pub fn new<P>(dir_path: P) -> Result<Self, Error>
     where
         P: AsRef<Path>,
     {
-        fn new<I: Input>(dir_path: PathBuf) -> Result<OnDiskCorpus<I>, Error> {
+        Self::_new(dir_path.as_ref(), Some(OnDiskMetadataFormat::JsonPretty))
-            fs::create_dir_all(&dir_path)?;
-            Ok(OnDiskCorpus {
-                inner: InMemoryCorpus::new(),
-                dir_path,
-                meta_format: None,
-            })
-        }
-        new(dir_path.as_ref().to_path_buf())
     }
 
-    /// Creates the [`OnDiskCorpus`] specifying the type of `Metadata` to be saved to disk.
+    /// Creates the [`OnDiskCorpus`] specifying the format in which `Metadata` will be saved to disk.
+    ///
     /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`.
-    pub fn new_save_meta(
+    pub fn with_meta_format<P>(
-        dir_path: PathBuf,
+        dir_path: P,
-        meta_format: Option<OnDiskMetadataFormat>,
+        meta_format: OnDiskMetadataFormat,
-    ) -> Result<Self, Error> {
+    ) -> Result<Self, Error>
-        fs::create_dir_all(&dir_path)?;
+    where
-        Ok(Self {
+        P: AsRef<Path>,
+    {
+        Self::_new(dir_path.as_ref(), Some(meta_format))
+    }
+
+    /// Creates an [`OnDiskCorpus`] that will not store .metadata files
+    ///
+    /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`.
+    pub fn no_meta<P>(dir_path: P) -> Result<Self, Error>
+    where
+        P: AsRef<Path>,
+    {
+        Self::_new(dir_path.as_ref(), None)
+    }
+
+    /// Private fn to crate a new corpus at the given (non-generic) path with the given optional `meta_format`
+    fn _new(dir_path: &Path, meta_format: Option<OnDiskMetadataFormat>) -> Result<Self, Error> {
+        fs::create_dir_all(dir_path)?;
+        Ok(OnDiskCorpus {
             inner: InMemoryCorpus::new(),
-            dir_path,
+            dir_path: dir_path.into(),
             meta_format,
         })
     }
@@ -193,7 +224,7 @@ where
                     break self.dir_path.join(file);
                 }
 
-                file = format!("{}-{ctr}", &file_orig);
+                file = format!("{file_orig}-{ctr}");
                 ctr += 1;
             };
 
@@ -224,6 +255,10 @@ where
                 OnDiskMetadataFormat::Postcard => postcard::to_allocvec(&ondisk_meta)?,
                 OnDiskMetadataFormat::Json => serde_json::to_vec(&ondisk_meta)?,
                 OnDiskMetadataFormat::JsonPretty => serde_json::to_vec_pretty(&ondisk_meta)?,
+                #[cfg(feature = "gzip")]
+                OnDiskMetadataFormat::JsonGzip => GzipCompressor::new(0)
+                    .compress(&serde_json::to_vec_pretty(&ondisk_meta)?)?
+                    .unwrap(),
             };
             tmpfile.write_all(&serialized)?;
             fs::rename(&tmpfile_name, &filename)?;
@@ -249,6 +284,7 @@ where
         Ok(())
     }
 }
+
 #[cfg(feature = "python")]
 /// `OnDiskCorpus` Python bindings
 pub mod pybind {

libafl/src/feedbacks/mod.rs

@@ -949,7 +949,7 @@ impl TimeFeedback {
 
     /// Creates a new [`TimeFeedback`], deciding if the given [`TimeObserver`] value of a run is interesting.
     #[must_use]
-    pub fn new_with_observer(observer: &TimeObserver) -> Self {
+    pub fn with_observer(observer: &TimeObserver) -> Self {
         Self {
             exec_time: None,
             name: observer.name().to_string(),
@@ -1021,7 +1021,7 @@ where
 
     /// Creates a new [`TimeFeedback`], deciding if the given [`ListObserver`] value of a run is interesting.
     #[must_use]
-    pub fn new_with_observer(observer: &ListObserver<T>) -> Self {
+    pub fn with_observer(observer: &ListObserver<T>) -> Self {
         Self {
             name: observer.name().to_string(),
             last_addr: 0,

libafl_concolic/symcc_runtime/src/filter/coverage.rs

@@ -160,7 +160,7 @@ where
 {
     /// Creates a new `HitmapFilter` using the given map and the [`DefaultHasher`].
     pub fn new(hitcounts_map: M) -> Self {
-        Self::new_with_default_hasher_builder(hitcounts_map)
+        Self::with_default_hasher_builder(hitcounts_map)
     }
 }
 
@@ -170,8 +170,8 @@ where
     H: Hasher + Default,
 {
     /// Creates a new `HitmapFilter` using the given map and [`Hasher`] (as type argument) using the [`BuildHasherDefault`].
-    pub fn new_with_default_hasher_builder(hitcounts_map: M) -> Self {
+    pub fn with_default_hasher_builder(hitcounts_map: M) -> Self {
-        Self::new_with_build_hasher(hitcounts_map, BuildHasherDefault::default())
+        Self::with_build_hasher(hitcounts_map, BuildHasherDefault::default())
     }
 }
 
@@ -181,7 +181,7 @@ where
     BH: BuildHasher,
 {
     /// Creates a new `HitmapFilter` using the given map and [`BuildHasher`] (as type argument).
-    pub fn new_with_build_hasher(hitcounts_map: M, build_hasher: BH) -> Self {
+    pub fn with_build_hasher(hitcounts_map: M, build_hasher: BH) -> Self {
         Self {
             hitcounts_map,
             build_hasher,

libafl_nyx/src/helper.rs

@@ -40,7 +40,7 @@ impl NyxHelper {
         parallel_mode: bool,
         parent_cpu_id: Option<u32>,
     ) -> Result<Self, Error> {
-        NyxHelper::new_with_initial_timeout(
+        NyxHelper::with_initial_timeout(
             target_dir,
             cpu_id,
             snap_mode,
@@ -52,7 +52,7 @@ impl NyxHelper {
     /// create `NyxProcess` and do basic settings
     /// It will convert instance to parent or child using `parent_cpu_id` when set`parallel_mode`
     /// will fail if initial connection takes more than `initial_timeout` seconds
-    pub fn new_with_initial_timeout(
+    pub fn with_initial_timeout(
         target_dir: &Path,
         cpu_id: u32,
         snap_mode: bool,

libafl_sugar/src/forkserver.rs

@@ -138,7 +138,7 @@ impl<'a, const MAP_SIZE: usize> ForkserverBytesCoverageSugar<'a, MAP_SIZE> {
                 // New maximization map feedback linked to the edges observer and the feedback state
                 MaxMapFeedback::new_tracking(&edges_observer, true, false),
                 // Time feedback, this one does not need a feedback state
-                TimeFeedback::new_with_observer(&time_observer)
+                TimeFeedback::with_observer(&time_observer)
             );
 
             // A feedback to choose if an input is a solution or not

libafl_sugar/src/inmemory.rs

@@ -157,7 +157,7 @@ where
                 // New maximization map feedback linked to the edges observer and the feedback state
                 MaxMapFeedback::new_tracking(&edges_observer, true, false),
                 // Time feedback, this one does not need a feedback state
-                TimeFeedback::new_with_observer(&time_observer)
+                TimeFeedback::with_observer(&time_observer)
             );
 
             // A feedback to choose if an input is a solution or not

libafl_sugar/src/qemu.rs

@@ -168,7 +168,7 @@ where
                 // New maximization map feedback linked to the edges observer and the feedback state
                 MaxMapFeedback::new_tracking(&edges_observer, true, false),
                 // Time feedback, this one does not need a feedback state
-                TimeFeedback::new_with_observer(&time_observer)
+                TimeFeedback::with_observer(&time_observer)
             );
 
             // A feedback to choose if an input is a solution or not