libafl-fuzz: fix not loading seeds recursively from directories (#2532)

* libafl-fuzz: fix not loading seeds recursively from directories

* add walk_initial_inputs to State

* libafl-fuzz: add afl++ style seed filename when copying initial files

* typo

fuzzers/others/libafl-fuzz/src/corpus.rs

@@ -82,11 +82,7 @@ fn parse_time_line(line: &str) -> Result<u64, Error> {
         .map_err(|_| Error::illegal_state("invalid stats file"))
 }
 
-pub fn check_autoresume(
+pub fn check_autoresume(fuzzer_dir: &Path, auto_resume: bool) -> Result<Flock<File>, Error> {
-    fuzzer_dir: &Path,
-    intial_inputs: &PathBuf,
-    auto_resume: bool,
-) -> Result<Flock<File>, Error> {
     if !fuzzer_dir.exists() {
         std::fs::create_dir(fuzzer_dir)?;
     }
@@ -129,11 +125,7 @@ pub fn check_autoresume(
             return Err(Error::illegal_state("The job output directory already exists and contains results! use AFL_AUTORESUME=true or provide \"-\" for -i "));
         }
     }
-    if auto_resume {
+    if !auto_resume {
-        // TODO: once the queue stuff is implemented finish the rest of the function
-        // see afl-fuzz-init.c line 1898 onwards. Gotta copy and delete shit
-        // No usable test cases in './output/default/_resume'
-    } else {
         let queue_dir = fuzzer_dir.join("queue");
         let hangs_dir = fuzzer_dir.join("hangs");
         let crashes_dir = fuzzer_dir.join("crashes");
@@ -141,24 +133,6 @@ pub fn check_autoresume(
         create_dir_if_not_exists(&crashes_dir).expect("should be able to create crashes dir");
         create_dir_if_not_exists(&hangs_dir).expect("should be able to create hangs dir");
         create_dir_if_not_exists(&queue_dir).expect("should be able to create queue dir");
-        // Copy all our seeds to queue
-        for file in std::fs::read_dir(intial_inputs)? {
-            let path = file?.path();
-            let cpy_res = std::fs::copy(
-                &path,
-                queue_dir.join(path.file_name().ok_or(Error::illegal_state(format!(
-                    "file {} in input directory does not have a filename",
-                    path.display()
-                )))?),
-            );
-            match cpy_res {
-                Err(e) if e.kind() == io::ErrorKind::InvalidInput => {
-                    println!("skipping {} since it is not a regular file", path.display());
-                }
-                Err(e) => return Err(e.into()),
-                Ok(_) => {}
-            }
-        }
     }
     Ok(file)
 }

fuzzers/others/libafl-fuzz/src/fuzzer.rs

@@ -250,6 +250,39 @@ where
         .build(tuple_list!(time_observer, edges_observer))
         .unwrap();
 
+    let queue_dir = fuzzer_dir.join("queue");
+    if opt.auto_resume {
+        // TODO - see afl-fuzz-init.c line 1898 onwards
+    } else {
+        // If we aren't auto resuming, copy all the files to our queue directory.
+        let mut id = 0;
+        state.walk_initial_inputs(&[opt.input_dir.clone()], |path: &PathBuf| {
+            let mut filename = path
+                .file_name()
+                .ok_or(Error::illegal_state(format!(
+                    "file {} in input directory does not have a filename",
+                    path.display()
+                )))?
+                .to_str()
+                .ok_or(Error::illegal_state(format!(
+                    "file {} in input directory does not have a legal filename",
+                    path.display()
+                )))?
+                .to_string();
+            filename = format!("id:{id:0>6},time:0,execs:0,orig:{filename}");
+            let cpy_res = std::fs::copy(&path, queue_dir.join(filename));
+            match cpy_res {
+                Err(e) if e.kind() == std::io::ErrorKind::InvalidInput => {
+                    println!("skipping {} since it is not a regular file", path.display());
+                }
+                Err(e) => return Err(e.into()),
+                Ok(_) => {
+                    id += 1;
+                }
+            }
+            Ok(())
+        })?;
+    }
     // Load our seeds.
     if state.must_load_initial_inputs() {
         state
@@ -257,7 +290,7 @@ where
                 &mut fuzzer,
                 &mut executor,
                 &mut restarting_mgr,
-                &[fuzzer_dir.join("queue")],
+                &[queue_dir],
                 &core_id,
                 opt.cores.as_ref().expect("invariant; should never occur"),
             )

fuzzers/others/libafl-fuzz/src/main.rs

@@ -77,7 +77,7 @@ fn main() {
         .main_run_client(|state: Option<_>, mgr: _, core_id: CoreId| {
             println!("run primary client on core {}", core_id.0);
             let fuzzer_dir = opt.output_dir.join("fuzzer_main");
-            check_autoresume(&fuzzer_dir, &opt.input_dir, opt.auto_resume).unwrap();
+            let _ = check_autoresume(&fuzzer_dir, opt.auto_resume).unwrap();
             let res = run_client(state, mgr, &fuzzer_dir, core_id, &opt, true);
             let _ = remove_main_node_file(&fuzzer_dir);
             res
@@ -87,7 +87,7 @@ fn main() {
             let fuzzer_dir = opt
                 .output_dir
                 .join(format!("fuzzer_secondary_{}", core_id.0));
-            check_autoresume(&fuzzer_dir, &opt.input_dir, opt.auto_resume).unwrap();
+            let _ = check_autoresume(&fuzzer_dir, opt.auto_resume).unwrap();
             run_client(state, mgr, &fuzzer_dir, core_id, &opt, false)
         })
         .cores(&opt.cores.clone().expect("invariant; should never occur"))

libafl/src/state/mod.rs

@@ -260,7 +260,7 @@ pub struct StdState<I, C, R, SC> {
     /// Remaining initial inputs to load, if any
     remaining_initial_files: Option<Vec<PathBuf>>,
     #[cfg(feature = "std")]
-    /// Remaining initial inputs to load, if any
+    /// symlinks we have already traversed when loading `remaining_initial_files`
     dont_reenter: Option<Vec<PathBuf>>,
     #[cfg(feature = "std")]
     /// If inputs have been processed for multicore loading
@@ -798,6 +798,28 @@ where
         Ok(())
     }
 
+    /// Recursively walk supplied corpus directories
+    pub fn walk_initial_inputs<F>(
+        &mut self,
+        in_dirs: &[PathBuf],
+        mut closure: F,
+    ) -> Result<(), Error>
+    where
+        F: FnMut(&PathBuf) -> Result<(), Error>,
+    {
+        self.canonicalize_input_dirs(in_dirs)?;
+        loop {
+            match self.next_file() {
+                Ok(path) => {
+                    closure(&path)?;
+                }
+                Err(Error::IteratorEnd(_, _)) => break,
+                Err(e) => return Err(e),
+            }
+        }
+        self.reset_initial_files_state();
+        Ok(())
+    }
     /// Loads all intial inputs, even if they are not considered `interesting`.
     /// This is rarely the right method, use `load_initial_inputs`,
     /// and potentially fix your `Feedback`, instead.