collect first n error inputs as objectives, fix configs

fuzzers/FRET/Cargo.toml

@@ -69,3 +69,4 @@ rand = "0.5"
 clap = { version = "4.4.11", features = ["derive"] }
 csv = "1.3.0"
 log = "0.4"
+simple_moving_average = "1.0.2"

fuzzers/FRET/benchmark/plot_all_benchmarks.sh

@@ -1,12 +1,13 @@
-[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_all.png ] && Rscript plot_multi.r remote waters_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_seq_all.png ] && Rscript plot_multi.r remote waters_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
-[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_int_all.png ] && Rscript plot_multi.r remote waters_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_seq_int_all.png ] && Rscript plot_multi.r remote waters_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
-[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_all.png ] && Rscript plot_multi.r remote watersv2_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_seq_all.png ] && Rscript plot_multi.r remote watersv2_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
-[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_int_all.png ] && Rscript plot_multi.r remote watersv2_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_seq_int_all.png ] && Rscript plot_multi.r remote watersv2_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
-[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspart_all.png ] && Rscript plot_multi.r remote waters_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_par_all.png ] && Rscript plot_multi.r remote waters_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
-[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspart_int_all.png ] && Rscript plot_multi.r remote waters_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_par_int_all.png ] && Rscript plot_multi.r remote waters_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
-[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspartv2_all.png ] && Rscript plot_multi.r remote watersv2_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_par_all.png ] && Rscript plot_multi.r remote watersv2_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
-[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspartv2_int_all.png ] && Rscript plot_multi.r remote watersv2_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_par_int_all.png ] && Rscript plot_multi.r remote watersv2_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
 [ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/interact_all.png ] && Rscript plot_multi.r remote interact ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
 [ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/interact_int_all.png ] && Rscript plot_multi.r remote interact_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
 [ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/release_all.png ] && Rscript plot_multi.r remote release ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
+[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/copter_all.png ] && Rscript plot_multi.r remote copter ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
 wait

fuzzers/FRET/benchmark/target_symbols.csv

@@ -31,5 +31,5 @@ gen3,main_minimal,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#1000
 interact,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,
 interact_int,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#1000
 release,main_release,FUZZ_INPUT,4096,trigger_Qemu_break,T3,0#10000;1#5000;2#2000;3#3000
-copter,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#5000
+copter,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#5000
 

fuzzers/FRET/src/fuzzer.rs

@@ -448,8 +448,8 @@ let run_client = |state: Option<_>, mut mgr, _core_id| {
             MaxMapFeedback::new(&stg_coverage_observer)
         );
 
-        // A feedback to choose if an input is a solution or not
+        // A feedback to choose if an input is producing an error
-        let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new(), SystraceErrorFeedback::new(cli.dump_cases || matches!(cli.command, Commands::Fuzz{..})));
+        let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new(), SystraceErrorFeedback::new(cli.dump_cases || matches!(cli.command, Commands::Fuzz{..}), Some(10)));
 
         // If not restarting, create a State from scratch
         let mut state = state.unwrap_or_else(|| {

fuzzers/FRET/src/systemstate/feedbacks.rs

@@ -250,7 +250,8 @@ impl DumpSystraceFeedback
 pub struct SystraceErrorFeedback
 {
     name: Cow<'static, str>,
-    dump_case: bool
+    dump_case: bool,
+    max_reports: Option<usize>,
 }
 
 impl<S> Feedback<S> for SystraceErrorFeedback
@@ -271,7 +272,14 @@ where
     {
         let observer = observers.match_name::<QemuSystemStateObserver<S::Input>>("systemstate")
             .expect("QemuSystemStateObserver not found");
-        Ok(self.dump_case&&!observer.success)
+        let is_err = (!observer.success || observer.do_report);
+        if let Some(m) = self.max_reports {
+            if m <= 0 {return Ok(false);}
+            if is_err {
+                self.max_reports = Some(m-1);
+            }
+        }
+        Ok(self.dump_case&&is_err)
     }
     /// Append to the testcase the generated metadata in case of a new corpus item
     #[inline]
@@ -297,7 +305,7 @@ impl Named for SystraceErrorFeedback
 impl SystraceErrorFeedback
 {
     #[must_use]
-    pub fn new(dump_case: bool) -> Self {
+    pub fn new(dump_case: bool, max_reports: Option<usize>) -> Self {
-        Self {name: Cow::from(String::from("SystraceErrorFeedback")), dump_case}
+        Self {name: Cow::from(String::from("SystraceErrorFeedback")), dump_case, max_reports}
     }
 }

fuzzers/FRET/src/systemstate/observers.rs

@@ -41,6 +41,7 @@ pub struct QemuSystemStateObserver<I>
     pub last_reads: Vec<HashSet<u32>>,
     pub last_input: I,
     pub job_instances: Vec<(u64, u64, String)>,
+    pub do_report: bool,
     pub worst_job_instances: HashMap<String, u64>,
     pub select_task: Option<String>,
     pub success: bool,
@@ -77,7 +78,7 @@ where
                 let releases = get_releases(&self.last_trace, &self.last_states);
                 // println!("Releases: {:?}",&releases);
                 let jobs_done =  JOBS_DONE.split_off(0);
-                self.job_instances = get_release_response_pairs(&releases, &jobs_done);
+                (self.job_instances, self.do_report) = get_release_response_pairs(&releases, &jobs_done);
                 // println!("Instances: {:?}",&self.job_instances);
                 let observer = &self;
                 let mut worst_case_per_task = HashMap::new();
@@ -139,7 +140,7 @@ impl<I> HasLen for QemuSystemStateObserver<I>
 impl<I> QemuSystemStateObserver<I>
 where I: Default {
     pub fn new(select_task: &Option<String>) -> Self {
-        Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), worst_job_instances: HashMap::new(), select_task: select_task.clone(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false, job_instances: vec![]}
+        Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), worst_job_instances: HashMap::new(), do_report: false, select_task: select_task.clone(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false, job_instances: vec![]}
     }
     pub fn last_runtime(&self) -> u64 {
         self.select_task.as_ref().map(|x| self.worst_job_instances.get(x).unwrap_or(&0).clone()).unwrap_or(unsafe{libafl_qemu::sys::icount_get_raw()})
@@ -341,7 +342,8 @@ fn get_releases(trace: &Vec<ExecInterval>, states: &HashMap<u64, ReducedFreeRTOS
     ret
 }
 
-fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)>) -> Vec<(u64, u64, String)> {
+fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)>) -> (Vec<(u64, u64, String)>, bool) {
+    let mut maybe_error = false;
     let mut ret = Vec::new();
     let mut ready : HashMap<&String, u64> = HashMap::new();
     let mut last_response : HashMap<&String, u64> = HashMap::new();
@@ -357,6 +359,7 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)
                 if let Some(peek_resp) = d.peek() {
                     if peek_resp.0 > peek_rel.0 {  // multiple releases before response
                         // It is unclear which release is real
+                        // maybe_error = true;
                         // eprintln!("Task {} released multiple times before response ({:.1}ms and {:.1}ms)", peek_rel.1, crate::time::clock::tick_to_time(ready[&peek_rel.1]).as_micros()/1000, crate::time::clock::tick_to_time(peek_rel.0).as_micros()/1000);
                         // ready.insert(&peek_rel.1, peek_rel.0);
                         r.next();
@@ -374,7 +377,8 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)
             if ready.contains_key(&next_resp.1) {
                 if ready[&next_resp.1] >= next_resp.0 {
                     if let Some(lr) = last_response.get(&next_resp.1) {
-                        eprintln!("Task {} response at {:.1}ms before next release at {:.1}ms. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(ready[&next_resp.1]).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0);
+                        maybe_error = true;
+                        // eprintln!("Task {} response at {:.1}ms before next release at {:.1}ms. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(ready[&next_resp.1]).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0);
                         // Sometimes a task is released immediately after a response. This might not be detected.
                         // Assume that the release occured with the last response
                         ret.push((*lr, next_resp.0, next_resp.1.clone()));
@@ -389,8 +393,9 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)
                     ready.remove(&next_resp.1);
                 }
             } else {
+                maybe_error = true;
                 if let Some(lr) = last_response.get(&next_resp.1) {
-                    eprintln!("Task {} response at {:.1}ms not found in ready list. Fallback to last release at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0);
+                    // eprintln!("Task {} response at {:.1}ms not found in ready list. Fallback to last release at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0);
                     // Sometimes a task is released immediately after a response. This might not be detected.
                     // Assume that the release occured with the last response
                     ret.push((*lr, next_resp.0, next_resp.1.clone()));
@@ -401,7 +406,7 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)
             }
         } else {
             // TODO: should remaining released tasks be counted as finished?
-            return ret;
+            return (ret,maybe_error);
         }
     }
 }