diff --git a/fuzzers/FRET/Cargo.toml b/fuzzers/FRET/Cargo.toml index 1ead64566c..b3742608e9 100644 --- a/fuzzers/FRET/Cargo.toml +++ b/fuzzers/FRET/Cargo.toml @@ -69,3 +69,4 @@ rand = "0.5" clap = { version = "4.4.11", features = ["derive"] } csv = "1.3.0" log = "0.4" +simple_moving_average = "1.0.2" diff --git a/fuzzers/FRET/benchmark/plot_all_benchmarks.sh b/fuzzers/FRET/benchmark/plot_all_benchmarks.sh index ae8354d8b6..9a0d671dd1 100644 --- a/fuzzers/FRET/benchmark/plot_all_benchmarks.sh +++ b/fuzzers/FRET/benchmark/plot_all_benchmarks.sh @@ -1,12 +1,13 @@ -[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_all.png ] && Rscript plot_multi.r remote waters_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & -[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_int_all.png ] && Rscript plot_multi.r remote waters_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & -[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_all.png ] && Rscript plot_multi.r remote watersv2_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & -[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_int_all.png ] && Rscript plot_multi.r remote watersv2_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & -[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspart_all.png ] && Rscript plot_multi.r remote waters_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & -[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspart_int_all.png ] && Rscript plot_multi.r remote waters_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & -[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspartv2_all.png ] && Rscript plot_multi.r remote watersv2_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & -[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspartv2_int_all.png ] && Rscript plot_multi.r remote watersv2_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_seq_all.png ] && Rscript plot_multi.r remote waters_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_seq_int_all.png ] && Rscript plot_multi.r remote waters_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_seq_all.png ] && Rscript plot_multi.r remote watersv2_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_seq_int_all.png ] && Rscript plot_multi.r remote watersv2_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_par_all.png ] && Rscript plot_multi.r remote waters_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_par_int_all.png ] && Rscript plot_multi.r remote waters_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_par_all.png ] && Rscript plot_multi.r remote watersv2_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_par_int_all.png ] && Rscript plot_multi.r remote watersv2_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & [ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/interact_all.png ] && Rscript plot_multi.r remote interact ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & [ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/interact_int_all.png ] && Rscript plot_multi.r remote interact_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & [ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/release_all.png ] && Rscript plot_multi.r remote release ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/copter_all.png ] && Rscript plot_multi.r remote copter ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & wait \ No newline at end of file diff --git a/fuzzers/FRET/benchmark/target_symbols.csv b/fuzzers/FRET/benchmark/target_symbols.csv index 61eb4b7fdd..97299ca60b 100644 --- a/fuzzers/FRET/benchmark/target_symbols.csv +++ b/fuzzers/FRET/benchmark/target_symbols.csv @@ -31,5 +31,5 @@ gen3,main_minimal,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#1000 interact,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break,NONE, interact_int,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#1000 release,main_release,FUZZ_INPUT,4096,trigger_Qemu_break,T3,0#10000;1#5000;2#2000;3#3000 -copter,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#5000 +copter,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#5000 diff --git a/fuzzers/FRET/src/fuzzer.rs b/fuzzers/FRET/src/fuzzer.rs index f1a283a2a2..6d188e2e18 100644 --- a/fuzzers/FRET/src/fuzzer.rs +++ b/fuzzers/FRET/src/fuzzer.rs @@ -448,8 +448,8 @@ let run_client = |state: Option<_>, mut mgr, _core_id| { MaxMapFeedback::new(&stg_coverage_observer) ); - // A feedback to choose if an input is a solution or not - let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new(), SystraceErrorFeedback::new(cli.dump_cases || matches!(cli.command, Commands::Fuzz{..}))); + // A feedback to choose if an input is producing an error + let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new(), SystraceErrorFeedback::new(cli.dump_cases || matches!(cli.command, Commands::Fuzz{..}), Some(10))); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { diff --git a/fuzzers/FRET/src/systemstate/feedbacks.rs b/fuzzers/FRET/src/systemstate/feedbacks.rs index ef282ccbf8..35c9748d03 100644 --- a/fuzzers/FRET/src/systemstate/feedbacks.rs +++ b/fuzzers/FRET/src/systemstate/feedbacks.rs @@ -250,7 +250,8 @@ impl DumpSystraceFeedback pub struct SystraceErrorFeedback { name: Cow<'static, str>, - dump_case: bool + dump_case: bool, + max_reports: Option, } impl Feedback for SystraceErrorFeedback @@ -271,7 +272,14 @@ where { let observer = observers.match_name::>("systemstate") .expect("QemuSystemStateObserver not found"); - Ok(self.dump_case&&!observer.success) + let is_err = (!observer.success || observer.do_report); + if let Some(m) = self.max_reports { + if m <= 0 {return Ok(false);} + if is_err { + self.max_reports = Some(m-1); + } + } + Ok(self.dump_case&&is_err) } /// Append to the testcase the generated metadata in case of a new corpus item #[inline] @@ -297,7 +305,7 @@ impl Named for SystraceErrorFeedback impl SystraceErrorFeedback { #[must_use] - pub fn new(dump_case: bool) -> Self { - Self {name: Cow::from(String::from("SystraceErrorFeedback")), dump_case} + pub fn new(dump_case: bool, max_reports: Option) -> Self { + Self {name: Cow::from(String::from("SystraceErrorFeedback")), dump_case, max_reports} } } \ No newline at end of file diff --git a/fuzzers/FRET/src/systemstate/observers.rs b/fuzzers/FRET/src/systemstate/observers.rs index 30ff8b2e4c..72bb722caa 100644 --- a/fuzzers/FRET/src/systemstate/observers.rs +++ b/fuzzers/FRET/src/systemstate/observers.rs @@ -41,6 +41,7 @@ pub struct QemuSystemStateObserver pub last_reads: Vec>, pub last_input: I, pub job_instances: Vec<(u64, u64, String)>, + pub do_report: bool, pub worst_job_instances: HashMap, pub select_task: Option, pub success: bool, @@ -77,7 +78,7 @@ where let releases = get_releases(&self.last_trace, &self.last_states); // println!("Releases: {:?}",&releases); let jobs_done = JOBS_DONE.split_off(0); - self.job_instances = get_release_response_pairs(&releases, &jobs_done); + (self.job_instances, self.do_report) = get_release_response_pairs(&releases, &jobs_done); // println!("Instances: {:?}",&self.job_instances); let observer = &self; let mut worst_case_per_task = HashMap::new(); @@ -139,7 +140,7 @@ impl HasLen for QemuSystemStateObserver impl QemuSystemStateObserver where I: Default { pub fn new(select_task: &Option) -> Self { - Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), worst_job_instances: HashMap::new(), select_task: select_task.clone(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false, job_instances: vec![]} + Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), worst_job_instances: HashMap::new(), do_report: false, select_task: select_task.clone(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false, job_instances: vec![]} } pub fn last_runtime(&self) -> u64 { self.select_task.as_ref().map(|x| self.worst_job_instances.get(x).unwrap_or(&0).clone()).unwrap_or(unsafe{libafl_qemu::sys::icount_get_raw()}) @@ -341,7 +342,8 @@ fn get_releases(trace: &Vec, states: &HashMap, resp: &Vec<(u64, String)>) -> Vec<(u64, u64, String)> { +fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)>) -> (Vec<(u64, u64, String)>, bool) { + let mut maybe_error = false; let mut ret = Vec::new(); let mut ready : HashMap<&String, u64> = HashMap::new(); let mut last_response : HashMap<&String, u64> = HashMap::new(); @@ -357,6 +359,7 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String) if let Some(peek_resp) = d.peek() { if peek_resp.0 > peek_rel.0 { // multiple releases before response // It is unclear which release is real + // maybe_error = true; // eprintln!("Task {} released multiple times before response ({:.1}ms and {:.1}ms)", peek_rel.1, crate::time::clock::tick_to_time(ready[&peek_rel.1]).as_micros()/1000, crate::time::clock::tick_to_time(peek_rel.0).as_micros()/1000); // ready.insert(&peek_rel.1, peek_rel.0); r.next(); @@ -374,7 +377,8 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String) if ready.contains_key(&next_resp.1) { if ready[&next_resp.1] >= next_resp.0 { if let Some(lr) = last_response.get(&next_resp.1) { - eprintln!("Task {} response at {:.1}ms before next release at {:.1}ms. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(ready[&next_resp.1]).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0); + maybe_error = true; + // eprintln!("Task {} response at {:.1}ms before next release at {:.1}ms. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(ready[&next_resp.1]).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0); // Sometimes a task is released immediately after a response. This might not be detected. // Assume that the release occured with the last response ret.push((*lr, next_resp.0, next_resp.1.clone())); @@ -389,8 +393,9 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String) ready.remove(&next_resp.1); } } else { + maybe_error = true; if let Some(lr) = last_response.get(&next_resp.1) { - eprintln!("Task {} response at {:.1}ms not found in ready list. Fallback to last release at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0); + // eprintln!("Task {} response at {:.1}ms not found in ready list. Fallback to last release at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0); // Sometimes a task is released immediately after a response. This might not be detected. // Assume that the release occured with the last response ret.push((*lr, next_resp.0, next_resp.1.clone())); @@ -401,7 +406,7 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String) } } else { // TODO: should remaining released tasks be counted as finished? - return ret; + return (ret,maybe_error); } } }