SYS-OSS/FRET-LibAFL
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

collect first n error inputs as objectives, fix configs

Browse Source
This commit is contained in:
Alwin Berger 2024-10-18 13:08:13 +02:00
parent bf827c077f
commit 735fc3e144
6 changed files with 36 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fuzzers/FRET/Cargo.toml
View File

@ -69,3 +69,4 @@ rand = "0.5"
clap = { version = "4.4.11", features = ["derive"] }
csv = "1.3.0"
log = "0.4"
simple_moving_average = "1.0.2"

17
fuzzers/FRET/benchmark/plot_all_benchmarks.sh
View File

@ -1,12 +1,13 @@
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_all.png ] && Rscript plot_multi.r remote waters_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_int_all.png ] && Rscript plot_multi.r remote waters_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_all.png ] && Rscript plot_multi.r remote watersv2_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_int_all.png ] && Rscript plot_multi.r remote watersv2_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspart_all.png ] && Rscript plot_multi.r remote waters_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspart_int_all.png ] && Rscript plot_multi.r remote waters_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspartv2_all.png ] && Rscript plot_multi.r remote watersv2_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waterspartv2_int_all.png ] && Rscript plot_multi.r remote watersv2_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_seq_all.png ] && Rscript plot_multi.r remote waters_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_seq_int_all.png ] && Rscript plot_multi.r remote waters_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_seq_all.png ] && Rscript plot_multi.r remote watersv2_seq ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_seq_int_all.png ] && Rscript plot_multi.r remote watersv2_seq_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_par_all.png ] && Rscript plot_multi.r remote waters_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/waters_par_int_all.png ] && Rscript plot_multi.r remote waters_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_par_all.png ] && Rscript plot_multi.r remote watersv2_par ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/watersv2_par_int_all.png ] && Rscript plot_multi.r remote watersv2_par_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/interact_all.png ] && Rscript plot_multi.r remote interact ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/interact_int_all.png ] && Rscript plot_multi.r remote interact_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/release_all.png ] && Rscript plot_multi.r remote release ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
[ ! -f ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote/copter_all.png ] && Rscript plot_multi.r remote copter ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote &
wait

2
fuzzers/FRET/benchmark/target_symbols.csv
View File

@ -31,5 +31,5 @@ gen3,main_minimal,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#1000
interact,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,
interact_int,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#1000
release,main_release,FUZZ_INPUT,4096,trigger_Qemu_break,T3,0#10000;1#5000;2#2000;3#3000
copter,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,NONE,0#5000
copter,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#5000

1 kernel main_function input_symbol input_size return_function select_task interrupts
31 interact main_interact FUZZ_INPUT 4096 trigger_Qemu_break NONE
32 interact_int main_interact FUZZ_INPUT 4096 trigger_Qemu_break NONE 0#1000
33 release main_release FUZZ_INPUT 4096 trigger_Qemu_break T3 0#10000;1#5000;2#2000;3#3000
34 copter main_osek FUZZ_INPUT 4096 trigger_Qemu_break NONE FC 0#5000
35

4
fuzzers/FRET/src/fuzzer.rs
View File

@ -448,8 +448,8 @@ let run_client = |state: Option<_>, mut mgr, _core_id| {
MaxMapFeedback::new(&stg_coverage_observer)
);
// A feedback to choose if an input is a solution or not
let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new(), SystraceErrorFeedback::new(cli.dump_cases || matches!(cli.command, Commands::Fuzz{..})));
// A feedback to choose if an input is producing an error
let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new(), SystraceErrorFeedback::new(cli.dump_cases || matches!(cli.command, Commands::Fuzz{..}), Some(10)));
// If not restarting, create a State from scratch
let mut state = state.unwrap_or_else(|| {

16
fuzzers/FRET/src/systemstate/feedbacks.rs
View File

@ -250,7 +250,8 @@ impl DumpSystraceFeedback
pub struct SystraceErrorFeedback
{
name: Cow<'static, str>,
dump_case: bool
dump_case: bool,
max_reports: Option<usize>,
}
impl<S> Feedback<S> for SystraceErrorFeedback
@ -271,7 +272,14 @@ where
{
let observer = observers.match_name::<QemuSystemStateObserver<S::Input>>("systemstate")
.expect("QemuSystemStateObserver not found");
Ok(self.dump_case&&!observer.success)
let is_err = (!observer.success || observer.do_report);
if let Some(m) = self.max_reports {
if m <= 0 {return Ok(false);}
if is_err {
self.max_reports = Some(m-1);
}
}
Ok(self.dump_case&&is_err)
}
/// Append to the testcase the generated metadata in case of a new corpus item
#[inline]
@ -297,7 +305,7 @@ impl Named for SystraceErrorFeedback
impl SystraceErrorFeedback
{
#[must_use]
pub fn new(dump_case: bool) -> Self {
Self {name: Cow::from(String::from("SystraceErrorFeedback")), dump_case}
pub fn new(dump_case: bool, max_reports: Option<usize>) -> Self {
Self {name: Cow::from(String::from("SystraceErrorFeedback")), dump_case, max_reports}
}
}

17
fuzzers/FRET/src/systemstate/observers.rs
View File

@ -41,6 +41,7 @@ pub struct QemuSystemStateObserver<I>
pub last_reads: Vec<HashSet<u32>>,
pub last_input: I,
pub job_instances: Vec<(u64, u64, String)>,
pub do_report: bool,
pub worst_job_instances: HashMap<String, u64>,
pub select_task: Option<String>,
pub success: bool,
@ -77,7 +78,7 @@ where
let releases = get_releases(&self.last_trace, &self.last_states);
// println!("Releases: {:?}",&releases);
let jobs_done = JOBS_DONE.split_off(0);
self.job_instances = get_release_response_pairs(&releases, &jobs_done);
(self.job_instances, self.do_report) = get_release_response_pairs(&releases, &jobs_done);
// println!("Instances: {:?}",&self.job_instances);
let observer = &self;
let mut worst_case_per_task = HashMap::new();
@ -139,7 +140,7 @@ impl<I> HasLen for QemuSystemStateObserver<I>
impl<I> QemuSystemStateObserver<I>
where I: Default {
pub fn new(select_task: &Option<String>) -> Self {
Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), worst_job_instances: HashMap::new(), select_task: select_task.clone(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false, job_instances: vec![]}
Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), worst_job_instances: HashMap::new(), do_report: false, select_task: select_task.clone(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false, job_instances: vec![]}
}
pub fn last_runtime(&self) -> u64 {
self.select_task.as_ref().map(|x| self.worst_job_instances.get(x).unwrap_or(&0).clone()).unwrap_or(unsafe{libafl_qemu::sys::icount_get_raw()})
@ -341,7 +342,8 @@ fn get_releases(trace: &Vec<ExecInterval>, states: &HashMap<u64, ReducedFreeRTOS
ret
}
fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)>) -> Vec<(u64, u64, String)> {
fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)>) -> (Vec<(u64, u64, String)>, bool) {
let mut maybe_error = false;
let mut ret = Vec::new();
let mut ready : HashMap<&String, u64> = HashMap::new();
let mut last_response : HashMap<&String, u64> = HashMap::new();
@ -357,6 +359,7 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)
if let Some(peek_resp) = d.peek() {
if peek_resp.0 > peek_rel.0 { // multiple releases before response
// It is unclear which release is real
// maybe_error = true;
// eprintln!("Task {} released multiple times before response ({:.1}ms and {:.1}ms)", peek_rel.1, crate::time::clock::tick_to_time(ready[&peek_rel.1]).as_micros()/1000, crate::time::clock::tick_to_time(peek_rel.0).as_micros()/1000);
// ready.insert(&peek_rel.1, peek_rel.0);
r.next();
@ -374,7 +377,8 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)
if ready.contains_key(&next_resp.1) {
if ready[&next_resp.1] >= next_resp.0 {
if let Some(lr) = last_response.get(&next_resp.1) {
eprintln!("Task {} response at {:.1}ms before next release at {:.1}ms. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(ready[&next_resp.1]).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0);
maybe_error = true;
// eprintln!("Task {} response at {:.1}ms before next release at {:.1}ms. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(ready[&next_resp.1]).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0);
// Sometimes a task is released immediately after a response. This might not be detected.
// Assume that the release occured with the last response
ret.push((*lr, next_resp.0, next_resp.1.clone()));
@ -389,8 +393,9 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)
ready.remove(&next_resp.1);
}
} else {
maybe_error = true;
if let Some(lr) = last_response.get(&next_resp.1) {
eprintln!("Task {} response at {:.1}ms not found in ready list. Fallback to last release at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0);
// eprintln!("Task {} response at {:.1}ms not found in ready list. Fallback to last release at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0);
// Sometimes a task is released immediately after a response. This might not be detected.
// Assume that the release occured with the last response
ret.push((*lr, next_resp.0, next_resp.1.clone()));
@ -401,7 +406,7 @@ fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)
}
} else {
// TODO: should remaining released tasks be counted as finished?
return ret;
return (ret,maybe_error);
}
}
}