From a13dca6f39a270a09ad165c0c05d5c43fc97337b Mon Sep 17 00:00:00 2001 From: Alwin Berger Date: Mon, 16 Dec 2024 16:00:18 +0100 Subject: [PATCH] abstract SystemTraceData --- fuzzers/FRET/src/fuzzer.rs | 30 +- fuzzers/FRET/src/systemstate/feedbacks.rs | 522 +++++--- fuzzers/FRET/src/systemstate/helpers.rs | 463 +------ fuzzers/FRET/src/systemstate/mod.rs | 261 +--- fuzzers/FRET/src/systemstate/mutational.rs | 73 +- fuzzers/FRET/src/systemstate/observers.rs | 627 +-------- fuzzers/FRET/src/systemstate/schedulers.rs | 16 +- fuzzers/FRET/src/systemstate/stg.rs | 148 ++- .../freertos/bindings.rs} | 36 +- .../src/systemstate/target_os/freertos/mod.rs | 548 ++++++++ .../target_os/freertos/qemu_module.rs | 1154 +++++++++++++++++ fuzzers/FRET/src/systemstate/target_os/mod.rs | 110 ++ fuzzers/FRET/src/time/clock.rs | 17 +- fuzzers/FRET/src/time/worst.rs | 6 +- libafl/src/fuzzer/mod.rs | 4 +- libafl_qemu/libafl_qemu_build/src/bindings.rs | 1 + 16 files changed, 2400 insertions(+), 1616 deletions(-) rename fuzzers/FRET/src/systemstate/{freertos.rs => target_os/freertos/bindings.rs} (73%) create mode 100644 fuzzers/FRET/src/systemstate/target_os/freertos/mod.rs create mode 100644 fuzzers/FRET/src/systemstate/target_os/freertos/qemu_module.rs create mode 100644 fuzzers/FRET/src/systemstate/target_os/mod.rs diff --git a/fuzzers/FRET/src/fuzzer.rs b/fuzzers/FRET/src/fuzzer.rs index 587aa48270..37cdcb41c0 100644 --- a/fuzzers/FRET/src/fuzzer.rs +++ b/fuzzers/FRET/src/fuzzer.rs @@ -14,7 +14,7 @@ elf::EasyElf, emu::Emulator, modules::{edges::{self}, FilterList}, GuestAddr, Gu }; use rand::{SeedableRng, StdRng, Rng}; use crate::{ - systemstate::{self, feedbacks::{DumpSystraceFeedback, SystraceErrorFeedback}, helpers::{get_function_range, load_symbol, try_load_symbol, QemuSystemStateHelper}, mutational::{input_bytes_to_interrupt_times, InterruptShiftStage, STGSnippetStage}, observers::QemuSystemStateObserver, schedulers::{GenerationScheduler, LongestTraceScheduler}, stg::{stg_map_mut_slice, GraphMaximizerCorpusScheduler, STGEdge, STGNode, StgFeedback, MAX_STG_NUM}}, time::{ + systemstate::{self, feedbacks::{DumpSystraceFeedback, SystraceErrorFeedback}, helpers::{get_function_range, load_symbol, try_load_symbol}, mutational::{input_bytes_to_interrupt_times, InterruptShiftStage, STGSnippetStage}, observers::QemuSystemStateObserver, schedulers::{GenerationScheduler, LongestTraceScheduler}, stg::{stg_map_mut_slice, GraphMaximizerCorpusScheduler, STGEdge, STGNode, StgFeedback, MAX_STG_NUM}, target_os::freertos::{qemu_module::FreeRTOSSystemStateHelper, FreeRTOSSystem}}, time::{ clock::{ClockTimeFeedback, IcHist, QemuClockIncreaseFeedback, QemuClockObserver, FUZZ_START_TIMESTAMP, QEMU_ICOUNT_SHIFT, QEMU_ISNS_PER_USEC}, qemustate::QemuStateRestoreHelper, worst::{AlwaysTrueFeedback, ExecTimeIncFeedback, RateLimitedMonitor, TimeMaximizerCorpusScheduler, TimeProbMassScheduler, TimeStateMaximizerCorpusScheduler} } }; @@ -123,7 +123,7 @@ macro_rules! do_dump_stg { if $cli.dump_graph { let dump_path = $cli.dump_name.clone().unwrap().with_extension(if $c=="" {"dot"} else {$c}); println!("Dumping graph to {:?}", &dump_path); - if let Some(md) = $state.named_metadata_map_mut().get_mut::("stgfeedbackstate") { + if let Some(md) = $state.named_metadata_map_mut().get_mut::>("stgfeedbackstate") { let out = md.graph.map(|_i,x| x.color_print(), |_i,x| x.color_print()); let outs = Dot::with_config(&out, &[]).to_string(); let outs = outs.replace("\\\"","\""); @@ -245,17 +245,17 @@ let mut api_ranges = get_all_fn_symbol_ranges(&elf, api_range); println!("APP functions:"); let app_fn_ranges = get_all_fn_symbol_ranges(&elf, app_range.clone()); -let mut isr_ranges : HashMap> = systemstate::helpers::ISR_SYMBOLS.iter().filter_map(|x| (api_ranges.get(&x.to_string()).map(|y| (x.to_string(),y.clone())))).collect(); -systemstate::helpers::ISR_SYMBOLS.iter().for_each(|x| {let _ =(app_fn_ranges.get(&x.to_string()).map(|y| (x.to_string(),y.clone()))).map(|y| isr_ranges.insert(y.0,y.1));}); // add used defined isr +let mut isr_ranges : HashMap> = systemstate::target_os::freertos::ISR_SYMBOLS.iter().filter_map(|x| (api_ranges.get(&x.to_string()).map(|y| (x.to_string(),y.clone())))).collect(); +systemstate::target_os::freertos::ISR_SYMBOLS.iter().for_each(|x| {let _ =(app_fn_ranges.get(&x.to_string()).map(|y| (x.to_string(),y.clone()))).map(|y| isr_ranges.insert(y.0,y.1));}); // add used defined isr let denylist : Vec<_> =isr_ranges.values().map(|x| x.clone()).collect(); let denylist = FilterList::DenyList(denylist); // do not count isr jumps, which are useless #[cfg(feature = "observe_systemstate")] -let mut isr_addreses : HashMap = systemstate::helpers::ISR_SYMBOLS.iter().filter_map(|x| (api_ranges.remove(&x.to_string()).map(|y| (y.start,x.to_string())))).collect(); +let mut isr_addreses : HashMap = systemstate::target_os::freertos::ISR_SYMBOLS.iter().filter_map(|x| (api_ranges.remove(&x.to_string()).map(|y| (y.start,x.to_string())))).collect(); #[cfg(feature = "observe_systemstate")] -systemstate::helpers::ISR_SYMBOLS.iter().for_each(|x| {let _ =(app_fn_ranges.get(&x.to_string()).map(|y| (x.to_string(),y.clone()))).map(|y| isr_addreses.insert(y.1.start, y.0));}); // add used defined isr +systemstate::target_os::freertos::ISR_SYMBOLS.iter().for_each(|x| {let _ =(app_fn_ranges.get(&x.to_string()).map(|y| (x.to_string(),y.clone()))).map(|y| isr_addreses.insert(y.1.start, y.0));}); // add used defined isr #[cfg(feature = "observe_systemstate")] -for i in systemstate::helpers::ISR_SYMBOLS { +for i in systemstate::target_os::freertos::ISR_SYMBOLS { if isr_ranges.get(&i.to_string()).is_none() { if let Some(fr) = get_function_range(&elf, i) { isr_addreses.insert(fr.start, i.to_string()); @@ -408,13 +408,13 @@ let run_client = |state: Option<_>, mut mgr, _core_id| { )}.track_indices(); #[cfg(feature = "observe_systemstate")] - let systemstate_observer = QemuSystemStateObserver::new(&cli.select_task); + let systemstate_observer = QemuSystemStateObserver::<_,FreeRTOSSystem>::new(&cli.select_task); // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR let mut feedback = feedback_or!( // Time feedback, this one does not need a feedback state - ClockTimeFeedback::new_with_observer(&clock_time_observer, &cli.select_task) + ClockTimeFeedback::::new_with_observer(&clock_time_observer, &cli.select_task) ); #[cfg(feature = "feed_genetic")] let mut feedback = feedback_or!( @@ -437,12 +437,12 @@ let run_client = |state: Option<_>, mut mgr, _core_id| { #[cfg(all(feature = "observe_systemstate"))] let mut feedback = feedback_or!( feedback, - DumpSystraceFeedback::with_dump(if cli.dump_traces {cli.dump_name.clone().map(|x| x.with_extension("trace.ron"))} else {None}) + DumpSystraceFeedback::::with_dump(if cli.dump_traces {cli.dump_name.clone().map(|x| x.with_extension("trace.ron"))} else {None}) ); #[cfg(feature = "trace_stg")] let mut feedback = feedback_or!( feedback, - StgFeedback::new(if cli.dump_graph {cli.dump_name.clone()} else {None}) + StgFeedback::::new(if cli.dump_graph {cli.dump_name.clone()} else {None}) ); #[cfg(feature = "feed_stg_edge")] let mut feedback = feedback_or!( @@ -451,7 +451,7 @@ let run_client = |state: Option<_>, mut mgr, _core_id| { ); // A feedback to choose if an input is producing an error - let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new(), SystraceErrorFeedback::new(matches!(cli.command, Commands::Fuzz{..}), Some(10))); + let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new(), SystraceErrorFeedback::::new(matches!(cli.command, Commands::Fuzz{..}), Some(10))); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { @@ -491,7 +491,7 @@ let run_client = |state: Option<_>, mut mgr, _core_id| { let qhelpers = tuple_list!(); #[cfg(feature = "observe_systemstate")] - let qhelpers = (QemuSystemStateHelper::new(api_addreses,api_ranges,isr_addreses,isr_ranges,input_addr..(input_addr+unsafe { MAX_INPUT_SIZE } as u32),curr_tcb_pointer,task_queue_addr,task_delay_addr,task_delay_overflow_addr,scheduler_lock,scheduler_running, critical_section,input_counter_ptr,app_range.clone(), job_done_addr), qhelpers); + let qhelpers = (FreeRTOSSystemStateHelper::new(api_addreses,api_ranges,isr_addreses,isr_ranges,input_addr..(input_addr+unsafe { MAX_INPUT_SIZE } as u32),curr_tcb_pointer,task_queue_addr,task_delay_addr,task_delay_overflow_addr,scheduler_lock,scheduler_running, critical_section,input_counter_ptr,app_range.clone(), job_done_addr), qhelpers); #[cfg(feature = "observe_edges")] let qhelpers = (QemuEdgeCoverageHelper::new(denylist, QemuFilterList::None), qhelpers); let qhelpers = (QemuStateRestoreHelper::with_fast(initial_snap), qhelpers); @@ -526,9 +526,9 @@ let run_client = |state: Option<_>, mut mgr, _core_id| { let stages = (systemstate::report::SchedulerStatsStage::default(),()); let stages = (StdMutationalStage::new(mutator), stages); #[cfg(feature = "mutate_stg")] - let mut stages = (STGSnippetStage::new(input_addr), stages); + let mut stages = (STGSnippetStage::<_,_,_,FreeRTOSSystem>::new(input_addr), stages); #[cfg(feature = "fuzz_int")] - let mut stages = (InterruptShiftStage::new(&interrupt_config), stages); + let mut stages = (InterruptShiftStage::<_,_,_,FreeRTOSSystem>::new(&interrupt_config), stages); if let Commands::Showmap { input } = cli.command.clone() { let s = input.as_os_str(); diff --git a/fuzzers/FRET/src/systemstate/feedbacks.rs b/fuzzers/FRET/src/systemstate/feedbacks.rs index 229d4ac0a4..ffbe67d227 100644 --- a/fuzzers/FRET/src/systemstate/feedbacks.rs +++ b/fuzzers/FRET/src/systemstate/feedbacks.rs @@ -1,41 +1,48 @@ -use libafl::SerdeAny; -use libafl::prelude::UsesInput; -use libafl::common::HasNamedMetadata; -use std::path::PathBuf; use crate::time::clock::QemuClockObserver; -use libafl::corpus::Testcase; -use std::collections::hash_map::DefaultHasher; -use std::hash::Hasher; -use std::hash::Hash; -use libafl::events::EventFirer; -use libafl::state::MaybeHasClientPerfMonitor; -use libafl::prelude::State; -use libafl::feedbacks::Feedback; -use libafl_bolts::Named; -use libafl::Error; use hashbrown::HashMap; -use libafl::{executors::ExitKind, observers::ObserversTuple, common::HasMetadata}; +use libafl::common::HasNamedMetadata; +use libafl::corpus::Testcase; +use libafl::events::EventFirer; +use libafl::feedbacks::Feedback; +use libafl::prelude::State; +use libafl::prelude::UsesInput; +use libafl::state::MaybeHasClientPerfMonitor; +use libafl::Error; +use libafl::{common::HasMetadata, executors::ExitKind, observers::ObserversTuple}; +use libafl_bolts::Named; use serde::{Deserialize, Serialize}; +use std::collections::hash_map::DefaultHasher; +use std::hash::Hash; +use std::hash::Hasher; +use std::path::PathBuf; -use super::ExecInterval; -use super::ReducedFreeRTOSSystemState; -use super::FreeRTOSSystemStateMetadata; use super::observers::QemuSystemStateObserver; +use super::target_os::TargetSystem; +use super::ExecInterval; +use libafl_bolts::prelude::SerdeAny; use std::borrow::Cow; +use std::marker::PhantomData; +use crate::systemstate::target_os::*; use libafl::prelude::StateInitializer; //============================= Feedback /// Shared Metadata for a systemstateFeedback -#[derive(Debug, Serialize, Deserialize, SerdeAny, Clone)] -pub struct SystemStateFeedbackState +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct SystemStateFeedbackState +where + SYS: TargetSystem, { name: Cow<'static, str>, - known_traces: HashMap, // encounters,ticks,length - longest: Vec, + known_traces: HashMap, // encounters,ticks,length + longest: Vec, } -impl Named for SystemStateFeedbackState +libafl_bolts::impl_serdeany!(SystemStateFeedbackState); + +impl Named for SystemStateFeedbackState +where + SYS: TargetSystem, { #[inline] fn name(&self) -> &Cow<'static, str> { @@ -43,195 +50,265 @@ impl Named for SystemStateFeedbackState } } -impl Default for SystemStateFeedbackState +impl Default for SystemStateFeedbackState +where + SYS: TargetSystem, { fn default() -> Self { - Self {name: Cow::from("systemstate".to_string()), known_traces: HashMap::new(), longest: Vec::new() } + Self { + name: Cow::from("systemstate".to_string()), + known_traces: HashMap::new(), + longest: Vec::new(), + } } } -/// A Feedback reporting novel System-State Transitions. Depends on [`QemuSystemStateObserver`] -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct NovelSystemStateFeedback +// /// A Feedback reporting novel System-State Transitions. Depends on [`QemuSystemStateObserver`] +// #[derive(Serialize, Deserialize, Clone, Debug)] +// pub struct NovelSystemStateFeedback +// { +// name: Cow<'static, str>, +// last_trace: Option>, +// // known_traces: HashMap, +// } + +// impl StateInitializer for NovelSystemStateFeedback {} + +// impl Feedback for NovelSystemStateFeedback +// where +// S: State + UsesInput + MaybeHasClientPerfMonitor + HasNamedMetadata, +// S::Input: Default, +// EM: EventFirer, +// OT: ObserversTuple, +// { +// fn is_interesting( +// &mut self, +// state: &mut S, +// _manager: &mut EM, +// _input: &I, +// observers: &OT, +// _exit_kind: &ExitKind, +// ) -> Result +// where +// { +// let observer : &QemuSystemStateObserver = observers.match_name::>("systemstate") +// .expect("QemuSystemStateObserver not found"); +// let clock_observer = observers.match_name::("clocktime") //TODO not fixed +// .expect("QemuClockObserver not found"); +// let feedbackstate = match state +// .named_metadata_map_mut() +// .get_mut::("systemstate") { +// Some(s) => s, +// Option::None => { +// let n=SystemStateFeedbackState::default(); +// state.named_metadata_map_mut().insert("systemstate",n); +// state.named_metadata_map_mut().get_mut::("systemstate").unwrap() +// } +// }; +// #[cfg(feature = "trace_job_response_times")] +// let last_runtime = observer.last_runtime(); +// #[cfg(not(feature = "trace_job_response_times"))] +// let last_runtime = clock_observer.last_runtime(); +// // let feedbackstate = state +// // .feedback_states_mut() +// // .match_name_mut::("systemstate") +// // .unwrap(); +// // Do Stuff +// let mut hasher = DefaultHasher::new(); +// observer.last_run.hash(&mut hasher); +// let somehash = hasher.finish(); +// let mut is_novel = false; +// let mut takes_longer = false; +// match feedbackstate.known_traces.get_mut(&somehash) { +// Option::None => { +// is_novel = true; +// feedbackstate.known_traces.insert(somehash,(1,last_runtime,observer.last_run.len())); +// } +// Some(s) => { +// s.0+=1; +// if s.1 < last_runtime { +// s.1 = last_runtime; +// takes_longer = true; +// } +// } +// } +// if observer.last_run.len() > feedbackstate.longest.len() { +// feedbackstate.longest=observer.last_run.clone(); +// } +// self.last_trace = Some(observer.last_run.clone()); +// // if (!is_novel) { println!("not novel") }; +// Ok(is_novel | takes_longer) +// } + +// /// Append to the testcase the generated metadata in case of a new corpus item +// #[inline] +// fn append_metadata(&mut self, _state: &mut S, _manager: &mut EM, _observers: &OT, testcase: &mut Testcase) -> Result<(), Error> { +// let a = self.last_trace.take(); +// match a { +// Some(s) => testcase.metadata_map_mut().insert(FreeRTOSSystemStateMetadata::new(s)), +// Option::None => (), +// } +// Ok(()) +// } + +// /// Discard the stored metadata in case that the testcase is not added to the corpus +// #[inline] +// fn discard_metadata(&mut self, _state: &mut S, _input: &I) -> Result<(), Error> { +// self.last_trace = None; +// Ok(()) +// } +// } + +// impl Named for NovelSystemStateFeedback +// { +// #[inline] +// fn name(&self) -> &Cow<'static, str> { +// &self.name +// } +// } + +// impl Default for NovelSystemStateFeedback +// { +// fn default() -> Self { +// Self {name: Cow::from("NovelSystemStateFeedback".to_string()), last_trace: None } +// } +// } + +//=========================== Debugging Feedback +/// A [`Feedback`] meant to dump the system-traces for debugging. Depends on [`QemuSystemStateObserver`] +#[derive(Debug)] +pub struct DumpSystraceFeedback +where + SYS: TargetSystem, { name: Cow<'static, str>, - last_trace: Option>, - // known_traces: HashMap, + dumpfile: Option, + dump_metadata: bool, + select_task: Option, // TODO: use some global config for this + phantom: PhantomData, } -impl StateInitializer for NovelSystemStateFeedback {} +impl StateInitializer for DumpSystraceFeedback where SYS: TargetSystem {} -impl Feedback for NovelSystemStateFeedback +impl Feedback for DumpSystraceFeedback where - S: State + UsesInput + MaybeHasClientPerfMonitor + HasNamedMetadata, - S::Input: Default, + S: State + UsesInput + MaybeHasClientPerfMonitor + HasMetadata, EM: EventFirer, OT: ObserversTuple, + SYS: TargetSystem, { fn is_interesting( &mut self, state: &mut S, _manager: &mut EM, _input: &I, - observers: &OT, + _observers: &OT, _exit_kind: &ExitKind, ) -> Result - where - { - let observer : &QemuSystemStateObserver = observers.match_name::>("systemstate") - .expect("QemuSystemStateObserver not found"); - let clock_observer = observers.match_name::("clocktime") //TODO not fixed - .expect("QemuClockObserver not found"); - let feedbackstate = match state - .named_metadata_map_mut() - .get_mut::("systemstate") { - Some(s) => s, - Option::None => { - let n=SystemStateFeedbackState::default(); - state.named_metadata_map_mut().insert("systemstate",n); - state.named_metadata_map_mut().get_mut::("systemstate").unwrap() - } - }; - #[cfg(feature = "trace_job_response_times")] - let last_runtime = observer.last_runtime(); - #[cfg(not(feature = "trace_job_response_times"))] - let last_runtime = clock_observer.last_runtime(); - // let feedbackstate = state - // .feedback_states_mut() - // .match_name_mut::("systemstate") - // .unwrap(); - // Do Stuff - let mut hasher = DefaultHasher::new(); - observer.last_run.hash(&mut hasher); - let somehash = hasher.finish(); - let mut is_novel = false; - let mut takes_longer = false; - match feedbackstate.known_traces.get_mut(&somehash) { - Option::None => { - is_novel = true; - feedbackstate.known_traces.insert(somehash,(1,last_runtime,observer.last_run.len())); - } - Some(s) => { - s.0+=1; - if s.1 < last_runtime { - s.1 = last_runtime; - takes_longer = true; - } - } - } - if observer.last_run.len() > feedbackstate.longest.len() { - feedbackstate.longest=observer.last_run.clone(); - } - self.last_trace = Some(observer.last_run.clone()); - // if (!is_novel) { println!("not novel") }; - Ok(is_novel | takes_longer) - } +where { + if self.dumpfile.is_none() { + return Ok(false); + }; + let trace = state + .metadata::() + .expect("TraceData not found"); - /// Append to the testcase the generated metadata in case of a new corpus item - #[inline] - fn append_metadata(&mut self, _state: &mut S, _manager: &mut EM, _observers: &OT, testcase: &mut Testcase) -> Result<(), Error> { - let a = self.last_trace.take(); - match a { - Some(s) => testcase.metadata_map_mut().insert(FreeRTOSSystemStateMetadata::new(s)), - Option::None => (), - } - Ok(()) - } - - /// Discard the stored metadata in case that the testcase is not added to the corpus - #[inline] - fn discard_metadata(&mut self, _state: &mut S, _input: &I) -> Result<(), Error> { - self.last_trace = None; - Ok(()) - } -} - -impl Named for NovelSystemStateFeedback -{ - #[inline] - fn name(&self) -> &Cow<'static, str> { - &self.name - } -} - -impl Default for NovelSystemStateFeedback -{ - fn default() -> Self { - Self {name: Cow::from("NovelSystemStateFeedback".to_string()), last_trace: None } - } -} - -//=========================== Debugging Feedback -/// A [`Feedback`] meant to dump the system-traces for debugging. Depends on [`QemuSystemStateObserver`] -#[derive(Debug)] -pub struct DumpSystraceFeedback -{ - name: Cow<'static, str>, - dumpfile: Option, - dump_metadata: bool, - last_states: Option>, - last_trace: Option>, -} - -impl StateInitializer for DumpSystraceFeedback {} - -impl Feedback for DumpSystraceFeedback -where - S: State + UsesInput + MaybeHasClientPerfMonitor, - EM: EventFirer, - OT: ObserversTuple -{ - fn is_interesting( - &mut self, - _state: &mut S, - _manager: &mut EM, - _input: &I, - observers: &OT, - _exit_kind: &ExitKind, - ) -> Result - where - { - if self.dumpfile.is_none() {return Ok(false)}; - let observer = observers.match_name::>("systemstate") - .expect("QemuSystemStateObserver not found"); - let names : Vec = observer.last_run.iter().map(|x| x.current_task.task_name.clone()).collect(); + let names: Vec = trace + .states() + .iter() + .map(|x| x.current_task().task_name().clone()) + .collect(); match &self.dumpfile { Some(s) => { - let per_task_metadata = if let Some(worst_instance) = observer.job_instances.iter().filter(|x| Some(&x.name) == observer.select_task.as_ref()).max_by(|a,b| (a.response-a.release).cmp(&(b.response-b.release))) { - // extract computation time spent in each task and abb - let t : Vec<_> = observer.last_trace.iter().filter(|x| x.start_tick < worst_instance.response && x.end_tick > worst_instance.release ).cloned().collect(); - // task_name -> addr -> (count, time) - let mut ret : HashMap> = HashMap::new(); - let mut t2 = t.clone(); - t2.sort_by_key(|x| x.get_task_name_unchecked()); - t2.chunk_by_mut(|x,y| x.get_task_name_unchecked() == y.get_task_name_unchecked()).for_each(|x| { - x.sort_by_key(|y| y.abb.as_ref().unwrap().start); - x.chunk_by(|y,z| y.abb.as_ref().unwrap().start == z.abb.as_ref().unwrap().start).for_each(|y| { - match ret.get_mut(&y[0].get_task_name_unchecked()) { - Option::None => { - ret.insert(y[0].get_task_name_unchecked(), HashMap::from([(y[0].abb.as_ref().unwrap().start, (y.len(), y.iter().filter(|x| x.is_abb_end()).count(), y.iter().map(|z| z.get_exec_time()).sum::<_>()))])); - } - Some(x) => { - x.insert(y[0].abb.as_ref().unwrap().start, (y.len(), y.iter().filter(|x| x.is_abb_end()).count(), y.iter().map(|z| z.get_exec_time()).sum())); - } + let per_task_metadata = if let Some(worst_instance) = trace.worst_jobs_per_task_by_time().get(self.select_task.as_ref().unwrap_or(&"".to_string())) + { + // extract computation time spent in each task and abb + let t: Vec<_> = trace.intervals() + .iter() + .filter(|x| { + x.start_tick < worst_instance.response + && x.end_tick > worst_instance.release + }) + .cloned() + .collect(); + // task_name -> addr -> (count, time) + let mut ret: HashMap> = + HashMap::new(); + let mut t2 = t.clone(); + t2.sort_by_key(|x| x.get_task_name_unchecked()); + t2.chunk_by_mut(|x, y| { + x.get_task_name_unchecked() == y.get_task_name_unchecked() + }) + .for_each(|x| { + x.sort_by_key(|y| y.abb.as_ref().unwrap().start); + x.chunk_by(|y, z| { + y.abb.as_ref().unwrap().start == z.abb.as_ref().unwrap().start + }) + .for_each(|y| { + match ret.get_mut(&y[0].get_task_name_unchecked()) { + Option::None => { + ret.insert( + y[0].get_task_name_unchecked(), + HashMap::from([( + y[0].abb.as_ref().unwrap().start, + ( + y.len(), + y.iter().filter(|x| x.is_abb_end()).count(), + y.iter().map(|z| z.get_exec_time()).sum::<_>(), + ), + )]), + ); } - }); + Some(x) => { + x.insert( + y[0].abb.as_ref().unwrap().start, + ( + y.len(), + y.iter().filter(|x| x.is_abb_end()).count(), + y.iter().map(|z| z.get_exec_time()).sum(), + ), + ); + } + } }); - // dbg!(&ret); - ret - } else {HashMap::new()}; - std::fs::write(s,ron::to_string(&(&observer.last_trace,&observer.last_states,&observer.job_instances,per_task_metadata)).expect("Error serializing hashmap")).expect("Can not dump to file"); - self.dumpfile = None - }, - Option::None => if self.dump_metadata {println!("{:?}\n{:?}",observer.last_run,names);} + }); + // dbg!(&ret); + ret + } else { + HashMap::new() + }; + std::fs::write( + s, + ron::to_string(&( + &trace, + per_task_metadata, + )) + .expect("Error serializing hashmap"), + ) + .expect("Can not dump to file"); + self.dumpfile = None + } + Option::None => { + if self.dump_metadata { + println!("{:?}\n{:?}", trace, names); + } + } }; // if self.dump_metadata {self.last_trace=Some(observer.last_trace.clone());} Ok(false) } /// Append to the testcase the generated metadata in case of a new corpus item #[inline] - fn append_metadata(&mut self, _state: &mut S, _manager: &mut EM, _observers: &OT, _testcase: &mut Testcase) -> Result<(), Error> { - if !self.dump_metadata {return Ok(());} + fn append_metadata( + &mut self, + _state: &mut S, + _manager: &mut EM, + _observers: &OT, + _testcase: &mut Testcase, + ) -> Result<(), Error> { + if !self.dump_metadata { + return Ok(()); + } // let a = self.last_trace.take(); // match a { // Some(s) => testcase.metadata_map_mut().insert(FreeRTOSSystemStateMetadata::new(s)), @@ -243,12 +320,13 @@ where /// Discard the stored metadata in case that the testcase is not added to the corpus #[inline] fn discard_metadata(&mut self, _state: &mut S, _input: &I) -> Result<(), Error> { - self.last_trace = None; Ok(()) } } -impl Named for DumpSystraceFeedback +impl Named for DumpSystraceFeedback +where + SYS: TargetSystem, { #[inline] fn name(&self) -> &Cow<'static, str> { @@ -256,39 +334,62 @@ impl Named for DumpSystraceFeedback } } -impl DumpSystraceFeedback +impl DumpSystraceFeedback +where + SYS: TargetSystem, { /// Creates a new [`DumpSystraceFeedback`] #[allow(unused)] pub fn new() -> Self { - Self {name: Cow::from("Dumpsystemstate".to_string()), dumpfile: None, dump_metadata: false, last_trace: None, last_states: None } + Self { + name: Cow::from("Dumpsystemstate".to_string()), + dumpfile: None, + dump_metadata: false, + phantom: PhantomData, + select_task: None, + } } #[allow(unused)] pub fn with_dump(dumpfile: Option) -> Self { - Self {name: Cow::from("Dumpsystemstate".to_string()), dumpfile: dumpfile, dump_metadata: false, last_trace: None, last_states: None} + Self { + name: Cow::from("Dumpsystemstate".to_string()), + dumpfile: dumpfile, + dump_metadata: false, + phantom: PhantomData, + select_task: None, + } } #[allow(unused)] pub fn metadata_only() -> Self { - Self {name: Cow::from("Dumpsystemstate".to_string()), dumpfile: None, dump_metadata: true, last_trace: None, last_states: None} + Self { + name: Cow::from("Dumpsystemstate".to_string()), + dumpfile: None, + dump_metadata: true, + phantom: PhantomData, + select_task: None, + } } } - #[derive(Debug, Default)] -pub struct SystraceErrorFeedback +pub struct SystraceErrorFeedback +where + SYS: TargetSystem, { name: Cow<'static, str>, dump_case: bool, max_reports: Option, + phantom: std::marker::PhantomData, } -impl StateInitializer for SystraceErrorFeedback {} +impl StateInitializer for SystraceErrorFeedback where SYS: TargetSystem {} -impl Feedback for SystraceErrorFeedback +impl Feedback for SystraceErrorFeedback where S: State + UsesInput + MaybeHasClientPerfMonitor, EM: EventFirer, - OT: ObserversTuple + OT: ObserversTuple, + SYS: TargetSystem, { fn is_interesting( &mut self, @@ -298,20 +399,22 @@ where observers: &OT, _exit_kind: &ExitKind, ) -> Result - where - { +where { #[cfg(feature = "trace_stg")] { - let observer = observers.match_name::>("systemstate") + let observer = observers + .match_name::>("systemstate") .expect("QemuSystemStateObserver not found"); let is_err = (!observer.success || observer.do_report); if let Some(m) = self.max_reports { - if m <= 0 {return Ok(false);} + if m <= 0 { + return Ok(false); + } if is_err { - self.max_reports = Some(m-1); + self.max_reports = Some(m - 1); } } - return Ok(self.dump_case&&is_err); + return Ok(self.dump_case && is_err); } #[cfg(not(feature = "trace_stg"))] { @@ -320,7 +423,13 @@ where } /// Append to the testcase the generated metadata in case of a new corpus item #[inline] - fn append_metadata(&mut self, _state: &mut S, _manager: &mut EM, _observers: &OT, _testcase: &mut Testcase) -> Result<(), Error> { + fn append_metadata( + &mut self, + _state: &mut S, + _manager: &mut EM, + _observers: &OT, + _testcase: &mut Testcase, + ) -> Result<(), Error> { Ok(()) } @@ -331,7 +440,9 @@ where } } -impl Named for SystraceErrorFeedback +impl Named for SystraceErrorFeedback +where + SYS: TargetSystem, { #[inline] fn name(&self) -> &Cow<'static, str> { @@ -339,10 +450,17 @@ impl Named for SystraceErrorFeedback } } -impl SystraceErrorFeedback +impl SystraceErrorFeedback +where + SYS: TargetSystem, { #[must_use] pub fn new(dump_case: bool, max_reports: Option) -> Self { - Self {name: Cow::from(String::from("SystraceErrorFeedback")), dump_case, max_reports} + Self { + name: Cow::from(String::from("SystraceErrorFeedback")), + dump_case, + max_reports, + phantom: std::marker::PhantomData, + } } -} \ No newline at end of file +} diff --git a/fuzzers/FRET/src/systemstate/helpers.rs b/fuzzers/FRET/src/systemstate/helpers.rs index 1370ef83b9..e3228ad2a9 100644 --- a/fuzzers/FRET/src/systemstate/helpers.rs +++ b/fuzzers/FRET/src/systemstate/helpers.rs @@ -15,14 +15,6 @@ use libafl_qemu::modules::{EmulatorModule, EmulatorModuleTuple}; use libafl_qemu::sys::TCGTemp; use libafl_qemu::qemu::MemAccessInfo; -use crate::systemstate::RawFreeRTOSSystemState; -use crate::systemstate::CURRENT_SYSTEMSTATE_VEC; -use crate::systemstate::NUM_PRIOS; -use super::freertos::void_ptr; -use super::freertos::TCB_t; -use super::freertos::rtos_struct::List_Item_struct; -use super::freertos::rtos_struct::*; -use super::freertos; use super::CaptureEvent; use libafl_qemu::EmulatorModules; use libafl::prelude::ObserversTuple; @@ -31,13 +23,6 @@ use libafl::prelude::ObserversTuple; //============================= API symbols -pub const ISR_SYMBOLS : &'static [&'static str] = &[ -// ISRs -"Reset_Handler","Default_Handler","Default_Handler2","Default_Handler3","Default_Handler4","Default_Handler5","Default_Handler6","vPortSVCHandler","xPortPendSVHandler","xPortSysTickHandler","ISR_0_Handler", "ISR_1_Handler", "ISR_2_Handler", "ISR_3_Handler", "ISR_4_Handler", "ISR_5_Handler", "ISR_6_Handler", "ISR_7_Handler", "ISR_8_Handler", "ISR_9_Handler", "ISR_10_Handler", "ISR_11_Handler", "ISR_12_Handler", "ISR_13_Handler" -]; -pub const USR_ISR_SYMBOLS : &'static [&'static str] = &[ - "ISR_0_Handler", "ISR_1_Handler", "ISR_2_Handler", "ISR_3_Handler", "ISR_4_Handler", "ISR_5_Handler", "ISR_6_Handler", "ISR_7_Handler", "ISR_8_Handler", "ISR_9_Handler", "ISR_10_Handler", "ISR_11_Handler", "ISR_12_Handler", "ISR_13_Handler" -]; /// Read ELF program headers to resolve physical load addresses. fn virt2phys(vaddr: GuestPhysAddr, tab: &EasyElf) -> GuestPhysAddr { @@ -97,454 +82,10 @@ pub fn get_function_range(elf: &EasyElf, symbol: &str) -> Option, - api_fn_ranges: Vec<(String, std::ops::Range)>, - // Address of interrupt routines - isr_addrs: HashMap, - isr_ranges: Vec<(String, std::ops::Range)>, - input_mem: Range, - tcb_addr: GuestAddr, - ready_queues: GuestAddr, - delay_queue: GuestAddr, - delay_queue_overflow: GuestAddr, - scheduler_lock_addr: GuestAddr, - scheduler_running_addr: GuestAddr, - critical_addr: GuestAddr, - input_counter: Option, - app_range: Range, - job_done_addrs: GuestAddr, -} - -impl QemuSystemStateHelper { - #[must_use] - pub fn new( - api_fn_addrs: HashMap, - api_fn_ranges: Vec<(String, std::ops::Range)>, - isr_addrs: HashMap, - isr_ranges: Vec<(String, std::ops::Range)>, - input_mem: Range, - tcb_addr: GuestAddr, - ready_queues: GuestAddr, - delay_queue: GuestAddr, - delay_queue_overflow: GuestAddr, - scheduler_lock_addr: GuestAddr, - scheduler_running_addr: GuestAddr, - critical_addr: GuestAddr, - input_counter: Option, - app_range: Range, - job_done_addrs: GuestAddr, - ) -> Self { - QemuSystemStateHelper { - api_fn_addrs, - api_fn_ranges, - isr_addrs, - isr_ranges, - input_mem, - tcb_addr: tcb_addr, - ready_queues: ready_queues, - delay_queue, - delay_queue_overflow, - scheduler_lock_addr, - scheduler_running_addr, - critical_addr, - input_counter: input_counter, - app_range, - job_done_addrs, - } - } -} - -impl EmulatorModule for QemuSystemStateHelper -where - S: UsesInput + Unpin, -{ - fn first_exec(&mut self, _emulator_modules: &mut EmulatorModules, _state: &mut S) - where - ET: EmulatorModuleTuple, - { - // for wp in self.api_fn_addrs.keys() { - // _hooks.instruction(*wp, Hook::Function(exec_syscall_hook::), false); - // } - for wp in self.isr_addrs.keys() { - _emulator_modules.instructions(*wp, Hook::Function(exec_isr_hook::), false); - } - _emulator_modules.jmps(Hook::Function(gen_jmp_is_syscall::), Hook::Function(trace_jmp::)); - #[cfg(feature = "trace_job_response_times")] - _emulator_modules.instructions(self.job_done_addrs, Hook::Function(job_done_hook::), false); - #[cfg(feature = "trace_reads")] - _emulator_modules.reads(Hook::Function(gen_read_is_input::), Hook::Empty,Hook::Empty,Hook::Empty,Hook::Empty,Hook::Function(trace_reads::)); - unsafe { INPUT_MEM = self.input_mem.clone() }; - } - - // TODO: refactor duplicate code - fn pre_exec( - &mut self, - _emulator_modules: &mut EmulatorModules, - _state: &mut S, - _input: &S::Input, - ) where - ET: EmulatorModuleTuple, - { - unsafe { - CURRENT_SYSTEMSTATE_VEC.clear(); - JOBS_DONE.clear(); - } - } - - fn post_exec( - &mut self, - _emulator_modules: &mut EmulatorModules, - _state: &mut S, - _input: &S::Input, - _observers: &mut OT, - _exit_kind: &mut ExitKind, - ) where - OT: ObserversTuple, - ET: EmulatorModuleTuple, - { - trigger_collection(&_emulator_modules.qemu(),(0, 0), CaptureEvent::End, self); - unsafe { - let c = _emulator_modules.qemu().cpu_from_index(0); - let pc = c.read_reg::(15).unwrap(); - if CURRENT_SYSTEMSTATE_VEC.len() == 0 {return;} - CURRENT_SYSTEMSTATE_VEC[CURRENT_SYSTEMSTATE_VEC.len()-1].edge = (pc,0); - CURRENT_SYSTEMSTATE_VEC[CURRENT_SYSTEMSTATE_VEC.len()-1].capture_point = (CaptureEvent::End,"Breakpoint".to_string()); - } - // Find the first ISREnd of vPortSVCHandler and drop anything before - unsafe { - let mut index = 0; - while index < CURRENT_SYSTEMSTATE_VEC.len() { - if CaptureEvent::ISREnd == CURRENT_SYSTEMSTATE_VEC[index].capture_point.0 && CURRENT_SYSTEMSTATE_VEC[index].capture_point.1 == "xPortPendSVHandler" { - break; - } - index += 1; - } - CURRENT_SYSTEMSTATE_VEC.drain(..index); - } - } - - type ModuleAddressFilter = NopAddressFilter; - - type ModulePageFilter = NopPageFilter; - - fn address_filter(&self) -> &Self::ModuleAddressFilter { - todo!() - } - - fn address_filter_mut(&mut self) -> &mut Self::ModuleAddressFilter { - todo!() - } - - fn page_filter(&self) -> &Self::ModulePageFilter { - todo!() - } - - fn page_filter_mut(&mut self) -> &mut Self::ModulePageFilter { - todo!() - } -} - -fn read_freertos_list(systemstate : &mut RawFreeRTOSSystemState, emulator: &libafl_qemu::Qemu, target: GuestAddr) -> (freertos::List_t, bool) { - let read : freertos::List_t = freertos::emu_lookup::lookup(emulator, target); - let listbytes : GuestAddr = GuestAddr::try_from(std::mem::size_of::()).unwrap(); - - let mut next_index = read.pxIndex; - for _j in 0..read.uxNumberOfItems { - // always jump over the xListEnd marker - if (target..target+listbytes).contains(&next_index) { - let next_item : freertos::MiniListItem_t = freertos::emu_lookup::lookup(emulator, next_index); - let new_next_index=next_item.pxNext; - systemstate.dumping_ground.insert(next_index,List_MiniItem_struct(next_item)); - next_index = new_next_index; - } - let next_item : freertos::ListItem_t = freertos::emu_lookup::lookup(emulator, next_index); - // println!("Item at {}: {:?}",next_index,next_item); - if next_item.pvContainer != target { - // the list is being modified, abort by setting the list empty - eprintln!("Warning: attempted to read a list that is being modified"); - let mut read=read; - read.uxNumberOfItems = 0; - return (read, false); - } - // assert_eq!(next_item.pvContainer,target); - let new_next_index=next_item.pxNext; - let next_tcb : TCB_t= freertos::emu_lookup::lookup(emulator,next_item.pvOwner); - // println!("TCB at {}: {:?}",next_item.pvOwner,next_tcb); - systemstate.dumping_ground.insert(next_item.pvOwner,TCB_struct(next_tcb.clone())); - systemstate.dumping_ground.insert(next_index,List_Item_struct(next_item)); - next_index=new_next_index; - } - // Handle edge case where the end marker was not included yet - if (target..target+listbytes).contains(&next_index) { - let next_item : freertos::MiniListItem_t = freertos::emu_lookup::lookup(emulator, next_index); - systemstate.dumping_ground.insert(next_index,List_MiniItem_struct(next_item)); - } - return (read, true); -} - -#[inline] -fn trigger_collection(emulator: &libafl_qemu::Qemu, edge: (GuestAddr, GuestAddr), event: CaptureEvent, h: &QemuSystemStateHelper) { - let listbytes : GuestAddr = GuestAddr::try_from(std::mem::size_of::()).unwrap(); - let mut systemstate = RawFreeRTOSSystemState::default(); - - match event { - CaptureEvent::APIStart => { - let s = h.api_fn_addrs.get(&edge.1).unwrap(); - systemstate.capture_point=(CaptureEvent::APIStart, s.to_string()); - }, - CaptureEvent::APIEnd => { - let s = h.api_fn_addrs.get(&edge.0).unwrap(); - systemstate.capture_point=(CaptureEvent::APIEnd, s.to_string()); - }, - CaptureEvent::ISRStart => { - let s = h.isr_addrs.get(&edge.1).unwrap(); - systemstate.capture_point=(CaptureEvent::ISRStart, s.to_string()); - }, - CaptureEvent::ISREnd => { - let s = h.isr_addrs.get(&edge.0).unwrap(); - systemstate.capture_point=(CaptureEvent::ISREnd, s.to_string()); - }, - CaptureEvent::End => {systemstate.capture_point=(CaptureEvent::End, "".to_string());}, - CaptureEvent::Undefined => (), - } - - if systemstate.capture_point.0 == CaptureEvent::Undefined { - // println!("Not found: {:#x} {:#x}", edge.0.unwrap_or(0), edge.1.unwrap_or(0)); - } - systemstate.edge = ((edge.0),(edge.1)); - - systemstate.qemu_tick = get_icount(emulator); - - let mut buf : [u8; 4] = [0,0,0,0]; - match h.input_counter { - Some(s) => unsafe { emulator.read_mem(s, &mut buf); }, - Option::None => (), - }; - systemstate.input_counter = GuestAddr::from_le_bytes(buf); - - let curr_tcb_addr : freertos::void_ptr = freertos::emu_lookup::lookup(emulator, h.tcb_addr); - if curr_tcb_addr == 0 { - return; - }; - - // println!("{:?}",std::str::from_utf8(¤t_tcb.pcTaskName)); - let critical : void_ptr = freertos::emu_lookup::lookup(emulator, h.critical_addr); - let suspended : void_ptr = freertos::emu_lookup::lookup(emulator, h.scheduler_lock_addr); - let _running : void_ptr = freertos::emu_lookup::lookup(emulator, h.scheduler_running_addr); - - systemstate.current_tcb = freertos::emu_lookup::lookup(emulator,curr_tcb_addr); - // During ISRs it is only safe to extract structs if they are not currently being modified - if systemstate.capture_point.0==CaptureEvent::APIStart || systemstate.capture_point.0==CaptureEvent::APIEnd || (critical == 0 && suspended == 0 ) { - // Extract delay list - let mut target : GuestAddr = h.delay_queue; - target = freertos::emu_lookup::lookup(emulator, target); - let _temp = read_freertos_list(&mut systemstate, emulator, target); - systemstate.delay_list = _temp.0; - systemstate.read_invalid |= !_temp.1; - - // Extract delay list overflow - let mut target : GuestAddr = h.delay_queue_overflow; - target = freertos::emu_lookup::lookup(emulator, target); - let _temp = read_freertos_list(&mut systemstate, emulator, target); - systemstate.delay_list_overflow = _temp.0; - systemstate.read_invalid |= !_temp.1; - - // Extract suspended tasks (infinite wait), seems broken, always appreas to be modified - // let mut target : GuestAddr = h.suspended_queue; - // target = freertos::emu_lookup::lookup(emulator, target); - // systemstate.suspended_list = read_freertos_list(&mut systemstate, emulator, target); - - // Extract priority lists - for i in 0..NUM_PRIOS { - let target : GuestAddr = listbytes*GuestAddr::try_from(i).unwrap()+h.ready_queues; - let _temp = read_freertos_list(&mut systemstate, emulator, target); - systemstate.prio_ready_lists[i] = _temp.0; - systemstate.read_invalid |= !_temp.1; - } - } else { - systemstate.read_invalid = true; - } - systemstate.mem_reads = unsafe { MEM_READ.take().unwrap_or_default() }; - - - - unsafe { CURRENT_SYSTEMSTATE_VEC.push(systemstate); } -} - -//============================= Trace job response times - -pub static mut JOBS_DONE : Vec<(u64, String)> = vec![]; - -pub fn job_done_hook( - hooks: &mut EmulatorModules, - _state: Option<&mut S>, - _pc: GuestAddr, -) -where - S: UsesInput, - QT: EmulatorModuleTuple, -{ - let emulator = hooks.qemu(); - let h = hooks.modules().match_first_type::().expect("QemuSystemHelper not found in helper tupel"); - let curr_tcb_addr : freertos::void_ptr = freertos::emu_lookup::lookup(&emulator, h.tcb_addr); - if curr_tcb_addr == 0 { - return; - }; - let current_tcb : TCB_t = freertos::emu_lookup::lookup(&emulator,curr_tcb_addr); - let tmp = unsafe {std::mem::transmute::<[i8; 10],[u8; 10]>(current_tcb.pcTaskName)}; - let name : String = std::str::from_utf8(&tmp).expect("TCB name was not utf8").chars().filter(|x| *x != '\0').collect::(); - unsafe { JOBS_DONE.push((get_icount(&emulator), name)); } -} - -//============================= Trace interrupt service routines - -pub fn exec_isr_hook( - hooks: &mut EmulatorModules, - _state: Option<&mut S>, - pc: GuestAddr, -) -where - S: UsesInput, - QT: EmulatorModuleTuple, -{ - let emulator = hooks.qemu(); - let h = hooks.modules().match_first_type::().expect("QemuSystemHelper not found in helper tupel"); - let src = read_rec_return_stackframe(&emulator, 0xfffffffc); - trigger_collection(&emulator, (src, pc), CaptureEvent::ISRStart, h); - // println!("Exec ISR Call {:#x} {:#x} {}", src, pc, get_icount(emulator)); -} - -//============================= Trace syscalls and returns - -pub fn gen_jmp_is_syscall( - hooks: &mut EmulatorModules, - _state: Option<&mut S>, - src: GuestAddr, - dest: GuestAddr, -) -> Option -where - S: UsesInput, - QT: EmulatorModuleTuple, -{ - if let Some(h) = hooks.modules().match_first_type::() { - if h.app_range.contains(&src) && !h.app_range.contains(&dest) && in_any_range(&h.isr_ranges,src).is_none() { - if let Some(_) = in_any_range(&h.api_fn_ranges,dest) { - // println!("New jmp {:x} {:x}", src, dest); - // println!("API Call Edge {:x} {:x}", src, dest); - return Some(1); - // TODO: trigger collection right here - // otherwise there can be a race-condition, where LAST_API_CALL is set before the api starts, if the interrupt handler calls an api function, it will misidentify the callsite of that api call - } - } else if dest == 0 { // !h.app_range.contains(&src) && - if let Some(_) = in_any_range(&h.api_fn_ranges, src) { - // println!("API Return Edge {:#x}", src); - return Some(2); - } - if let Some(_) = in_any_range(&h.isr_ranges, src) { - // println!("ISR Return Edge {:#x}", src); - return Some(3); - } - } - } - return None; -} - -pub fn trace_jmp( - hooks: &mut EmulatorModules, - _state: Option<&mut S>, - src: GuestAddr, mut dest: GuestAddr, id: u64 -) -where - S: UsesInput, - QT: EmulatorModuleTuple, -{ - let h = hooks.modules().match_first_type::().expect("QemuSystemHelper not found in helper tupel"); - let emulator = hooks.qemu(); - if id == 1 { // API call - trigger_collection(&emulator, (src, dest), CaptureEvent::APIStart, h); - // println!("Exec API Call {:#x} {:#x} {}", src, dest, get_icount(emulator)); - } else if id == 2 { // API return - // Ignore returns to other APIs or ISRs. We only account for the first call depth of API calls from user space. - if in_any_range(&h.api_fn_ranges, dest).is_none() && in_any_range(&h.isr_ranges, dest).is_none() { - - let mut edge = (0, 0); - edge.0=in_any_range(&h.api_fn_ranges, src).unwrap().start; - edge.1=dest; - - trigger_collection(&emulator, edge, CaptureEvent::APIEnd, h); - // println!("Exec API Return Edge {:#x} {:#x} {}", src, dest, get_icount(emulator)); - } - } else if id == 3 { // ISR return - dest = read_rec_return_stackframe(&emulator, dest); - - let mut edge = (0, 0); - edge.0=in_any_range(&h.isr_ranges, src).unwrap().start; - edge.1=dest; - - trigger_collection(&emulator, edge, CaptureEvent::ISREnd, h); - // println!("Exec ISR Return Edge {:#x} {:#x} {}", src, dest, get_icount(emulator)); - } -} - -//============================= Read Hooks -#[allow(unused)] -pub fn gen_read_is_input( - hooks: &mut EmulatorModules, - _state: Option<&mut S>, - pc: GuestAddr, - _addr: *mut TCGTemp, - _info: MemAccessInfo, -) -> Option -where - S: UsesInput, - QT: EmulatorModuleTuple, -{ - if let Some(h) = hooks.modules().match_first_type::() { - if h.app_range.contains(&pc) { - // println!("gen_read {:x}", pc); - return Some(1); - } - } - return None; -} - -static mut INPUT_MEM : Range = 0..0; -static mut MEM_READ : Option> = None; - -#[allow(unused)] -pub fn trace_reads( - hooks: &mut EmulatorModules, - _state: Option<&mut S>, - _id: u64, - addr: GuestAddr, - _size: usize -) -where - S: UsesInput, - QT: EmulatorModuleTuple, -{ - if unsafe { INPUT_MEM.contains(&addr) } { - let emulator = hooks.qemu(); - let mut buf : [u8; 1] = [0]; - unsafe {emulator.read_mem(addr, &mut buf);} - if unsafe { MEM_READ.is_none() } { - unsafe { MEM_READ = Some(Vec::from([(addr, buf[0])])) }; - } else { - unsafe { MEM_READ.as_mut().unwrap().push((addr, buf[0])) }; - } - // println!("exec_read {:x} {}", addr, size); - } -} //============================= Utility functions -fn get_icount(emulator : &libafl_qemu::Qemu) -> u64 { +pub fn get_icount(emulator : &libafl_qemu::Qemu) -> u64 { unsafe { // TODO: investigate why can_do_io is not set sometimes, as this is just a workaround let c = emulator.cpu_from_index(0); @@ -556,7 +97,7 @@ fn get_icount(emulator : &libafl_qemu::Qemu) -> u64 { } } -fn read_rec_return_stackframe(emu : &libafl_qemu::Qemu, lr : GuestAddr) -> GuestAddr { +pub fn read_rec_return_stackframe(emu : &libafl_qemu::Qemu, lr : GuestAddr) -> GuestAddr { let lr_ = lr & u32::MAX-1; if lr_ == 0xfffffffc || lr_ == 0xFFFFFFF8 || lr_ == 0xFFFFFFF0 { unsafe { diff --git a/fuzzers/FRET/src/systemstate/mod.rs b/fuzzers/FRET/src/systemstate/mod.rs index 6195208020..51f108c174 100644 --- a/fuzzers/FRET/src/systemstate/mod.rs +++ b/fuzzers/FRET/src/systemstate/mod.rs @@ -10,9 +10,6 @@ use hashbrown::HashMap; use serde::{Deserialize, Serialize}; use itertools::Itertools; -use freertos::TCB_t; - -pub mod freertos; pub mod helpers; pub mod observers; pub mod feedbacks; @@ -20,9 +17,7 @@ pub mod schedulers; pub mod stg; pub mod mutational; pub mod report; - -// Constants -const NUM_PRIOS: usize = 15; +pub mod target_os; //============================= Struct definitions @@ -43,154 +38,10 @@ pub enum CaptureEvent { - ReducedFreeRTOSSystemState: Generalized state of the system, without execution context - ExecInterval: Some interval of execution between instants - AtomicBasicBlock: A single-entry multiple-exit region between api calls. May be used referenced in multiple intervals. - - JobInstance: A single execution of a task, records the place and input read - - TaskJob: Generalized Job instance, records the worst inputs seen so far + - RTOSJob: A single execution of a task, records the place and input read + - RTOSTask: Generalized Job instance, records the worst inputs seen so far */ -// ============================= State info - -/// Raw info Dump from Qemu -#[derive(Debug, Default)] -pub struct RawFreeRTOSSystemState { - qemu_tick: u64, - current_tcb: TCB_t, - prio_ready_lists: [freertos::List_t; NUM_PRIOS], - delay_list: freertos::List_t, - delay_list_overflow: freertos::List_t, - dumping_ground: HashMap, - read_invalid: bool, - input_counter: u32, - edge: (GuestAddr,GuestAddr), - capture_point: (CaptureEvent,String), - mem_reads: Vec<(u32, u8)> -} -/// List of system state dumps from EmulatorModules -static mut CURRENT_SYSTEMSTATE_VEC: Vec = vec![]; - -/// A reduced version of freertos::TCB_t -#[derive(Debug, Default, Serialize, Deserialize, Clone)] -pub struct RefinedTCB { - pub task_name: String, - pub priority: u32, - pub base_priority: u32, - mutexes_held: u32, - // notify_value: u32, - notify_state: u8, -} - -impl PartialEq for RefinedTCB { - fn eq(&self, other: &Self) -> bool { - let ret = self.task_name == other.task_name && - self.priority == other.priority && - self.base_priority == other.base_priority; - #[cfg(feature = "do_hash_notify_state")] - let ret = ret && self.notify_state == other.notify_state; - ret - } -} - -impl Hash for RefinedTCB { - fn hash(&self, state: &mut H) { - self.task_name.hash(state); - self.priority.hash(state); - self.mutexes_held.hash(state); - #[cfg(feature = "do_hash_notify_state")] - self.notify_state.hash(state); - // self.notify_value.hash(state); - } -} - -impl RefinedTCB { - pub fn from_tcb(input: &TCB_t) -> Self { - unsafe { - let tmp = std::mem::transmute::<[i8; 10],[u8; 10]>(input.pcTaskName); - let name : String = std::str::from_utf8(&tmp).expect("TCB name was not utf8").chars().filter(|x| *x != '\0').collect::(); - Self { - task_name: name, - priority: input.uxPriority, - base_priority: input.uxBasePriority, - mutexes_held: input.uxMutexesHeld, - // notify_value: input.ulNotifiedValue[0], - notify_state: input.ucNotifyState[0], - } - } - } - pub fn from_tcb_owned(input: TCB_t) -> Self { - unsafe { - let tmp = std::mem::transmute::<[i8; 10],[u8; 10]>(input.pcTaskName); - let name : String = std::str::from_utf8(&tmp).expect("TCB name was not utf8").chars().filter(|x| *x != '\0').collect::(); - Self { - task_name: name, - priority: input.uxPriority, - base_priority: input.uxBasePriority, - mutexes_held: input.uxMutexesHeld, - // notify_value: input.ulNotifiedValue[0], - notify_state: input.ucNotifyState[0], - } - } - } -} - -/// Reduced information about a systems state, without any execution context -#[derive(Debug, Default, Serialize, Deserialize, Clone)] -pub struct ReducedFreeRTOSSystemState { - // pub tick: u64, - pub current_task: RefinedTCB, - ready_list_after: Vec, - delay_list_after: Vec, - read_invalid: bool, - // edge: (Option,Option), - // pub capture_point: (CaptureEvent,String), - // input_counter: u32 -} -impl PartialEq for ReducedFreeRTOSSystemState { - fn eq(&self, other: &Self) -> bool { - self.current_task == other.current_task && self.ready_list_after == other.ready_list_after && - self.delay_list_after == other.delay_list_after && self.read_invalid == other.read_invalid - // && self.edge == other.edge - // && self.capture_point == other.capture_point - } -} - -impl Hash for ReducedFreeRTOSSystemState { - fn hash(&self, state: &mut H) { - self.current_task.hash(state); - self.ready_list_after.hash(state); - self.delay_list_after.hash(state); - self.read_invalid.hash(state); - } -} -impl ReducedFreeRTOSSystemState { - // fn get_tick(&self) -> u64 { - // self.tick - // } - - pub fn print_lists(&self) -> String { - let mut ret = String::from("+"); - for j in self.ready_list_after.iter() { - ret.push_str(format!(" {}", j.task_name).as_str()); - } - ret.push_str("\n-"); - for j in self.delay_list_after.iter() { - ret.push_str(format!(" {}", j.task_name).as_str()); - } - ret - } - pub fn get_hash(&self) -> u64 { - let mut h = DefaultHasher::new(); - self.hash(&mut h); - h.finish() - } -} - -impl fmt::Display for ReducedFreeRTOSSystemState { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let ready = self.ready_list_after.iter().map(|x| x.task_name.clone()).collect::>().join(" "); - let delay = self.delay_list_after.iter().map(|x| x.task_name.clone()).collect::>().join(" "); - write!(f, "Valid: {} | Current: {} | Ready: {} | Delay: {}", u32::from(!self.read_invalid), self.current_task.task_name, ready, delay) - } -} - // ============================= Interval info // #[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] @@ -209,13 +60,13 @@ pub struct ExecInterval { pub start_capture: (CaptureEvent, String), pub end_capture: (CaptureEvent, String), pub level: u8, - tick_spend_preempted: u64, + // tick_spend_preempted: u64, pub abb: Option } impl ExecInterval { pub fn get_exec_time(&self) -> u64 { - self.end_tick-self.start_tick-self.tick_spend_preempted + self.end_tick-self.start_tick//-self.tick_spend_preempted } pub fn is_valid(&self) -> bool { self.start_tick != 0 || self.end_tick != 0 @@ -226,18 +77,18 @@ impl ExecInterval { } /// Attach this interval to the later one, keep a record of the time spend preempted - pub fn try_unite_with_later_interval(&mut self, later_interval : &mut Self) -> bool { - if self.end_state!=later_interval.start_state || self.abb!=later_interval.abb || !self.is_valid() || !later_interval.is_valid() { - return false; - } - // assert_eq!(self.end_state, later_interval.start_state); - // assert_eq!(self.abb, later_interval.abb); - later_interval.tick_spend_preempted += self.tick_spend_preempted + (later_interval.start_tick-self.end_tick); - later_interval.start_tick = self.start_tick; - later_interval.start_state = self.start_state; - self.invaildate(); - return true; - } + // pub fn try_unite_with_later_interval(&mut self, later_interval : &mut Self) -> bool { + // if self.end_state!=later_interval.start_state || self.abb!=later_interval.abb || !self.is_valid() || !later_interval.is_valid() { + // return false; + // } + // // assert_eq!(self.end_state, later_interval.start_state); + // // assert_eq!(self.abb, later_interval.abb); + // later_interval.tick_spend_preempted += self.tick_spend_preempted + (later_interval.start_tick-self.end_tick); + // later_interval.start_tick = self.start_tick; + // later_interval.start_state = self.start_state; + // self.invaildate(); + // return true; + // } pub fn get_hash_index(&self) -> (u64, u64) { return (self.start_state, self.abb.as_ref().expect("ABB not set").get_hash()) @@ -357,20 +208,13 @@ impl AtomicBasicBlock { } -fn get_task_names(trace: &Vec) -> HashSet { - let mut ret: HashSet<_, _> = HashSet::new(); - for state in trace { - ret.insert(state.current_task.task_name.to_string()); - } - ret -} libafl_bolts::impl_serdeany!(AtomicBasicBlock); // ============================= Job instances #[derive(Debug, Default, Serialize, Deserialize, Clone)] -pub struct JobInstance { +pub struct RTOSJob { pub name: String, pub mem_reads: Vec<(u32, u8)>, pub release: u64, @@ -381,18 +225,18 @@ pub struct JobInstance { hash_cache: u64 } -impl PartialEq for JobInstance { +impl PartialEq for RTOSJob { fn eq(&self, other: &Self) -> bool { self.abbs == other.abbs } } -impl Eq for JobInstance {} -impl Hash for JobInstance { +impl Eq for RTOSJob {} +impl Hash for RTOSJob { fn hash(&self, state: &mut H) { self.abbs.hash(state); } } -impl JobInstance { +impl RTOSJob { pub fn get_hash(&mut self) -> u64 { if self.hash_cache == 0 { let mut s = DefaultHasher::new(); @@ -415,7 +259,7 @@ impl JobInstance { // ============================= Generalized job instances #[derive(Debug, Default, Serialize, Deserialize, Clone)] -pub struct TaskJob { +pub struct RTOSTask { pub name: String, pub worst_bytes: Vec, pub woet_ticks: u64, @@ -424,18 +268,18 @@ pub struct TaskJob { hash_cache: u64 } -impl PartialEq for TaskJob { +impl PartialEq for RTOSTask { fn eq(&self, other: &Self) -> bool { self.abbs == other.abbs } } -impl Eq for TaskJob {} -impl Hash for TaskJob { +impl Eq for RTOSTask {} +impl Hash for RTOSTask { fn hash(&self, state: &mut H) { self.abbs.hash(state); } } -impl TaskJob { +impl RTOSTask { pub fn get_hash(&mut self) -> u64 { if self.hash_cache == 0 { let mut s = DefaultHasher::new(); @@ -453,7 +297,7 @@ impl TaskJob { self.hash_cache } } - pub fn try_update(&mut self, other: &JobInstance) -> bool { + pub fn try_update(&mut self, other: &RTOSJob) -> bool { assert_eq!(self.get_hash(), other.get_hash_cached()); let mut ret = false; if other.exec_ticks > self.woet_ticks { @@ -464,7 +308,7 @@ impl TaskJob { } ret } - pub fn from_instance(input: &JobInstance) -> Self { + pub fn from_instance(input: &RTOSJob) -> Self { let c = input.get_hash_cached(); Self { name: input.name.clone(), @@ -475,7 +319,7 @@ impl TaskJob { hash_cache: c } } - pub fn map_bytes_onto(&self, input: &JobInstance, offset: Option) -> Vec<(u32,u8)> { + pub fn map_bytes_onto(&self, input: &RTOSJob, offset: Option) -> Vec<(u32,u8)> { if input.mem_reads.len() == 0 {return vec![];} let ret = input.mem_reads.iter().take(self.worst_bytes.len()).enumerate().filter_map(|(idx,(addr,oldbyte))| if self.worst_bytes[idx]!=*oldbyte {Some((*addr-offset.unwrap_or_default(), self.worst_bytes[idx]))} else {None}).collect(); // eprintln!("Mapped: {:?}", ret); @@ -485,48 +329,3 @@ impl TaskJob { // ============================= Per testcase metadata - -// Wrapper around Vec to attach as Metadata -#[derive(Debug, Default, Serialize, Deserialize, Clone)] -pub struct FreeRTOSSystemStateMetadata { - pub inner: Vec, - // TODO: Add abbs and memory reads - trace_length: usize, - indices: Vec, // Hashed enumeration of States - tcref: isize, -} -impl FreeRTOSSystemStateMetadata { - pub fn new(inner: Vec) -> Self{ - let tmp = inner.iter().enumerate().map(|x| compute_hash(x) as usize).collect(); - Self {trace_length: inner.len(), inner: inner, indices: tmp, tcref: 0} - } -} -pub fn compute_hash(obj: T) -> u64 -where - T: Hash -{ - let mut s = DefaultHasher::new(); - obj.hash(&mut s); - s.finish() -} - -// impl AsSlice for FreeRTOSSystemStateMetadata { -// /// Convert the slice of system-states to a slice of hashes over enumerated states -// fn as_slice(&self) -> &[usize] { -// self.indices.as_slice() -// } - -// type Entry = usize; -// } - -impl HasRefCnt for FreeRTOSSystemStateMetadata { - fn refcnt(&self) -> isize { - self.tcref - } - - fn refcnt_mut(&mut self) -> &mut isize { - &mut self.tcref - } -} - -libafl_bolts::impl_serdeany!(FreeRTOSSystemStateMetadata); diff --git a/fuzzers/FRET/src/systemstate/mutational.rs b/fuzzers/FRET/src/systemstate/mutational.rs index f861032899..3b35728aa5 100644 --- a/fuzzers/FRET/src/systemstate/mutational.rs +++ b/fuzzers/FRET/src/systemstate/mutational.rs @@ -13,13 +13,13 @@ use libafl::{ }; use libafl::prelude::State; use petgraph::{graph::NodeIndex, graph::{self, DiGraph}}; -use crate::{time::clock::{IcHist, QEMU_ISNS_PER_USEC}, fuzzer::{DO_NUM_INTERRUPT, FIRST_INT, MAX_NUM_INTERRUPT}, systemstate::{stg::{STGFeedbackState, STGNodeMetadata}, CaptureEvent, ExecInterval, FreeRTOSSystemStateMetadata, ReducedFreeRTOSSystemState}}; +use crate::{time::clock::{IcHist, QEMU_ISNS_PER_USEC}, fuzzer::{DO_NUM_INTERRUPT, FIRST_INT, MAX_NUM_INTERRUPT}, systemstate::{stg::{STGFeedbackState, STGNodeMetadata}, CaptureEvent, ExecInterval}}; use libafl::state::HasCurrentTestcase; use std::borrow::Cow; use simple_moving_average::SMA; -use super::{stg::{STGEdge, STGNode}, JobInstance}; +use super::{stg::{STGEdge, STGNode}, target_os::TargetSystem, RTOSJob}; // pub static mut MINIMUM_INTER_ARRIVAL_TIME : u32 = 1000 /*us*/ * QEMU_ISNS_PER_USEC; // one isn per 2**4 ns @@ -68,21 +68,33 @@ pub fn interrupt_times_to_input_bytes(interrupt_times: &[u32]) -> Vec { //======================= Custom mutator -fn is_interrupt_handler(graph: &DiGraph, node: NodeIndex) -> bool { +fn is_interrupt_handler(graph: &DiGraph, STGEdge>, node: NodeIndex) -> bool +where + SYS: TargetSystem, +{ graph.edges_directed(node as NodeIndex, petgraph::Direction::Incoming).any(|x| x.weight().event == CaptureEvent::ISRStart) } -fn has_interrupt_handler_non_systick(graph: &DiGraph, node: NodeIndex) -> bool { +fn has_interrupt_handler_non_systick(graph: &DiGraph, STGEdge>, node: NodeIndex) -> bool +where + SYS: TargetSystem, +{ graph.edges_directed(node as NodeIndex, petgraph::Direction::Outgoing).any(|x| x.weight().event == CaptureEvent::ISRStart && x.weight().name!="xPortSysTickHandler") } -fn is_candidate_for_new_branches(graph: &DiGraph, node: NodeIndex) -> bool { +fn is_candidate_for_new_branches(graph: &DiGraph, STGEdge>, node: NodeIndex) -> bool +where + SYS: TargetSystem, +{ !has_interrupt_handler_non_systick(graph, node) && !is_interrupt_handler(graph, node) } // TODO: this can be much more efficient, if the graph stored snapshots of the state and input progress was tracked /// Determines if a given node in the state transition graph (STG) is a candidate for introducing new branches. -pub fn try_force_new_branches(interrupt_ticks : &[u32], fbs: &STGFeedbackState, meta: &STGNodeMetadata, config: (usize, u32)) -> Option> { +pub fn try_force_new_branches(interrupt_ticks : &[u32], fbs: &STGFeedbackState, meta: &STGNodeMetadata, config: (usize, u32)) -> Option> +where + SYS: TargetSystem, +{ let mut new = false; let mut new_interrupt_times = Vec::new(); for (num,&interrupt_time) in interrupt_ticks.iter().enumerate() { @@ -112,14 +124,14 @@ pub fn try_force_new_branches(interrupt_ticks : &[u32], fbs: &STGFeedbackState, /// The default mutational stage #[derive(Clone, Debug)] -pub struct InterruptShiftStage { +pub struct InterruptShiftStage { #[allow(clippy::type_complexity)] - phantom: PhantomData<(E, EM, Z)>, + phantom: PhantomData<(E, EM, Z, SYS)>, interrup_config: Vec<(usize,u32)>, success: simple_moving_average::SingleSumSMA } -impl InterruptShiftStage +impl InterruptShiftStage where E: UsesState, EM: UsesState, @@ -135,7 +147,7 @@ static mut num_stage_execs : u64 = 0; static mut sum_reruns : u64 = 0; static mut sum_interesting_reruns : u64 = 0; -impl InterruptShiftStage +impl InterruptShiftStage where E: UsesState, EM: UsesState, @@ -144,9 +156,10 @@ where Z::State: MaybeHasClientPerfMonitor + HasCorpus + HasRand + HasMetadata + HasNamedMetadata, ::Input: Input, Z::State: UsesInput>, - I: HasMutatorBytes + Default + I: HasMutatorBytes + Default, + SYS: TargetSystem, { - fn report_stats(&self, state: &mut as UsesState>::State, manager: &mut EM) { + fn report_stats(&self, state: &mut as libafl::state::UsesState>::State, manager: &mut EM) { unsafe { let _ = manager.fire( state, @@ -163,7 +176,7 @@ where } } -impl Stage for InterruptShiftStage +impl Stage for InterruptShiftStage where E: UsesState, EM: UsesState, @@ -172,6 +185,7 @@ where <::State as HasCorpus>::Corpus: Corpus, //delete me EM: EventFirer, I: Default + Input + HasMutatorBytes, + SYS: TargetSystem, { fn perform( &mut self, @@ -236,7 +250,7 @@ where else if choice <= 75 { // 0.5 * 0.25 = 12.5% of cases let feedbackstate = match state .named_metadata_map() - .get::("stgfeedbackstate") { + .get::>("stgfeedbackstate") { Some(s) => s, Option::None => { panic!("STGfeedbackstate not visible") @@ -439,7 +453,7 @@ where } } -impl UsesState for InterruptShiftStage +impl UsesState for InterruptShiftStage where E: UsesState, EM: UsesState, @@ -450,7 +464,10 @@ where } -pub fn try_worst_snippets(bytes : &[u8], fbs: &STGFeedbackState, meta: &STGNodeMetadata) -> Option> { +pub fn try_worst_snippets(bytes : &[u8], fbs: &STGFeedbackState, meta: &STGNodeMetadata) -> Option> +where + SYS: TargetSystem, +{ let mut new = false; let mut ret = Vec::new(); for (num,interval) in meta.intervals().iter().enumerate() { @@ -466,25 +483,26 @@ static mut num_snippet_success : u64 = 0; /// The default mutational stage #[derive(Clone, Debug, Default)] -pub struct STGSnippetStage { +pub struct STGSnippetStage { #[allow(clippy::type_complexity)] - phantom: PhantomData<(E, EM, Z)>, + phantom: PhantomData<(E, EM, Z, SYS)>, input_addr: u32 } -impl STGSnippetStage +impl STGSnippetStage where E: UsesState, EM: UsesState, Z: Evaluator, Z::State: MaybeHasClientPerfMonitor + HasCorpus + HasRand, + SYS: TargetSystem, { pub fn new(input_addr: u32) -> Self { Self { phantom: PhantomData, input_addr } } } -impl STGSnippetStage +impl STGSnippetStage where E: UsesState, EM: UsesState, @@ -493,9 +511,10 @@ where Z::State: MaybeHasClientPerfMonitor + HasCorpus + HasRand + HasMetadata + HasNamedMetadata, ::Input: Input, Z::State: UsesInput>, - I: HasMutatorBytes + Default + I: HasMutatorBytes + Default, + SYS: TargetSystem, { - fn report_stats(&self, state: &mut as UsesState>::State, manager: &mut EM) { + fn report_stats(&self, state: &mut as UsesState>::State, manager: &mut EM) { unsafe { let _ = manager.fire( state, @@ -512,7 +531,7 @@ where } } -impl Stage for STGSnippetStage +impl Stage for STGSnippetStage where E: UsesState, EM: UsesState, @@ -523,7 +542,8 @@ where Z::State: UsesInput>, I: HasMutatorBytes + Default, Z::State: HasCurrentTestcase+HasCorpus+HasCurrentCorpusId, - ::Corpus: Corpus> + ::Corpus: Corpus>, + SYS: TargetSystem, { fn perform( &mut self, @@ -546,7 +566,7 @@ where if let Some(meta) = current_case.metadata_map().get::() { let feedbackstate = match state .named_metadata_map() - .get::("stgfeedbackstate") { + .get::>("stgfeedbackstate") { Some(s) => s, Option::None => { panic!("STGfeedbackstate not visible") @@ -590,12 +610,13 @@ where } } -impl UsesState for STGSnippetStage +impl UsesState for STGSnippetStage where E: UsesState, EM: UsesState, Z: Evaluator, Z::State: MaybeHasClientPerfMonitor + HasCorpus + HasRand, + SYS: TargetSystem, { type State = Z::State; } \ No newline at end of file diff --git a/fuzzers/FRET/src/systemstate/observers.rs b/fuzzers/FRET/src/systemstate/observers.rs index f92593ddfb..233c002399 100644 --- a/fuzzers/FRET/src/systemstate/observers.rs +++ b/fuzzers/FRET/src/systemstate/observers.rs @@ -15,18 +15,13 @@ use std::rc::Rc; use std::cell::RefCell; use std::collections::VecDeque; use std::borrow::Cow; +use itertools::Itertools; -use super::helpers::USR_ISR_SYMBOLS; -use super::JobInstance; +use super::target_os::TargetSystem; +use super::RTOSJob; use super::{ AtomicBasicBlock, ExecInterval}; -use super::{ - CURRENT_SYSTEMSTATE_VEC, - RawFreeRTOSSystemState, - RefinedTCB, - ReducedFreeRTOSSystemState, - freertos::{List_t, TCB_t, rtos_struct, rtos_struct::*}, - helpers::JOBS_DONE, -}; +use crate::systemstate::target_os::SystemState; +use crate::systemstate::target_os::*; //============================= Observer @@ -34,120 +29,76 @@ use super::{ /// that will get updated by the target. #[derive(Serialize, Deserialize, Debug)] #[allow(clippy::unsafe_derive_deserialize)] -pub struct QemuSystemStateObserver +pub struct QemuSystemStateObserver +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, { - pub last_run: Vec, - pub last_states: HashMap, - pub last_trace: Vec, - pub last_reads: Vec>, - pub last_input: I, - pub job_instances: Vec, + last_run: Vec, + last_states: HashMap, + last_trace: Vec, + last_reads: Vec>, + last_input: I, + job_instances: Vec, pub do_report: bool, - pub worst_job_instances: HashMap, + worst_job_instances: HashMap, pub select_task: Option, pub success: bool, name: Cow<'static, str>, } -impl Observer for QemuSystemStateObserver + +impl Observer for QemuSystemStateObserver where S: UsesInput + HasMetadata, S::Input: Default, I: Clone, + SYS: TargetSystem, { #[inline] fn pre_exec(&mut self, _state: &mut S, _input: &I) -> Result<(), Error> { - unsafe {CURRENT_SYSTEMSTATE_VEC.clear(); } Ok(()) } #[inline] - fn post_exec(&mut self, _state: &mut S, _input: &I, _exit_kind: &ExitKind) -> Result<(), Error> { - // unsafe {self.last_run = invalidate_ineffective_isr(refine_system_states(&mut CURRENT_SYSTEMSTATE_VEC));} - unsafe { - let temp = refine_system_states(CURRENT_SYSTEMSTATE_VEC.split_off(0)); - // fix_broken_trace(&mut temp.1); - self.last_run = temp.0.clone(); - // println!("{:?}",temp); - let temp = states2intervals(temp.0, temp.1); - self.last_trace = temp.0; - self.last_reads = temp.1; - self.last_states = temp.2; - self.success = temp.3; - #[cfg(feature="trace_job_response_times")] - { - let metadata =_state.metadata_map_mut(); - let releases = get_releases(&self.last_trace, &self.last_states); - // println!("Releases: {:?}",&releases); - let jobs_done = JOBS_DONE.split_off(0); - let (job_instances, do_report) = get_release_response_pairs(&releases, &jobs_done); - self.do_report = do_report; - let job_instances = job_instances.into_iter().map(|x| { - let intervals = self.last_trace.iter().enumerate().filter(|y| y.1.start_tick <= x.1 && y.1.end_tick >= x.0 && x.2 == y.1.get_task_name_unchecked()).map(|(idx,x)| (x, &self.last_reads[idx])).collect::>(); - let (abbs, rest) : (Vec<_>, Vec<_>) = intervals.chunk_by(|a,b| a.0.abb.as_ref().unwrap().instance_eq(b.0.abb.as_ref().unwrap())).into_iter().map(|intervals| (intervals[0].0.abb.as_ref().unwrap().clone(), (intervals.iter().fold(0, |sum, x| sum+x.0.get_exec_time()), intervals.iter().fold(Vec::new(), |mut sum, x| {sum.extend(x.1.iter()); sum})))).unzip(); - let (ticks_per_abb, mem_reads) : (Vec<_>, Vec<_>) = rest.into_iter().unzip(); - JobInstance { - name: x.2.clone(), - mem_reads: mem_reads.into_iter().flatten().collect(), // TODO: add read values - release: x.0, - response: x.1, - exec_ticks: ticks_per_abb.iter().sum(), - ticks_per_abb: ticks_per_abb, - abbs: abbs, - hash_cache: 0 - } - }).collect::>(); - // println!("Instances: {:?}",&job_instances); - self.job_instances = job_instances; - let observer = &self; - let mut worst_case_per_task : HashMap = HashMap::new(); - observer.job_instances.iter().for_each(|x| { - if worst_case_per_task.get(&x.name).is_some() { - let old = worst_case_per_task.get_mut(&x.name).unwrap(); - if x.exec_ticks > old.exec_ticks { - old.exec_ticks=x.exec_ticks; - } - } else { - worst_case_per_task.insert(x.name.clone(), x.clone()); - } - }); - self.worst_job_instances = worst_case_per_task; - // copy-paste form clock observer - { - let hist = metadata.get_mut::(); - let timestamp = SystemTime::now().duration_since(unsafe {FUZZ_START_TIMESTAMP}).unwrap().as_millis(); - match hist { - Option::None => { - metadata.insert(IcHist(vec![(self.last_runtime(), timestamp)], - (self.last_runtime(), timestamp))); - } - Some(v) => { - v.0.push((self.last_runtime(), timestamp)); - if v.1.0 < self.last_runtime() { - v.1 = (self.last_runtime(), timestamp); - } - } + fn post_exec(&mut self, state: &mut S, _input: &I, _exit_kind: &ExitKind) -> Result<(), Error> { + // let trace =state.metadata::().expect("TraceData not found"); + + // copy-paste form clock observer + { + let hist = state.metadata_mut::(); + let timestamp = SystemTime::now().duration_since(unsafe {FUZZ_START_TIMESTAMP}).unwrap().as_millis(); + match hist { + Err(_) => { + state.add_metadata(IcHist(vec![(self.last_runtime(), timestamp)], + (self.last_runtime(), timestamp))); + } + Ok(v) => { + v.0.push((self.last_runtime(), timestamp)); + if v.1.0 < self.last_runtime() { + v.1 = (self.last_runtime(), timestamp); } } } } - // let abbs = extract_abbs_from_trace(&self.last_run); - // println!("{:?}",abbs); - // let abbs = trace_to_state_abb(&self.last_run); - // println!("{:?}",abbs); - self.last_input=_input.clone(); Ok(()) } } -impl Named for QemuSystemStateObserver +impl Named for QemuSystemStateObserver +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, { fn name(&self) -> &Cow<'static, str> { &self.name } } -impl HasLen for QemuSystemStateObserver +impl HasLen for QemuSystemStateObserver +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, { #[inline] fn len(&self) -> usize { @@ -155,8 +106,11 @@ impl HasLen for QemuSystemStateObserver } } -impl QemuSystemStateObserver -where I: Default { +impl QemuSystemStateObserver +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, + I: Default { pub fn new(select_task: &Option) -> Self { Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), worst_job_instances: HashMap::new(), do_report: false, select_task: select_task.clone(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false, job_instances: vec![]} } @@ -164,491 +118,16 @@ where I: Default { self.select_task.as_ref().map(|x| self.worst_job_instances.get(x).map(|y| y.response-y.release).unwrap_or(0).clone()).unwrap_or(unsafe{libafl_qemu::sys::icount_get_raw()}) } } -impl Default for QemuSystemStateObserver -where I: Default { +impl Default for QemuSystemStateObserver +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, +I: Default { fn default() -> Self { Self::new(&None) } } -//============================= Parsing helpers - -/// Parse a List_t containing TCB_t into Vec from cache. Consumes the elements from cache -fn tcb_list_to_vec_cached(list: List_t, dump: &mut HashMap) -> Vec -{ - let mut ret : Vec = Vec::new(); - if list.uxNumberOfItems == 0 {return ret;} - let last_list_item = match dump.remove(&list.pxIndex).expect("List_t entry was not in Hashmap") { - List_Item_struct(li) => li, - List_MiniItem_struct(mli) => match dump.remove(&mli.pxNext).expect("MiniListItem pointer invaild") { - List_Item_struct(li) => li, - _ => panic!("MiniListItem of a non empty List does not point to ListItem"), - }, - _ => panic!("List_t entry was not a ListItem"), - }; - let mut next_index = last_list_item.pxNext; - let last_tcb = match dump.remove(&last_list_item.pvOwner).expect("ListItem Owner not in Hashmap") { - TCB_struct(t) => t, - _ => panic!("List content does not equal type"), - }; - for _ in 0..list.uxNumberOfItems-1 { - let next_list_item = match dump.remove(&next_index).expect("List_t entry was not in Hashmap") { - List_Item_struct(li) => li, - List_MiniItem_struct(mli) => match dump.remove(&mli.pxNext).expect("MiniListItem pointer invaild") { - List_Item_struct(li) => li, - _ => panic!("MiniListItem of a non empty List does not point to ListItem"), - }, - _ => panic!("List_t entry was not a ListItem"), - }; - match dump.remove(&next_list_item.pvOwner).expect("ListItem Owner not in Hashmap") { - TCB_struct(t) => {ret.push(t)}, - _ => panic!("List content does not equal type"), - } - next_index=next_list_item.pxNext; - } - ret.push(last_tcb); - ret -} - -/// Drains a List of raw SystemStates to produce a refined trace -/// returns: -/// - a Vec of ReducedFreeRTOSSystemStates -/// - a Vec of metadata tuples (qemu_tick, capture_event, capture_name, edge, mem_reads) -fn refine_system_states(mut input: Vec) -> (Vec, Vec<(u64, CaptureEvent, String, (u32, u32), Vec<(u32, u8)>)>) { - let mut ret = (Vec::<_>::new(), Vec::<_>::new()); - for mut i in input.drain(..) { - let cur = RefinedTCB::from_tcb_owned(i.current_tcb); - // println!("Refine: {} {:?} {:?} {:x}-{:x}", cur.task_name, i.capture_point.0, i.capture_point.1.to_string(), i.edge.0, i.edge.1); - // collect ready list - let mut collector = Vec::::new(); - for j in i.prio_ready_lists.into_iter().rev() { - let mut tmp = tcb_list_to_vec_cached(j,&mut i.dumping_ground).iter().map(|x| RefinedTCB::from_tcb(x)).collect(); - collector.append(&mut tmp); - } - // collect delay list - let mut delay_list : Vec:: = tcb_list_to_vec_cached(i.delay_list, &mut i.dumping_ground).iter().map(|x| RefinedTCB::from_tcb(x)).collect(); - let mut delay_list_overflow : Vec:: = tcb_list_to_vec_cached(i.delay_list_overflow, &mut i.dumping_ground).iter().map(|x| RefinedTCB::from_tcb(x)).collect(); - delay_list.append(&mut delay_list_overflow); - delay_list.sort_by(|a,b| a.task_name.cmp(&b.task_name)); - - ret.0.push(ReducedFreeRTOSSystemState { - current_task: cur, - ready_list_after: collector, - delay_list_after: delay_list, - read_invalid: i.read_invalid, - // input_counter: i.input_counter,//+IRQ_INPUT_BYTES_NUMBER, - }); - ret.1.push((i.qemu_tick, i.capture_point.0, i.capture_point.1.to_string(), i.edge, i.mem_reads)); - } - return ret; -} - - -// Find all task release times. -fn get_releases(trace: &Vec, states: &HashMap) -> Vec<(u64, String)> { - let mut ret = Vec::new(); - let mut initial_released = false; - for (_n, i) in trace.iter().enumerate() { - // The first release starts from xPortPendSVHandler - if !initial_released && i.start_capture.0 == CaptureEvent::ISREnd && i.start_capture.1 == "xPortPendSVHandler" { - let start_state = states.get(&i.start_state).expect("State not found"); - initial_released = true; - start_state.ready_list_after.iter().for_each(|x| { - ret.push((i.start_tick, x.task_name.clone())); - }); - continue; - } - // A timed release is SysTickHandler isr block that moves a task from the delay list to the ready list. - if i.start_capture.0 == CaptureEvent::ISRStart && ( i.start_capture.1 == "xPortSysTickHandler" || USR_ISR_SYMBOLS.contains(&i.start_capture.1.as_str()) ) { - // detect race-conditions, get start and end state from the nearest valid intervals - if states.get(&i.start_state).map(|x| x.read_invalid).unwrap_or(true) { - let mut start_index=None; - for n in 1.._n { - if let Some(interval_start) = trace.get(_n-n) { - let start_state = states.get(&interval_start.start_state).unwrap(); - if !start_state.read_invalid { - start_index = Some(_n-n); - break; - } - } else {break;} - }; - let mut end_index=None; - for n in (_n+1)..trace.len() { - if let Some(interval_end) = trace.get(n) { - let end_state = states.get(&interval_end.end_state).unwrap(); - if !end_state.read_invalid { - end_index = Some(n); - break; - } - } else {break;} - }; - if let Some(Some(start_state)) = start_index.map(|x| states.get(&trace[x].start_state)) { - if let Some(Some(end_state)) = end_index.map(|x| states.get(&trace[x].end_state)) { - end_state.ready_list_after.iter().for_each(|x| { - if x.task_name != end_state.current_task.task_name && x.task_name != start_state.current_task.task_name && !start_state.ready_list_after.iter().any(|y| x.task_name == y.task_name) { - ret.push((i.end_tick, x.task_name.clone())); - } - }); - } - } - } else - // canonical case, userspace -> isr -> userspace - if i.end_capture.0 == CaptureEvent::ISREnd { - let start_state = states.get(&i.start_state).expect("State not found"); - let end_state = states.get(&i.end_state).expect("State not found"); - end_state.ready_list_after.iter().for_each(|x| { - if x.task_name != end_state.current_task.task_name && x.task_name != start_state.current_task.task_name && !start_state.ready_list_after.iter().any(|y| x.task_name == y.task_name) { - ret.push((i.end_tick, x.task_name.clone())); - } - }); - // start_state.delay_list_after.iter().for_each(|x| { - // if !end_state.delay_list_after.iter().any(|y| x.task_name == y.task_name) { - // ret.push((i.end_tick, x.task_name.clone())); - // } - // }); - } else if i.end_capture.0 == CaptureEvent::ISRStart { - // Nested interrupts. Fast-forward to the end of the original interrupt, or the first valid state thereafter - // TODO: this may cause the same release to be registered multiple times - let mut isr_has_ended = false; - let start_state = states.get(&i.start_state).expect("State not found"); - for n in (_n+1)..trace.len() { - if let Some(interval_end) = trace.get(n) { - if interval_end.end_capture.1 == i.start_capture.1 || isr_has_ended { - let end_state = states.get(&interval_end.end_state).unwrap(); - isr_has_ended = true; - if !end_state.read_invalid { - end_state.ready_list_after.iter().for_each(|x| { - if x.task_name != end_state.current_task.task_name && x.task_name != start_state.current_task.task_name && !start_state.ready_list_after.iter().any(|y| x.task_name == y.task_name) { - ret.push((i.end_tick, x.task_name.clone())); - } - }); - break; - } - } - } else {break;} - }; - // if let Some(interval_end) = trace.get(_n+2) { - // if interval_end.start_capture.0 == CaptureEvent::ISREnd && interval_end.end_capture.0 == CaptureEvent::ISREnd && interval_end.end_capture.1 == i.start_capture.1 { - // let start_state = states.get(&i.start_state).expect("State not found"); - // let end_state = states.get(&interval_end.end_state).expect("State not found"); - // end_state.ready_list_after.iter().for_each(|x| { - // if x.task_name != end_state.current_task.task_name && x.task_name != start_state.current_task.task_name && !start_state.ready_list_after.iter().any(|y| x.task_name == y.task_name) { - // ret.push((i.end_tick, x.task_name.clone())); - // } - // }); - // } - // } - } - } - // Release driven by an API call. This produces a lot of false positives, as a job may block multiple times per instance. Despite this, aperiodic jobs not be modeled otherwise. If we assume the first release is the real one, we can filter out the rest. - if i.start_capture.0 == CaptureEvent::APIStart { - let api_start_state = states.get(&i.start_state).expect("State not found"); - let api_end_state = { - let mut end_index = _n; - for n in (_n)..trace.len() { - if trace[n].end_capture.0 == CaptureEvent::APIEnd || trace[n].end_capture.0 == CaptureEvent::End { - end_index = n; - break; - } else if n > _n && trace[n].level == 0 { // API Start -> ISR Start+End -> APP Continue - end_index = n-1; // any return to a regular app block is a fair point of comparison for the ready list, because scheduling has been performed - break; - } - }; - states.get(&trace[end_index].end_state).expect("State not found") - }; - api_end_state.ready_list_after.iter().for_each(|x| { - if x.task_name != api_start_state.current_task.task_name && !api_start_state.ready_list_after.iter().any(|y| x.task_name == y.task_name) { - ret.push((i.end_tick, x.task_name.clone())); - // eprintln!("Task {} released by API call at {:.1}ms", x.task_name, crate::time::clock::tick_to_time(i.end_tick).as_micros() as f32/1000.0); - } - }); - } - } - ret -} - -fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)>) -> (Vec<(u64, u64, String)>, bool) { - let mut maybe_error = false; - let mut ret = Vec::new(); - let mut ready : HashMap<&String, u64> = HashMap::new(); - let mut last_response : HashMap<&String, u64> = HashMap::new(); - let mut r = rel.iter().peekable(); - let mut d = resp.iter().peekable(); - loop { - while let Some(peek_rel) = r.peek() { - // Fill releases as soon as possible - if !ready.contains_key(&peek_rel.1) { - ready.insert(&peek_rel.1, peek_rel.0); - r.next(); - } else { - if let Some(peek_resp) = d.peek() { - if peek_resp.0 > peek_rel.0 { // multiple releases before response - // It is unclear which release is real - // maybe_error = true; - // eprintln!("Task {} released multiple times before response ({:.1}ms and {:.1}ms)", peek_rel.1, crate::time::clock::tick_to_time(ready[&peek_rel.1]).as_micros()/1000, crate::time::clock::tick_to_time(peek_rel.0).as_micros()/1000); - // ready.insert(&peek_rel.1, peek_rel.0); - r.next(); - } else { - // releases have overtaken responses, wait until the ready list clears up a bit - break; - } - } else { - // no more responses - break; - } - } - } - if let Some(next_resp) = d.next() { - if ready.contains_key(&next_resp.1) { - if ready[&next_resp.1] >= next_resp.0 { - if let Some(lr) = last_response.get(&next_resp.1) { - if u128::abs_diff(crate::time::clock::tick_to_time(next_resp.0).as_micros(), crate::time::clock::tick_to_time(*lr).as_micros()) > 500 { // tolerate pending notifications for 500us - maybe_error = true; - // eprintln!("Task {} response at {:.1}ms before next release at {:.1}ms. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(ready[&next_resp.1]).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0); - } - // Sometimes a task is released immediately after a response. This might not be detected. - // Assume that the release occured with the last response - ret.push((*lr, next_resp.0, next_resp.1.clone())); - last_response.insert(&next_resp.1, next_resp.0); - } else { - maybe_error = true; - // eprintln!("Task {} released after response", next_resp.1); - } - } else { - // assert!(peek_resp.0 >= ready[&peek_resp.1]); - last_response.insert(&next_resp.1, next_resp.0); - ret.push((ready[&next_resp.1], next_resp.0, next_resp.1.clone())); - ready.remove(&next_resp.1); - } - } else { - if let Some(lr) = last_response.get(&next_resp.1) { - if u128::abs_diff(crate::time::clock::tick_to_time(next_resp.0).as_micros(), crate::time::clock::tick_to_time(*lr).as_micros()) > 1000 { // tolerate pending notifications for 1ms - // maybe_error = true; - // eprintln!("Task {} response at {:.1}ms not found in ready list. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0); - } - // Sometimes a task is released immediately after a response (e.g. pending notification). This might not be detected. - // Assume that the release occured with the last response - ret.push((*lr, next_resp.0, next_resp.1.clone())); - last_response.insert(&next_resp.1, next_resp.0); - } else { - maybe_error = true; - // eprintln!("Task {} response at {:.1}ms not found in ready list", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0); - } - } - } else { - // TODO: should remaining released tasks be counted as finished? - return (ret,maybe_error); - } - } -} - -/// Transform the states and metadata into a list of ExecIntervals, along with a HashMap of states, a list of HashSets marking memory reads and a bool indicating success -/// returns: -/// - a Vec of ExecIntervals -/// - a Vec of HashSets marking memory reads during these intervals -/// - a HashMap of ReducedFreeRTOSSystemStates by hash -/// - a bool indicating success -fn states2intervals(trace: Vec, meta: Vec<(u64, CaptureEvent, String, (u32, u32), Vec<(u32, u8)>)>) -> (Vec, Vec>, HashMap, bool) { - if trace.len() == 0 {return (Vec::new(), Vec::new(), HashMap::new(), true);} - let mut isr_stack : VecDeque = VecDeque::from([]); // 2+ = ISR, 1 = systemcall, 0 = APP. Trace starts with an ISREnd and executes the app - - - let mut level_of_task : HashMap<&str, u8> = HashMap::new(); - - let mut ret: Vec = vec![]; - let mut reads: Vec> = vec![]; - let mut edges: Vec<(u32, u32)> = vec![]; - let mut last_hash : u64 = trace[0].get_hash(); - let mut table : HashMap = HashMap::new(); - table.insert(last_hash, trace[0].clone()); - for i in 0..trace.len()-1 { - let curr_name = trace[i].current_task.task_name.as_str(); - // let mut interval_name = curr_name; // Name of the interval, either the task name or the isr/api funtion name - let level = match meta[i].1 { - CaptureEvent::APIEnd => { // API end always exits towards the app - if !level_of_task.contains_key(curr_name) { - level_of_task.insert(curr_name, 0); - } - *level_of_task.get_mut(curr_name).unwrap()=0; - 0 - }, - CaptureEvent::APIStart => { // API start can only be called in the app - if !level_of_task.contains_key(curr_name) { // Should not happen, apps start from an ISR End. Some input exibited this behavior for unknown reasons - level_of_task.insert(curr_name, 0); - } - *level_of_task.get_mut(curr_name).unwrap()=1; - // interval_name = &meta[i].2; - 1 - }, - CaptureEvent::ISREnd => { - // special case where the next block is an app start - if !level_of_task.contains_key(curr_name) { - level_of_task.insert(curr_name, 0); - } - // nested isr, TODO: Test level > 2 - if isr_stack.len() > 1 { - // interval_name = ""; // We can't know which isr is running - isr_stack.pop_back().unwrap(); - *isr_stack.back().unwrap() - } else { - isr_stack.pop_back(); - // possibly go back to an api call that is still running for this task - if level_of_task.get(curr_name).unwrap() == &1 { - // interval_name = ""; // We can't know which api is running - } - *level_of_task.get(curr_name).unwrap() - } - }, - CaptureEvent::ISRStart => { - // special case for isrs which do not capture their end - // if meta[i].2 == "ISR_0_Handler" { - // &2 - // } else { - // regular case - // interval_name = &meta[i].2; - if isr_stack.len() > 0 { - let l = *isr_stack.back().unwrap(); - isr_stack.push_back(l+1); - l+1 - } else { - isr_stack.push_back(2); - 2 - } - // } - } - _ => 100 - }; - // if trace[i].2 == CaptureEvent::End {break;} - let next_hash=trace[i+1].get_hash(); - if !table.contains_key(&next_hash) { - table.insert(next_hash, trace[i+1].clone()); - } - ret.push(ExecInterval{ - start_tick: meta[i].0, - end_tick: meta[i+1].0, - start_state: last_hash, - end_state: next_hash, - start_capture: (meta[i].1, meta[i].2.clone()), - end_capture: (meta[i+1].1, meta[i+1].2.clone()), - level: level, - tick_spend_preempted: 0, - abb: None - }); - reads.push(meta[i+1].4.clone()); - last_hash = next_hash; - edges.push((meta[i].3.1, meta[i+1].3.0)); - } - let t = add_abb_info(&mut ret, &table, &edges); - (ret, reads, table, t) -} - -/// Marks which abbs were executed at each interval -fn add_abb_info(trace: &mut Vec, table: &HashMap, edges: &Vec<(u32, u32)>) -> bool { - let mut id_count = 0; - let mut ret = true; - let mut task_has_started : HashSet = HashSet::new(); - let mut wip_abb_trace : Vec>> = vec![]; - // let mut open_abb_at_this_task_or_level : HashMap<(u8,&str),usize> = HashMap::new(); - let mut open_abb_at_this_ret_addr_and_task : HashMap<(u32,&str),usize> = HashMap::new(); - - for i in 0..trace.len() { - let curr_name = &table[&trace[i].start_state].current_task.task_name; - // let last : Option<&usize> = last_abb_start_of_task.get(&curr_name); - - // let open_abb = open_abb_at_this_task_or_level.get(&(trace[i].level, if trace[i].level<2 {&curr_name} else {""})).to_owned(); // apps/apis are differentiated by task name, isrs by nested level - let open_abb = open_abb_at_this_ret_addr_and_task.get(&(edges[i].0, if trace[i].level<2 {&curr_name} else {""})).to_owned(); // apps/apis are differentiated by task name, isrs by nested level - - // println!("Edge {:x}-{:x}", edges[i].0.unwrap_or(0xffff), edges[i].1.unwrap_or(0xffff)); - - match trace[i].start_capture.0 { - // generic api abb start - CaptureEvent::APIStart => { - // assert_eq!(open_abb, None); - ret &= open_abb.is_none(); - open_abb_at_this_ret_addr_and_task.insert((edges[i].1, if trace[i].level<2 {&curr_name} else {""}), i); - wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock{start: edges[i].0, ends: HashSet::new(), level: if trace[i].level<2 {trace[i].level} else {2}, instance_id: id_count, instance_name: Some(trace[i].start_capture.1.clone())}))); - id_count+=1; - }, - // generic isr abb start - CaptureEvent::ISRStart => { - // assert_eq!(open_abb, None); - ret &= open_abb.is_none(); - open_abb_at_this_ret_addr_and_task.insert((edges[i].1, if trace[i].level<2 {&curr_name} else {""}), i); - wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock{start: edges[i].0, ends: HashSet::new(), level: if trace[i].level<2 {trace[i].level} else {2}, instance_id: id_count, instance_name: Some(trace[i].start_capture.1.clone())}))); - id_count+=1; - }, - // generic app abb start - CaptureEvent::APIEnd => { - // assert_eq!(open_abb, None); - ret &= open_abb.is_none(); - open_abb_at_this_ret_addr_and_task.insert((edges[i].1, if trace[i].level<2 {&curr_name} else {""}), i); - wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock{start: edges[i].0, ends: HashSet::new(), level: if trace[i].level<2 {trace[i].level} else {2}, instance_id: id_count, instance_name: if trace[i].level<2 {Some(curr_name.clone())} else {None}}))); - id_count+=1; - }, - // generic continued blocks - CaptureEvent::ISREnd => { - // special case app abb start - if trace[i].start_capture.1=="xPortPendSVHandler" && !task_has_started.contains(curr_name) { - // assert_eq!(open_abb, None); - ret &= open_abb.is_none(); - wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock{start: 0, ends: HashSet::new(), level: if trace[i].level<2 {trace[i].level} else {2}, instance_id: id_count, instance_name: Some(curr_name.clone())}))); - id_count+=1; - open_abb_at_this_ret_addr_and_task.insert((edges[i].1, if trace[i].level<2 {&curr_name} else {""}), i); - task_has_started.insert(curr_name.clone()); - } else { - if let Some(last) = open_abb_at_this_ret_addr_and_task.get(&(edges[i].0, if trace[i].level<2 {&curr_name} else {""})) { - let last = last.clone(); // required to drop immutable reference - wip_abb_trace.push(wip_abb_trace[last].clone()); - // if the abb is interrupted again, it will need to continue at edge[i].1 - open_abb_at_this_ret_addr_and_task.remove(&(edges[i].0, if trace[i].level<2 {&curr_name} else {""})); - open_abb_at_this_ret_addr_and_task.insert((edges[i].1, if trace[i].level<2 {&curr_name} else {""}), last); // order matters! - } else { - // panic!(); - // println!("Continued block with no start {} {} {:?} {:?} {:x}-{:x} {} {}", curr_name, trace[i].start_tick, trace[i].start_capture, trace[i].end_capture, edges[i].0, edges[i].1, task_has_started.contains(curr_name),trace[i].level); - // println!("{:x?}", open_abb_at_this_ret_addr_and_task); - ret = false; - wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock{start: edges[i].1, ends: HashSet::new(), level: if trace[i].level<2 {trace[i].level} else {2}, instance_id: id_count, instance_name: if trace[i].level<1 {Some(curr_name.clone())} else {None}}))); - id_count+=1; - } - } - }, - _ => panic!("Undefined block start") - } - match trace[i].end_capture.0 { - // generic app abb end - CaptureEvent::APIStart => { - let _t = &wip_abb_trace[i]; - RefCell::borrow_mut(&*wip_abb_trace[i]).ends.insert(edges[i].1); - open_abb_at_this_ret_addr_and_task.remove(&(edges[i].1, if trace[i].level<2 {&curr_name} else {""})); - }, - // generic api abb end - CaptureEvent::APIEnd => { - RefCell::borrow_mut(&*wip_abb_trace[i]).ends.insert(edges[i].1); - open_abb_at_this_ret_addr_and_task.remove(&(edges[i].1, if trace[i].level<2 {&curr_name} else {""})); - }, - // generic isr abb end - CaptureEvent::ISREnd => { - RefCell::borrow_mut(&*wip_abb_trace[i]).ends.insert(edges[i].1); - open_abb_at_this_ret_addr_and_task.remove(&(edges[i].1, if trace[i].level<2 {&curr_name} else {""})); - }, - // end anything - CaptureEvent::End => { - RefCell::borrow_mut(&*wip_abb_trace[i]).ends.insert(edges[i].1); - open_abb_at_this_ret_addr_and_task.remove(&(edges[i].1, if trace[i].level<2 {&curr_name} else {""})); - }, - CaptureEvent::ISRStart => (), - _ => panic!("Undefined block end") - } - // println!("{} {} {:x}-{:x} {:x}-{:x} {:?} {:?} {}",curr_name, trace[i].level, edges[i].0, edges[i].1, ((*wip_abb_trace[i])).borrow().start, ((*wip_abb_trace[i])).borrow().ends.iter().next().unwrap_or(&0xffff), trace[i].start_capture, trace[i].end_capture, trace[i].start_tick); - // println!("{:x?}", open_abb_at_this_ret_addr_and_task); - } - // drop(open_abb_at_this_task_or_level); - - for i in 0..trace.len() { - trace[i].abb = Some((*wip_abb_trace[i]).borrow().clone()); - } - return ret; -} // /// restore the isr/api begin/end invariant diff --git a/fuzzers/FRET/src/systemstate/schedulers.rs b/fuzzers/FRET/src/systemstate/schedulers.rs index b7fb9598ab..f6b352c754 100644 --- a/fuzzers/FRET/src/systemstate/schedulers.rs +++ b/fuzzers/FRET/src/systemstate/schedulers.rs @@ -14,7 +14,7 @@ use libafl::{ use crate::time::worst::MaxTimeFavFactor; -use super::{stg::STGNodeMetadata, FreeRTOSSystemStateMetadata}; +use super::{stg::STGNodeMetadata, target_os::*}; /// A state metadata holding a map of favoreds testcases for each map entry #[derive(Debug, Serialize, Deserialize, SerdeAny, Default)] @@ -33,22 +33,24 @@ impl LongestTracesMetadata { /// corpus that exercise all the requested features (e.g. all the coverage seen so far) /// prioritizing [`Testcase`]`s` using [`TestcaseScore`] #[derive(Debug, Clone)] -pub struct LongestTraceScheduler { +pub struct LongestTraceScheduler { base: CS, skip_non_favored_prob: f64, + phantom: PhantomData, } -impl UsesState for LongestTraceScheduler +impl UsesState for LongestTraceScheduler where CS: UsesState, { type State = CS::State; } -impl Scheduler for LongestTraceScheduler +impl Scheduler for LongestTraceScheduler where CS: UsesState + Scheduler, CS::State: HasCorpus + HasMetadata + HasRand, + SYS: TargetSystem, { /// Add an entry to the corpus and return its index fn on_add(&mut self, state: &mut CS::State, idx: CorpusId) -> Result<(), Error> { @@ -56,7 +58,7 @@ where .get(idx)? .borrow() .metadata_map() - .get::().map_or(0, |x| x.trace_length); + .get::().map_or(0, |x| x.trace_length()); self.get_update_trace_length(state,l); self.base.on_add(state, idx) } @@ -115,10 +117,11 @@ where } } -impl LongestTraceScheduler +impl LongestTraceScheduler where CS: UsesState + Scheduler, CS::State: HasCorpus + HasMetadata + HasRand, + SYS: TargetSystem, { pub fn get_update_trace_length(&self, state: &mut CS::State, par: usize) -> u64 { // Create a new top rated meta if not existing @@ -136,6 +139,7 @@ where Self { base, skip_non_favored_prob: DEFAULT_SKIP_NON_FAVORED_PROB, + phantom: PhantomData, } } } diff --git a/fuzzers/FRET/src/systemstate/stg.rs b/fuzzers/FRET/src/systemstate/stg.rs index 5bf5e575e3..cfddc9664f 100644 --- a/fuzzers/FRET/src/systemstate/stg.rs +++ b/fuzzers/FRET/src/systemstate/stg.rs @@ -2,7 +2,7 @@ use hashbrown::HashSet; use libafl::inputs::Input; /// Feedbacks organizing SystemStates as a graph -use libafl::SerdeAny; +use libafl_bolts::prelude::SerdeAny; use libafl_bolts::ownedref::OwnedMutSlice; use petgraph::graph::EdgeIndex; use libafl::prelude::UsesInput; @@ -11,6 +11,7 @@ use libafl::state::UsesState; use libafl::prelude::State; use libafl::schedulers::MinimizerScheduler; use libafl_bolts::HasRefCnt; +use serde::de::DeserializeOwned; use std::path::PathBuf; use libafl::corpus::Testcase; use std::collections::hash_map::DefaultHasher; @@ -24,14 +25,15 @@ use libafl::Error; use hashbrown::HashMap; use libafl::{executors::ExitKind, observers::ObserversTuple, common::HasMetadata}; use serde::{Deserialize, Serialize}; +use std::marker::PhantomData; +use super::target_os::SystemState; use super::AtomicBasicBlock; use super::CaptureEvent; use super::ExecInterval; -use super::JobInstance; -use super::ReducedFreeRTOSSystemState; +use super::RTOSJob; use super::observers::QemuSystemStateObserver; -use super::TaskJob; +use super::RTOSTask; use petgraph::prelude::DiGraph; use petgraph::graph::NodeIndex; use petgraph::Direction; @@ -46,19 +48,25 @@ use std::ops::Deref; use std::ops::DerefMut; use std::rc::Rc; use petgraph::visit::EdgeRef; +use crate::systemstate::target_os::*; use libafl::prelude::StateInitializer; //============================= Data Structures #[derive(Serialize, Deserialize, Clone, Debug, Default, Hash)] -pub struct STGNode +#[serde(bound = "SYS: Serialize, for<'de2> SYS: Deserialize<'de2>")] +pub struct STGNode +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, { - base: ReducedFreeRTOSSystemState, + base: SYS::State, abb: AtomicBasicBlock, } -impl STGNode { +impl STGNode +where SYS: TargetSystem { pub fn _pretty_print(&self) -> String { - format!("{}\nl{} {:x}-{:x}\n{}", self.base.current_task.task_name, self.abb.level, self.abb.start, self.abb.ends.iter().next().unwrap_or_else(||&0xFFFF), self.base.print_lists()) + format!("{}\nl{} {:x}-{:x}\n{}", self.base.current_task().task_name(), self.abb.level, self.abb.start, self.abb.ends.iter().next().unwrap_or_else(||&0xFFFF), self.base.print_lists()) } pub fn color_print(&self) -> String { let color = match self.abb.level { @@ -70,10 +78,10 @@ impl STGNode { let message = match self.abb.level { 1 => format!("API Call"), 2 => format!("ISR"), - 0 => format!("Task: {}",self.base.current_task.task_name), + 0 => format!("Task: {}",self.base.current_task().task_name()), _ => format!(""), }; - let mut label = format!("{}\nABB: {:x}-{:x}\nHash:{:X}\n{}", message, self.abb.start, self.abb.ends.iter().next().unwrap_or_else(||&0xFFFF), self.base.get_hash()>>48, self.base.print_lists()); + let mut label = format!("{}\nABB: {:x}-{:x}\nHash:{:X}\n{}", message, self.abb.start, self.abb.ends.iter().next().unwrap_or_else(||&0xFFFF), compute_hash(&self.base)>>48, self.base.print_lists()); label.push_str(color); label } @@ -84,8 +92,11 @@ impl STGNode { s.finish() } } -impl PartialEq for STGNode { - fn eq(&self, other: &STGNode) -> bool { +impl PartialEq for STGNode +where + SYS: TargetSystem, +{ + fn eq(&self, other: &STGNode) -> bool { self.base==other.base } } @@ -139,13 +150,17 @@ impl Hash for STGEdge { } /// Shared Metadata for a systemstateFeedback -#[derive(Debug, Serialize, Deserialize, SerdeAny, Clone)] -pub struct STGFeedbackState +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(bound = "SYS: Serialize, for<'de2> SYS: Deserialize<'de2>")] +pub struct STGFeedbackState +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, { name: Cow<'static, str>, // aggregated traces as a graph - pub graph: DiGraph, - systemstate_index: HashMap, + pub graph: DiGraph, STGEdge>, + systemstate_index: HashMap, pub state_abb_hash_index: HashMap<(u64, u64), NodeIndex>, stgnode_index: HashMap, entrypoint: NodeIndex, @@ -156,18 +171,24 @@ pub struct STGFeedbackState worst_observed_per_stg_path: HashMap, worst_abb_exec_count: HashMap, // Metadata about job instances - pub worst_task_jobs: HashMap, + pub worst_task_jobs: HashMap, } -impl Default for STGFeedbackState { - fn default() -> STGFeedbackState { - let mut graph = DiGraph::new(); - let mut entry = STGNode::default(); - entry.base.current_task.task_name="Start".to_string(); - let mut exit = STGNode::default(); - exit.base.current_task.task_name="End".to_string(); +libafl_bolts::impl_serdeany!(STGFeedbackState); - let systemstate_index = HashMap::from([(entry.base.get_hash(), entry.base.clone()), (exit.base.get_hash(), exit.base.clone())]); +impl Default for STGFeedbackState +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, +{ + fn default() -> STGFeedbackState { + let mut graph = DiGraph::new(); + let mut entry : STGNode = STGNode::default(); + *(entry.base.current_task_mut().task_name_mut())="Start".to_string(); + let mut exit : STGNode = STGNode::default(); + *(exit.base.current_task_mut().task_name_mut())="End".to_string(); + + let systemstate_index = HashMap::from([(compute_hash(&entry.base), entry.base.clone()), (compute_hash(&exit.base), exit.base.clone())]); let h_entry = entry.get_hash(); let h_exit = exit.get_hash(); @@ -175,7 +196,7 @@ impl Default for STGFeedbackState { let entrypoint = graph.add_node(entry.clone()); let exitpoint = graph.add_node(exit.clone()); - let state_abb_hash_index = HashMap::from([((entry.base.get_hash(), entry.abb.get_hash()), entrypoint), ((exit.base.get_hash(), exit.abb.get_hash()), exitpoint)]); + let state_abb_hash_index = HashMap::from([((compute_hash(&entry.base), entry.abb.get_hash()), entrypoint), ((compute_hash(&exit.base), exit.abb.get_hash()), exitpoint)]); let index = HashMap::from([(h_entry, entrypoint), (h_exit, exitpoint)]); @@ -196,7 +217,9 @@ impl Default for STGFeedbackState { } } -impl Named for STGFeedbackState +impl Named for STGFeedbackState +where + SYS: TargetSystem, { #[inline] fn name(&self) -> &Cow<'static, str> { @@ -213,12 +236,12 @@ pub struct STGNodeMetadata { aggregate: u64, top_abb_counts: Vec, intervals: Vec, - jobs: Vec, + jobs: Vec, indices: Vec, tcref: isize, } impl STGNodeMetadata { - pub fn new(nodes: Vec, edges: Vec, abb_trace: Vec, abbs_pathhash: u64, aggregate: u64, top_abb_counts: Vec, intervals: Vec, jobs: Vec) -> Self { + pub fn new(nodes: Vec, edges: Vec, abb_trace: Vec, abbs_pathhash: u64, aggregate: u64, top_abb_counts: Vec, intervals: Vec, jobs: Vec) -> Self { #[allow(unused)] let mut indices : Vec<_> = vec![]; #[cfg(feature = "sched_stg_edge")] @@ -267,7 +290,7 @@ impl STGNodeMetadata { &self.intervals } - pub fn jobs(&self) -> &Vec { + pub fn jobs(&self) -> &Vec { &self.jobs } } @@ -344,7 +367,11 @@ pub unsafe fn stg_map_mut_slice<'a>() -> OwnedMutSlice<'a, u16> { /// A Feedback reporting novel System-State Transitions. Depends on [`QemuSystemStateObserver`] #[derive(Serialize, Deserialize, Clone, Debug, Default)] -pub struct StgFeedback +#[serde(bound = "SYS: Serialize, for<'de2> SYS: Deserialize<'de2>")] +pub struct StgFeedback +where + SYS: TargetSystem, + for<'de2> SYS: Deserialize<'de2>, { name: Cow<'static, str>, last_node_trace: Option>, @@ -354,8 +381,9 @@ pub struct StgFeedback last_abbs_hash: Option, // only set, if it was interesting last_aggregate_hash: Option, // only set, if it was interesting last_top_abb_hashes: Option>, // only set, if it was interesting - last_job_trace: Option>, // only set, if it was interesting - dump_path: Option + last_job_trace: Option>, // only set, if it was interesting + dump_path: Option, + _phantom_data: PhantomData, } #[cfg(feature = "feed_stg")] const INTEREST_EDGE : bool = true; @@ -425,7 +453,10 @@ fn execinterval_to_abb_instances(trace: &Vec, read_trace: &Vec StgFeedback +where + SYS: TargetSystem, +{ pub fn new(dump_name: Option) -> Self { // Self {name: String::from("STGFeedback"), last_node_trace: None, last_edge_trace: None, last_intervals: None } let mut s = Self::default(); @@ -442,7 +473,7 @@ impl StgFeedback { /// newly discovered node? /// side effect: /// the graph gets new nodes and edge - fn update_stg_interval(trace: &Vec, read_trace: &Vec>, table: &HashMap, fbs: &mut STGFeedbackState) -> (Vec<(NodeIndex, u64)>, Vec<(EdgeIndex, u64)>, bool, bool) { + fn update_stg_interval(trace: &Vec, read_trace: &Vec>, table: &HashMap, fbs: &mut STGFeedbackState) -> (Vec<(NodeIndex, u64)>, Vec<(EdgeIndex, u64)>, bool, bool) { let mut return_node_trace = vec![(fbs.entrypoint, 0)]; // Assuming entrypoint timestamp is 0 let mut return_edge_trace = vec![]; let mut interesting = false; @@ -453,14 +484,14 @@ impl StgFeedback { let mut instance_time = execinterval_to_abb_instances(trace, read_trace); // add all missing state+abb combinations to the graph for (_i,interval) in trace.iter().enumerate() { // Iterate intervals - let node = STGNode {base: table[&interval.start_state].clone(), abb: interval.abb.as_ref().unwrap().clone()}; + let node : STGNode = STGNode {base: table[&interval.start_state].clone(), abb: interval.abb.as_ref().unwrap().clone()}; let h_node = node.get_hash(); let next_idx = if let Some(idx) = fbs.stgnode_index.get(&h_node) { // already present *idx } else { // not present - let h = (node.base.get_hash(), node.abb.get_hash()); + let h = (compute_hash(&node.base), node.abb.get_hash()); let idx = fbs.graph.add_node(node); fbs.stgnode_index.insert(h_node, idx); fbs.state_abb_hash_index.insert(h, idx); @@ -524,14 +555,18 @@ impl StgFeedback { } } -impl StateInitializer for StgFeedback {} +impl StateInitializer for StgFeedback +where + SYS: TargetSystem, +{} -impl Feedback for StgFeedback +impl Feedback for StgFeedback where - S: State + UsesInput + MaybeHasClientPerfMonitor + HasNamedMetadata, + S: State + UsesInput + MaybeHasClientPerfMonitor + HasNamedMetadata + HasMetadata, S::Input: Default, EM: EventFirer, OT: ObserversTuple, + SYS: TargetSystem, { #[allow(clippy::wrong_self_convention)] fn is_interesting( @@ -545,7 +580,9 @@ where where ::Input: Default, { - let observer = observers.match_name::::Input>>("systemstate") + // TODO: don't remove metadata. work around ownership issues + let trace = state.remove_metadata::().expect("TraceData not found"); + let observer = observers.match_name::::Input, SYS>>("systemstate") .expect("QemuSystemStateObserver not found"); let clock_observer = observers.match_name::("clocktime") .expect("QemuClockObserver not found"); @@ -555,21 +592,22 @@ where let last_runtime = clock_observer.last_runtime(); let feedbackstate = match state .named_metadata_map_mut() - .get_mut::("stgfeedbackstate") { + .get_mut::>("stgfeedbackstate") { Some(s) => s, Option::None => { - let n=STGFeedbackState::default(); + let n=STGFeedbackState::::default(); state.named_metadata_map_mut().insert("stgfeedbackstate",n); - state.named_metadata_map_mut().get_mut::("stgfeedbackstate").unwrap() + state.named_metadata_map_mut().get_mut::>("stgfeedbackstate").unwrap() } }; // --------------------------------- Update STG - let (mut nodetrace, mut edgetrace, mut interesting, mut updated) = StgFeedback::update_stg_interval(&observer.last_trace, &observer.last_reads, &observer.last_states, feedbackstate); + let (mut nodetrace, mut edgetrace, mut interesting, mut updated) = StgFeedback::update_stg_interval(trace.intervals(), &trace.mem_reads(), trace.states_map(), feedbackstate); + let worst_jobs = trace.worst_jobs_per_task_by_time(); #[cfg(feature = "trace_job_response_times")] - let worst_target_instance = observer.job_instances.iter().filter(|x| Some(x.name.clone()) == observer.select_task).max_by(|a,b| (a.response-a.release).cmp(&(b.response-b.release))); + let worst_target_instance = worst_jobs.get(observer.select_task.as_ref().unwrap_or(&String::new())); #[cfg(feature = "trace_job_response_times")] if let Some(worst_instance) = worst_target_instance { @@ -586,17 +624,17 @@ where set_observer_map(&edgetrace.iter().map(|x| x.0).collect::>()); // --------------------------------- Update job instances - for i in observer.worst_job_instances.iter() { + for i in trace.worst_jobs_per_task_by_time().iter() { interesting |= INTEREST_JOB_INSTANCE && if let Some(x) = feedbackstate.worst_task_jobs.get_mut(&i.1.get_hash_cached()) { // eprintln!("Job instance already present"); x.try_update(i.1) } else { // eprintln!("New Job instance"); - feedbackstate.worst_task_jobs.insert(i.1.get_hash_cached(), TaskJob::from_instance(&i.1)); + feedbackstate.worst_task_jobs.insert(i.1.get_hash_cached(), RTOSTask::from_instance(&i.1)); true } }; - self.last_job_trace = Some(observer.job_instances.clone()); + self.last_job_trace = Some(trace.jobs().clone()); // dbg!(&observer.job_instances); { @@ -619,11 +657,11 @@ where #[cfg(feature = "trace_job_response_times")] let tmp = { if let Some(worst_instance) = worst_target_instance { - let t = observer.last_trace.iter().filter(|x| x.start_tick < worst_instance.response && x.end_tick > worst_instance.release ).cloned().collect(); - StgFeedback::abbs_in_exec_order(&t) + let t = trace.intervals().iter().filter(|x| x.start_tick < worst_instance.response && x.end_tick > worst_instance.release ).cloned().collect(); + StgFeedback::::abbs_in_exec_order(&t) } else { if observer.select_task.is_none() { // if nothing was selected, just take the whole trace, otherwise there is nothing interesting here - StgFeedback::abbs_in_exec_order(&observer.last_trace) + StgFeedback::::abbs_in_exec_order(trace.intervals()) } else { Vec::new() } @@ -684,7 +722,7 @@ where // fs::write("./mystg.dot",outs).expect("Failed to write graph"); self.last_node_trace = Some(nodetrace.into_iter().map(|x| x.0).collect::>()); self.last_edge_trace = Some(edgetrace.into_iter().map(|x| x.0).collect::>()); - self.last_intervals = Some(observer.last_trace.clone()); + self.last_intervals = Some(trace.intervals().clone()); self.last_abb_trace = Some(tmp); if let Some(dp) = &self.dump_path { @@ -716,7 +754,9 @@ where Ok(()) } } -impl Named for StgFeedback +impl Named for StgFeedback +where + SYS: TargetSystem, { #[inline] fn name(&self) -> &Cow<'static, str> { diff --git a/fuzzers/FRET/src/systemstate/freertos.rs b/fuzzers/FRET/src/systemstate/target_os/freertos/bindings.rs similarity index 73% rename from fuzzers/FRET/src/systemstate/freertos.rs rename to fuzzers/FRET/src/systemstate/target_os/freertos/bindings.rs index ff13f686db..493733ad4f 100644 --- a/fuzzers/FRET/src/systemstate/freertos.rs +++ b/fuzzers/FRET/src/systemstate/target_os/freertos/bindings.rs @@ -85,38 +85,4 @@ pub struct tskTaskControlBlock { } pub type tskTCB = tskTaskControlBlock; pub type TCB_t = tskTCB; -/*========== End of generated Code =============*/ - -pub trait emu_lookup { - fn lookup(emu: &Qemu, addr: ::std::os::raw::c_uint) -> Self; -} - - -#[derive(Debug, Copy, Clone, Serialize, Deserialize)] -pub enum rtos_struct { - TCB_struct(TCB_t), - List_struct(List_t), - List_Item_struct(ListItem_t), - List_MiniItem_struct(MiniListItem_t), -} - -#[macro_export] -macro_rules! impl_emu_lookup { - ($struct_name:ident) => { - impl $crate::systemstate::freertos::emu_lookup for $struct_name { - fn lookup(emu: &Qemu, addr: ::std::os::raw::c_uint) -> $struct_name { - let mut tmp : [u8; std::mem::size_of::<$struct_name>()] = [0u8; std::mem::size_of::<$struct_name>()]; - unsafe { - emu.read_mem(addr.into(), &mut tmp); - std::mem::transmute::<[u8; std::mem::size_of::<$struct_name>()], $struct_name>(tmp) - } - } - } - }; -} -impl_emu_lookup!(TCB_t); -impl_emu_lookup!(List_t); -impl_emu_lookup!(ListItem_t); -impl_emu_lookup!(MiniListItem_t); -impl_emu_lookup!(void_ptr); -impl_emu_lookup!(TaskStatus_t); +/*========== End of generated Code =============*/ \ No newline at end of file diff --git a/fuzzers/FRET/src/systemstate/target_os/freertos/mod.rs b/fuzzers/FRET/src/systemstate/target_os/freertos/mod.rs new file mode 100644 index 0000000000..d30cbac8c1 --- /dev/null +++ b/fuzzers/FRET/src/systemstate/target_os/freertos/mod.rs @@ -0,0 +1,548 @@ +use libafl_qemu::GuestAddr; +use qemu_module::{FreeRTOSSystemStateHelper, MEM_READ}; +use serde::{Deserialize, Serialize}; + +use crate::{ + impl_emu_lookup, + systemstate::{helpers::get_icount, CaptureEvent}, +}; + +pub mod bindings; +pub mod qemu_module; +use bindings::*; + +use super::QemuLookup; +use crate::systemstate::target_os::*; + +// Constants +const NUM_PRIOS: usize = 15; + +//============================================================================= Outside interface + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct FreeRTOSSystem { + pub raw_trace: Vec, +} + +impl TargetSystem for FreeRTOSSystem { + type State = FreeRTOSSystemState; + type TCB = RefinedTCB; + type TraceData = FreeRTOSTraceMetadata; +} + +impl TaskControlBlock for RefinedTCB { + fn task_name(&self) -> &String { + &self.task_name + } + fn task_name_mut(&mut self) -> &mut String { + &mut self.task_name + } +} + +impl SystemState for FreeRTOSSystemState { + type TCB = RefinedTCB; + + fn current_task(&self) -> &Self::TCB { + &self.current_task + } + + fn get_ready_lists(&self) -> &Vec { + &self.ready_list_after + } + + fn get_delay_list(&self) -> &Vec { + &self.delay_list_after + } + + fn print_lists(&self) -> String { + self.print_lists() + } + + fn current_task_mut(&mut self) -> &mut Self::TCB { + &mut self.current_task + } + + // fn get_edge(&self) -> (GuestAddr, GuestAddr) { + // (self.edge.0, self.edge.1) + // } + + // fn get_capture_point(&self) -> (CaptureEvent, String) { + // self.capture_point.clone() + // } + + // fn get_mem_reads(&self) -> &Vec<(u32, u8)> { + // &self.mem_reads + // } +} + +//============================================================================= Data structures + +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +pub enum FreeRTOSStruct { + TCB_struct(TCB_t), + List_struct(List_t), + List_Item_struct(ListItem_t), + List_MiniItem_struct(MiniListItem_t), +} + +impl_emu_lookup!(TCB_t); +impl_emu_lookup!(List_t); +impl_emu_lookup!(ListItem_t); +impl_emu_lookup!(MiniListItem_t); +impl_emu_lookup!(void_ptr); +impl_emu_lookup!(TaskStatus_t); + +pub const ISR_SYMBOLS: &'static [&'static str] = &[ + // ISRs + "Reset_Handler", + "Default_Handler", + "Default_Handler2", + "Default_Handler3", + "Default_Handler4", + "Default_Handler5", + "Default_Handler6", + "vPortSVCHandler", + "xPortPendSVHandler", + "xPortSysTickHandler", + "ISR_0_Handler", + "ISR_1_Handler", + "ISR_2_Handler", + "ISR_3_Handler", + "ISR_4_Handler", + "ISR_5_Handler", + "ISR_6_Handler", + "ISR_7_Handler", + "ISR_8_Handler", + "ISR_9_Handler", + "ISR_10_Handler", + "ISR_11_Handler", + "ISR_12_Handler", + "ISR_13_Handler", +]; +pub const USR_ISR_SYMBOLS: &'static [&'static str] = &[ + "ISR_0_Handler", + "ISR_1_Handler", + "ISR_2_Handler", + "ISR_3_Handler", + "ISR_4_Handler", + "ISR_5_Handler", + "ISR_6_Handler", + "ISR_7_Handler", + "ISR_8_Handler", + "ISR_9_Handler", + "ISR_10_Handler", + "ISR_11_Handler", + "ISR_12_Handler", + "ISR_13_Handler", +]; + +//============================================================================= Helper functions + +fn read_freertos_list( + systemstate: &mut RawFreeRTOSSystemState, + emulator: &libafl_qemu::Qemu, + target: GuestAddr, +) -> (List_t, bool) { + let read: List_t = QemuLookup::lookup(emulator, target); + let listbytes: GuestAddr = GuestAddr::try_from(std::mem::size_of::()).unwrap(); + + let mut next_index = read.pxIndex; + for _j in 0..read.uxNumberOfItems { + // always jump over the xListEnd marker + if (target..target + listbytes).contains(&next_index) { + let next_item: MiniListItem_t = QemuLookup::lookup(emulator, next_index); + let new_next_index = next_item.pxNext; + systemstate + .dumping_ground + .insert(next_index, FreeRTOSStruct::List_MiniItem_struct(next_item)); + next_index = new_next_index; + } + let next_item: ListItem_t = QemuLookup::lookup(emulator, next_index); + // println!("Item at {}: {:?}",next_index,next_item); + if next_item.pvContainer != target { + // the list is being modified, abort by setting the list empty + eprintln!("Warning: attempted to read a list that is being modified"); + let mut read = read; + read.uxNumberOfItems = 0; + return (read, false); + } + // assert_eq!(next_item.pvContainer,target); + let new_next_index = next_item.pxNext; + let next_tcb: TCB_t = QemuLookup::lookup(emulator, next_item.pvOwner); + // println!("TCB at {}: {:?}",next_item.pvOwner,next_tcb); + systemstate.dumping_ground.insert( + next_item.pvOwner, + FreeRTOSStruct::TCB_struct(next_tcb.clone()), + ); + systemstate + .dumping_ground + .insert(next_index, FreeRTOSStruct::List_Item_struct(next_item)); + next_index = new_next_index; + } + // Handle edge case where the end marker was not included yet + if (target..target + listbytes).contains(&next_index) { + let next_item: freertos::MiniListItem_t = QemuLookup::lookup(emulator, next_index); + systemstate + .dumping_ground + .insert(next_index, FreeRTOSStruct::List_MiniItem_struct(next_item)); + } + return (read, true); +} + +#[inline] +fn trigger_collection( + emulator: &libafl_qemu::Qemu, + edge: (GuestAddr, GuestAddr), + event: CaptureEvent, + h: &FreeRTOSSystemStateHelper, +) { + let listbytes: GuestAddr = + GuestAddr::try_from(std::mem::size_of::()).unwrap(); + let mut systemstate = RawFreeRTOSSystemState::default(); + + match event { + CaptureEvent::APIStart => { + let s = h.api_fn_addrs.get(&edge.1).unwrap(); + systemstate.capture_point = (CaptureEvent::APIStart, s.to_string()); + } + CaptureEvent::APIEnd => { + let s = h.api_fn_addrs.get(&edge.0).unwrap(); + systemstate.capture_point = (CaptureEvent::APIEnd, s.to_string()); + } + CaptureEvent::ISRStart => { + let s = h.isr_addrs.get(&edge.1).unwrap(); + systemstate.capture_point = (CaptureEvent::ISRStart, s.to_string()); + } + CaptureEvent::ISREnd => { + let s = h.isr_addrs.get(&edge.0).unwrap(); + systemstate.capture_point = (CaptureEvent::ISREnd, s.to_string()); + } + CaptureEvent::End => { + systemstate.capture_point = (CaptureEvent::End, "".to_string()); + } + CaptureEvent::Undefined => (), + } + + if systemstate.capture_point.0 == CaptureEvent::Undefined { + // println!("Not found: {:#x} {:#x}", edge.0.unwrap_or(0), edge.1.unwrap_or(0)); + } + systemstate.edge = ((edge.0), (edge.1)); + + systemstate.qemu_tick = get_icount(emulator); + + let mut buf: [u8; 4] = [0, 0, 0, 0]; + match h.input_counter { + Some(s) => unsafe { + emulator.read_mem(s, &mut buf); + }, + Option::None => (), + }; + systemstate.input_counter = GuestAddr::from_le_bytes(buf); + + let curr_tcb_addr: freertos::void_ptr = QemuLookup::lookup(emulator, h.tcb_addr); + if curr_tcb_addr == 0 { + return; + }; + + // println!("{:?}",std::str::from_utf8(¤t_tcb.pcTaskName)); + let critical: void_ptr = QemuLookup::lookup(emulator, h.critical_addr); + let suspended: void_ptr = QemuLookup::lookup(emulator, h.scheduler_lock_addr); + let _running: void_ptr = QemuLookup::lookup(emulator, h.scheduler_running_addr); + + systemstate.current_tcb = QemuLookup::lookup(emulator, curr_tcb_addr); + // During ISRs it is only safe to extract structs if they are not currently being modified + if systemstate.capture_point.0 == CaptureEvent::APIStart + || systemstate.capture_point.0 == CaptureEvent::APIEnd + || (critical == 0 && suspended == 0) + { + // Extract delay list + let mut target: GuestAddr = h.delay_queue; + target = QemuLookup::lookup(emulator, target); + let _temp = read_freertos_list(&mut systemstate, emulator, target); + systemstate.delay_list = _temp.0; + systemstate.read_invalid |= !_temp.1; + + // Extract delay list overflow + let mut target: GuestAddr = h.delay_queue_overflow; + target = QemuLookup::lookup(emulator, target); + let _temp = read_freertos_list(&mut systemstate, emulator, target); + systemstate.delay_list_overflow = _temp.0; + systemstate.read_invalid |= !_temp.1; + + // Extract suspended tasks (infinite wait), seems broken, always appreas to be modified + // let mut target : GuestAddr = h.suspended_queue; + // target = QemuLookup::lookup(emulator, target); + // systemstate.suspended_list = read_freertos_list(&mut systemstate, emulator, target); + + // Extract priority lists + for i in 0..NUM_PRIOS { + let target: GuestAddr = listbytes * GuestAddr::try_from(i).unwrap() + h.ready_queues; + let _temp = read_freertos_list(&mut systemstate, emulator, target); + systemstate.prio_ready_lists[i] = _temp.0; + systemstate.read_invalid |= !_temp.1; + } + } else { + systemstate.read_invalid = true; + } + systemstate.mem_reads = unsafe { MEM_READ.take().unwrap_or_default() }; + + unsafe { + CURRENT_SYSTEMSTATE_VEC.push(systemstate); + } +} + +/// Raw info Dump from Qemu +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct RawFreeRTOSSystemState { + qemu_tick: u64, + current_tcb: TCB_t, + prio_ready_lists: [freertos::List_t; NUM_PRIOS], + delay_list: freertos::List_t, + delay_list_overflow: freertos::List_t, + dumping_ground: HashMap, + read_invalid: bool, + input_counter: u32, + edge: (GuestAddr, GuestAddr), + capture_point: (CaptureEvent, String), + mem_reads: Vec<(u32, u8)>, +} +/// List of system state dumps from EmulatorModules +static mut CURRENT_SYSTEMSTATE_VEC: Vec = vec![]; + +/// A reduced version of freertos::TCB_t +#[derive(Debug, Default, Serialize, Deserialize, Clone)] +pub struct RefinedTCB { + pub task_name: String, + pub priority: u32, + pub base_priority: u32, + mutexes_held: u32, + // notify_value: u32, + notify_state: u8, +} + +impl PartialEq for RefinedTCB { + fn eq(&self, other: &Self) -> bool { + let ret = self.task_name == other.task_name + && self.priority == other.priority + && self.base_priority == other.base_priority; + #[cfg(feature = "do_hash_notify_state")] + let ret = ret && self.notify_state == other.notify_state; + ret + } +} + +impl Hash for RefinedTCB { + fn hash(&self, state: &mut H) { + self.task_name.hash(state); + self.priority.hash(state); + self.mutexes_held.hash(state); + #[cfg(feature = "do_hash_notify_state")] + self.notify_state.hash(state); + // self.notify_value.hash(state); + } +} + +impl RefinedTCB { + pub fn from_tcb(input: &TCB_t) -> Self { + unsafe { + let tmp = std::mem::transmute::<[i8; 10], [u8; 10]>(input.pcTaskName); + let name: String = std::str::from_utf8(&tmp) + .expect("TCB name was not utf8") + .chars() + .filter(|x| *x != '\0') + .collect::(); + Self { + task_name: name, + priority: input.uxPriority, + base_priority: input.uxBasePriority, + mutexes_held: input.uxMutexesHeld, + // notify_value: input.ulNotifiedValue[0], + notify_state: input.ucNotifyState[0], + } + } + } + pub fn from_tcb_owned(input: TCB_t) -> Self { + unsafe { + let tmp = std::mem::transmute::<[i8; 10], [u8; 10]>(input.pcTaskName); + let name: String = std::str::from_utf8(&tmp) + .expect("TCB name was not utf8") + .chars() + .filter(|x| *x != '\0') + .collect::(); + Self { + task_name: name, + priority: input.uxPriority, + base_priority: input.uxBasePriority, + mutexes_held: input.uxMutexesHeld, + // notify_value: input.ulNotifiedValue[0], + notify_state: input.ucNotifyState[0], + } + } + } +} + +/// Reduced information about a systems state, without any execution context +#[derive(Debug, Default, Serialize, Deserialize, Clone)] +pub struct FreeRTOSSystemState { + current_task: RefinedTCB, + ready_list_after: Vec, + delay_list_after: Vec, + read_invalid: bool, +} +impl PartialEq for FreeRTOSSystemState { + fn eq(&self, other: &Self) -> bool { + self.current_task == other.current_task + && self.ready_list_after == other.ready_list_after + && self.delay_list_after == other.delay_list_after + && self.read_invalid == other.read_invalid + } +} + +impl Hash for FreeRTOSSystemState { + fn hash(&self, state: &mut H) { + self.current_task.hash(state); + self.ready_list_after.hash(state); + self.delay_list_after.hash(state); + self.read_invalid.hash(state); + } +} +impl FreeRTOSSystemState { + // fn get_tick(&self) -> u64 { + // self.tick + // } + + pub fn print_lists(&self) -> String { + let mut ret = String::from("+"); + for j in self.ready_list_after.iter() { + ret.push_str(format!(" {}", j.task_name).as_str()); + } + ret.push_str("\n-"); + for j in self.delay_list_after.iter() { + ret.push_str(format!(" {}", j.task_name).as_str()); + } + ret + } + pub fn get_hash(&self) -> u64 { + let mut h = DefaultHasher::new(); + self.hash(&mut h); + h.finish() + } +} + +impl fmt::Display for FreeRTOSSystemState { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let ready = self + .ready_list_after + .iter() + .map(|x| x.task_name.clone()) + .collect::>() + .join(" "); + let delay = self + .delay_list_after + .iter() + .map(|x| x.task_name.clone()) + .collect::>() + .join(" "); + write!( + f, + "Valid: {} | Current: {} | Ready: {} | Delay: {}", + u32::from(!self.read_invalid), + self.current_task.task_name, + ready, + delay + ) + } +} + +#[derive(Debug, Default, Serialize, Deserialize, Clone)] +pub(crate)struct FreeRTOSSystemStateContext { + pub qemu_tick: u64, + pub capture_point: (CaptureEvent, String), + pub edge: (GuestAddr, GuestAddr), + pub mem_reads: Vec<(u32, u8)>, +} + + +#[derive(Debug, Default, Serialize, Deserialize, Clone)] +pub struct FreeRTOSTraceMetadata +{ + trace_map: HashMap::State>, + intervals: Vec, + mem_reads: Vec>, + jobs: Vec, + trace_length: usize, + indices: Vec, // Hashed enumeration of States + tcref: isize, +} +impl FreeRTOSTraceMetadata +{ + pub fn new(trace: Vec<::State>, intervals: Vec, mem_reads: Vec>, jobs: Vec) -> Self { + let hashes : Vec<_> = trace + .iter() + .map(|x| compute_hash(&x) as usize) + .collect(); + let trace_map = HashMap::from_iter(trace.into_iter().zip(hashes.iter()).map(|(x, y)| (*y as u64, x))); + Self { + trace_length: hashes.len(), // TODO make this configurable + trace_map: trace_map, + intervals: intervals, + mem_reads: mem_reads, + jobs: jobs, + indices: hashes, + tcref: 0, + } + } +} + +impl HasRefCnt for FreeRTOSTraceMetadata +{ + fn refcnt(&self) -> isize { + self.tcref + } + + fn refcnt_mut(&mut self) -> &mut isize { + &mut self.tcref + } +} + +impl SystemTraceData for FreeRTOSTraceMetadata +{ + type State = FreeRTOSSystemState; + + fn states(&self) -> Vec<&Self::State> { + self.indices.iter().map(|x| self.trace_map.get(&(*x as u64)).unwrap()).collect() + } + + fn intervals(&self) -> &Vec { + &self.intervals + } + + fn jobs(&self) -> &Vec { + &self.jobs + } + + fn trace_length(&self) -> usize { + self.trace_length + } + + fn mem_reads(&self) -> &Vec> { + &self.mem_reads + } + + fn states_map(&self) -> &HashMap { + &self.trace_map + } +} + +libafl_bolts::impl_serdeany!(FreeRTOSTraceMetadata); +libafl_bolts::impl_serdeany!(RefinedTCB); +libafl_bolts::impl_serdeany!(FreeRTOSSystemState); +libafl_bolts::impl_serdeany!(FreeRTOSSystem); + +fn get_task_names(trace: &Vec) -> HashSet { + let mut ret: HashSet<_, _> = HashSet::new(); + for state in trace { + ret.insert(state.current_task.task_name.to_string()); + } + ret +} diff --git a/fuzzers/FRET/src/systemstate/target_os/freertos/qemu_module.rs b/fuzzers/FRET/src/systemstate/target_os/freertos/qemu_module.rs new file mode 100644 index 0000000000..36ba6e5758 --- /dev/null +++ b/fuzzers/FRET/src/systemstate/target_os/freertos/qemu_module.rs @@ -0,0 +1,1154 @@ +use std::{cell::RefCell, collections::VecDeque, ops::Range, rc::Rc}; + +use freertos::{FreeRTOSTraceMetadata, USR_ISR_SYMBOLS}; +use hashbrown::HashMap; + +use libafl::{ + inputs::UsesInput, + prelude::{ExitKind, ObserversTuple}, HasMetadata, +}; +use libafl_qemu::{ + modules::{EmulatorModule, EmulatorModuleTuple, NopAddressFilter, NopPageFilter}, + sys::TCGTemp, + EmulatorModules, GuestAddr, Hook, MemAccessInfo, +}; + +use crate::systemstate::{ + helpers::{get_icount, in_any_range, read_rec_return_stackframe}, + target_os::{freertos::FreeRTOSStruct::*, *}, + AtomicBasicBlock, CaptureEvent, RTOSJob, +}; + +use super::{ + bindings::{self, *}, + compute_hash, trigger_collection, ExecInterval, FreeRTOSStruct, FreeRTOSSystemState, + FreeRTOSSystemStateContext, RawFreeRTOSSystemState, RefinedTCB, CURRENT_SYSTEMSTATE_VEC, +}; + +//============================= Qemu Helper + +/// A Qemu Helper with reads FreeRTOS specific structs from Qemu whenever certain syscalls occur, also inject inputs +#[derive(Debug)] +pub struct FreeRTOSSystemStateHelper { + // Address of API functions + pub api_fn_addrs: HashMap, + pub api_fn_ranges: Vec<(String, std::ops::Range)>, + // Address of interrupt routines + pub isr_addrs: HashMap, + pub isr_ranges: Vec<(String, std::ops::Range)>, + pub input_mem: Range, + pub tcb_addr: GuestAddr, + pub ready_queues: GuestAddr, + pub delay_queue: GuestAddr, + pub delay_queue_overflow: GuestAddr, + pub scheduler_lock_addr: GuestAddr, + pub scheduler_running_addr: GuestAddr, + pub critical_addr: GuestAddr, + pub input_counter: Option, + pub app_range: Range, + pub job_done_addrs: GuestAddr, +} + +impl FreeRTOSSystemStateHelper { + #[must_use] + pub fn new( + api_fn_addrs: HashMap, + api_fn_ranges: Vec<(String, std::ops::Range)>, + isr_addrs: HashMap, + isr_ranges: Vec<(String, std::ops::Range)>, + input_mem: Range, + tcb_addr: GuestAddr, + ready_queues: GuestAddr, + delay_queue: GuestAddr, + delay_queue_overflow: GuestAddr, + scheduler_lock_addr: GuestAddr, + scheduler_running_addr: GuestAddr, + critical_addr: GuestAddr, + input_counter: Option, + app_range: Range, + job_done_addrs: GuestAddr, + ) -> Self { + FreeRTOSSystemStateHelper { + api_fn_addrs, + api_fn_ranges, + isr_addrs, + isr_ranges, + input_mem, + tcb_addr: tcb_addr, + ready_queues: ready_queues, + delay_queue, + delay_queue_overflow, + scheduler_lock_addr, + scheduler_running_addr, + critical_addr, + input_counter: input_counter, + app_range, + job_done_addrs, + } + } +} + +impl EmulatorModule for FreeRTOSSystemStateHelper +where + S: UsesInput + Unpin + HasMetadata, +{ + fn first_exec(&mut self, emulator_modules: &mut EmulatorModules, _state: &mut S) + where + ET: EmulatorModuleTuple, + { + for wp in self.isr_addrs.keys() { + emulator_modules.instructions(*wp, Hook::Function(exec_isr_hook::), false); + } + emulator_modules.jmps( + Hook::Function(gen_jmp_is_syscall::), + Hook::Function(trace_jmp::), + ); + #[cfg(feature = "trace_job_response_times")] + emulator_modules.instructions( + self.job_done_addrs, + Hook::Function(job_done_hook::), + false, + ); + #[cfg(feature = "trace_reads")] + emulator_modules.reads( + Hook::Function(gen_read_is_input::), + Hook::Empty, + Hook::Empty, + Hook::Empty, + Hook::Empty, + Hook::Function(trace_reads::), + ); + unsafe { INPUT_MEM = self.input_mem.clone() }; + } + + // TODO: refactor duplicate code + fn pre_exec( + &mut self, + _emulator_modules: &mut EmulatorModules, + state: &mut S, + _input: &S::Input, + ) where + ET: EmulatorModuleTuple, + { + unsafe { + CURRENT_SYSTEMSTATE_VEC.clear(); + JOBS_DONE.clear(); + } + if state.has_metadata::() { + state.remove_metadata::(); + } + } + + fn post_exec( + &mut self, + emulator_modules: &mut EmulatorModules, + _state: &mut S, + _input: &S::Input, + _observers: &mut OT, + _exit_kind: &mut ExitKind, + ) where + OT: ObserversTuple, + ET: EmulatorModuleTuple, + { + if unsafe { CURRENT_SYSTEMSTATE_VEC.len() } == 0 { + eprintln!("No system states captured, aborting"); + return; + } + // Collect the final system state + trigger_collection(&emulator_modules.qemu(), (0, 0), CaptureEvent::End, self); + unsafe { + let c = emulator_modules.qemu().cpu_from_index(0); + let pc = c.read_reg::(15).unwrap(); + CURRENT_SYSTEMSTATE_VEC[CURRENT_SYSTEMSTATE_VEC.len() - 1].edge = (pc, 0); + CURRENT_SYSTEMSTATE_VEC[CURRENT_SYSTEMSTATE_VEC.len() - 1].capture_point = + (CaptureEvent::End, "Breakpoint".to_string()); + } + // Find the first ISREnd of vPortSVCHandler (start of the first task) and drop anything before + unsafe { + let mut index = 0; + while index < CURRENT_SYSTEMSTATE_VEC.len() { + if CaptureEvent::ISREnd == CURRENT_SYSTEMSTATE_VEC[index].capture_point.0 + && CURRENT_SYSTEMSTATE_VEC[index].capture_point.1 == "xPortPendSVHandler" + { + break; + } + index += 1; + } + drop(CURRENT_SYSTEMSTATE_VEC.drain(..index)); + if CURRENT_SYSTEMSTATE_VEC.len() == 1 { + eprintln!("No system states captured, aborting"); + return; + } + } + // Start refining the state trace + let (refined_states, metadata) = + refine_system_states(unsafe { CURRENT_SYSTEMSTATE_VEC.split_off(0) }); + let (intervals, mem_reads, dumped_states, success) = + states2intervals(refined_states.clone(), metadata); + #[cfg(not(feature = "trace_job_response_times"))] + let jobs = Vec::new(); + #[cfg(feature = "trace_job_response_times")] + let jobs = { + let releases = get_releases(&intervals, &dumped_states); + let responses = unsafe { JOBS_DONE.split_off(0) }; + let (job_spans, do_report) = get_release_response_pairs(&releases, &responses); + + let jobs : Vec = job_spans + .into_iter() + .map(|x| { + let intervals_of_job_x = intervals + .iter() + .enumerate() + .filter(|y| { + y.1.start_tick <= x.1 + && y.1.end_tick >= x.0 + && x.2 == y.1.get_task_name_unchecked() + }) + .map(|(idx, x)| (x, &mem_reads[idx])) + .collect::>(); + + let (abbs, rest): (Vec<_>, Vec<_>) = intervals_of_job_x + .chunk_by(|a, b| { + a.0.abb + .as_ref() + .unwrap() + .instance_eq(b.0.abb.as_ref().unwrap()) + }) + .into_iter() // group by abb + .map(|intervals| { + ( + intervals[0].0.abb.as_ref().unwrap().clone(), + ( + intervals.iter().fold(0, |sum, z| sum + z.0.get_exec_time()), + intervals.iter().fold(Vec::new(), |mut sum, z| { + sum.extend(z.1.iter()); + sum + }), + ), + ) + }) + .unzip(); + let (ticks_per_abb, mem_reads_per_abb): (Vec<_>, Vec<_>) = rest.into_iter().unzip(); + RTOSJob { + name: x.2, + mem_reads: mem_reads_per_abb.into_iter().flatten().collect(), // TODO: add read values + release: x.0, + response: x.1, + exec_ticks: ticks_per_abb.iter().sum(), + ticks_per_abb: ticks_per_abb, + abbs: abbs, + hash_cache: 0, + } + }) + .collect::>(); + jobs + }; + _state.add_metadata(FreeRTOSTraceMetadata::new(refined_states, intervals, mem_reads, jobs)); + } + + type ModuleAddressFilter = NopAddressFilter; + + type ModulePageFilter = NopPageFilter; + + fn address_filter(&self) -> &Self::ModuleAddressFilter { + todo!() + } + + fn address_filter_mut(&mut self) -> &mut Self::ModuleAddressFilter { + todo!() + } + + fn page_filter(&self) -> &Self::ModulePageFilter { + todo!() + } + + fn page_filter_mut(&mut self) -> &mut Self::ModulePageFilter { + todo!() + } +} + +//============================= Trace job response times + +pub static mut JOBS_DONE: Vec<(u64, String)> = vec![]; + +pub fn job_done_hook( + hooks: &mut EmulatorModules, + _state: Option<&mut S>, + _pc: GuestAddr, +) where + S: UsesInput, + QT: EmulatorModuleTuple, +{ + let emulator = hooks.qemu(); + let h = hooks + .modules() + .match_first_type::() + .expect("QemuSystemHelper not found in helper tupel"); + let curr_tcb_addr: bindings::void_ptr = super::QemuLookup::lookup(&emulator, h.tcb_addr); + if curr_tcb_addr == 0 { + return; + }; + let current_tcb: TCB_t = super::QemuLookup::lookup(&emulator, curr_tcb_addr); + let tmp = unsafe { std::mem::transmute::<[i8; 10], [u8; 10]>(current_tcb.pcTaskName) }; + let name: String = std::str::from_utf8(&tmp) + .expect("TCB name was not utf8") + .chars() + .filter(|x| *x != '\0') + .collect::(); + unsafe { + JOBS_DONE.push((get_icount(&emulator), name)); + } +} + +//============================= Trace interrupt service routines + +pub fn exec_isr_hook( + hooks: &mut EmulatorModules, + _state: Option<&mut S>, + pc: GuestAddr, +) where + S: UsesInput, + QT: EmulatorModuleTuple, +{ + let emulator = hooks.qemu(); + let h = hooks + .modules() + .match_first_type::() + .expect("QemuSystemHelper not found in helper tupel"); + let src = read_rec_return_stackframe(&emulator, 0xfffffffc); + trigger_collection(&emulator, (src, pc), CaptureEvent::ISRStart, h); + // println!("Exec ISR Call {:#x} {:#x} {}", src, pc, get_icount(emulator)); +} + +//============================= Trace syscalls and returns + +pub fn gen_jmp_is_syscall( + hooks: &mut EmulatorModules, + _state: Option<&mut S>, + src: GuestAddr, + dest: GuestAddr, +) -> Option +where + S: UsesInput, + QT: EmulatorModuleTuple, +{ + if let Some(h) = hooks + .modules() + .match_first_type::() + { + if h.app_range.contains(&src) + && !h.app_range.contains(&dest) + && in_any_range(&h.isr_ranges, src).is_none() + { + if let Some(_) = in_any_range(&h.api_fn_ranges, dest) { + // println!("New jmp {:x} {:x}", src, dest); + // println!("API Call Edge {:x} {:x}", src, dest); + return Some(1); + // TODO: trigger collection right here + // otherwise there can be a race-condition, where LAST_API_CALL is set before the api starts, if the interrupt handler calls an api function, it will misidentify the callsite of that api call + } + } else if dest == 0 { + // !h.app_range.contains(&src) && + if let Some(_) = in_any_range(&h.api_fn_ranges, src) { + // println!("API Return Edge {:#x}", src); + return Some(2); + } + if let Some(_) = in_any_range(&h.isr_ranges, src) { + // println!("ISR Return Edge {:#x}", src); + return Some(3); + } + } + } + return None; +} + +pub fn trace_jmp( + hooks: &mut EmulatorModules, + _state: Option<&mut S>, + src: GuestAddr, + mut dest: GuestAddr, + id: u64, +) where + S: UsesInput, + QT: EmulatorModuleTuple, +{ + let h = hooks + .modules() + .match_first_type::() + .expect("QemuSystemHelper not found in helper tupel"); + let emulator = hooks.qemu(); + if id == 1 { + // API call + trigger_collection(&emulator, (src, dest), CaptureEvent::APIStart, h); + // println!("Exec API Call {:#x} {:#x} {}", src, dest, get_icount(emulator)); + } else if id == 2 { + // API return + // Ignore returns to other APIs or ISRs. We only account for the first call depth of API calls from user space. + if in_any_range(&h.api_fn_ranges, dest).is_none() + && in_any_range(&h.isr_ranges, dest).is_none() + { + let mut edge = (0, 0); + edge.0 = in_any_range(&h.api_fn_ranges, src).unwrap().start; + edge.1 = dest; + + trigger_collection(&emulator, edge, CaptureEvent::APIEnd, h); + // println!("Exec API Return Edge {:#x} {:#x} {}", src, dest, get_icount(emulator)); + } + } else if id == 3 { + // ISR return + dest = read_rec_return_stackframe(&emulator, dest); + + let mut edge = (0, 0); + edge.0 = in_any_range(&h.isr_ranges, src).unwrap().start; + edge.1 = dest; + + trigger_collection(&emulator, edge, CaptureEvent::ISREnd, h); + // println!("Exec ISR Return Edge {:#x} {:#x} {}", src, dest, get_icount(emulator)); + } +} + +//============================= Read Hooks +#[allow(unused)] +pub fn gen_read_is_input( + hooks: &mut EmulatorModules, + _state: Option<&mut S>, + pc: GuestAddr, + _addr: *mut TCGTemp, + _info: MemAccessInfo, +) -> Option +where + S: UsesInput, + QT: EmulatorModuleTuple, +{ + if let Some(h) = hooks + .modules() + .match_first_type::() + { + if h.app_range.contains(&pc) { + // println!("gen_read {:x}", pc); + return Some(1); + } + } + return None; +} + +static mut INPUT_MEM: Range = 0..0; +pub static mut MEM_READ: Option> = None; + +#[allow(unused)] +pub fn trace_reads( + hooks: &mut EmulatorModules, + _state: Option<&mut S>, + _id: u64, + addr: GuestAddr, + _size: usize, +) where + S: UsesInput, + QT: EmulatorModuleTuple, +{ + if unsafe { INPUT_MEM.contains(&addr) } { + let emulator = hooks.qemu(); + let mut buf: [u8; 1] = [0]; + unsafe { + emulator.read_mem(addr, &mut buf); + } + if unsafe { MEM_READ.is_none() } { + unsafe { MEM_READ = Some(Vec::from([(addr, buf[0])])) }; + } else { + unsafe { MEM_READ.as_mut().unwrap().push((addr, buf[0])) }; + } + // println!("exec_read {:x} {}", addr, size); + } +} + +//============================= Parsing helpers + +/// Parse a List_t containing TCB_t into Vec from cache. Consumes the elements from cache +fn tcb_list_to_vec_cached(list: List_t, dump: &mut HashMap) -> Vec { + let mut ret: Vec = Vec::new(); + if list.uxNumberOfItems == 0 { + return ret; + } + let last_list_item = match dump + .remove(&list.pxIndex) + .expect("List_t entry was not in Hashmap") + { + List_Item_struct(li) => li, + List_MiniItem_struct(mli) => match dump + .remove(&mli.pxNext) + .expect("MiniListItem pointer invaild") + { + List_Item_struct(li) => li, + _ => panic!("MiniListItem of a non empty List does not point to ListItem"), + }, + _ => panic!("List_t entry was not a ListItem"), + }; + let mut next_index = last_list_item.pxNext; + let last_tcb = match dump + .remove(&last_list_item.pvOwner) + .expect("ListItem Owner not in Hashmap") + { + TCB_struct(t) => t, + _ => panic!("List content does not equal type"), + }; + for _ in 0..list.uxNumberOfItems - 1 { + let next_list_item = match dump + .remove(&next_index) + .expect("List_t entry was not in Hashmap") + { + List_Item_struct(li) => li, + List_MiniItem_struct(mli) => match dump + .remove(&mli.pxNext) + .expect("MiniListItem pointer invaild") + { + List_Item_struct(li) => li, + _ => panic!("MiniListItem of a non empty List does not point to ListItem"), + }, + _ => panic!("List_t entry was not a ListItem"), + }; + match dump + .remove(&next_list_item.pvOwner) + .expect("ListItem Owner not in Hashmap") + { + TCB_struct(t) => ret.push(t), + _ => panic!("List content does not equal type"), + } + next_index = next_list_item.pxNext; + } + ret.push(last_tcb); + ret +} + +//============================= State refinement + +/// Drains a List of raw SystemStates to produce a refined trace +/// returns: +/// - a Vec of FreeRTOSSystemState +/// - a Vec of FreeRTOSSystemStateContext (qemu_tick, (capture_event, capture_name), edge, mem_reads) +fn refine_system_states( + mut input: Vec, +) -> (Vec, Vec) { + let mut ret = (Vec::<_>::new(), Vec::<_>::new()); + for mut i in input.drain(..) { + let cur = RefinedTCB::from_tcb_owned(i.current_tcb); + // println!("Refine: {} {:?} {:?} {:x}-{:x}", cur.task_name, i.capture_point.0, i.capture_point.1.to_string(), i.edge.0, i.edge.1); + // collect ready list + let mut collector = Vec::::new(); + for j in i.prio_ready_lists.into_iter().rev() { + let mut tmp = tcb_list_to_vec_cached(j, &mut i.dumping_ground) + .iter() + .map(|x| RefinedTCB::from_tcb(x)) + .collect(); + collector.append(&mut tmp); + } + // collect delay list + let mut delay_list: Vec = + tcb_list_to_vec_cached(i.delay_list, &mut i.dumping_ground) + .iter() + .map(|x| RefinedTCB::from_tcb(x)) + .collect(); + let mut delay_list_overflow: Vec = + tcb_list_to_vec_cached(i.delay_list_overflow, &mut i.dumping_ground) + .iter() + .map(|x| RefinedTCB::from_tcb(x)) + .collect(); + delay_list.append(&mut delay_list_overflow); + delay_list.sort_by(|a, b| a.task_name.cmp(&b.task_name)); + + ret.0.push(FreeRTOSSystemState { + current_task: cur, + ready_list_after: collector, + delay_list_after: delay_list, + read_invalid: i.read_invalid, + // input_counter: i.input_counter,//+IRQ_INPUT_BYTES_NUMBER, + }); + ret.1.push(FreeRTOSSystemStateContext { + qemu_tick: i.qemu_tick, + capture_point: (i.capture_point.0, i.capture_point.1.to_string()), + edge: i.edge, + mem_reads: i.mem_reads, + }); + } + return ret; +} + +/// Transform the states and metadata into a list of ExecIntervals, along with a HashMap of states, a list of HashSets marking memory reads and a bool indicating success +/// returns: +/// - a Vec of ExecIntervals +/// - a Vec of HashSets marking memory reads during these intervals +/// - a HashMap of ReducedFreeRTOSSystemStates by hash +/// - a bool indicating success +fn states2intervals( + trace: Vec, + meta: Vec, +) -> ( + Vec, + Vec>, + HashMap, + bool, +) { + if trace.len() == 0 { + return (Vec::new(), Vec::new(), HashMap::new(), true); + } + let mut isr_stack: VecDeque = VecDeque::from([]); // 2+ = ISR, 1 = systemcall, 0 = APP. Trace starts with an ISREnd and executes the app + + let mut level_of_task: HashMap<&str, u8> = HashMap::new(); + + let mut ret: Vec = vec![]; + let mut reads: Vec> = vec![]; + let mut edges: Vec<(u32, u32)> = vec![]; + let mut last_hash: u64 = compute_hash(&trace[0]); + let mut table: HashMap = HashMap::new(); + table.insert(last_hash, trace[0].clone()); + for i in 0..trace.len() - 1 { + let curr_name = trace[i].current_task().task_name().as_str(); + // let mut interval_name = curr_name; // Name of the interval, either the task name or the isr/api funtion name + let level = match meta[i].capture_point.0 { + CaptureEvent::APIEnd => { + // API end always exits towards the app + if !level_of_task.contains_key(curr_name) { + level_of_task.insert(curr_name, 0); + } + *level_of_task.get_mut(curr_name).unwrap() = 0; + 0 + } + CaptureEvent::APIStart => { + // API start can only be called in the app + if !level_of_task.contains_key(curr_name) { + // Should not happen, apps start from an ISR End. Some input exibited this behavior for unknown reasons + level_of_task.insert(curr_name, 0); + } + *level_of_task.get_mut(curr_name).unwrap() = 1; + // interval_name = &meta[i].2; + 1 + } + CaptureEvent::ISREnd => { + // special case where the next block is an app start + if !level_of_task.contains_key(curr_name) { + level_of_task.insert(curr_name, 0); + } + // nested isr, TODO: Test level > 2 + if isr_stack.len() > 1 { + // interval_name = ""; // We can't know which isr is running + isr_stack.pop_back().unwrap(); + *isr_stack.back().unwrap() + } else { + isr_stack.pop_back(); + // possibly go back to an api call that is still running for this task + if level_of_task.get(curr_name).unwrap() == &1 { + // interval_name = ""; // We can't know which api is running + } + *level_of_task.get(curr_name).unwrap() + } + } + CaptureEvent::ISRStart => { + // special case for isrs which do not capture their end + // if meta[i].2 == "ISR_0_Handler" { + // &2 + // } else { + // regular case + // interval_name = &meta[i].2; + if isr_stack.len() > 0 { + let l = *isr_stack.back().unwrap(); + isr_stack.push_back(l + 1); + l + 1 + } else { + isr_stack.push_back(2); + 2 + } + // } + } + _ => 100, + }; + // if trace[i].2 == CaptureEvent::End {break;} + let next_hash = compute_hash(&trace[i + 1]); + if !table.contains_key(&next_hash) { + table.insert(next_hash, trace[i + 1].clone()); + } + ret.push(ExecInterval { + start_tick: meta[i].qemu_tick, + end_tick: meta[i + 1].qemu_tick, + start_state: last_hash, + end_state: next_hash, + start_capture: meta[i].capture_point.clone(), + end_capture: meta[i + 1].capture_point.clone(), + level: level, + abb: None, + }); + reads.push(meta[i + 1].mem_reads.clone()); + last_hash = next_hash; + edges.push((meta[i].edge.1, meta[i + 1].edge.0)); + } + let t = add_abb_info(&mut ret, &table, &edges); + (ret, reads, table, t) +} + +/// Marks which abbs were executed at each interval +fn add_abb_info( + trace: &mut Vec, + table: &HashMap, + edges: &Vec<(u32, u32)>, +) -> bool { + let mut id_count = 0; + let mut ret = true; + let mut task_has_started: HashSet = HashSet::new(); + let mut wip_abb_trace: Vec>> = vec![]; + // let mut open_abb_at_this_task_or_level : HashMap<(u8,&str),usize> = HashMap::new(); + let mut open_abb_at_this_ret_addr_and_task: HashMap<(u32, &str), usize> = HashMap::new(); + + for i in 0..trace.len() { + let curr_name = &table[&trace[i].start_state].current_task().task_name(); + // let last : Option<&usize> = last_abb_start_of_task.get(&curr_name); + + // let open_abb = open_abb_at_this_task_or_level.get(&(trace[i].level, if trace[i].level<2 {&curr_name} else {""})).to_owned(); // apps/apis are differentiated by task name, isrs by nested level + let open_abb = open_abb_at_this_ret_addr_and_task + .get(&(edges[i].0, if trace[i].level < 2 { &curr_name } else { "" })) + .to_owned(); // apps/apis are differentiated by task name, isrs by nested level + + // println!("Edge {:x}-{:x}", edges[i].0.unwrap_or(0xffff), edges[i].1.unwrap_or(0xffff)); + + match trace[i].start_capture.0 { + // generic api abb start + CaptureEvent::APIStart => { + // assert_eq!(open_abb, None); + ret &= open_abb.is_none(); + open_abb_at_this_ret_addr_and_task.insert( + (edges[i].1, if trace[i].level < 2 { &curr_name } else { "" }), + i, + ); + wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock { + start: edges[i].0, + ends: HashSet::new(), + level: if trace[i].level < 2 { + trace[i].level + } else { + 2 + }, + instance_id: id_count, + instance_name: Some(trace[i].start_capture.1.clone()), + }))); + id_count += 1; + } + // generic isr abb start + CaptureEvent::ISRStart => { + // assert_eq!(open_abb, None); + ret &= open_abb.is_none(); + open_abb_at_this_ret_addr_and_task.insert( + (edges[i].1, if trace[i].level < 2 { &curr_name } else { "" }), + i, + ); + wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock { + start: edges[i].0, + ends: HashSet::new(), + level: if trace[i].level < 2 { + trace[i].level + } else { + 2 + }, + instance_id: id_count, + instance_name: Some(trace[i].start_capture.1.clone()), + }))); + id_count += 1; + } + // generic app abb start + CaptureEvent::APIEnd => { + // assert_eq!(open_abb, None); + ret &= open_abb.is_none(); + open_abb_at_this_ret_addr_and_task.insert( + (edges[i].1, if trace[i].level < 2 { &curr_name } else { "" }), + i, + ); + wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock { + start: edges[i].0, + ends: HashSet::new(), + level: if trace[i].level < 2 { + trace[i].level + } else { + 2 + }, + instance_id: id_count, + instance_name: if trace[i].level < 2 { + Some(curr_name.clone().clone()) + } else { + None + }, + }))); + id_count += 1; + } + // generic continued blocks + CaptureEvent::ISREnd => { + // special case app abb start + if trace[i].start_capture.1 == "xPortPendSVHandler" + && !task_has_started.contains(curr_name.clone()) + { + // assert_eq!(open_abb, None); + ret &= open_abb.is_none(); + wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock { + start: 0, + ends: HashSet::new(), + level: if trace[i].level < 2 { + trace[i].level + } else { + 2 + }, + instance_id: id_count, + instance_name: Some(curr_name.clone().clone()), + }))); + id_count += 1; + open_abb_at_this_ret_addr_and_task.insert( + (edges[i].1, if trace[i].level < 2 { &curr_name } else { "" }), + i, + ); + task_has_started.insert(curr_name.clone().clone()); + } else { + if let Some(last) = open_abb_at_this_ret_addr_and_task + .get(&(edges[i].0, if trace[i].level < 2 { &curr_name } else { "" })) + { + let last = last.clone(); // required to drop immutable reference + wip_abb_trace.push(wip_abb_trace[last].clone()); + // if the abb is interrupted again, it will need to continue at edge[i].1 + open_abb_at_this_ret_addr_and_task.remove(&( + edges[i].0, + if trace[i].level < 2 { &curr_name } else { "" }, + )); + open_abb_at_this_ret_addr_and_task.insert( + (edges[i].1, if trace[i].level < 2 { &curr_name } else { "" }), + last, + ); // order matters! + } else { + // panic!(); + // println!("Continued block with no start {} {} {:?} {:?} {:x}-{:x} {} {}", curr_name, trace[i].start_tick, trace[i].start_capture, trace[i].end_capture, edges[i].0, edges[i].1, task_has_started.contains(curr_name),trace[i].level); + // println!("{:x?}", open_abb_at_this_ret_addr_and_task); + ret = false; + wip_abb_trace.push(Rc::new(RefCell::new(AtomicBasicBlock { + start: edges[i].1, + ends: HashSet::new(), + level: if trace[i].level < 2 { + trace[i].level + } else { + 2 + }, + instance_id: id_count, + instance_name: if trace[i].level < 1 { + Some(curr_name.clone().clone()) + } else { + None + }, + }))); + id_count += 1; + } + } + } + _ => panic!("Undefined block start"), + } + match trace[i].end_capture.0 { + // generic app abb end + CaptureEvent::APIStart => { + let _t = &wip_abb_trace[i]; + RefCell::borrow_mut(&*wip_abb_trace[i]) + .ends + .insert(edges[i].1); + open_abb_at_this_ret_addr_and_task + .remove(&(edges[i].1, if trace[i].level < 2 { &curr_name } else { "" })); + } + // generic api abb end + CaptureEvent::APIEnd => { + RefCell::borrow_mut(&*wip_abb_trace[i]) + .ends + .insert(edges[i].1); + open_abb_at_this_ret_addr_and_task + .remove(&(edges[i].1, if trace[i].level < 2 { &curr_name } else { "" })); + } + // generic isr abb end + CaptureEvent::ISREnd => { + RefCell::borrow_mut(&*wip_abb_trace[i]) + .ends + .insert(edges[i].1); + open_abb_at_this_ret_addr_and_task + .remove(&(edges[i].1, if trace[i].level < 2 { &curr_name } else { "" })); + } + // end anything + CaptureEvent::End => { + RefCell::borrow_mut(&*wip_abb_trace[i]) + .ends + .insert(edges[i].1); + open_abb_at_this_ret_addr_and_task + .remove(&(edges[i].1, if trace[i].level < 2 { &curr_name } else { "" })); + } + CaptureEvent::ISRStart => (), + _ => panic!("Undefined block end"), + } + // println!("{} {} {:x}-{:x} {:x}-{:x} {:?} {:?} {}",curr_name, trace[i].level, edges[i].0, edges[i].1, ((*wip_abb_trace[i])).borrow().start, ((*wip_abb_trace[i])).borrow().ends.iter().next().unwrap_or(&0xffff), trace[i].start_capture, trace[i].end_capture, trace[i].start_tick); + // println!("{:x?}", open_abb_at_this_ret_addr_and_task); + } + // drop(open_abb_at_this_task_or_level); + + for i in 0..trace.len() { + trace[i].abb = Some((*wip_abb_trace[i]).borrow().clone()); + } + return ret; +} + +//============================================= Task release times + +// Find all task release times. +fn get_releases( + trace: &Vec, + states: &HashMap, +) -> Vec<(u64, String)> { + let mut ret = Vec::new(); + let mut initial_released = false; + for (_n, i) in trace.iter().enumerate() { + // The first release starts from xPortPendSVHandler + if !initial_released + && i.start_capture.0 == CaptureEvent::ISREnd + && i.start_capture.1 == "xPortPendSVHandler" + { + let start_state = states.get(&i.start_state).expect("State not found"); + initial_released = true; + start_state.get_ready_lists().iter().for_each(|x| { + ret.push((i.start_tick, x.task_name().clone())); + }); + continue; + } + // A timed release is SysTickHandler isr block that moves a task from the delay list to the ready list. + if i.start_capture.0 == CaptureEvent::ISRStart + && (i.start_capture.1 == "xPortSysTickHandler" + || USR_ISR_SYMBOLS.contains(&i.start_capture.1.as_str())) + { + // detect race-conditions, get start and end state from the nearest valid intervals + if states + .get(&i.start_state) + .map(|x| x.read_invalid) + .unwrap_or(true) + { + let mut start_index = None; + for n in 1.._n { + if let Some(interval_start) = trace.get(_n - n) { + let start_state = states.get(&interval_start.start_state).unwrap(); + if !start_state.read_invalid { + start_index = Some(_n - n); + break; + } + } else { + break; + } + } + let mut end_index = None; + for n in (_n + 1)..trace.len() { + if let Some(interval_end) = trace.get(n) { + let end_state = states.get(&interval_end.end_state).unwrap(); + if !end_state.read_invalid { + end_index = Some(n); + break; + } + } else { + break; + } + } + if let Some(Some(start_state)) = + start_index.map(|x| states.get(&trace[x].start_state)) + { + if let Some(Some(end_state)) = + end_index.map(|x| states.get(&trace[x].end_state)) + { + end_state.ready_list_after.iter().for_each(|x| { + if x.task_name != end_state.current_task.task_name + && x.task_name != start_state.current_task.task_name + && !start_state + .ready_list_after + .iter() + .any(|y| x.task_name == y.task_name) + { + ret.push((i.end_tick, x.task_name.clone())); + } + }); + } + } + } else + // canonical case, userspace -> isr -> userspace + if i.end_capture.0 == CaptureEvent::ISREnd { + let start_state = states.get(&i.start_state).expect("State not found"); + let end_state = states.get(&i.end_state).expect("State not found"); + end_state.ready_list_after.iter().for_each(|x| { + if x.task_name != end_state.current_task.task_name + && x.task_name != start_state.current_task.task_name + && !start_state + .ready_list_after + .iter() + .any(|y| x.task_name == y.task_name) + { + ret.push((i.end_tick, x.task_name.clone())); + } + }); + // start_state.delay_list_after.iter().for_each(|x| { + // if !end_state.delay_list_after.iter().any(|y| x.task_name == y.task_name) { + // ret.push((i.end_tick, x.task_name.clone())); + // } + // }); + } else if i.end_capture.0 == CaptureEvent::ISRStart { + // Nested interrupts. Fast-forward to the end of the original interrupt, or the first valid state thereafter + // TODO: this may cause the same release to be registered multiple times + let mut isr_has_ended = false; + let start_state = states.get(&i.start_state).expect("State not found"); + for n in (_n + 1)..trace.len() { + if let Some(interval_end) = trace.get(n) { + if interval_end.end_capture.1 == i.start_capture.1 || isr_has_ended { + let end_state = states.get(&interval_end.end_state).unwrap(); + isr_has_ended = true; + if !end_state.read_invalid { + end_state.ready_list_after.iter().for_each(|x| { + if x.task_name != end_state.current_task.task_name + && x.task_name != start_state.current_task.task_name + && !start_state + .ready_list_after + .iter() + .any(|y| x.task_name == y.task_name) + { + ret.push((i.end_tick, x.task_name.clone())); + } + }); + break; + } + } + } else { + break; + } + } + // if let Some(interval_end) = trace.get(_n+2) { + // if interval_end.start_capture.0 == CaptureEvent::ISREnd && interval_end.end_capture.0 == CaptureEvent::ISREnd && interval_end.end_capture.1 == i.start_capture.1 { + // let start_state = states.get(&i.start_state).expect("State not found"); + // let end_state = states.get(&interval_end.end_state).expect("State not found"); + // end_state.ready_list_after.iter().for_each(|x| { + // if x.task_name != end_state.current_task.task_name && x.task_name != start_state.current_task.task_name && !start_state.ready_list_after.iter().any(|y| x.task_name == y.task_name) { + // ret.push((i.end_tick, x.task_name.clone())); + // } + // }); + // } + // } + } + } + // Release driven by an API call. This produces a lot of false positives, as a job may block multiple times per instance. Despite this, aperiodic jobs not be modeled otherwise. If we assume the first release is the real one, we can filter out the rest. + if i.start_capture.0 == CaptureEvent::APIStart { + let api_start_state = states.get(&i.start_state).expect("State not found"); + let api_end_state = { + let mut end_index = _n; + for n in (_n)..trace.len() { + if trace[n].end_capture.0 == CaptureEvent::APIEnd + || trace[n].end_capture.0 == CaptureEvent::End + { + end_index = n; + break; + } else if n > _n && trace[n].level == 0 { + // API Start -> ISR Start+End -> APP Continue + end_index = n - 1; // any return to a regular app block is a fair point of comparison for the ready list, because scheduling has been performed + break; + } + } + states + .get(&trace[end_index].end_state) + .expect("State not found") + }; + api_end_state.ready_list_after.iter().for_each(|x| { + if x.task_name != api_start_state.current_task.task_name + && !api_start_state + .ready_list_after + .iter() + .any(|y| x.task_name == y.task_name) + { + ret.push((i.end_tick, x.task_name.clone())); + // eprintln!("Task {} released by API call at {:.1}ms", x.task_name, crate::time::clock::tick_to_time(i.end_tick).as_micros() as f32/1000.0); + } + }); + } + } + ret +} + +fn get_release_response_pairs( + rel: &Vec<(u64, String)>, + resp: &Vec<(u64, String)>, +) -> (Vec<(u64, u64, String)>, bool) { + let mut maybe_error = false; + let mut ret = Vec::new(); + let mut ready: HashMap<&String, u64> = HashMap::new(); + let mut last_response: HashMap<&String, u64> = HashMap::new(); + let mut r = rel.iter().peekable(); + let mut d = resp.iter().peekable(); + loop { + while let Some(peek_rel) = r.peek() { + // Fill releases as soon as possible + if !ready.contains_key(&peek_rel.1) { + ready.insert(&peek_rel.1, peek_rel.0); + r.next(); + } else { + if let Some(peek_resp) = d.peek() { + if peek_resp.0 > peek_rel.0 { + // multiple releases before response + // It is unclear which release is real + // maybe_error = true; + // eprintln!("Task {} released multiple times before response ({:.1}ms and {:.1}ms)", peek_rel.1, crate::time::clock::tick_to_time(ready[&peek_rel.1]).as_micros()/1000, crate::time::clock::tick_to_time(peek_rel.0).as_micros()/1000); + // ready.insert(&peek_rel.1, peek_rel.0); + r.next(); + } else { + // releases have overtaken responses, wait until the ready list clears up a bit + break; + } + } else { + // no more responses + break; + } + } + } + if let Some(next_resp) = d.next() { + if ready.contains_key(&next_resp.1) { + if ready[&next_resp.1] >= next_resp.0 { + if let Some(lr) = last_response.get(&next_resp.1) { + if u128::abs_diff( + crate::time::clock::tick_to_time(next_resp.0).as_micros(), + crate::time::clock::tick_to_time(*lr).as_micros(), + ) > 500 + { + // tolerate pending notifications for 500us + maybe_error = true; + // eprintln!("Task {} response at {:.1}ms before next release at {:.1}ms. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(ready[&next_resp.1]).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0); + } + // Sometimes a task is released immediately after a response. This might not be detected. + // Assume that the release occured with the last response + ret.push((*lr, next_resp.0, next_resp.1.clone())); + last_response.insert(&next_resp.1, next_resp.0); + } else { + maybe_error = true; + // eprintln!("Task {} released after response", next_resp.1); + } + } else { + // assert!(peek_resp.0 >= ready[&peek_resp.1]); + last_response.insert(&next_resp.1, next_resp.0); + ret.push((ready[&next_resp.1], next_resp.0, next_resp.1.clone())); + ready.remove(&next_resp.1); + } + } else { + if let Some(lr) = last_response.get(&next_resp.1) { + if u128::abs_diff( + crate::time::clock::tick_to_time(next_resp.0).as_micros(), + crate::time::clock::tick_to_time(*lr).as_micros(), + ) > 1000 + { // tolerate pending notifications for 1ms + // maybe_error = true; + // eprintln!("Task {} response at {:.1}ms not found in ready list. Fallback to last response at {:.1}ms.", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0, crate::time::clock::tick_to_time(*lr).as_micros() as f32/1000.0); + } + // Sometimes a task is released immediately after a response (e.g. pending notification). This might not be detected. + // Assume that the release occured with the last response + ret.push((*lr, next_resp.0, next_resp.1.clone())); + last_response.insert(&next_resp.1, next_resp.0); + } else { + maybe_error = true; + // eprintln!("Task {} response at {:.1}ms not found in ready list", next_resp.1, crate::time::clock::tick_to_time(next_resp.0).as_micros() as f32/1000.0); + } + } + } else { + // TODO: should remaining released tasks be counted as finished? + return (ret, maybe_error); + } + } +} diff --git a/fuzzers/FRET/src/systemstate/target_os/mod.rs b/fuzzers/FRET/src/systemstate/target_os/mod.rs new file mode 100644 index 0000000000..a4a325c275 --- /dev/null +++ b/fuzzers/FRET/src/systemstate/target_os/mod.rs @@ -0,0 +1,110 @@ +use std::collections::hash_map::DefaultHasher; +use std::fmt; +use hashbrown::HashSet; +use libafl_bolts::prelude::SerdeAny; +use libafl_bolts::HasRefCnt; +use libafl_qemu::GuestAddr; +use libafl_qemu::Qemu; +use serde::de::DeserializeOwned; +use std::hash::Hasher; +use std::hash::Hash; +use hashbrown::HashMap; +use serde::{Deserialize, Serialize}; +use itertools::Itertools; +use std::fmt::Debug; + +use super::CaptureEvent; +use super::ExecInterval; +use super::RTOSJob; + +pub mod freertos; + +// Constants +const NUM_PRIOS: usize = 15; + +//============================= Trait definitions + +pub trait TargetSystem: Serialize + Sized + for<'a> Deserialize<'a> + Default + Debug + Clone + SerdeAny { + type State: SystemState; + type TCB: TaskControlBlock; + type TraceData: SystemTraceData; +} + +pub trait SystemState: Serialize + Sized + for<'a> Deserialize<'a> + Default + Debug + Hash + PartialEq + Clone + SerdeAny { + type TCB: TaskControlBlock; + + fn current_task(&self) -> &Self::TCB; + fn current_task_mut(&mut self) -> &mut Self::TCB; + fn get_ready_lists(&self) -> &Vec; + fn get_delay_list(&self) -> &Vec; + fn print_lists(&self) -> String; +} + +pub trait SystemTraceData: Serialize + Sized + for<'a> Deserialize<'a> + Default + Debug + Clone + SerdeAny + HasRefCnt { + type State: SystemState; + + fn states(&self) -> Vec<&Self::State>; + fn states_map(&self) -> &HashMap; + fn intervals(&self) -> &Vec; + fn mem_reads(&self) -> &Vec>; + fn jobs(&self) -> &Vec; + fn trace_length(&self) -> usize; + + #[inline] + fn worst_jobs_per_task_by(&self, pred: &dyn Fn(&RTOSJob,&RTOSJob) -> bool) -> HashMap { + self.jobs().iter().fold(HashMap::new(), |mut acc, next| { + match acc.get_mut(&next.name) { + Some(old) => { + if pred(old,next) { + *old=next.clone(); + } + }, + Option::None => { + acc.insert(next.name.clone(), next.clone()); + } + } + acc + }) + } + #[inline] + fn worst_jobs_per_task_by_time(&self) -> HashMap { + self.worst_jobs_per_task_by(&|old, x| x.exec_ticks > old.exec_ticks) + } +} + + +pub trait TaskControlBlock: Serialize + for<'a> Deserialize<'a> + Default + Debug + Hash + PartialEq + Clone + SerdeAny { + fn task_name(&self) -> &String; + fn task_name_mut(&mut self) -> &mut String; + // Define methods common to TCBs across different systems +} + +//============================= + +pub trait QemuLookup { + fn lookup(emu: &Qemu, addr: ::std::os::raw::c_uint) -> Self; +} + +#[macro_export] +macro_rules! impl_emu_lookup { + ($struct_name:ident) => { + impl $crate::systemstate::target_os::QemuLookup for $struct_name { + fn lookup(emu: &Qemu, addr: ::std::os::raw::c_uint) -> $struct_name { + let mut tmp : [u8; std::mem::size_of::<$struct_name>()] = [0u8; std::mem::size_of::<$struct_name>()]; + unsafe { + emu.read_mem(addr.into(), &mut tmp); + std::mem::transmute::<[u8; std::mem::size_of::<$struct_name>()], $struct_name>(tmp) + } + } + } + }; +} + +pub fn compute_hash(obj: &T) -> u64 +where + T: Hash, +{ + let mut s = DefaultHasher::new(); + obj.hash(&mut s); + s.finish() +} diff --git a/fuzzers/FRET/src/time/clock.rs b/fuzzers/FRET/src/time/clock.rs index 0347decea1..a70e1fc4c9 100644 --- a/fuzzers/FRET/src/time/clock.rs +++ b/fuzzers/FRET/src/time/clock.rs @@ -23,6 +23,7 @@ use std::path::PathBuf; use std::borrow::Cow; use crate::systemstate::observers::QemuSystemStateObserver; +use crate::systemstate::target_os::TargetSystem; pub static mut FUZZ_START_TIMESTAMP : SystemTime = UNIX_EPOCH; @@ -212,20 +213,22 @@ impl Default for QemuClockObserver { /// for this Feedback, the testcase is never interesting (use with an OR). /// It decides, if the given [`QemuClockObserver`] value of a run is interesting. #[derive(Serialize, Deserialize, Clone, Debug)] -pub struct ClockTimeFeedback { +pub struct ClockTimeFeedback { exec_time: Option, select_task: Option, name: Cow<'static, str>, + phantom: std::marker::PhantomData, } -impl StateInitializer for ClockTimeFeedback {} +impl StateInitializer for ClockTimeFeedback where SYS: TargetSystem {} -impl Feedback for ClockTimeFeedback +impl Feedback for ClockTimeFeedback where S: State + UsesInput + MaybeHasClientPerfMonitor + HasMetadata, ::Input: Default, EM: EventFirer, OT: ObserversTuple, + SYS: TargetSystem, { #[allow(clippy::wrong_self_convention)] fn is_interesting( @@ -241,7 +244,7 @@ where #[cfg(feature="trace_job_response_times")] { if self.select_task.is_some() { - let observer = observers.match_name::>("systemstate").unwrap(); + let observer = observers.match_name::>("systemstate").unwrap(); self.exec_time = Some(Duration::from_nanos(observer.last_runtime())); return Ok(false) } @@ -274,14 +277,14 @@ where } } -impl Named for ClockTimeFeedback { +impl Named for ClockTimeFeedback { #[inline] fn name(&self) -> &Cow<'static, str> { &self.name } } -impl ClockTimeFeedback { +impl ClockTimeFeedback { /// Creates a new [`ClockFeedback`], deciding if the value of a [`QemuClockObserver`] with the given `name` of a run is interesting. #[must_use] pub fn new(name: &'static str, select_task: Option) -> Self { @@ -289,6 +292,7 @@ impl ClockTimeFeedback { exec_time: None, select_task: select_task, name: Cow::from(name.to_string()), + phantom: std::marker::PhantomData, } } @@ -299,6 +303,7 @@ impl ClockTimeFeedback { exec_time: None, select_task: select_task.clone(), name: observer.name().clone(), + phantom: std::marker::PhantomData, } } } diff --git a/fuzzers/FRET/src/time/worst.rs b/fuzzers/FRET/src/time/worst.rs index fa8d70bb8a..3d9e41ad42 100644 --- a/fuzzers/FRET/src/time/worst.rs +++ b/fuzzers/FRET/src/time/worst.rs @@ -36,8 +36,8 @@ use libafl::{ Error, }; +use crate::systemstate::target_os::TargetSystem; use crate::time::clock::QemuClockObserver; -use crate::systemstate::FreeRTOSSystemStateMetadata; use libafl::prelude::StateInitializer; @@ -72,8 +72,8 @@ where pub type LenTimeMaximizerCorpusScheduler = MinimizerScheduler::State>, MapIndexesMetadata, O>; -pub type TimeStateMaximizerCorpusScheduler = - MinimizerScheduler::State>, FreeRTOSSystemStateMetadata, O>; +pub type TimeStateMaximizerCorpusScheduler = + MinimizerScheduler::State>, ::TraceData, O>; /// Multiply the testcase size with the execution time. /// This favors small and quick testcases. diff --git a/libafl/src/fuzzer/mod.rs b/libafl/src/fuzzer/mod.rs index 0bd6955ef5..551cebb602 100644 --- a/libafl/src/fuzzer/mod.rs +++ b/libafl/src/fuzzer/mod.rs @@ -313,7 +313,6 @@ where } let mut ret = None; - let mut last = current_time(); let monitor_timeout = STATS_TIMEOUT_DEFAULT; let starttime = std::time::Instant::now(); @@ -349,7 +348,6 @@ where time: std::time::Instant ) -> Result { let mut ret = None; - let mut last = current_time(); let monitor_timeout = STATS_TIMEOUT_DEFAULT; while std::time::Instant::now() < time { @@ -364,7 +362,7 @@ where if ret.is_none() { eprintln!("Warning: fuzzing loop ended with no last element"); - ret = Some(crate::corpus::CorpusId(0)); + ret = Some(CorpusId(0)); } Ok(ret.unwrap()) } diff --git a/libafl_qemu/libafl_qemu_build/src/bindings.rs b/libafl_qemu/libafl_qemu_build/src/bindings.rs index 0e15432081..eb92daf15c 100644 --- a/libafl_qemu/libafl_qemu_build/src/bindings.rs +++ b/libafl_qemu/libafl_qemu_build/src/bindings.rs @@ -181,6 +181,7 @@ pub fn generate( .allowlist_function("vm_start") .allowlist_function("qemu_main_loop") .allowlist_function("qemu_cleanup") + .allowlist_function("icount_get_raw") .blocklist_function("main_loop_wait") // bindgen issue #1313 .blocklist_type("siginfo_t") .raw_line("use libc::siginfo_t;")