WIP: per-task response times

fuzzers/FRET/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Alwin Berger <alwin.berger@tu-dortmund.de>"]
 edition = "2021"
 
 [features]
-default = ["std", "snapshot_restore", "snapshot_fast", "singlecore", "restarting", "do_hash_notify_state", "config_stg", "fuzz_int" ]
+default = ["std", "snapshot_restore", "snapshot_fast", "singlecore", "restarting", "do_hash_notify_state", "config_stg", "fuzz_int", "trace_job_response_times" ]
 std = []
 # Exec environemnt basics
 snapshot_restore = []
@@ -19,6 +19,7 @@ observe_edges = [] # observe cfg edges
 observe_hitcounts = [ "observe_edges" ] # reduces edge granularity
 observe_systemstate = []
 do_hash_notify_state = []
+trace_job_response_times = [ "trace_stg" ]
 trace_stg = [ "observe_systemstate" ]
 trace_reads = [ "trace_stg" ]
 # feedbacks
@@ -60,7 +61,7 @@ libafl_qemu = { path = "../../libafl_qemu/", features = ["arm", "systemmode"] }
 serde = { version = "1.0", default-features = false, features = ["alloc"] } # serialization lib
 serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
 hashbrown =  { version = "0.14.0", features = ["serde"] } # A faster hashmap, nostd compatible
-petgraph = { version="0.6.4", features = ["serde-1"] }
+petgraph = { version="0.6.5", features = ["serde-1"] }
 ron = "0.7" # write serialized data - including hashmaps
 rand = "0.5"
 clap = { version = "4.4.11", features = ["derive"] }

fuzzers/FRET/benchmark/Snakefile

@@ -1,6 +1,6 @@
 import csv
 import os
-def_flags="--no-default-features --features std,snapshot_restore,singlecore,restarting,do_hash_notify_state"
+def_flags="--no-default-features --features std,snapshot_restore,singlecore,restarting,do_hash_notify_state,trace_job_response_times"
 remote="remote/"
 RUNTIME=1800
 TARGET_REPS_A=2
@@ -200,7 +200,7 @@ rule tarnsform_trace:
     output:
         "{remote}timedump/{fuzzer}/{target}.{num}.trace.csv"
     shell:
-        "$(pwd)/../../../../state2gantt/target/debug/state2gantt {input} > {output[0]}"
+        "$(pwd)/../../../../state2gantt/target/debug/state2gantt {input} {output[0]} {output[0]}2"
 
 rule trace2gantt:
     input:
@@ -236,11 +236,11 @@ rule all_compare_afl_int:
 
 rule all_images:
     input:
-        expand("{remote}timedump/{fuzzer}/{target}.{num}.trace.csv.png",remote=remote, fuzzer=['afl','feedgeneration10','state'], target=['waters','watersv2'],num=range(0,3))
+        expand("{remote}timedump/{fuzzer}/{target}.{num}.trace.csv.png",remote=remote, fuzzer=['frafl','feedgeneration10','state'], target=['waters'],num=range(0,3))
 
 rule all_images_int:
     input:
-        expand("{remote}timedump/{fuzzer}/{target}.{num}.trace.csv.png",remote=remote, fuzzer=['afl_int','feedgeneration10_int','state_int'], target=['waters_int','watersv2_int'],num=range(0,3))
+        expand("{remote}timedump/{fuzzer}/{target}.{num}.trace.csv.png",remote=remote, fuzzer=['frafl_int','feedgeneration10_int','state_int'], target=['waters_int'],num=range(0,3))
 
 rule clusterfuzz:
     input:

fuzzers/FRET/benchmark/target_symbols.csv

@@ -1,32 +1,33 @@
-kernel,main_function,input_symbol,input_size,return_function
+kernel,main_function,input_symbol,input_size,return_function,target_task_name
-mpeg2,mpeg2_main,mpeg2_oldorgframe,90112,mpeg2_return
+mpeg2,mpeg2_main,mpeg2_oldorgframe,90112,mpeg2_return,
-audiobeam,audiobeam_main,audiobeam_input,11520,audiobeam_return
+audiobeam,audiobeam_main,audiobeam_input,11520,audiobeam_return,
-epic,epic_main,epic_image,4096,epic_return
+epic,epic_main,epic_image,4096,epic_return,
-dijkstra,dijkstra_main,dijkstra_AdjMatrix,10000,dijkstra_return
+dijkstra,dijkstra_main,dijkstra_AdjMatrix,10000,dijkstra_return,
-fft,fft_main,fft_twidtable,2046,fft_return
+fft,fft_main,fft_twidtable,2046,fft_return,
-bsort,bsort_main,bsort_Array,400,bsort_return
+bsort,bsort_main,bsort_Array,400,bsort_return,
-insertsort,insertsort_main,insertsort_a,400,insertsort_return
+insertsort,insertsort_main,insertsort_a,400,insertsort_return,
-g723_enc,g723_enc_main,g723_enc_INPUT,1024,g723_enc_return
+g723_enc,g723_enc_main,g723_enc_INPUT,1024,g723_enc_return,
-rijndael_dec,rijndael_dec_main,rijndael_dec_data,32768,rijndael_dec_return
+rijndael_dec,rijndael_dec_main,rijndael_dec_data,32768,rijndael_dec_return,
-rijndael_enc,rijndael_enc_main,rijndael_enc_data,31369,rijndael_enc_return
+rijndael_enc,rijndael_enc_main,rijndael_enc_data,31369,rijndael_enc_return,
-huff_dec,huff_dec_main,huff_dec_encoded,419,huff_dec_return
+huff_dec,huff_dec_main,huff_dec_encoded,419,huff_dec_return,
-huff_enc,huff_enc_main,huff_enc_plaintext,600,huff_enc_return
+huff_enc,huff_enc_main,huff_enc_plaintext,600,huff_enc_return,
-gsm_enc,gsm_enc_main,gsm_enc_pcmdata,6400,gsm_enc_return
+gsm_enc,gsm_enc_main,gsm_enc_pcmdata,6400,gsm_enc_return,
-tmr,main,FUZZ_INPUT,32,trigger_Qemu_break
+tmr,main,FUZZ_INPUT,32,trigger_Qemu_break,
-tacle_rtos,prvStage0,FUZZ_INPUT,604,trigger_Qemu_break
+tacle_rtos,prvStage0,FUZZ_INPUT,604,trigger_Qemu_break,
-lift,main_lift,FUZZ_INPUT,100,trigger_Qemu_break
+lift,main_lift,FUZZ_INPUT,100,trigger_Qemu_break,
-waters,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break
+waters,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,1129
-watersv2,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break
+watersv2,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,1129
-waterspart,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break
+waterspart,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,1129
-waterspartv2,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break
+waterspartv2,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,1129
-waters_int,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break
+waters_int,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,1129
-watersv2_int,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break
+watersv2_int,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,1129
-waterspart_int,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break
+waterspart_int,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,1129
-waterspartv2_int,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break
+waterspartv2_int,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,1129
-micro_branchless,main_branchless,FUZZ_INPUT,4,trigger_Qemu_break
+micro_branchless,main_branchless,FUZZ_INPUT,4,trigger_Qemu_break,
-micro_int,main_int,FUZZ_INPUT,16,trigger_Qemu_break
+micro_int,main_int,FUZZ_INPUT,16,trigger_Qemu_break,
-micro_longint,main_micro_longint,FUZZ_INPUT,16,trigger_Qemu_break
+micro_longint,main_micro_longint,FUZZ_INPUT,16,trigger_Qemu_break,
-minimal,main_minimal,FUZZ_INPUT,4096,trigger_Qemu_break
+minimal,main_minimal,FUZZ_INPUT,4096,trigger_Qemu_break,
-gen3,main_minimal,FUZZ_INPUT,4096,trigger_Qemu_break
+gen3,main_minimal,FUZZ_INPUT,4096,trigger_Qemu_break,
-interact,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break
+interact,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break,
-interact_int,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break
+interact_int,main_interact,FUZZ_INPUT,4096,trigger_Qemu_break,
+

fuzzers/FRET/src/cli.rs

@@ -34,6 +34,10 @@ pub struct Cli {
     #[arg(short='g', long)]
     pub dump_graph: bool,
 
+    /// select a task for measurments
+    #[arg(short='s', long)]
+    pub select_task: Option<String>,
+
     #[command(subcommand)]
     pub command: Commands,
 }

fuzzers/FRET/src/fuzzer.rs

@@ -219,6 +219,7 @@ let app_range = app_start..app_end;
 let api_start = load_symbol(&elf, "__API_CODE_START__", false);
 let api_end = load_symbol(&elf, "__API_CODE_END__", false);
 let api_range = api_start..api_end;
+let job_done_addr = load_symbol(&elf, "trigger_job_done", false);
 
 let breakpoint = elf
     .resolve_symbol(
@@ -386,13 +387,13 @@ let run_client = |state: Option<_>, mut mgr, _core_id| {
         )}.track_indices();
 
         #[cfg(feature = "observe_systemstate")]
-        let systemstate_observer = QemuSystemStateObserver::new();
+        let systemstate_observer = QemuSystemStateObserver::new(&cli.select_task);
 
         // Feedback to rate the interestingness of an input
         // This one is composed by two Feedbacks in OR
         let mut feedback = feedback_or!(
             // Time feedback, this one does not need a feedback state
-            ClockTimeFeedback::new_with_observer(&clock_time_observer)
+            ClockTimeFeedback::new_with_observer(&clock_time_observer, &cli.select_task)
         );
         #[cfg(feature = "feed_genetic")]
         let mut feedback = feedback_or!(
@@ -465,7 +466,7 @@ let run_client = |state: Option<_>, mut mgr, _core_id| {
 
         let qhelpers = tuple_list!();
         #[cfg(feature = "observe_systemstate")]
-        let qhelpers = (QemuSystemStateHelper::new(api_addreses,api_ranges,isr_addreses,isr_ranges,input_addr..(input_addr+unsafe { MAX_INPUT_SIZE } as u32),curr_tcb_pointer,task_queue_addr,task_delay_addr,task_delay_overflow_addr,scheduler_lock,scheduler_running, critical_section,input_counter_ptr,app_range.clone()), qhelpers);
+        let qhelpers = (QemuSystemStateHelper::new(api_addreses,api_ranges,isr_addreses,isr_ranges,input_addr..(input_addr+unsafe { MAX_INPUT_SIZE } as u32),curr_tcb_pointer,task_queue_addr,task_delay_addr,task_delay_overflow_addr,scheduler_lock,scheduler_running, critical_section,input_counter_ptr,app_range.clone(), job_done_addr), qhelpers);
         #[cfg(feature = "observe_edges")]
         let qhelpers = (QemuEdgeCoverageHelper::new(denylist, QemuFilterList::None), qhelpers);
         let qhelpers = (QemuStateRestoreHelper::with_fast(initial_snap), qhelpers);

fuzzers/FRET/src/lib.rs

@@ -3,6 +3,6 @@ mod fuzzer;
 #[cfg(target_os = "linux")]
 mod time;
 #[cfg(target_os = "linux")]
-mod systemstate;
+pub mod systemstate;
 #[cfg(target_os = "linux")]
 mod cli;

fuzzers/FRET/src/systemstate/feedbacks.rs

@@ -60,6 +60,7 @@ pub struct NovelSystemStateFeedback
 impl<S> Feedback<S> for NovelSystemStateFeedback
 where
     S: State + UsesInput + MaybeHasClientPerfMonitor + HasNamedMetadata,
+    S::Input: Default,
 {
     fn is_interesting<EM, OT>(
         &mut self,
@@ -71,9 +72,10 @@ where
     ) -> Result<bool, Error>
     where
         EM: EventFirer<State = S>,
-        OT: ObserversTuple<S>
+        OT: ObserversTuple<S>,
+        S::Input: Default
     {
-        let observer = observers.match_name::<QemuSystemStateObserver<S::Input>>("systemstate")
+        let observer : &QemuSystemStateObserver<S::Input> = observers.match_name::<QemuSystemStateObserver<S::Input>>("systemstate")
             .expect("QemuSystemStateObserver not found");
         let clock_observer = observers.match_name::<QemuClockObserver>("clocktime")  //TODO not fixed
             .expect("QemuClockObserver not found");
@@ -81,12 +83,16 @@ where
             .named_metadata_map_mut()
             .get_mut::<SystemStateFeedbackState>("systemstate") {
                 Some(s) => s,
-                None => {
+                Option::None => {
                     let n=SystemStateFeedbackState::default();
                     state.named_metadata_map_mut().insert("systemstate",n);
                     state.named_metadata_map_mut().get_mut::<SystemStateFeedbackState>("systemstate").unwrap()
                 }
             };
+        #[cfg(feature = "trace_job_response_times")]
+        let last_runtime = observer.last_runtime();
+        #[cfg(not(feature = "trace_job_response_times"))]
+        let last_runtime = clock_observer.last_runtime();
         // let feedbackstate = state
         //     .feedback_states_mut()
         //     .match_name_mut::<systemstateFeedbackState>("systemstate")
@@ -98,14 +104,14 @@ where
         let mut is_novel = false;
         let mut takes_longer = false;
         match feedbackstate.known_traces.get_mut(&somehash) {
-            None => {
+            Option::None => {
                 is_novel = true;
-                feedbackstate.known_traces.insert(somehash,(1,clock_observer.last_runtime(),observer.last_run.len()));
+                feedbackstate.known_traces.insert(somehash,(1,last_runtime,observer.last_run.len()));
             }
             Some(s) => {
                 s.0+=1;
-                if s.1 < clock_observer.last_runtime() {
+                if s.1 < last_runtime {
-                    s.1 = clock_observer.last_runtime();
+                    s.1 = last_runtime;
                     takes_longer = true;
                 }
             }
@@ -124,7 +130,7 @@ where
         let a = self.last_trace.take();
         match a {
             Some(s) => testcase.metadata_map_mut().insert(FreeRTOSSystemStateMetadata::new(s)),
-            None => (),
+            Option::None => (),
         }
         Ok(())
     }
@@ -186,10 +192,10 @@ where
         let names : Vec<String> = observer.last_run.iter().map(|x| x.current_task.task_name.clone()).collect();
         match &self.dumpfile {
             Some(s) => {
-                    std::fs::write(s,ron::to_string(&(&observer.last_trace,&observer.last_states)).expect("Error serializing hashmap")).expect("Can not dump to file");
+                    std::fs::write(s,ron::to_string(&(&observer.last_trace,&observer.last_states,&observer.job_instances)).expect("Error serializing hashmap")).expect("Can not dump to file");
                     self.dumpfile = None
                 },
-            None => if self.dump_metadata {println!("{:?}\n{:?}",observer.last_run,names);}
+            Option::None => if self.dump_metadata {println!("{:?}\n{:?}",observer.last_run,names);}
         };
         // if self.dump_metadata {self.last_trace=Some(observer.last_trace.clone());}
         Ok(false)

fuzzers/FRET/src/systemstate/helpers.rs

@@ -111,6 +111,7 @@ pub struct QemuSystemStateHelper {
     critical_addr: GuestAddr,
     input_counter: Option<GuestAddr>,
     app_range: Range<GuestAddr>,
+    job_done_addrs: GuestAddr,
 }
 
 impl QemuSystemStateHelper {
@@ -130,6 +131,7 @@ impl QemuSystemStateHelper {
         critical_addr: GuestAddr,
         input_counter: Option<GuestAddr>,
         app_range: Range<GuestAddr>,
+        job_done_addrs: GuestAddr,
     ) -> Self {
         QemuSystemStateHelper {
             api_fn_addrs,
@@ -146,6 +148,7 @@ impl QemuSystemStateHelper {
             critical_addr,
             input_counter: input_counter,
             app_range,
+            job_done_addrs,
         }
     }
 }
@@ -165,6 +168,8 @@ where
             _hooks.instruction(*wp, Hook::Function(exec_isr_hook::<QT, S>), false);
         }
         _hooks.jmps(Hook::Function(gen_jmp_is_syscall::<QT, S>), Hook::Function(trace_jmp::<QT, S>));
+        #[cfg(feature = "trace_job_response_times")]
+        _hooks.instruction(self.job_done_addrs, Hook::Function(job_done_hook::<QT, S>), false);
         #[cfg(feature = "trace_reads")]
         _hooks.reads(Hook::Function(gen_read_is_input::<QT, S>), Hook::Empty,Hook::Empty,Hook::Empty,Hook::Empty,Hook::Function(trace_reads::<QT, S>));
         unsafe { INPUT_MEM = self.input_mem.clone() };
@@ -314,6 +319,31 @@ fn trigger_collection(emulator: &libafl_qemu::Qemu, edge: (GuestAddr, GuestAddr)
     unsafe { CURRENT_SYSTEMSTATE_VEC.push(systemstate); }
 }
 
+//============================= Trace job response times
+
+pub static mut JOBS_DONE : Vec<(u64, String)> = vec![];
+
+pub fn job_done_hook<QT, S>(
+    hooks: &mut QemuHooks<QT, S>,
+    _state: Option<&mut S>,
+    _pc: GuestAddr,
+)
+where
+    S: UsesInput,
+    QT: QemuHelperTuple<S>,
+{
+    let emulator = hooks.qemu();
+    let h = hooks.helpers().match_first_type::<QemuSystemStateHelper>().expect("QemuSystemHelper not found in helper tupel");
+    let curr_tcb_addr : freertos::void_ptr = freertos::emu_lookup::lookup(emulator, h.tcb_addr);
+    if curr_tcb_addr == 0 {
+        return;
+    };
+    let current_tcb : TCB_t = freertos::emu_lookup::lookup(emulator,curr_tcb_addr);
+    let tmp = unsafe {std::mem::transmute::<[i8; 10],[u8; 10]>(current_tcb.pcTaskName)};
+    let name : String = std::str::from_utf8(&tmp).expect("TCB name was not utf8").chars().filter(|x| *x != '\0').collect::<String>();
+    unsafe { JOBS_DONE.push((get_icount(emulator), name)); }
+}
+
 //============================= Trace interrupt service routines
 
 pub fn exec_isr_hook<QT, S>(

fuzzers/FRET/src/systemstate/observers.rs

@@ -1,5 +1,6 @@
 use libafl::prelude::ExitKind;
 use libafl::prelude::UsesInput;
+use libafl::HasMetadata;
 use libafl_bolts::HasLen;
 use libafl_bolts::Named;
 use libafl::Error;
@@ -7,6 +8,9 @@ use libafl::observers::Observer;
 use serde::{Deserialize, Serialize};
 use hashbrown::{HashMap, HashSet};
 use crate::systemstate::CaptureEvent;
+use crate::time::clock::IcHist;
+use crate::time::clock::FUZZ_START_TIMESTAMP;
+use std::time::SystemTime;
 use std::rc::Rc;
 use std::cell::RefCell;
 use std::collections::VecDeque;
@@ -19,6 +23,7 @@ use super::{
     RefinedTCB,
     ReducedFreeRTOSSystemState,
     freertos::{List_t, TCB_t, rtos_struct, rtos_struct::*},
+    helpers::JOBS_DONE,
 };
 
 //============================= Observer
@@ -34,13 +39,17 @@ pub struct QemuSystemStateObserver<I>
     pub last_trace: Vec<ExecInterval>,
     pub last_reads: Vec<HashSet<u32>>,
     pub last_input: I,
+    pub job_instances: Vec<(u64, u64, String)>,
+    pub worst_job_instances: HashMap<String, u64>,
+    pub select_task: Option<String>,
     pub success: bool,
     name: Cow<'static, str>,
 }
 
 impl<S> Observer<S> for QemuSystemStateObserver<S::Input>
 where
-    S: UsesInput,
+    S: UsesInput + HasMetadata,
+    S::Input: Default
 {
     #[inline]
     fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> {
@@ -61,7 +70,46 @@ where
             self.last_reads = temp.1;
             self.last_states = temp.2;
             self.success = temp.3;
-            // println!("{:?}",temp);
+            #[cfg(feature="trace_job_response_times")]
+            {
+                let metadata =_state.metadata_map_mut();
+                let releases = get_releases(&self.last_trace, &self.last_states);
+                // println!("Releases: {:?}",&releases);
+                let jobs_done =  JOBS_DONE.split_off(0);
+                self.job_instances = get_release_response_pairs(&releases, &jobs_done);
+                // println!("Instances: {:?}",&self.job_instances);
+                let observer = &self;
+                let mut worst_case_per_task = HashMap::new();
+                observer.job_instances.iter().for_each(|x| {
+                    let time = (x.1-x.0);
+                    if worst_case_per_task.get(&x.2).is_some() {
+                        let old = worst_case_per_task.get_mut(&x.2).unwrap();
+                        if time > *old {
+                            *old=time;
+                        }
+                    } else {
+                        worst_case_per_task.insert(x.2.clone(), time);
+                    }
+                });
+                self.worst_job_instances = worst_case_per_task;
+                // copy-paste form clock observer
+                {
+                    let hist = metadata.get_mut::<IcHist>();
+                    let timestamp = SystemTime::now().duration_since(unsafe {FUZZ_START_TIMESTAMP}).unwrap().as_millis();
+                    match hist {
+                        Option::None => {
+                            metadata.insert(IcHist(vec![(self.last_runtime(), timestamp)],
+                            (self.last_runtime(), timestamp)));
+                        }
+                        Some(v) => {
+                            v.0.push((self.last_runtime(), timestamp));
+                            if v.1.0 < self.last_runtime() {
+                                v.1 = (self.last_runtime(), timestamp);
+                            }
+                        }
+                    }
+                }
+            }
         }
         // let abbs = extract_abbs_from_trace(&self.last_run);
         // println!("{:?}",abbs);
@@ -89,14 +137,17 @@ impl<I> HasLen for QemuSystemStateObserver<I>
 
 impl<I> QemuSystemStateObserver<I>
 where I: Default {
-    pub fn new() -> Self {
+    pub fn new(select_task: &Option<String>) -> Self {
-        Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false }
+        Self{last_run: vec![], last_trace: vec![], last_reads: vec![], last_input: I::default(), worst_job_instances: HashMap::new(), select_task: select_task.clone(), name: Cow::from("systemstate".to_string()), last_states: HashMap::new(), success: false, job_instances: vec![]}
+    }
+    pub fn last_runtime(&self) -> u64 {
+        self.select_task.as_ref().map(|x| self.worst_job_instances.get(x).unwrap_or(&unsafe{libafl_qemu::sys::icount_get_raw()}).clone()).unwrap_or(unsafe{libafl_qemu::sys::icount_get_raw()})
     }
 }
 impl<I> Default for QemuSystemStateObserver<I>
 where I: Default {
     fn default() -> Self {
-        Self::new()
+        Self::new(&None)
     }
 }
 
@@ -167,6 +218,75 @@ fn refine_system_states(mut input: Vec<RawFreeRTOSSystemState>) -> (Vec<ReducedF
     return ret;
 }
 
+
+// Find all task release times. A release is SysTickHandler isr block that moves a task from the delay list to the ready list
+fn get_releases(tarce: &Vec<ExecInterval>, states: &HashMap<u64, ReducedFreeRTOSSystemState>) -> Vec<(u64, String)> {
+    let mut ret = Vec::new();
+    let mut initial_released = false;
+    for (_n, i) in tarce.iter().enumerate() {
+        if !initial_released && i.start_capture.0 == CaptureEvent::ISREnd && i.start_capture.1 == "xPortPendSVHandler" {
+            let start_state = states.get(&i.start_state).expect("State not found");
+            initial_released = true;
+            start_state.ready_list_after.iter().for_each(|x| {
+                ret.push((i.start_tick, x.task_name.clone()));
+            });
+            continue;
+        }
+        if i.start_capture.0 == CaptureEvent::ISRStart && ( i.start_capture.1 == "xPortSysTickHandler" || i.start_capture.1 == "isr_starter" ) && i.end_capture.0 == CaptureEvent::ISREnd {
+            let start_state = states.get(&i.start_state).expect("State not found");
+            let end_state = states.get(&i.end_state).expect("State not found");
+            end_state.ready_list_after.iter().for_each(|x| {
+                if x.task_name != end_state.current_task.task_name && x.task_name != start_state.current_task.task_name && !start_state.ready_list_after.iter().any(|y| x.task_name == y.task_name) {
+                    ret.push((i.end_tick, x.task_name.clone()));
+                }
+            });
+            // start_state.delay_list_after.iter().for_each(|x| {
+            //     if !end_state.delay_list_after.iter().any(|y| x.task_name == y.task_name) {
+            //         ret.push((i.end_tick, x.task_name.clone()));
+            //     }
+            // });
+        }
+    }
+    ret
+}
+
+fn get_release_response_pairs(rel: &Vec<(u64, String)>, resp: &Vec<(u64, String)>) -> Vec<(u64, u64, String)> {
+    let mut ret = Vec::new();
+    let mut ready : HashMap<&String, u64> = HashMap::new();
+    let mut r = rel.iter().peekable();
+    let mut d = resp.iter().peekable();
+    loop {
+        while let Some(peek_rel) = r.peek() {
+            if !ready.contains_key(&peek_rel.1) {
+                ready.insert(&peek_rel.1, peek_rel.0);
+                r.next();
+            } else {
+                if let Some(peek_resp) = d.peek() {
+                    if peek_resp.0 >= peek_rel.0 {  // multiple releases before respopnse, only use the latest release
+                        eprintln!("Task {} released multiple times before response", peek_rel.1);
+                        ready.insert(&peek_rel.1, peek_rel.0);
+                        r.next();
+                    }
+                }
+                break;
+            }
+        }
+        if let Some(peek_resp) = d.next() {
+            if ready.contains_key(&peek_resp.1) {
+                assert!(peek_resp.0 >= ready[&peek_resp.1]);
+                ret.push((ready[&peek_resp.1], peek_resp.0, peek_resp.1.clone()));
+                ready.remove(&peek_resp.1);
+            }
+            else {
+                eprintln!("Task {} not found in ready list", peek_resp.1);
+            }
+        } else {
+            // TODO: should remaining released tasks be counted as finished?
+            return ret;
+        }
+    }
+}
+
 /// Transform the states and metadata into a list of ExecIntervals, along with a HashMap of states, a list of HashSets marking memory reads and a bool indicating success
 fn states2intervals(trace: Vec<ReducedFreeRTOSSystemState>, meta: Vec<(u64, CaptureEvent, String, (u32, u32), HashSet<u32>)>) -> (Vec<ExecInterval>, Vec<HashSet<u32>>, HashMap<u64, ReducedFreeRTOSSystemState>, bool) {
     if trace.len() == 0 {return (Vec::new(), Vec::new(), HashMap::new(), true);}

fuzzers/FRET/src/systemstate/stg.rs

@@ -490,6 +490,7 @@ impl StgFeedback {
 impl<S> Feedback<S> for StgFeedback
 where
     S: State + UsesInput + MaybeHasClientPerfMonitor + HasNamedMetadata,
+    S::Input: Default,
 {
     #[allow(clippy::wrong_self_convention)]
     fn is_interesting<EM, OT>(
@@ -503,11 +504,16 @@ where
     where
         EM: EventFirer<State = S>,
         OT: ObserversTuple<S>,
+        S::Input: Default,
     {
         let observer = observers.match_name::<QemuSystemStateObserver<S::Input>>("systemstate")
             .expect("QemuSystemStateObserver not found");
         let clock_observer = observers.match_name::<QemuClockObserver>("clocktime")
             .expect("QemuClockObserver not found");
+        #[cfg(feature = "trace_job_response_times")]
+        let last_runtime = observer.last_runtime();
+        #[cfg(not(feature = "trace_job_response_times"))]
+        let last_runtime = clock_observer.last_runtime();
         let feedbackstate = match state
             .named_metadata_map_mut()
             .get_mut::<STGFeedbackState>("stgfeedbackstate") {
@@ -524,13 +530,13 @@ where
         {
             let h = get_generic_hash(&edgetrace);
             if let Some(x) = feedbackstate.worst_observed_per_stg_path.get_mut(&h) {
-                let t = clock_observer.last_runtime();
+                let t = last_runtime;
                 if t > *x {
                     *x = t;
                     interesting |= INTEREST_PATH;
                 }
             } else {
-                feedbackstate.worst_observed_per_stg_path.insert(h, clock_observer.last_runtime());
+                feedbackstate.worst_observed_per_stg_path.insert(h, last_runtime);
                 updated = true;
                 interesting |= INTEREST_PATH;
             }
@@ -543,13 +549,13 @@ where
                 self.last_abbs_hash = Some(h);
                 // order of execution is relevant
                 if let Some(x) = feedbackstate.worst_observed_per_abb_path.get_mut(&h) {
-                    let t = clock_observer.last_runtime();
+                    let t = last_runtime;
                     if t > *x {
                         *x = t;
                         interesting |= INTEREST_ABBPATH;
                     }
                 } else {
-                    feedbackstate.worst_observed_per_abb_path.insert(h, clock_observer.last_runtime());
+                    feedbackstate.worst_observed_per_abb_path.insert(h, last_runtime);
                     interesting |= INTEREST_ABBPATH;
                 }
             }
@@ -574,13 +580,13 @@ where
 
                 self.last_aggregate_hash = Some(get_generic_hash(&_tmp));
                 if let Some(x) = feedbackstate.worst_observed_per_aggegated_path.get_mut(&_tmp) {
-                    let t = clock_observer.last_runtime();
+                    let t = last_runtime;
                     if t > *x {
                         *x = t;
                         interesting |= INTEREST_AGGREGATE;
                     }
                 } else {
-                    feedbackstate.worst_observed_per_aggegated_path.insert(_tmp, clock_observer.last_runtime());
+                    feedbackstate.worst_observed_per_aggegated_path.insert(_tmp, last_runtime);
                     interesting |= INTEREST_AGGREGATE;
                 }
             }

fuzzers/FRET/src/time/clock.rs

@@ -1,3 +1,4 @@
+use hashbrown::HashMap;
 use libafl_bolts::Named;
 use libafl::{
     executors::ExitKind,
@@ -21,6 +22,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use std::path::PathBuf;
 use std::borrow::Cow;
 
+use crate::systemstate::observers::QemuSystemStateObserver;
+
 pub static mut FUZZ_START_TIMESTAMP : SystemTime = UNIX_EPOCH;
 
 pub const QEMU_ICOUNT_SHIFT : u32 = 5;
@@ -140,14 +143,22 @@ where
             let hist = metadata.get_mut::<IcHist>();
             let timestamp = SystemTime::now().duration_since(unsafe {FUZZ_START_TIMESTAMP}).unwrap().as_millis();
             match hist {
-                None => {
+                Option::None => {
-                    metadata.insert(IcHist(vec![(self.end_tick - self.start_tick, timestamp)],
+                    #[cfg(not(feature="trace_job_response_times"))]
-                    (self.end_tick - self.start_tick, timestamp)));
+                    {
+                        metadata.insert(IcHist(vec![(self.end_tick - self.start_tick, timestamp)],
+                        (self.end_tick - self.start_tick, timestamp)));
+                    }
+                    #[cfg(feature="trace_job_response_times")]
+                    metadata.insert(IcHist(vec![],(0,timestamp)));
                 }
                 Some(v) => {
-                    v.0.push((self.end_tick - self.start_tick, timestamp));
+                    #[cfg(not(feature="trace_job_response_times"))]
-                    if v.1.0 < self.end_tick-self.start_tick {
+                    {
-                        v.1 = (self.end_tick - self.start_tick, timestamp);
+                        v.0.push((self.end_tick - self.start_tick, timestamp));
+                        if v.1.0 < self.end_tick-self.start_tick {
+                            v.1 = (self.end_tick - self.start_tick, timestamp);
+                        }
                     }
                     if v.0.len() >= 100 {
                         let mut file = OpenOptions::new()
@@ -193,6 +204,7 @@ impl Default for QemuClockObserver {
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct ClockTimeFeedback {
     exec_time: Option<Duration>,
+    select_task: Option<String>,
     name: Cow<'static, str>,
 }
 
@@ -213,6 +225,19 @@ where
         EM: EventFirer<State = S>,
         OT: ObserversTuple<S>,
     {
+        #[cfg(feature="trace_job_response_times")]
+        {
+            if let Some(t) = &self.select_task {
+                let observer = observers.match_name::<QemuSystemStateObserver<S::Input>>("systemstate").unwrap();
+                if let Some(time) = observer.worst_job_instances.get(t) {
+                    self.exec_time = Some(Duration::from_nanos(*time));
+                    return Ok(true);
+                } else {
+                    self.exec_time = Some(Duration::from_nanos(0));
+                    return Ok(true);
+                }
+            }
+        }
         // TODO Replace with match_name_type when stable
         let observer = observers.match_name::<QemuClockObserver>(self.name()).unwrap();
         self.exec_time = Some(Duration::from_nanos(observer.last_runtime() << QEMU_ICOUNT_SHIFT)); // Assume a somewhat realistic multiplier of clock, it does not matter
@@ -251,18 +276,20 @@ impl Named for ClockTimeFeedback {
 impl ClockTimeFeedback {
     /// Creates a new [`ClockFeedback`], deciding if the value of a [`QemuClockObserver`] with the given `name` of a run is interesting.
     #[must_use]
-    pub fn new(name: &'static str) -> Self {
+    pub fn new(name: &'static str, select_task: Option<String>) -> Self {
         Self {
             exec_time: None,
+            select_task: select_task,
             name: Cow::from(name.to_string()),
         }
     }
 
     /// Creates a new [`ClockFeedback`], deciding if the given [`QemuClockObserver`] value of a run is interesting.
     #[must_use]
-    pub fn new_with_observer(observer: &QemuClockObserver) -> Self {
+    pub fn new_with_observer(observer: &QemuClockObserver, select_task: &Option<String>) -> Self {
         Self {
             exec_time: None,
+            select_task: select_task.clone(),
             name: observer.name().clone(),
         }
     }