fix feedbacks

fuzzers/FRET/Cargo.toml

@@ -5,15 +5,17 @@ authors = ["Andrea Fioraldi <andreafioraldi@gmail.com>", "Dominik Maier <domenuk
 edition = "2021"
 
 [features]
-default = ["std", "snapshot_restore", "singlecore"]
+default = ["std", "snapshot_restore", "singlecore", "feed_longest", "feed_afl"]
 std = []
 snapshot_restore = []
 snapshot_fast = [ "snapshot_restore" ]
 singlecore = []
 trace_abbs = []
 systemstate = []
-systemgraph = [ "systemstate" ]
+feed_systemgraph = [ "systemstate" ]
-systemtrace = [ "systemstate" ]
+feed_systemtrace = [ "systemstate" ]
+feed_longest = [ ]
+feed_afl = [ ]
 
 [profile.release]
 lto = true

fuzzers/FRET/benchmark/Snakefile

@@ -1,28 +1,41 @@
 import csv
+def_flags="--no-default-features --features std,snapshot_restore,singlecore"
 
 rule build_random:
     output:
         directory("bins/target_random")
     shell:
-        "cargo build --target-dir {output}"
+        "cargo build --target-dir {output} {def_flags}"
+
+rule build_feedlongest:
+    output:
+        directory("bins/target_feedlongest")
+    shell:
+        "cargo build --target-dir {output} {def_flags},feed_longest"
+
+rule build_feedaflnolongest:
+    output:
+        directory("bins/target_feedaflnolongest")
+    shell:
+        "cargo build --target-dir {output} {def_flags},feed_afl"
 
 rule build_afl:
     output:
         directory("bins/target_afl")
     shell:
-        "cargo build --target-dir {output}"
+        "cargo build --target-dir {output} {def_flags},feed_afl,feed_longest"
 
 rule build_state:
     output:
         directory("bins/target_state")
     shell:
-        "cargo build --target-dir {output} --features systemtrace"
+        "cargo build --target-dir {output} {def_flags},feed_systemtrace"
 
 rule build_graph:
     output:
         directory("bins/target_graph")
     shell:
-        "cargo build --target-dir {output} --features systemgraph"
+        "cargo build --target-dir {output} {def_flags},feed_systemgraph"
 
 rule run_bench:
     input:
@@ -51,7 +64,7 @@ rule run_bench:
                 export SEED_RANDOM=1
                 export TIME_DUMP=$(pwd)/{output[0]}
                 export CASE_DUMP=$(pwd)/{output[2]}
-                export FUZZ_ITERS=7200
+                export FUZZ_ITERS=3600
                 export FUZZER=$(pwd)/{input[1]}/debug/fret
                 set +e
                 ../fuzzer.sh > {output[1]} 2>&1 
@@ -63,4 +76,8 @@ rule run_bench:
 
 rule all_periodic:
     input:
-        expand("timedump/{fuzzer}/{target}.{num}", fuzzer=['random','afl','state','graph'], target=['waters'],num=range(0,10))
+        expand("timedump/{fuzzer}/{target}.{num}", fuzzer=['random','afl','state','graph'], target=['waters'],num=range(0,10))
+
+rule all_compare_afl_longest:
+    input:
+        expand("timedump/{fuzzer}/{target}.{num}", fuzzer=['afl','feedlongest','feedaflnolongest'], target=['waters'],num=range(0,10))

fuzzers/FRET/benchmark/plot_multi.r

@@ -0,0 +1,163 @@
+library("mosaic")
+args = commandArgs(trailingOnly=TRUE)
+
+#myolors=c("#339933","#0066ff","#993300") # grün, balu, rot
+myolors=c("dark green","dark blue","dark red", "yellow") # grün, balu, rot
+
+if (length(args)==0) {
+  runtype="timedump"
+  target="waters"
+  filename_1=sprintf("%s.png",target)
+  filename_2=sprintf("%s_maxline.png",target)
+  filename_3=sprintf("%s_hist.png",target)
+} else {
+  runtype=args[1]
+  target=args[2]
+  filename_1=sprintf("%s.png",args[2])
+  filename_2=sprintf("%s_maxline.png",args[2])
+  filename_3=sprintf("%s_hist.png",args[2])
+  # filename_1=args[3]
+}
+
+library("dplyr")
+COLORS <- c("dark grey","blue", "red", "green", "magenta", "orange", "cyan", "pink", "black", "orange")
+BENCHDIR=sprintf("~/code/FRET/LibAFL/fuzzers/FRET/benchmark/%s",runtype)
+BASENAMES=Filter(function(x) x!="",list.dirs(BENCHDIR,full.names=FALSE))
+PATTERNS=c(".*.0",
+           ".*.1",
+           ".*.2",
+           ".*.3",
+           ".*.4",
+           ".*.5",
+           ".*.6",
+           ".*.7",
+           ".*.8",
+           ".*.9")
+
+# Trimm a list of data frames to common length
+trim_data <- function(input,len=NULL) {
+  if (is.null(len)) {
+    len <- min(sapply(input, function(v) dim(v)[1]))
+  }
+  return(lapply(input, function(d) slice_head(d,n=len)))
+}
+
+
+length_of_data <- function(input) {
+  min(sapply(input, function(v) dim(v)[1]))
+}
+
+
+# Takes a flat list
+trace2maxline <- function(tr) {
+  maxline = tr
+  for (var in seq_len(length(maxline))[2:length(maxline)]) {
+    maxline[var] = max(maxline[var],maxline[var-1])
+  }
+  #plot(seq_len(length(maxline)),maxline,"l",xlab="Index",ylab="WOET")
+  return(maxline)
+}
+
+# Take a list of data frames, output same form but maxlines
+data2maxlines <- function(tr) {
+  min_length <- min(sapply(tr, function(v) dim(v)[1]))
+  maxline <- tr
+  for (var in seq_len(length(tr))) {
+    maxline[[var]][[1]]=trace2maxline(tr[[var]][[1]])
+  }
+  return(maxline)
+}
+# Take a multi-column data frame, output same form but maxlines
+frame2maxlines <- function(tr) {
+  for (var in seq_len(length(tr))) {
+    tr[[var]]=trace2maxline(tr[[var]])
+  }
+  return(tr)
+}
+
+maxlines2plot <- function(maxlines) {
+  min_length <- min(sapply(maxlines, function(v) dim(v)[1]))
+  ml_cut <- lapply(maxlines, function(v) v[1:min_length,])
+  plot(seq_len(min_length),unlist(ml_cut[[1]]),"l",xlab="Index",ylab="WOET",col=COLORS[1],ylim=c(best, worst))
+  for (ml in seq_len(length(maxlines))[2:length(maxlines)]) {
+    lines(seq_len(min_length),unlist(ml_cut[ml]),"l",col=COLORS[ml])  
+  }
+  lines(seq_len(min_length),rep_len(best, min_length),col="black")
+  lines(seq_len(min_length),rep_len(best_success, min_length),col="black")
+  lines(seq_len(min_length),rep_len(worst_trace, min_length),col="green")
+  lines(seq_len(min_length),rep_len(worst, min_length),col="black")
+  legend("bottomright",legend=lapply(maxlines,names),col = COLORS[1:length(maxlines)], lwd=2)
+}
+
+frame2plot <- function(maxlines,colors=NULL,draw_extreme=TRUE,doint=FALSE,over_name=NULL) {
+  if (is.null(colors)) {
+    colors <- COLORS
+  }
+  min_length <- dim(maxlines)[1]
+  cur_worst <- worst
+  if (doint) {
+    cur_worst <- worst_int
+  }
+  plot(seq_len(min_length),unlist(maxlines[[1]]),"l",xlab="Execution",ylab="Worst observed response time",col=colors[1],ylim=c(min(best,min(maxlines)), max(cur_worst,max(maxlines))))
+  for (ml in seq_len(length(maxlines))[2:length(maxlines)]) {
+    lines(seq_len(min_length),unlist(maxlines[ml]),"l",col=colors[ml])  
+  }
+  if (draw_extreme) {
+    if (!doint) {
+      nam <- c(names(maxlines),"worst case") #,"best case"
+      col <- c(colors[1:length(maxlines)],"black") #,"black"
+      ltp <- c(rep_len("solid",length(maxlines)),"longdash") # ,"longdash"
+      lines(seq_len(min_length),rep_len(worst, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1])
+      
+      #nam <- c(names(maxlines),"worst trace","worst case") #,"best case"
+      #col <- c(colors[1:length(maxlines)],"black","black") #,"black"
+      #ltp <- c(rep_len("solid",length(maxlines)),"dotted","longdash") # ,"longdash"
+      
+      #lines(seq_len(min_length),rep_len(best, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1])
+      #lines(seq_len(min_length),rep_len(best_success, min_length),col="black")
+      
+      #lines(seq_len(min_length),rep_len(worst_trace, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1])
+      #lines(seq_len(min_length),rep_len(worst, min_length),col=col[length(maxlines)+2],lty=ltp[length(maxlines)+2])
+    } else {
+      nam <- c(names(maxlines),"worst case")
+      col <- c(colors[1:length(maxlines)],"black")
+      ltp <- c(rep_len("solid",length(maxlines)),"longdash")
+      lines(seq_len(min_length),rep_len(worst_int, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1])
+    }
+  } else {
+    nam <- names(maxlines)
+    col <- colors[1:length(maxlines)]
+    ltp <- rep_len("solid",length(maxlines))
+  }
+  if (is.null(over_name)) {
+    legend("bottomright", legend=nam, col=col, lty=ltp)
+  } else {
+    legend("bottomright", legend=over_name, col=col, lty=ltp)
+  }
+}
+
+all_maxlines = c()
+for (bn in BASENAMES) {
+  runtypefiles <- list.files(file.path(BENCHDIR,bn),pattern="\\.[0-9]$",full.names = TRUE)
+  runtypetables = lapply(runtypefiles, function(x) read.table(x, quote="\"", comment.char="", col.names=c(bn)))
+  runtypetables = trim_data(runtypetables)
+  list_of_maxlines = data2maxlines(runtypetables)
+  mean_maxline<-Reduce(function(a,b) a+b,list_of_maxlines,0)/length(runtypetables)
+  all_maxlines=append(all_maxlines,mean_maxline)
+}
+min_length <- min(sapply(all_maxlines, length))
+all_maxlines=lapply(all_maxlines, function(v) v[1:min_length])
+one_frame<-data.frame(all_maxlines)
+one_frame[length(one_frame)+1] <- seq_len(length(one_frame[[1]]))
+names(one_frame)[length(one_frame)] <- 'iters'
+
+ylow=min(one_frame[1:length(one_frame)-1])
+yhigh=max(one_frame[1:length(one_frame)-1])
+
+plot(c(1,length(one_frame[[1]])),c(ylow,yhigh), col='white', xlab="iters", ylab="wcet", pch='.')
+
+typenames = names(one_frame)[which(names(one_frame) != 'iters')]
+for (t in seq_len(length(typenames))) {
+  points(one_frame[c('iters',typenames[t])], col=myolors[t], pch='.')
+}
+legend("bottomright", legend=typenames, col=myolors, lty="solid")

fuzzers/FRET/src/fuzzer.rs

@@ -214,26 +214,33 @@ pub fn fuzz() {
 
         // Feedback to rate the interestingness of an input
         // This one is composed by two Feedbacks in OR
+        let mut feedback = feedback_or!(
+            // Time feedback, this one does not need a feedback state
+            ClockTimeFeedback::new_with_observer(&clock_time_observer)
+        );
+        #[cfg(feature = "feed_afl")]
         let mut feedback = feedback_or!(
             // New maximization map feedback linked to the edges observer and the feedback state
             MaxMapFeedback::new_tracking(&edges_observer, true, true),
-            // QemuClockIncreaseFeedback::default(),
+            feedback
-            // Time feedback, this one does not need a feedback state
-            ClockTimeFeedback::new_with_observer(&clock_time_observer),
-            // Feedback to reward any input which increses the execution time
-            ExecTimeIncFeedback::new()
         );
-        #[cfg(all(feature = "systemstate",not(any(feature = "systemgraph",feature = "systemtrace"))))]
+        #[cfg(feature = "feed_longest")]
+        let mut feedback = feedback_or!(
+            // Feedback to reward any input which increses the execution time
+            ExecTimeIncFeedback::new(),
+            feedback
+        );
+        #[cfg(all(feature = "systemstate",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace"))))]
         let mut feedback = feedback_or!(
             DumpSystraceFeedback::with_dump(env::var("TRACE_DUMP").ok().map(PathBuf::from)),
             feedback
         );
-        #[cfg(feature = "systemtrace")]
+        #[cfg(feature = "feed_systemtrace")]
         let mut feedback = feedback_or!(
             NovelSystemStateFeedback::default(),
             feedback
         );
-        #[cfg(feature = "systemgraph")]
+        #[cfg(feature = "feed_systemgraph")]
         let mut feedback = feedback_or!(
             SysMapFeedback::default(),
             feedback
@@ -262,11 +269,13 @@ pub fn fuzz() {
         });
 
         // A minimization+queue policy to get testcasess from the corpus
-        #[cfg(not(any(feature = "systemgraph",feature = "systemtrace")))]
+        #[cfg(not(all(feature = "feed_afl",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace")))))]
+        let scheduler = QueueScheduler::new();
+        #[cfg(all(feature = "feed_afl",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace"))))]
         let scheduler = TimeMaximizerCorpusScheduler::new(QueueScheduler::new());
-        #[cfg(feature = "systemtrace")]
+        #[cfg(feature = "feed_systemtrace")]
         let scheduler = TimeStateMaximizerCorpusScheduler::new(QueueScheduler::new());
-        #[cfg(feature = "systemgraph")]
+        #[cfg(feature = "feed_systemgraph")]
         let scheduler = GraphMaximizerCorpusScheduler::new(QueueScheduler::new());
 
         // A fuzzer with feedbacks and a corpus scheduler
@@ -424,7 +433,7 @@ pub fn fuzz() {
                             },
                             None => (),
                         }
-                        #[cfg(feature = "systemgraph")]
+                        #[cfg(feature = "feed_systemgraph")]
                         {
                             let mut gd = String::from(&td);
                             gd.push_str(".graph");

fuzzers/FRET/src/worst.rs

@@ -293,15 +293,19 @@ where
             .expect("QemuClockObserver not found");
         if observer.last_runtime() > self.longest_time {
             self.longest_time = observer.last_runtime();
+            self.last_is_longest = true;
+            Ok(true)
+        } else {
+            self.last_is_longest = false;
+            Ok(false)
         }
-        self.last_is_longest = observer.last_runtime() > self.longest_time;
-        Ok(observer.last_runtime() > self.longest_time)
     }
     fn append_metadata(
         &mut self,
         _state: &mut S,
         testcase: &mut Testcase<<S as UsesInput>::Input>,
     ) -> Result<(), Error> {
+        #[cfg(feature = "feed_afl")]
         if self.last_is_longest {
             let mim : Option<&mut MapIndexesMetadata>= testcase.metadata_mut().get_mut();
             // pretend that the longest input alone excercises some non-existing edge, to keep it relevant