diff --git a/fuzzers/FRET/Cargo.toml b/fuzzers/FRET/Cargo.toml index de1e903ac9..13778f3b90 100644 --- a/fuzzers/FRET/Cargo.toml +++ b/fuzzers/FRET/Cargo.toml @@ -5,15 +5,17 @@ authors = ["Andrea Fioraldi ", "Dominik Maier {output[1]} 2>&1 @@ -63,4 +76,8 @@ rule run_bench: rule all_periodic: input: - expand("timedump/{fuzzer}/{target}.{num}", fuzzer=['random','afl','state','graph'], target=['waters'],num=range(0,10)) \ No newline at end of file + expand("timedump/{fuzzer}/{target}.{num}", fuzzer=['random','afl','state','graph'], target=['waters'],num=range(0,10)) + +rule all_compare_afl_longest: + input: + expand("timedump/{fuzzer}/{target}.{num}", fuzzer=['afl','feedlongest','feedaflnolongest'], target=['waters'],num=range(0,10)) \ No newline at end of file diff --git a/fuzzers/FRET/benchmark/plot_multi.r b/fuzzers/FRET/benchmark/plot_multi.r new file mode 100644 index 0000000000..424f5cc2f2 --- /dev/null +++ b/fuzzers/FRET/benchmark/plot_multi.r @@ -0,0 +1,163 @@ +library("mosaic") +args = commandArgs(trailingOnly=TRUE) + +#myolors=c("#339933","#0066ff","#993300") # grün, balu, rot +myolors=c("dark green","dark blue","dark red", "yellow") # grün, balu, rot + +if (length(args)==0) { + runtype="timedump" + target="waters" + filename_1=sprintf("%s.png",target) + filename_2=sprintf("%s_maxline.png",target) + filename_3=sprintf("%s_hist.png",target) +} else { + runtype=args[1] + target=args[2] + filename_1=sprintf("%s.png",args[2]) + filename_2=sprintf("%s_maxline.png",args[2]) + filename_3=sprintf("%s_hist.png",args[2]) + # filename_1=args[3] +} + +library("dplyr") +COLORS <- c("dark grey","blue", "red", "green", "magenta", "orange", "cyan", "pink", "black", "orange") +BENCHDIR=sprintf("~/code/FRET/LibAFL/fuzzers/FRET/benchmark/%s",runtype) +BASENAMES=Filter(function(x) x!="",list.dirs(BENCHDIR,full.names=FALSE)) +PATTERNS=c(".*.0", + ".*.1", + ".*.2", + ".*.3", + ".*.4", + ".*.5", + ".*.6", + ".*.7", + ".*.8", + ".*.9") + +# Trimm a list of data frames to common length +trim_data <- function(input,len=NULL) { + if (is.null(len)) { + len <- min(sapply(input, function(v) dim(v)[1])) + } + return(lapply(input, function(d) slice_head(d,n=len))) +} + + +length_of_data <- function(input) { + min(sapply(input, function(v) dim(v)[1])) +} + + +# Takes a flat list +trace2maxline <- function(tr) { + maxline = tr + for (var in seq_len(length(maxline))[2:length(maxline)]) { + maxline[var] = max(maxline[var],maxline[var-1]) + } + #plot(seq_len(length(maxline)),maxline,"l",xlab="Index",ylab="WOET") + return(maxline) +} + +# Take a list of data frames, output same form but maxlines +data2maxlines <- function(tr) { + min_length <- min(sapply(tr, function(v) dim(v)[1])) + maxline <- tr + for (var in seq_len(length(tr))) { + maxline[[var]][[1]]=trace2maxline(tr[[var]][[1]]) + } + return(maxline) +} +# Take a multi-column data frame, output same form but maxlines +frame2maxlines <- function(tr) { + for (var in seq_len(length(tr))) { + tr[[var]]=trace2maxline(tr[[var]]) + } + return(tr) +} + +maxlines2plot <- function(maxlines) { + min_length <- min(sapply(maxlines, function(v) dim(v)[1])) + ml_cut <- lapply(maxlines, function(v) v[1:min_length,]) + plot(seq_len(min_length),unlist(ml_cut[[1]]),"l",xlab="Index",ylab="WOET",col=COLORS[1],ylim=c(best, worst)) + for (ml in seq_len(length(maxlines))[2:length(maxlines)]) { + lines(seq_len(min_length),unlist(ml_cut[ml]),"l",col=COLORS[ml]) + } + lines(seq_len(min_length),rep_len(best, min_length),col="black") + lines(seq_len(min_length),rep_len(best_success, min_length),col="black") + lines(seq_len(min_length),rep_len(worst_trace, min_length),col="green") + lines(seq_len(min_length),rep_len(worst, min_length),col="black") + legend("bottomright",legend=lapply(maxlines,names),col = COLORS[1:length(maxlines)], lwd=2) +} + +frame2plot <- function(maxlines,colors=NULL,draw_extreme=TRUE,doint=FALSE,over_name=NULL) { + if (is.null(colors)) { + colors <- COLORS + } + min_length <- dim(maxlines)[1] + cur_worst <- worst + if (doint) { + cur_worst <- worst_int + } + plot(seq_len(min_length),unlist(maxlines[[1]]),"l",xlab="Execution",ylab="Worst observed response time",col=colors[1],ylim=c(min(best,min(maxlines)), max(cur_worst,max(maxlines)))) + for (ml in seq_len(length(maxlines))[2:length(maxlines)]) { + lines(seq_len(min_length),unlist(maxlines[ml]),"l",col=colors[ml]) + } + if (draw_extreme) { + if (!doint) { + nam <- c(names(maxlines),"worst case") #,"best case" + col <- c(colors[1:length(maxlines)],"black") #,"black" + ltp <- c(rep_len("solid",length(maxlines)),"longdash") # ,"longdash" + lines(seq_len(min_length),rep_len(worst, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1]) + + #nam <- c(names(maxlines),"worst trace","worst case") #,"best case" + #col <- c(colors[1:length(maxlines)],"black","black") #,"black" + #ltp <- c(rep_len("solid",length(maxlines)),"dotted","longdash") # ,"longdash" + + #lines(seq_len(min_length),rep_len(best, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1]) + #lines(seq_len(min_length),rep_len(best_success, min_length),col="black") + + #lines(seq_len(min_length),rep_len(worst_trace, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1]) + #lines(seq_len(min_length),rep_len(worst, min_length),col=col[length(maxlines)+2],lty=ltp[length(maxlines)+2]) + } else { + nam <- c(names(maxlines),"worst case") + col <- c(colors[1:length(maxlines)],"black") + ltp <- c(rep_len("solid",length(maxlines)),"longdash") + lines(seq_len(min_length),rep_len(worst_int, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1]) + } + } else { + nam <- names(maxlines) + col <- colors[1:length(maxlines)] + ltp <- rep_len("solid",length(maxlines)) + } + if (is.null(over_name)) { + legend("bottomright", legend=nam, col=col, lty=ltp) + } else { + legend("bottomright", legend=over_name, col=col, lty=ltp) + } +} + +all_maxlines = c() +for (bn in BASENAMES) { + runtypefiles <- list.files(file.path(BENCHDIR,bn),pattern="\\.[0-9]$",full.names = TRUE) + runtypetables = lapply(runtypefiles, function(x) read.table(x, quote="\"", comment.char="", col.names=c(bn))) + runtypetables = trim_data(runtypetables) + list_of_maxlines = data2maxlines(runtypetables) + mean_maxline<-Reduce(function(a,b) a+b,list_of_maxlines,0)/length(runtypetables) + all_maxlines=append(all_maxlines,mean_maxline) +} +min_length <- min(sapply(all_maxlines, length)) +all_maxlines=lapply(all_maxlines, function(v) v[1:min_length]) +one_frame<-data.frame(all_maxlines) +one_frame[length(one_frame)+1] <- seq_len(length(one_frame[[1]])) +names(one_frame)[length(one_frame)] <- 'iters' + +ylow=min(one_frame[1:length(one_frame)-1]) +yhigh=max(one_frame[1:length(one_frame)-1]) + +plot(c(1,length(one_frame[[1]])),c(ylow,yhigh), col='white', xlab="iters", ylab="wcet", pch='.') + +typenames = names(one_frame)[which(names(one_frame) != 'iters')] +for (t in seq_len(length(typenames))) { + points(one_frame[c('iters',typenames[t])], col=myolors[t], pch='.') +} +legend("bottomright", legend=typenames, col=myolors, lty="solid") diff --git a/fuzzers/FRET/src/fuzzer.rs b/fuzzers/FRET/src/fuzzer.rs index 5dd33d2e9f..e0e2b60ae9 100644 --- a/fuzzers/FRET/src/fuzzer.rs +++ b/fuzzers/FRET/src/fuzzer.rs @@ -214,26 +214,33 @@ pub fn fuzz() { // Feedback to rate the interestingness of an input // This one is composed by two Feedbacks in OR + let mut feedback = feedback_or!( + // Time feedback, this one does not need a feedback state + ClockTimeFeedback::new_with_observer(&clock_time_observer) + ); + #[cfg(feature = "feed_afl")] let mut feedback = feedback_or!( // New maximization map feedback linked to the edges observer and the feedback state MaxMapFeedback::new_tracking(&edges_observer, true, true), - // QemuClockIncreaseFeedback::default(), - // Time feedback, this one does not need a feedback state - ClockTimeFeedback::new_with_observer(&clock_time_observer), - // Feedback to reward any input which increses the execution time - ExecTimeIncFeedback::new() + feedback ); - #[cfg(all(feature = "systemstate",not(any(feature = "systemgraph",feature = "systemtrace"))))] + #[cfg(feature = "feed_longest")] + let mut feedback = feedback_or!( + // Feedback to reward any input which increses the execution time + ExecTimeIncFeedback::new(), + feedback + ); + #[cfg(all(feature = "systemstate",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace"))))] let mut feedback = feedback_or!( DumpSystraceFeedback::with_dump(env::var("TRACE_DUMP").ok().map(PathBuf::from)), feedback ); - #[cfg(feature = "systemtrace")] + #[cfg(feature = "feed_systemtrace")] let mut feedback = feedback_or!( NovelSystemStateFeedback::default(), feedback ); - #[cfg(feature = "systemgraph")] + #[cfg(feature = "feed_systemgraph")] let mut feedback = feedback_or!( SysMapFeedback::default(), feedback @@ -262,11 +269,13 @@ pub fn fuzz() { }); // A minimization+queue policy to get testcasess from the corpus - #[cfg(not(any(feature = "systemgraph",feature = "systemtrace")))] + #[cfg(not(all(feature = "feed_afl",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace")))))] + let scheduler = QueueScheduler::new(); + #[cfg(all(feature = "feed_afl",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace"))))] let scheduler = TimeMaximizerCorpusScheduler::new(QueueScheduler::new()); - #[cfg(feature = "systemtrace")] + #[cfg(feature = "feed_systemtrace")] let scheduler = TimeStateMaximizerCorpusScheduler::new(QueueScheduler::new()); - #[cfg(feature = "systemgraph")] + #[cfg(feature = "feed_systemgraph")] let scheduler = GraphMaximizerCorpusScheduler::new(QueueScheduler::new()); // A fuzzer with feedbacks and a corpus scheduler @@ -424,7 +433,7 @@ pub fn fuzz() { }, None => (), } - #[cfg(feature = "systemgraph")] + #[cfg(feature = "feed_systemgraph")] { let mut gd = String::from(&td); gd.push_str(".graph"); diff --git a/fuzzers/FRET/src/worst.rs b/fuzzers/FRET/src/worst.rs index 827be83076..d4b4d5b280 100644 --- a/fuzzers/FRET/src/worst.rs +++ b/fuzzers/FRET/src/worst.rs @@ -293,15 +293,19 @@ where .expect("QemuClockObserver not found"); if observer.last_runtime() > self.longest_time { self.longest_time = observer.last_runtime(); + self.last_is_longest = true; + Ok(true) + } else { + self.last_is_longest = false; + Ok(false) } - self.last_is_longest = observer.last_runtime() > self.longest_time; - Ok(observer.last_runtime() > self.longest_time) } fn append_metadata( &mut self, _state: &mut S, testcase: &mut Testcase<::Input>, ) -> Result<(), Error> { + #[cfg(feature = "feed_afl")] if self.last_is_longest { let mim : Option<&mut MapIndexesMetadata>= testcase.metadata_mut().get_mut(); // pretend that the longest input alone excercises some non-existing edge, to keep it relevant