SYS-OSS/FRET-LibAFL
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix feedbacks

Browse Source
This commit is contained in:
Alwin Berger 2023-02-19 18:38:31 +01:00
parent f7a05d2a7c
commit dfe4f713b9
5 changed files with 218 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
fuzzers/FRET/Cargo.toml
View File

@ -5,15 +5,17 @@ authors = ["Andrea Fioraldi <andreafioraldi@gmail.com>", "Dominik Maier <domenuk
edition = "2021"
[features]
default = ["std", "snapshot_restore", "singlecore"]
default = ["std", "snapshot_restore", "singlecore", "feed_longest", "feed_afl"]
std = []
snapshot_restore = []
snapshot_fast = [ "snapshot_restore" ]
singlecore = []
trace_abbs = []
systemstate = []
systemgraph = [ "systemstate" ]
systemtrace = [ "systemstate" ]
feed_systemgraph = [ "systemstate" ]
feed_systemtrace = [ "systemstate" ]
feed_longest = [ ]
feed_afl = [ ]
[profile.release]
lto = true

27
fuzzers/FRET/benchmark/Snakefile
View File

@ -1,28 +1,41 @@
import csv
def_flags="--no-default-features --features std,snapshot_restore,singlecore"
rule build_random:
output:
directory("bins/target_random")
shell:
"cargo build --target-dir {output}"
"cargo build --target-dir {output} {def_flags}"
rule build_feedlongest:
output:
directory("bins/target_feedlongest")
shell:
"cargo build --target-dir {output} {def_flags},feed_longest"
rule build_feedaflnolongest:
output:
directory("bins/target_feedaflnolongest")
shell:
"cargo build --target-dir {output} {def_flags},feed_afl"
rule build_afl:
output:
directory("bins/target_afl")
shell:
"cargo build --target-dir {output}"
"cargo build --target-dir {output} {def_flags},feed_afl,feed_longest"
rule build_state:
output:
directory("bins/target_state")
shell:
"cargo build --target-dir {output} --features systemtrace"
"cargo build --target-dir {output} {def_flags},feed_systemtrace"
rule build_graph:
output:
directory("bins/target_graph")
shell:
"cargo build --target-dir {output} --features systemgraph"
"cargo build --target-dir {output} {def_flags},feed_systemgraph"
rule run_bench:
input:
@ -51,7 +64,7 @@ rule run_bench:
export SEED_RANDOM=1
export TIME_DUMP=$(pwd)/{output[0]}
export CASE_DUMP=$(pwd)/{output[2]}
export FUZZ_ITERS=7200
export FUZZ_ITERS=3600
export FUZZER=$(pwd)/{input[1]}/debug/fret
set +e
../fuzzer.sh > {output[1]} 2>&1
@ -64,3 +77,7 @@ rule run_bench:
rule all_periodic:
input:
expand("timedump/{fuzzer}/{target}.{num}", fuzzer=['random','afl','state','graph'], target=['waters'],num=range(0,10))
rule all_compare_afl_longest:
input:
expand("timedump/{fuzzer}/{target}.{num}", fuzzer=['afl','feedlongest','feedaflnolongest'], target=['waters'],num=range(0,10))

163
fuzzers/FRET/benchmark/plot_multi.r Normal file
View File

@ -0,0 +1,163 @@
library("mosaic")
args = commandArgs(trailingOnly=TRUE)
#myolors=c("#339933","#0066ff","#993300") # grün, balu, rot
myolors=c("dark green","dark blue","dark red", "yellow") # grün, balu, rot
if (length(args)==0) {
runtype="timedump"
target="waters"
filename_1=sprintf("%s.png",target)
filename_2=sprintf("%s_maxline.png",target)
filename_3=sprintf("%s_hist.png",target)
} else {
runtype=args[1]
target=args[2]
filename_1=sprintf("%s.png",args[2])
filename_2=sprintf("%s_maxline.png",args[2])
filename_3=sprintf("%s_hist.png",args[2])
# filename_1=args[3]
}
library("dplyr")
COLORS <- c("dark grey","blue", "red", "green", "magenta", "orange", "cyan", "pink", "black", "orange")
BENCHDIR=sprintf("~/code/FRET/LibAFL/fuzzers/FRET/benchmark/%s",runtype)
BASENAMES=Filter(function(x) x!="",list.dirs(BENCHDIR,full.names=FALSE))
PATTERNS=c(".*.0",
".*.1",
".*.2",
".*.3",
".*.4",
".*.5",
".*.6",
".*.7",
".*.8",
".*.9")
# Trimm a list of data frames to common length
trim_data <- function(input,len=NULL) {
if (is.null(len)) {
len <- min(sapply(input, function(v) dim(v)[1]))
}
return(lapply(input, function(d) slice_head(d,n=len)))
}
length_of_data <- function(input) {
min(sapply(input, function(v) dim(v)[1]))
}
# Takes a flat list
trace2maxline <- function(tr) {
maxline = tr
for (var in seq_len(length(maxline))[2:length(maxline)]) {
maxline[var] = max(maxline[var],maxline[var-1])
}
#plot(seq_len(length(maxline)),maxline,"l",xlab="Index",ylab="WOET")
return(maxline)
}
# Take a list of data frames, output same form but maxlines
data2maxlines <- function(tr) {
min_length <- min(sapply(tr, function(v) dim(v)[1]))
maxline <- tr
for (var in seq_len(length(tr))) {
maxline[[var]][[1]]=trace2maxline(tr[[var]][[1]])
}
return(maxline)
}
# Take a multi-column data frame, output same form but maxlines
frame2maxlines <- function(tr) {
for (var in seq_len(length(tr))) {
tr[[var]]=trace2maxline(tr[[var]])
}
return(tr)
}
maxlines2plot <- function(maxlines) {
min_length <- min(sapply(maxlines, function(v) dim(v)[1]))
ml_cut <- lapply(maxlines, function(v) v[1:min_length,])
plot(seq_len(min_length),unlist(ml_cut[[1]]),"l",xlab="Index",ylab="WOET",col=COLORS[1],ylim=c(best, worst))
for (ml in seq_len(length(maxlines))[2:length(maxlines)]) {
lines(seq_len(min_length),unlist(ml_cut[ml]),"l",col=COLORS[ml])
}
lines(seq_len(min_length),rep_len(best, min_length),col="black")
lines(seq_len(min_length),rep_len(best_success, min_length),col="black")
lines(seq_len(min_length),rep_len(worst_trace, min_length),col="green")
lines(seq_len(min_length),rep_len(worst, min_length),col="black")
legend("bottomright",legend=lapply(maxlines,names),col = COLORS[1:length(maxlines)], lwd=2)
}
frame2plot <- function(maxlines,colors=NULL,draw_extreme=TRUE,doint=FALSE,over_name=NULL) {
if (is.null(colors)) {
colors <- COLORS
}
min_length <- dim(maxlines)[1]
cur_worst <- worst
if (doint) {
cur_worst <- worst_int
}
plot(seq_len(min_length),unlist(maxlines[[1]]),"l",xlab="Execution",ylab="Worst observed response time",col=colors[1],ylim=c(min(best,min(maxlines)), max(cur_worst,max(maxlines))))
for (ml in seq_len(length(maxlines))[2:length(maxlines)]) {
lines(seq_len(min_length),unlist(maxlines[ml]),"l",col=colors[ml])
}
if (draw_extreme) {
if (!doint) {
nam <- c(names(maxlines),"worst case") #,"best case"
col <- c(colors[1:length(maxlines)],"black") #,"black"
ltp <- c(rep_len("solid",length(maxlines)),"longdash") # ,"longdash"
lines(seq_len(min_length),rep_len(worst, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1])
#nam <- c(names(maxlines),"worst trace","worst case") #,"best case"
#col <- c(colors[1:length(maxlines)],"black","black") #,"black"
#ltp <- c(rep_len("solid",length(maxlines)),"dotted","longdash") # ,"longdash"
#lines(seq_len(min_length),rep_len(best, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1])
#lines(seq_len(min_length),rep_len(best_success, min_length),col="black")
#lines(seq_len(min_length),rep_len(worst_trace, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1])
#lines(seq_len(min_length),rep_len(worst, min_length),col=col[length(maxlines)+2],lty=ltp[length(maxlines)+2])
} else {
nam <- c(names(maxlines),"worst case")
col <- c(colors[1:length(maxlines)],"black")
ltp <- c(rep_len("solid",length(maxlines)),"longdash")
lines(seq_len(min_length),rep_len(worst_int, min_length),col=col[length(maxlines)+1],lty=ltp[length(maxlines)+1])
}
} else {
nam <- names(maxlines)
col <- colors[1:length(maxlines)]
ltp <- rep_len("solid",length(maxlines))
}
if (is.null(over_name)) {
legend("bottomright", legend=nam, col=col, lty=ltp)
} else {
legend("bottomright", legend=over_name, col=col, lty=ltp)
}
}
all_maxlines = c()
for (bn in BASENAMES) {
runtypefiles <- list.files(file.path(BENCHDIR,bn),pattern="\\.[0-9]$",full.names = TRUE)
runtypetables = lapply(runtypefiles, function(x) read.table(x, quote="\"", comment.char="", col.names=c(bn)))
runtypetables = trim_data(runtypetables)
list_of_maxlines = data2maxlines(runtypetables)
mean_maxline<-Reduce(function(a,b) a+b,list_of_maxlines,0)/length(runtypetables)
all_maxlines=append(all_maxlines,mean_maxline)
}
min_length <- min(sapply(all_maxlines, length))
all_maxlines=lapply(all_maxlines, function(v) v[1:min_length])
one_frame<-data.frame(all_maxlines)
one_frame[length(one_frame)+1] <- seq_len(length(one_frame[[1]]))
names(one_frame)[length(one_frame)] <- 'iters'
ylow=min(one_frame[1:length(one_frame)-1])
yhigh=max(one_frame[1:length(one_frame)-1])
plot(c(1,length(one_frame[[1]])),c(ylow,yhigh), col='white', xlab="iters", ylab="wcet", pch='.')
typenames = names(one_frame)[which(names(one_frame) != 'iters')]
for (t in seq_len(length(typenames))) {
points(one_frame[c('iters',typenames[t])], col=myolors[t], pch='.')
}
legend("bottomright", legend=typenames, col=myolors, lty="solid")

33
fuzzers/FRET/src/fuzzer.rs
View File

@ -214,26 +214,33 @@ pub fn fuzz() {
// Feedback to rate the interestingness of an input
// This one is composed by two Feedbacks in OR
let mut feedback = feedback_or!(
// Time feedback, this one does not need a feedback state
ClockTimeFeedback::new_with_observer(&clock_time_observer)
);
#[cfg(feature = "feed_afl")]
let mut feedback = feedback_or!(
// New maximization map feedback linked to the edges observer and the feedback state
MaxMapFeedback::new_tracking(&edges_observer, true, true),
// QemuClockIncreaseFeedback::default(),
// Time feedback, this one does not need a feedback state
ClockTimeFeedback::new_with_observer(&clock_time_observer),
// Feedback to reward any input which increses the execution time
ExecTimeIncFeedback::new()
feedback
);
#[cfg(all(feature = "systemstate",not(any(feature = "systemgraph",feature = "systemtrace"))))]
#[cfg(feature = "feed_longest")]
let mut feedback = feedback_or!(
// Feedback to reward any input which increses the execution time
ExecTimeIncFeedback::new(),
feedback
);
#[cfg(all(feature = "systemstate",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace"))))]
let mut feedback = feedback_or!(
DumpSystraceFeedback::with_dump(env::var("TRACE_DUMP").ok().map(PathBuf::from)),
feedback
);
#[cfg(feature = "systemtrace")]
#[cfg(feature = "feed_systemtrace")]
let mut feedback = feedback_or!(
NovelSystemStateFeedback::default(),
feedback
);
#[cfg(feature = "systemgraph")]
#[cfg(feature = "feed_systemgraph")]
let mut feedback = feedback_or!(
SysMapFeedback::default(),
feedback
@ -262,11 +269,13 @@ pub fn fuzz() {
});
// A minimization+queue policy to get testcasess from the corpus
#[cfg(not(any(feature = "systemgraph",feature = "systemtrace")))]
#[cfg(not(all(feature = "feed_afl",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace")))))]
let scheduler = QueueScheduler::new();
#[cfg(all(feature = "feed_afl",not(any(feature = "feed_systemgraph",feature = "feed_systemtrace"))))]
let scheduler = TimeMaximizerCorpusScheduler::new(QueueScheduler::new());
#[cfg(feature = "systemtrace")]
#[cfg(feature = "feed_systemtrace")]
let scheduler = TimeStateMaximizerCorpusScheduler::new(QueueScheduler::new());
#[cfg(feature = "systemgraph")]
#[cfg(feature = "feed_systemgraph")]
let scheduler = GraphMaximizerCorpusScheduler::new(QueueScheduler::new());
// A fuzzer with feedbacks and a corpus scheduler
@ -424,7 +433,7 @@ pub fn fuzz() {
},
None => (),
}
#[cfg(feature = "systemgraph")]
#[cfg(feature = "feed_systemgraph")]
{
let mut gd = String::from(&td);
gd.push_str(".graph");

8
fuzzers/FRET/src/worst.rs
View File

@ -293,15 +293,19 @@ where
.expect("QemuClockObserver not found");
if observer.last_runtime() > self.longest_time {
self.longest_time = observer.last_runtime();
self.last_is_longest = true;
Ok(true)
} else {
self.last_is_longest = false;
Ok(false)
}
self.last_is_longest = observer.last_runtime() > self.longest_time;
Ok(observer.last_runtime() > self.longest_time)
}
fn append_metadata(
&mut self,
_state: &mut S,
testcase: &mut Testcase<<S as UsesInput>::Input>,
) -> Result<(), Error> {
#[cfg(feature = "feed_afl")]
if self.last_is_longest {
let mim : Option<&mut MapIndexesMetadata>= testcase.metadata_mut().get_mut();
// pretend that the longest input alone excercises some non-existing edge, to keep it relevant