diff --git a/fuzzers/FRET/benchmark/Snakefile b/fuzzers/FRET/benchmark/Snakefile index 3b6378cac1..b886ff4770 100644 --- a/fuzzers/FRET/benchmark/Snakefile +++ b/fuzzers/FRET/benchmark/Snakefile @@ -1,10 +1,10 @@ import csv import os -def_flags="--no-default-features --features std,snapshot_restore,singlecore,restarting" -remote="mnt/" -RUNTIME=21600 -TARGET_REPS_A=20 -TARGET_REPS_B=10 +def_flags="--no-default-features --features std,snapshot_restore,singlecore,restarting,run_until_saturation" +remote="timedump_253048_1873f6_all/" +RUNTIME=10 +TARGET_REPS_A=2 +TARGET_REPS_B=2 NUM_NODES=2 REP_PER_NODE_A=int(TARGET_REPS_A/NUM_NODES) REP_PER_NODE_B=int(TARGET_REPS_B/NUM_NODES) @@ -48,6 +48,12 @@ rule build_state: shell: "cargo build --target-dir {output} {def_flags},feed_systemtrace" +rule build_nohashstate: + output: + directory("bins/target_nohashstate") + shell: + "cargo build --target-dir {output} {def_flags},feed_systemtrace,no_hash_state" + rule build_graph: output: directory("bins/target_graph") @@ -72,6 +78,12 @@ rule build_state_int: shell: "cargo build --target-dir {output} {def_flags},feed_systemtrace,fuzz_int" +rule build_nohashstate_int: + output: + directory("bins/target_nohashstate_int") + shell: + "cargo build --target-dir {output} {def_flags},feed_systemtrace,fuzz_int,no_hash_state" + rule build_frafl_int: output: directory("bins/target_frafl_int") diff --git a/fuzzers/FRET/benchmark/plot_multi.r b/fuzzers/FRET/benchmark/plot_multi.r index 9631b27986..58377dd24a 100644 --- a/fuzzers/FRET/benchmark/plot_multi.r +++ b/fuzzers/FRET/benchmark/plot_multi.r @@ -12,7 +12,7 @@ args = commandArgs(trailingOnly=TRUE) if (length(args)==0) { runtype="timedump_253048_1873f6_all/timedump" - target="waters" + target="waters_int" outputpath="~/code/FRET/LibAFL/fuzzers/FRET/benchmark/" #MY_SELECTION <- c('state', 'afl', 'graph', 'random') SAVE_FILE=TRUE @@ -38,9 +38,9 @@ PATTERNS="%s.[0-9]*$" #RIBBON='span' RIBBON='both' DRAW_WC = worst_case > 0 -LEGEND_POS="topleft" +LEGEND_POS="topright" #LEGEND_POS="bottomright" -CONTINUE_LINE_TO_END=TRUE +CONTINUE_LINE_TO_END=FALSE # https://www.r-bloggers.com/2013/04/how-to-change-the-alpha-value-of-colours-in-r/ alpha <- function(col, alpha=1){ @@ -103,6 +103,9 @@ trace2maxpoints <- function(tr) { minval = tr[i,1] } } + tmp = tr[dim(tr)[1],] + tmp[1] = minval[1] + collect = rbind(collect,tmp) return(collect) } @@ -170,8 +173,8 @@ all_maxlines <- foreach (rtt=all_runtypetables) %do% { } #runtypetables_sampled = lapply(rtt, function(v) sample_maxpoints(v, all_points)[1]) tmp_frame <- Reduce(cbind, runtypetables_sampled) - statframe <- data.frame(rowMeans(tmp_frame),apply(tmp_frame, 1, sd),apply(tmp_frame, 1, min),apply(tmp_frame, 1, max)) - names(statframe) <- c(bn, sprintf("%s_sd",bn), sprintf("%s_min",bn), sprintf("%s_max",bn)) + statframe <- data.frame(rowMeans(tmp_frame),apply(tmp_frame, 1, sd),apply(tmp_frame, 1, min),apply(tmp_frame, 1, max), apply(tmp_frame, 1, median)) + names(statframe) <- c(bn, sprintf("%s_sd",bn), sprintf("%s_min",bn), sprintf("%s_max",bn), sprintf("%s_med",bn)) #statframe[sprintf("%s_times",bn)] = all_points round(statframe) #all_maxlines = c(all_maxlines, list(round(statframe))) @@ -182,6 +185,7 @@ names(one_frame)[length(one_frame)] <- 'time' typenames = names(one_frame)[which(names(one_frame) != 'time')] typenames = typenames[which(!endsWith(typenames, "_sd"))] +typenames = typenames[which(!endsWith(typenames, "_med"))] ylow=min(one_frame[typenames]) yhigh=max(one_frame[typenames],worst_case) typenames = typenames[which(!endsWith(typenames, "_min"))] @@ -205,6 +209,7 @@ plotting <- function(selection, filename, MY_COLORS_) { # filter out names of iters and sd cols typenames = names(one_frame)[which(names(one_frame) != 'times')] typenames = typenames[which(!endsWith(typenames, "_sd"))] +typenames = typenames[which(!endsWith(typenames, "_med"))] typenames = typenames[which(!endsWith(typenames, "_min"))] typenames = typenames[which(!endsWith(typenames, "_max"))] typenames = selection[which(selection %in% typenames)] @@ -220,34 +225,61 @@ par(oma=c(0,0,0,0)) plot(c(1,max(one_frame['time'])),c(ylow,yhigh), col='white', xlab="Time [h]", ylab="WORT [insn]", pch='.') for (t in seq_len(length(typenames))) { - proj = one_frame[seq(1, dim(one_frame)[1], by=max(1, length(one_frame[[1]])/(10*w_))),] + #proj = one_frame[seq(1, dim(one_frame)[1], by=max(1, length(one_frame[[1]])/(10*w_))),] #points(proj[c('iters',typenames[t])], col=MY_COLORS_[t], pch='.') avglines = ml2lines(one_frame[c(typenames[t],'time')],all_max_points[typenames[t]]) - lines(avglines, col=MY_COLORS_[t]) - if (exists("RIBBON") && ( RIBBON=='both' || RIBBON=='span')) { - milines = ml2lines(one_frame[c(sprintf("%s_min",typenames[t]),'time')],all_max_points[typenames[t]]) - malines = ml2lines(one_frame[c(sprintf("%s_max",typenames[t]),'time')],all_max_points[typenames[t]]) - lines(milines, col=MY_COLORS_[t], lty='dashed') + #lines(avglines, col=MY_COLORS_[t]) + medlines = ml2lines(one_frame[c(sprintf("%s_med",typenames[t]),'time')],all_max_points[typenames[t]]) + lines(medlines, col=MY_COLORS_[t], lty='solid') + milines = NULL + malines = NULL + milines = ml2lines(one_frame[c(sprintf("%s_min",typenames[t]),'time')],all_max_points[typenames[t]]) + malines = ml2lines(one_frame[c(sprintf("%s_max",typenames[t]),'time')],all_max_points[typenames[t]]) + if (exists("RIBBON") && ( RIBBON=='max' )) { + #lines(milines, col=MY_COLORS_[t], lty='dashed') lines(malines, col=MY_COLORS_[t], lty='dashed') #points(proj[c('iters',sprintf("%s_min",typenames[t]))], col=MY_COLORS_[t], pch='.') #points(proj[c('iters',sprintf("%s_max",typenames[t]))], col=MY_COLORS_[t], pch='.') } if (exists("RIBBON") && RIBBON != '') { - for (i in seq_len(dim(proj)[1])) { - row = proj[i,] - x_ <- row['time'][[1]] - y_ <- row[typenames[t]][[1]] - sd_ <- row[sprintf("%s_sd",typenames[t])][[1]] - min_ <- row[sprintf("%s_min",typenames[t])][[1]] - max_ <- row[sprintf("%s_max",typenames[t])][[1]] - if (exists("RIBBON")) { - switch (RIBBON, - 'sd' = arrows(x_, y_-sd_, x_, y_+sd_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.03)), - 'both' = arrows(x_, y_-sd_, x_, y_+sd_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.05)), - 'span' = #arrows(x_, min_, x_, max_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.03)) - ) + for (i in seq_len(dim(avglines)[1]-1)) { + if (RIBBON=='both') { + # draw boxes + x_l <- milines[i,][['X']] + x_r <- milines[i+1,][['X']] + y_l <- milines[i,][['Y']] + y_h <- malines[i,][['Y']] + rect(x_l, y_l, x_r, y_h, col=alpha(MY_COLORS_[t], alpha=0.1), lwd=0) } - #arrows(x_, y_-sd_, x_, y_+sd_, length=0.05, angle=90, code=3, col=alpha(MY_COLORS[t], alpha=0.1)) + if (FALSE && RIBBON=='span') { + # draw boxes + x_l <- milines[i,][['X']] + x_r <- milines[i+1,][['X']] + y_l <- milines[i,][['Y']] + y_h <- malines[i,][['Y']] + rect(x_l, y_l, x_r, y_h, col=alpha(MY_COLORS_[t], alpha=0.1), lwd=0) + } + #if (FALSE && RIBBON=='both' || RIBBON=='sd') { + # # draw sd + # x_l <- avglines[i,][['X']] + # x_r <- avglines[i+1,][['X']] + # y_l <- avglines[i,][['Y']]-one_frame[ceiling(i/2),][[sprintf("%s_sd",typenames[t])]] + # y_h <- avglines[i,][['Y']]+one_frame[ceiling(i/2),][[sprintf("%s_sd",typenames[t])]] + # if (x_r != x_l) { + # rect(x_l, y_l, x_r, y_h, col=alpha(MY_COLORS_[t], alpha=0.1), lwd=0) + # } + #} + #sd_ <- row[sprintf("%s_sd",typenames[t])][[1]] + #min_ <- row[sprintf("%s_min",typenames[t])][[1]] + #max_ <- row[sprintf("%s_max",typenames[t])][[1]] + #if (exists("RIBBON")) { + # switch (RIBBON, + # 'sd' = arrows(x_, y_-sd_, x_, y_+sd_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.03)), + # 'both' = arrows(x_, y_-sd_, x_, y_+sd_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.05)), + # 'span' = #arrows(x_, min_, x_, max_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.03)) + # ) + #} + ##arrows(x_, y_-sd_, x_, y_+sd_, length=0.05, angle=90, code=3, col=alpha(MY_COLORS[t], alpha=0.1)) } } } @@ -266,20 +298,30 @@ stopCluster(cl) par(mar=c(3.8,3.8,0,0)) par(oma=c(0,0,0,0)) + +#RIBBON='both' +#MY_SELECTION = c('state_int','generation100_int') +#MY_SELECTION = c('state_int','frafl_int') + if (exists("MY_SELECTION")) { - plotting(MY_SELECTION, 'custom', MY_COLORS) + plotting(MY_SELECTION, 'custom', MY_COLORS[c(1,2)]) } else { - MY_SELECTION=c('state', 'afl', 'random', 'feedlongest', 'feedgeneration', 'feedgeneration10') + # MY_SELECTION=c('state', 'afl', 'random', 'feedlongest', 'feedgeneration', 'feedgeneration10') #MY_SELECTION=c('state_int', 'afl_int', 'random_int', 'feedlongest_int', 'feedgeneration_int', 'feedgeneration10_int') #MY_SELECTION=c('state', 'frAFL', 'statenohash', 'feedgeneration10') #MY_SELECTION=c('state_int', 'frAFL_int', 'statenohash_int', 'feedgeneration10_int') MY_SELECTION=typenames - RIBBON='span' + RIBBON='both' for (i in seq_len(length(MY_SELECTION))) { n <- MY_SELECTION[i] plotting(c(n), n, c(MY_COLORS[i])) } - RIBBON='span' + RIBBON='max' plotting(MY_SELECTION,'all', MY_COLORS) } +for (t in seq_len(length(typenames))) { + li = one_frame[dim(one_frame)[1],] + pear = (li[[typenames[[t]]]]-li[[sprintf("%s_med",typenames[[t]])]])/li[[sprintf("%s_sd",typenames[[t]])]] + print(sprintf("%s pearson: %g",typenames[[t]],pear)) +}