eval script wrangeling

fuzzers/FRET/benchmark/Snakefile

@@ -1,10 +1,10 @@
 import csv
 import os
-def_flags="--no-default-features --features std,snapshot_restore,singlecore,restarting"
+def_flags="--no-default-features --features std,snapshot_restore,singlecore,restarting,run_until_saturation"
-remote="mnt/"
+remote="timedump_253048_1873f6_all/"
-RUNTIME=21600
+RUNTIME=10
-TARGET_REPS_A=20
+TARGET_REPS_A=2
-TARGET_REPS_B=10
+TARGET_REPS_B=2
 NUM_NODES=2
 REP_PER_NODE_A=int(TARGET_REPS_A/NUM_NODES)
 REP_PER_NODE_B=int(TARGET_REPS_B/NUM_NODES)
@@ -48,6 +48,12 @@ rule build_state:
     shell:
         "cargo build --target-dir {output} {def_flags},feed_systemtrace"
 
+rule build_nohashstate:
+    output:
+        directory("bins/target_nohashstate")
+    shell:
+        "cargo build --target-dir {output} {def_flags},feed_systemtrace,no_hash_state"
+
 rule build_graph:
     output:
         directory("bins/target_graph")
@@ -72,6 +78,12 @@ rule build_state_int:
     shell:
         "cargo build --target-dir {output} {def_flags},feed_systemtrace,fuzz_int"
 
+rule build_nohashstate_int:
+    output:
+        directory("bins/target_nohashstate_int")
+    shell:
+        "cargo build --target-dir {output} {def_flags},feed_systemtrace,fuzz_int,no_hash_state"
+
 rule build_frafl_int:
     output:
         directory("bins/target_frafl_int")

fuzzers/FRET/benchmark/plot_multi.r

@@ -12,7 +12,7 @@ args = commandArgs(trailingOnly=TRUE)
 
 if (length(args)==0) {
   runtype="timedump_253048_1873f6_all/timedump"
-  target="waters"
+  target="waters_int"
   outputpath="~/code/FRET/LibAFL/fuzzers/FRET/benchmark/"
   #MY_SELECTION <- c('state', 'afl', 'graph', 'random')
   SAVE_FILE=TRUE
@@ -38,9 +38,9 @@ PATTERNS="%s.[0-9]*$"
 #RIBBON='span'
 RIBBON='both'
 DRAW_WC = worst_case > 0
-LEGEND_POS="topleft"
+LEGEND_POS="topright"
 #LEGEND_POS="bottomright"
-CONTINUE_LINE_TO_END=TRUE
+CONTINUE_LINE_TO_END=FALSE
 
 # https://www.r-bloggers.com/2013/04/how-to-change-the-alpha-value-of-colours-in-r/
 alpha <- function(col, alpha=1){
@@ -103,6 +103,9 @@ trace2maxpoints <- function(tr) {
       minval = tr[i,1]
     }
   }
+  tmp = tr[dim(tr)[1],]
+  tmp[1] = minval[1]
+  collect = rbind(collect,tmp)
   return(collect)
 }
 
@@ -170,8 +173,8 @@ all_maxlines <- foreach (rtt=all_runtypetables) %do% {
   }
   #runtypetables_sampled = lapply(rtt, function(v) sample_maxpoints(v, all_points)[1])
   tmp_frame <- Reduce(cbind, runtypetables_sampled)
-  statframe <- data.frame(rowMeans(tmp_frame),apply(tmp_frame, 1, sd),apply(tmp_frame, 1, min),apply(tmp_frame, 1, max))
+  statframe <- data.frame(rowMeans(tmp_frame),apply(tmp_frame, 1, sd),apply(tmp_frame, 1, min),apply(tmp_frame, 1, max), apply(tmp_frame, 1, median))
-  names(statframe) <- c(bn, sprintf("%s_sd",bn), sprintf("%s_min",bn), sprintf("%s_max",bn))
+  names(statframe) <- c(bn, sprintf("%s_sd",bn), sprintf("%s_min",bn), sprintf("%s_max",bn), sprintf("%s_med",bn))
   #statframe[sprintf("%s_times",bn)] = all_points
   round(statframe)
   #all_maxlines = c(all_maxlines, list(round(statframe))) 
@@ -182,6 +185,7 @@ names(one_frame)[length(one_frame)] <- 'time'
 
 typenames = names(one_frame)[which(names(one_frame) != 'time')]
 typenames = typenames[which(!endsWith(typenames, "_sd"))]
+typenames = typenames[which(!endsWith(typenames, "_med"))]
 ylow=min(one_frame[typenames])
 yhigh=max(one_frame[typenames],worst_case)
 typenames = typenames[which(!endsWith(typenames, "_min"))]
@@ -205,6 +209,7 @@ plotting <- function(selection, filename, MY_COLORS_) {
 # filter out names of iters and sd cols
 typenames = names(one_frame)[which(names(one_frame) != 'times')]
 typenames = typenames[which(!endsWith(typenames, "_sd"))]
+typenames = typenames[which(!endsWith(typenames, "_med"))]
 typenames = typenames[which(!endsWith(typenames, "_min"))]
 typenames = typenames[which(!endsWith(typenames, "_max"))]
 typenames = selection[which(selection %in% typenames)]
@@ -220,34 +225,61 @@ par(oma=c(0,0,0,0))
 plot(c(1,max(one_frame['time'])),c(ylow,yhigh), col='white', xlab="Time [h]", ylab="WORT [insn]", pch='.')
 
 for (t in seq_len(length(typenames))) {
-  proj = one_frame[seq(1, dim(one_frame)[1], by=max(1, length(one_frame[[1]])/(10*w_))),]
+  #proj = one_frame[seq(1, dim(one_frame)[1], by=max(1, length(one_frame[[1]])/(10*w_))),]
   #points(proj[c('iters',typenames[t])], col=MY_COLORS_[t], pch='.')
   avglines = ml2lines(one_frame[c(typenames[t],'time')],all_max_points[typenames[t]])
-  lines(avglines, col=MY_COLORS_[t])
+  #lines(avglines, col=MY_COLORS_[t])
-  if (exists("RIBBON") && ( RIBBON=='both' || RIBBON=='span')) {
+  medlines = ml2lines(one_frame[c(sprintf("%s_med",typenames[t]),'time')],all_max_points[typenames[t]])
-    milines = ml2lines(one_frame[c(sprintf("%s_min",typenames[t]),'time')],all_max_points[typenames[t]])
+  lines(medlines, col=MY_COLORS_[t], lty='solid')
-    malines = ml2lines(one_frame[c(sprintf("%s_max",typenames[t]),'time')],all_max_points[typenames[t]])
+  milines = NULL
-    lines(milines, col=MY_COLORS_[t], lty='dashed')
+  malines = NULL
+  milines = ml2lines(one_frame[c(sprintf("%s_min",typenames[t]),'time')],all_max_points[typenames[t]])
+  malines = ml2lines(one_frame[c(sprintf("%s_max",typenames[t]),'time')],all_max_points[typenames[t]])
+  if (exists("RIBBON") && ( RIBBON=='max' )) {
+    #lines(milines, col=MY_COLORS_[t], lty='dashed')
     lines(malines, col=MY_COLORS_[t], lty='dashed')
     #points(proj[c('iters',sprintf("%s_min",typenames[t]))], col=MY_COLORS_[t], pch='.')
     #points(proj[c('iters',sprintf("%s_max",typenames[t]))], col=MY_COLORS_[t], pch='.')
   }
   if (exists("RIBBON") && RIBBON != '') {
-  for (i in seq_len(dim(proj)[1])) {
+  for (i in seq_len(dim(avglines)[1]-1)) {
-    row = proj[i,]
+    if (RIBBON=='both') {
-    x_ <- row['time'][[1]]
+      # draw boxes
-    y_ <- row[typenames[t]][[1]]
+      x_l <- milines[i,][['X']]
-    sd_ <- row[sprintf("%s_sd",typenames[t])][[1]]
+      x_r <- milines[i+1,][['X']]
-    min_ <- row[sprintf("%s_min",typenames[t])][[1]]
+      y_l <- milines[i,][['Y']]
-    max_ <- row[sprintf("%s_max",typenames[t])][[1]]
+      y_h <- malines[i,][['Y']]
-    if (exists("RIBBON")) {
+      rect(x_l, y_l, x_r, y_h, col=alpha(MY_COLORS_[t], alpha=0.1), lwd=0)
-      switch (RIBBON,
-        'sd' = arrows(x_, y_-sd_, x_, y_+sd_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.03)),
-        'both' = arrows(x_, y_-sd_, x_, y_+sd_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.05)),
-        'span' = #arrows(x_, min_, x_, max_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.03))
-      )
     }
-    #arrows(x_, y_-sd_, x_, y_+sd_, length=0.05, angle=90, code=3, col=alpha(MY_COLORS[t], alpha=0.1))
+    if (FALSE && RIBBON=='span') {
+      # draw boxes
+      x_l <- milines[i,][['X']]
+      x_r <- milines[i+1,][['X']]
+      y_l <- milines[i,][['Y']]
+      y_h <- malines[i,][['Y']]
+      rect(x_l, y_l, x_r, y_h, col=alpha(MY_COLORS_[t], alpha=0.1), lwd=0)
+    }
+    #if (FALSE && RIBBON=='both' || RIBBON=='sd') {
+    #  # draw sd
+    #  x_l <- avglines[i,][['X']]
+    #  x_r <- avglines[i+1,][['X']]
+    #  y_l <- avglines[i,][['Y']]-one_frame[ceiling(i/2),][[sprintf("%s_sd",typenames[t])]]
+    #  y_h <- avglines[i,][['Y']]+one_frame[ceiling(i/2),][[sprintf("%s_sd",typenames[t])]]
+    #  if (x_r != x_l) {
+    #    rect(x_l, y_l, x_r, y_h, col=alpha(MY_COLORS_[t], alpha=0.1), lwd=0) 
+    #  }
+    #}
+    #sd_ <- row[sprintf("%s_sd",typenames[t])][[1]]
+    #min_ <- row[sprintf("%s_min",typenames[t])][[1]]
+    #max_ <- row[sprintf("%s_max",typenames[t])][[1]]
+    #if (exists("RIBBON")) {
+    #  switch (RIBBON,
+    #    'sd' = arrows(x_, y_-sd_, x_, y_+sd_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.03)),
+    #    'both' = arrows(x_, y_-sd_, x_, y_+sd_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.05)),
+    #    'span' = #arrows(x_, min_, x_, max_, length=0, angle=90, code=3, col=alpha(MY_COLORS_[t], alpha=0.03))
+    #  )
+    #}
+    ##arrows(x_, y_-sd_, x_, y_+sd_, length=0.05, angle=90, code=3, col=alpha(MY_COLORS[t], alpha=0.1))
   }
   }
 }
@@ -266,20 +298,30 @@ stopCluster(cl)
 
 par(mar=c(3.8,3.8,0,0))
 par(oma=c(0,0,0,0))
+
+#RIBBON='both'
+#MY_SELECTION = c('state_int','generation100_int')
+#MY_SELECTION = c('state_int','frafl_int')
+
 if (exists("MY_SELECTION")) {
-  plotting(MY_SELECTION, 'custom', MY_COLORS)
+  plotting(MY_SELECTION, 'custom', MY_COLORS[c(1,2)])
 } else {
-  MY_SELECTION=c('state', 'afl', 'random', 'feedlongest', 'feedgeneration', 'feedgeneration10')
+  # MY_SELECTION=c('state', 'afl', 'random', 'feedlongest', 'feedgeneration', 'feedgeneration10')
   #MY_SELECTION=c('state_int', 'afl_int', 'random_int', 'feedlongest_int', 'feedgeneration_int', 'feedgeneration10_int')
   #MY_SELECTION=c('state', 'frAFL', 'statenohash', 'feedgeneration10')
   #MY_SELECTION=c('state_int', 'frAFL_int', 'statenohash_int', 'feedgeneration10_int')
   MY_SELECTION=typenames
-  RIBBON='span'
+  RIBBON='both'
   for (i in seq_len(length(MY_SELECTION))) {
     n <- MY_SELECTION[i]
     plotting(c(n), n, c(MY_COLORS[i]))
   }
-  RIBBON='span'
+  RIBBON='max'
   plotting(MY_SELECTION,'all', MY_COLORS)
 }
 
+for (t in seq_len(length(typenames))) {
+  li = one_frame[dim(one_frame)[1],]
+  pear = (li[[typenames[[t]]]]-li[[sprintf("%s_med",typenames[[t]])]])/li[[sprintf("%s_sd",typenames[[t]])]]
+  print(sprintf("%s pearson: %g",typenames[[t]],pear))
+}