From 1df4bba604b836f8af9e6422662b9d2b62b8cd31 Mon Sep 17 00:00:00 2001 From: Alwin Berger Date: Fri, 21 Feb 2025 18:36:32 +0100 Subject: [PATCH] config updates --- fuzzers/FRET/benchmark/.gitignore | 1 + fuzzers/FRET/benchmark/Snakefile | 151 ++++++++++++++-------- fuzzers/FRET/benchmark/build_all_demos.sh | 4 +- fuzzers/FRET/benchmark/plot_all_traces.sh | 2 +- fuzzers/FRET/benchmark/plot_sqlite.r | 29 ++++- fuzzers/FRET/benchmark/target_symbols.csv | 6 +- 6 files changed, 131 insertions(+), 62 deletions(-) diff --git a/fuzzers/FRET/benchmark/.gitignore b/fuzzers/FRET/benchmark/.gitignore index 52eedad332..a6a7e9ee50 100644 --- a/fuzzers/FRET/benchmark/.gitignore +++ b/fuzzers/FRET/benchmark/.gitignore @@ -10,3 +10,4 @@ bins .snakemake *.zip *.tar.* +*.sqlite \ No newline at end of file diff --git a/fuzzers/FRET/benchmark/Snakefile b/fuzzers/FRET/benchmark/Snakefile index c2497da43b..b4237929da 100644 --- a/fuzzers/FRET/benchmark/Snakefile +++ b/fuzzers/FRET/benchmark/Snakefile @@ -1,129 +1,150 @@ import csv import os +envvars: + "BENCHDIR" def_flags="--release --no-default-features --features std,snapshot_fast,restarting,do_hash_notify_state,do_hash_notify_value,fuzz_int,trace_job_response_times" -remote="remote/" -RUNTIME=1800 -NUM_ITERS=2 +benchdir=os.environ["BENCHDIR"] +RUNTIME=1 + +rule copy_kernel: + input: + "build/{target}.elf" + output: + "{benchdir}/build/{target}.elf" + shell: + "mkdir -p {benchdir}/build && cp {input} {output}" + +rule rebuild_qemu: + shell: + "unset CUSTOM_QEMU_NO_BUILD CUSTOM_QEMU_NO_CONFIGURE && cargo build" rule build_default: input: "../Cargo.toml", "../src" output: - directory("bins/target_default") + directory("{benchdir}/bins/target_default") shell: "cargo build --target-dir {output} {def_flags}" rule build_showmap: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_showmap") + directory("{benchdir}/bins/target_showmap") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_stg" rule build_random: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_random") + directory("{benchdir}/bins/target_random") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},feed_longest" rule build_frafl: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_frafl") + directory("{benchdir}/bins/target_frafl") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_frafl,feed_longest" rule build_afl: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_afl") + directory("{benchdir}/bins/target_afl") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_afl" rule build_stg: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_stg") + directory("{benchdir}/bins/target_stg") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_stg" +rule build_stgwoet: + input: + "{benchdir}/bins/target_default" + output: + directory("{benchdir}/bins/target_stgwoet") + shell: + "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_stg_woet" + rule build_stg_abbpath: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_stg_abbpath") + directory("{benchdir}/bins/target_stg_abbpath") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_stg_abbpath" rule build_stg_edge: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_stg_edge") + directory("{benchdir}/bins/target_stg_edge") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_stg_edge" rule build_feedgeneration1: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_feedgeneration1") + directory("{benchdir}/bins/target_feedgeneration1") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},feed_genetic,gensize_1" rule build_feedgeneration10: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_feedgeneration10") + directory("{benchdir}/bins/target_feedgeneration10") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},feed_genetic,gensize_10" rule build_feedgeneration100: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_feedgeneration100") + directory("{benchdir}/bins/target_feedgeneration100") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_genetic,gensize_100" rule build_genetic100: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_genetic100") + directory("{benchdir}/bins/target_genetic100") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_genetic,mutate_stg,gensize_100" rule build_feedgeneration1000: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_feedgeneration1000") + directory("{benchdir}/bins/target_feedgeneration1000") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_genetic,gensize_1000" rule build_genetic1000: input: - "bins/target_default" + "{benchdir}/bins/target_default" output: - directory("bins/target_genetic1000") + directory("{benchdir}/bins/target_genetic1000") shell: "cp -r -a --reflink=auto {input} {output} && cargo build --target-dir {output} {def_flags},config_genetic,mutate_stg,gensize_1000" rule run_bench: input: - "build/{target}.elf", - "bins/target_{fuzzer}" + "{benchdir}/build/{target}.elf", + "{benchdir}/bins/target_{fuzzer}" output: - multiext("timedump/{fuzzer}/{target}#{num}", ".time", ".log") # , ".case" + multiext("{benchdir}/timedump/{fuzzer}/{target}#{num}", ".time", ".log") # , ".case" run: with open('target_symbols.csv') as csvfile: reader = csv.DictReader(csvfile) @@ -141,8 +162,8 @@ rule run_bench: export RUST_BACKTRACE=1 mkdir -p $(dirname {output[0]}) set +e - echo $(pwd)/{input[1]}/release/fret -n $(pwd)/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv fuzz --random -t {RUNTIME} -s {wildcards.num} - $(pwd)/{input[1]}/release/fret -n $(pwd)/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv fuzz --random -t {RUNTIME} -s {wildcards.num} > {output[1]} 2>&1 + echo $(pwd)/{input[1]}/release/fret -n $(pwd)/{benchdir}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv fuzz --random -t {RUNTIME} -s {wildcards.num} + $(pwd)/{input[1]}/release/fret -n $(pwd)/{benchdir}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv fuzz --random -t {RUNTIME} -s {wildcards.num} > {output[1]} 2>&1 exit 0 """ else: @@ -150,20 +171,20 @@ rule run_bench: export RUST_BACKTRACE=1 mkdir -p $(dirname {output[0]}) set +e - echo $(pwd)/{input[1]}/release/fret -n $(pwd)/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv fuzz -t {RUNTIME} -s {wildcards.num} - $(pwd)/{input[1]}/release/fret -n $(pwd)/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv fuzz -t {RUNTIME} -s {wildcards.num} > {output[1]} 2>&1 + echo $(pwd)/{input[1]}/release/fret -n $(pwd)/{benchdir}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv fuzz -t {RUNTIME} -s {wildcards.num} + $(pwd)/{input[1]}/release/fret -n $(pwd)/{benchdir}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv fuzz -t {RUNTIME} -s {wildcards.num} > {output[1]} 2>&1 exit 0 """ shell(script) rule run_showmap: input: - "{remote}build/{target}.elf", - "bins/target_showmap", - "{remote}timedump/{fuzzer}/{target}#{num}.case" + "{benchdir}/build/{target}.elf", + "{benchdir}/bins/target_showmap", + "{benchdir}/timedump/{fuzzer}/{target}#{num}.case" output: - "{remote}timedump/{fuzzer}/{target}#{num}_case.trace.ron", - "{remote}timedump/{fuzzer}/{target}#{num}_case.time", + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.trace.ron", + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.time", run: with open('target_symbols.csv') as csvfile: reader = csv.DictReader(csvfile) @@ -180,21 +201,21 @@ rule run_showmap: export FUZZER=$(pwd)/{input[1]}/release/fret mkdir -p $(dirname {output}) set +e - echo $FUZZER -n $(pwd)/{remote}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num}_case -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv showmap -i {input[2]} - $FUZZER -n $(pwd)/{remote}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num}_case -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv showmap -i {input[2]} + echo $FUZZER -n $(pwd)/{benchdir}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num}_case -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv showmap -i {input[2]} + $FUZZER -n $(pwd)/{benchdir}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num}_case -s {select_task} -t -a -r -g -k {input[0]} -c ./target_symbols.csv showmap -i {input[2]} exit 0 """ if wildcards.fuzzer.find('random') >= 0: script="export FUZZ_RANDOM=1\n"+script shell(script) -rule tarnsform_trace: +rule transform_trace: input: - "{remote}timedump/{fuzzer}/{target}#{num}_case.trace.ron", + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.trace.ron", output: - "{remote}timedump/{fuzzer}/{target}#{num}_case.jobs.csv", - "{remote}timedump/{fuzzer}/{target}#{num}_case.resp.csv", - "{remote}timedump/{fuzzer}/{target}#{num}_case.abbs.csv" + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.jobs.csv", + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.resp.csv", + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.abbs.csv" run: with open('target_symbols.csv') as csvfile: reader = csv.DictReader(csvfile) @@ -215,17 +236,39 @@ rule tarnsform_trace: rule trace2gantt: input: - "{remote}timedump/{fuzzer}/{target}#{num}_case.jobs.csv", - "{remote}timedump/{fuzzer}/{target}#{num}_case.resp.csv" + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.jobs.csv", + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.resp.csv" output: - "{remote}timedump/{fuzzer}/{target}#{num}_case.jobs.html", + "{benchdir}/timedump/{fuzzer}/{target}#{num}_case.jobs.html", shell: "Rscript $(pwd)/../../../../state2gantt/plot_response.r {input[0]} {input[1]} html" rule quicktest: + params: + benchdir=benchdir input: - expand("timedump/{fuzzer}/{target}{variant}#{num}.time", fuzzer=['feedgeneration100', 'stg'], target=['copter'], variant=['_seq_full', '_par_full', '_seq_stateful_full', '_par_stateful_full', '_seq_dataflow_full'], num=range(0,int( 3 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg', 'random'], target=['polycopter'], variant=['_seq_dataflow_full', '_par_dataflow_full'], num=range(0,int( 1 ))), + +rule minimal_set: + params: + benchdir=benchdir + input: + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 5 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['stgwoet'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 4 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['waters'], variant=['_seq_full', '_seq_int', '_seq_bytes'], num=range(0,int( 5 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['release'], variant=['_seq_full', '_seq_int'], num=range(0,int( 5 ))), + rule all_bins: + params: + benchdir=benchdir input: - expand("bins/target_{target}",target=['random','frafl','stg','feedgeneration100','feedgeneration1000','genetic100','genetic1000']) + expand("{benchdir}/bins/target_{target}", benchdir=benchdir, target=['random','frafl','stg','stgwoet','feedgeneration100','genetic100']) + +rule clean: + shell: + "rm -rf {benchdir}/timedump" + +rule full_clean: + shell: + "rm -rf {benchdir}/bins || rm -rf {benchdir}/timedump" \ No newline at end of file diff --git a/fuzzers/FRET/benchmark/build_all_demos.sh b/fuzzers/FRET/benchmark/build_all_demos.sh index 5a7d52fe0a..917ec0ab19 100644 --- a/fuzzers/FRET/benchmark/build_all_demos.sh +++ b/fuzzers/FRET/benchmark/build_all_demos.sh @@ -73,7 +73,7 @@ export PARTITION_INPUT=0 export IGNORE_INTERNAL_STATE=1 export IGNORE_INTERRUPTS=0 IGNORE_BYTES=0 SUFFIX="_seq_dataflow_full" export SPECIAL_CFLAGS="-DCOPTER_DATAFLOW=1" -build COPTER_DEMO $SUFFIX +build POLYCOPTER_DEMO $SUFFIX unset SPECIAL_CFLAGS # stateless + dataflow @@ -81,5 +81,5 @@ export PARTITION_INPUT=1 export IGNORE_INTERNAL_STATE=1 export IGNORE_INTERRUPTS=0 IGNORE_BYTES=0 SUFFIX="_par_dataflow_full" export SPECIAL_CFLAGS="-DCOPTER_DATAFLOW=1" -build COPTER_DEMO $SUFFIX +build POLYCOPTER_DEMO $SUFFIX unset SPECIAL_CFLAGS diff --git a/fuzzers/FRET/benchmark/plot_all_traces.sh b/fuzzers/FRET/benchmark/plot_all_traces.sh index e035c02266..e9426f07a2 100644 --- a/fuzzers/FRET/benchmark/plot_all_traces.sh +++ b/fuzzers/FRET/benchmark/plot_all_traces.sh @@ -22,7 +22,7 @@ do # if [ ! -f "$P.html" ]; then # ~/code/FRET/state2gantt/driver.sh "$T" # fi -done < <(find ./remote/timedump -maxdepth 2 -type 'f' -iregex '.*\.case') +done < <(find $BENCHDIR/timedump -maxdepth 2 -type 'f' -iregex '.*\.case') echo "${PLOTS[@]}" snakemake -c 6 --rerun-incomplete --keep-incomplete "${PLOTS[@]}" diff --git a/fuzzers/FRET/benchmark/plot_sqlite.r b/fuzzers/FRET/benchmark/plot_sqlite.r index fd6221b88b..3664359564 100644 --- a/fuzzers/FRET/benchmark/plot_sqlite.r +++ b/fuzzers/FRET/benchmark/plot_sqlite.r @@ -4,6 +4,11 @@ library("DBI") args = commandArgs(trailingOnly=TRUE) +KNOWN_WCRT <- list( + watersc14_par_full=242454, + watersc14_seq_full=242454 + ) + # Read the first command line argument as an sqlite file if (length(args) > 0) { sqlite_file <- args[1] @@ -46,9 +51,16 @@ draw_plot <- function(data, casename) { data[[n]]$sdiv <- data[[n]]$sdiv / ISNS_PER_US } + wcrt = KNOWN_WCRT[[casename]] + if (!is.null(wcrt)) { + wcrt = wcrt / ISNS_PER_US + } else { + wcrt = 0 + } + # draw limits max_x <- max(sapply(data, function(tbl) max(tbl$timestamp, na.rm = TRUE))) - max_y <- max(sapply(data, function(tbl) max(tbl$max, na.rm = TRUE))) + max_y <- max(wcrt,max(sapply(data, function(tbl) max(tbl$max, na.rm = TRUE)))) min_y <- min(sapply(data, function(tbl) min(tbl$min, na.rm = TRUE))) # plot setup @@ -70,9 +82,18 @@ draw_plot <- function(data, casename) { lines(milines, col=MY_COLORS[[n]], lty='dashed') } - legend(LEGEND_POS, legend=names(data),#"bottomright", - col=c(MY_COLORS[1:length(data)],"black"), - lty=c(rep("solid",length(data)),"dotted")) + legend_names <- names(data) + legend_colors <- c(MY_COLORS[1:length(data)],"black") + legend_styles <- c(rep("solid",length(data)),"dotted") + + if (wcrt > 0) { + abline(h=wcrt, col='black', lty='dotted') + legend_names <- c(names(data), "WCRT") + } + + legend(LEGEND_POS, legend=legend_names,#"bottomright", + col=legend_colors, + lty=legend_styles) par(las = 2, mar = c(10, 5, 1, 1)) dev.off() diff --git a/fuzzers/FRET/benchmark/target_symbols.csv b/fuzzers/FRET/benchmark/target_symbols.csv index fa719fcb0d..3728ccd939 100644 --- a/fuzzers/FRET/benchmark/target_symbols.csv +++ b/fuzzers/FRET/benchmark/target_symbols.csv @@ -40,4 +40,8 @@ copter_par_stateful_bytes,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC, copter_seq_dataflow_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 copter_par_dataflow_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 polycopter_par_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 -polycopter_seq_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 \ No newline at end of file +polycopter_seq_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 +polycopter_par_dataflow_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 +polycopter_seq_dataflow_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 +watersc14_par_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000 +watersc14_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000 \ No newline at end of file