From a7becb403efbd485f859f3b54d5dc74794f54c26 Mon Sep 17 00:00:00 2001 From: Alwin Berger Date: Mon, 17 Jun 2024 10:31:48 +0200 Subject: [PATCH] update helper scripts --- fuzzers/FRET/benchmark/Snakefile | 21 ++++++++++++------- fuzzers/FRET/benchmark/build_all_demos.sh | 17 +++++++++++++++ fuzzers/FRET/benchmark/logtail.sh | 8 +++++++ fuzzers/FRET/benchmark/plot_all_benchmarks.sh | 7 +++++++ fuzzers/FRET/benchmark/plot_all_traces.sh | 14 +++++++++++++ fuzzers/FRET/benchmark/plot_multi.r | 14 ++++++++++--- fuzzers/FRET/tests/run_test.sh | 9 ++++++-- 7 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 fuzzers/FRET/benchmark/build_all_demos.sh create mode 100644 fuzzers/FRET/benchmark/logtail.sh create mode 100644 fuzzers/FRET/benchmark/plot_all_benchmarks.sh create mode 100644 fuzzers/FRET/benchmark/plot_all_traces.sh diff --git a/fuzzers/FRET/benchmark/Snakefile b/fuzzers/FRET/benchmark/Snakefile index c316ae3659..bc2340f68e 100644 --- a/fuzzers/FRET/benchmark/Snakefile +++ b/fuzzers/FRET/benchmark/Snakefile @@ -1,7 +1,7 @@ import csv import os def_flags="--no-default-features --features std,snapshot_restore,singlecore,restarting,do_hash_notify_state" -remote="timedump_253048_1873f6_all/" +remote="remote/" RUNTIME=1800 TARGET_REPS_A=2 TARGET_REPS_B=2 @@ -16,7 +16,7 @@ rule build_showmap: output: directory("bins/target_showmap") shell: - "cargo build --target-dir {output} {def_flags},systemstate" + "cargo build --target-dir {output} {def_flags},config_stg" rule build_random: output: @@ -52,7 +52,7 @@ rule build_showmap_int: output: directory("bins/target_showmap_int") shell: - "cargo build --target-dir {output} {def_flags},systemstate,fuzz_int" + "cargo build --target-dir {output} {def_flags},config_stg,fuzz_int" rule build_random_int: output: @@ -165,8 +165,8 @@ rule run_showmap: "bins/target_showmap_int", "{remote}timedump/{fuzzer}/{target}#{num}.case" output: - "{remote}timedump/{fuzzer}/{target}#{num}.trace.ron", - "{remote}timedump/{fuzzer}/{target}#{num}.case.time", + "{remote}timedump/{fuzzer}/{target}#{num}_case.trace.ron", + "{remote}timedump/{fuzzer}/{target}#{num}_case.time", run: with open('target_symbols.csv') as csvfile: reader = csv.DictReader(csvfile) @@ -186,8 +186,8 @@ rule run_showmap: script+=""" mkdir -p $(dirname {output}) set +e - echo $(pwd)/{input[1]}/debug/fret -n $(pwd)/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -t -a -r -k {input[0]} -c ./target_symbols.csv showmap -i {input[3]} - $(pwd)/{input[1]}/debug/fret -n $(pwd)/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num} -t -a -r -k {input[0]} -c ./target_symbols.csv showmap -i {input[3]} + echo $(pwd)/{input[1]}/debug/fret -n $(pwd)/{remote}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num}_case -t -a -r -g -k {input[0]} -c ./target_symbols.csv showmap -i {input[3]} + $(pwd)/{input[1]}/debug/fret -n $(pwd)/{remote}/timedump/{wildcards.fuzzer}/{wildcards.target}#{wildcards.num}_case -t -a -r -g -k {input[0]} -c ./target_symbols.csv showmap -i {input[3]} exit 0 """ if wildcards.fuzzer.find('random') >= 0: @@ -258,6 +258,13 @@ rule all_new: expand("timedump/{fuzzer}/{target}#{num}.time", fuzzer=['random', 'stgpath'], target=['waters', 'watersv2'],num=range(0,3)), expand("timedump/{fuzzer}/{target}#{num}.time", fuzzer=['random_int', 'stgpath_int'], target=['waters_int', 'watersv2_int'],num=range(0,3)) +rule all_showmap: + input: + expand("{remote}timedump/{fuzzer}/{target}#{num}_case.trace.ron",remote=remote, fuzzer=['frafl', 'stg'], target=['watersv2'],num=range(2,3)), + expand("{remote}timedump/{fuzzer}/{target}#{num}_case.trace.ron",remote=remote, fuzzer=['frafl_int', 'stg_int'], target=['watersv2_int'],num=range(0,3)), + expand("{remote}timedump/{fuzzer}/{target}#{num}_case.trace.ron",remote=remote, fuzzer=['random', 'stgpath'], target=['watersv2'],num=range(0,1)), + expand("{remote}timedump/{fuzzer}/{target}#{num}_case.trace.ron",remote=remote, fuzzer=['random_int', 'stgpath_int'], target=['watersv2_int'],num=range(0,1)) + rule all_bins: diff --git a/fuzzers/FRET/benchmark/build_all_demos.sh b/fuzzers/FRET/benchmark/build_all_demos.sh new file mode 100644 index 0000000000..7a79349f90 --- /dev/null +++ b/fuzzers/FRET/benchmark/build_all_demos.sh @@ -0,0 +1,17 @@ +make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC clean && make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC WATERS_DEMO=1 INTERRUPT_ACTIVATION=1 +cp ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC/build/RTOSDemo.axf build/waters_int.elf + +make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC clean && make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC WATERS_DEMO=1 INTERRUPT_ACTIVATION=0 +cp ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC/build/RTOSDemo.axf build/waters.elf + +make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC clean && make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC WATERSV2_DEMO=1 INTERRUPT_ACTIVATION=1 +cp ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC/build/RTOSDemo.axf build/watersv2_int.elf + +make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC clean && make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC WATERSV2_DEMO=1 INTERRUPT_ACTIVATION=0 +cp ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC/build/RTOSDemo.axf build/watersv2.elf + +make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC clean && make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC INTERACT_DEMO=1 INTERRUPT_ACTIVATION=1 +cp ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC/build/RTOSDemo.axf build/interact_int.elf + +make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC clean && make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC INTERACT_DEMO=1 INTERRUPT_ACTIVATION=0 +cp ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC/build/RTOSDemo.axf build/interact.elf diff --git a/fuzzers/FRET/benchmark/logtail.sh b/fuzzers/FRET/benchmark/logtail.sh new file mode 100644 index 0000000000..56bc049ae6 --- /dev/null +++ b/fuzzers/FRET/benchmark/logtail.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +find $1 -type 'f' -iname "${2}#*.log" | while IFS="" read -r p || [ -n "$p" ] +do + LINE=$(tail -n 100 $p | grep -io "run time: .* corpus: [0-9]*" | tail -n 1) + echo $p: $LINE + LINE=$(grep -i "interesting corpus elements" $p | tail -n 1) + echo $p: $LINE +done diff --git a/fuzzers/FRET/benchmark/plot_all_benchmarks.sh b/fuzzers/FRET/benchmark/plot_all_benchmarks.sh new file mode 100644 index 0000000000..e5ffadd510 --- /dev/null +++ b/fuzzers/FRET/benchmark/plot_all_benchmarks.sh @@ -0,0 +1,7 @@ +Rscript plot_multi.r remote waters ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +Rscript plot_multi.r remote waters_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +Rscript plot_multi.r remote watersv2 ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +Rscript plot_multi.r remote watersv2_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +Rscript plot_multi.r remote interact ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +Rscript plot_multi.r remote interact_int ~/code/FRET/LibAFL/fuzzers/FRET/benchmark/remote & +wait \ No newline at end of file diff --git a/fuzzers/FRET/benchmark/plot_all_traces.sh b/fuzzers/FRET/benchmark/plot_all_traces.sh new file mode 100644 index 0000000000..001451e833 --- /dev/null +++ b/fuzzers/FRET/benchmark/plot_all_traces.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +find ./remote/timedump -type 'f' -iregex '.*case' | while IFS="" read -r p || [ -n "$p" ] +do + N=$(dirname "$p")/$(basename -s .case "$p") + T="${N}_case.trace.ron" + P="${N}_case" + echo $N + if [ ! -f "$T" ]; then + snakemake -c1 "$T" + fi + if [ ! -f "$P.html" ]; then + ~/code/FRET/state2gantt/driver.sh "$T" + fi +done diff --git a/fuzzers/FRET/benchmark/plot_multi.r b/fuzzers/FRET/benchmark/plot_multi.r index 2ca1a84ca4..337e9a2ef5 100644 --- a/fuzzers/FRET/benchmark/plot_multi.r +++ b/fuzzers/FRET/benchmark/plot_multi.r @@ -12,7 +12,10 @@ args = commandArgs(trailingOnly=TRUE) if (length(args)==0) { runtype="remote" - target="waters_int" + #target="waters" + target="watersv2" + #target="waters_int" + #target="watersv2_int" outputpath="~/code/FRET/LibAFL/fuzzers/FRET/benchmark/" #MY_SELECTION <- c('state', 'afl', 'graph', 'random') SAVE_FILE=TRUE @@ -20,8 +23,13 @@ if (length(args)==0) { runtype=args[1] target=args[2] outputpath=args[3] - MY_SELECTION <- args[4:length(args)] + #MY_SELECTION <- args[4:length(args)] + #if (length(MY_SELECTION) == 0) + # MY_SELECTION<-NULL SAVE_FILE=TRUE + print(runtype) + print(target) + print(outputpath) } worst_cases <- list(waters=0, waters_int=0, tmr=405669, micro_longint=0, gen3=0) worst_case <- worst_cases[[target]] @@ -218,7 +226,7 @@ if (length(typenames) == 0) {return()} h_ = 500 w_ = h_*4/3 -if (SAVE_FILE) {png(file=sprintf("%s%s_%s.png",outputpath,target,filename), width=w_, height=h_)} +if (SAVE_FILE) {png(file=sprintf("%s/%s_%s.png",outputpath,target,filename), width=w_, height=h_)} par(mar=c(4,4,1,1)) par(oma=c(0,0,0,0)) diff --git a/fuzzers/FRET/tests/run_test.sh b/fuzzers/FRET/tests/run_test.sh index 5e5a1419da..11c5bf3a33 100644 --- a/fuzzers/FRET/tests/run_test.sh +++ b/fuzzers/FRET/tests/run_test.sh @@ -18,5 +18,10 @@ if [[ -n "$(diff -q demo.example.state.ron dump/demo.trace.ron)" ]]; then echo " # cargo build --no-default-features --features std,snapshot_restore,singlecore,feed_afl,observer_hitcounts,systemstate,trace_abbs if [[ -n "$(diff -q demo.example.abb.ron dump/demo.trace.ron)" ]]; then echo "ABB not reproducible!"; else echo "ABB Reproducible"; fi -# ../target/debug/fret -k ../benchmark/build/minimal.elf -c ../benchmark/target_symbols.csv -n ./dump/minimal -tar fuzz -t 100 -s 123 -# ../target/debug/fret -k ../benchmark/build/minimal.elf -c ../benchmark/target_symbols.csv -n ./dump/minimal_worst -tr showmap -i ./dump/minimal.case \ No newline at end of file +# ../target/debug/fret -k ../benchmark/build/minimal.elf -c ../benchmark/target_symbols.csv -n ./dump/minimal -tar fuzz -t 20 -s 123 +# ../target/debug/fret -k ../benchmark/build/minimal.elf -c ../benchmark/target_symbols.csv -n ./dump/minimal_worst -tr showmap -i ./dump/minimal.case + +# Test fuzzing using systemtraces +cargo build --no-default-features --features std,snapshot_restore,singlecore,feed_systemtrace + +../target/debug/fret -k ../benchmark/build/waters.elf -c ../benchmark/target_symbols.csv -n ./dump/waters -tar fuzz -t 10 -s 123 \ No newline at end of file