diff --git a/fuzzers/FRET/benchmark/Snakefile b/fuzzers/FRET/benchmark/Snakefile index 8fea2d12f1..4a7d2edbbd 100644 --- a/fuzzers/FRET/benchmark/Snakefile +++ b/fuzzers/FRET/benchmark/Snakefile @@ -4,7 +4,7 @@ envvars: "BENCHDIR" def_flags="--release --no-default-features --features std,snapshot_fast,restarting,do_hash_notify_state,do_hash_notify_value,fuzz_int,trace_job_response_times" benchdir=os.environ["BENCHDIR"] -RUNTIME=1 +RUNTIME=(3600*24) rule copy_kernel: input: @@ -247,16 +247,31 @@ rule quicktest: params: benchdir=benchdir input: - expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg', 'random'], target=['polycopter'], variant=['_seq_dataflow_full', '_par_dataflow_full'], num=range(0,int( 1 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg', 'random'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 1 ))), -rule minimal_set: +rule set128: params: benchdir=benchdir input: - expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 5 ))), - expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['stgwoet'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 4 ))), - expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['waters'], variant=['_seq_full', '_seq_int', '_seq_bytes'], num=range(0,int( 5 ))), - expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['release'], variant=['_seq_full', '_seq_int'], num=range(0,int( 5 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['release'], variant=['_seq_full', '_seq_int'], num=range(0,int( 13 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['release'], variant=['_seq_full', '_seq_int'], num=range(0,int( 3 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 13 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 3 ))), + +rule set64: + params: + benchdir=benchdir + input: + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 10 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 2 ))), + + +rule set48: + params: + benchdir=benchdir + input: + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['waters'], variant=['_seq_int', '_seq_bytes'], num=range(0,int( 7 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_int', '_seq_bytes'], num=range(0,int( 3 ))), rule all_bins: diff --git a/fuzzers/FRET/benchmark/build_all_demos.sh b/fuzzers/FRET/benchmark/build_all_demos.sh index 917ec0ab19..25ed297aff 100644 --- a/fuzzers/FRET/benchmark/build_all_demos.sh +++ b/fuzzers/FRET/benchmark/build_all_demos.sh @@ -2,6 +2,9 @@ build () { make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC clean && make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC $1=1 IGNORE_INTERRUPTS=$IGNORE_INTERRUPTS IGNORE_BYTES=$IGNORE_BYTES IGNORE_INTERNAL_STATE=$IGNORE_INTERNAL_STATE cp ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC/build/RTOSDemo.axf build/$(echo $1 | cut -d_ -f1 | tr '[:upper:]' '[:lower:]')$2.elf } +# INSERT_WC=1 + +mkdir -p build # Sequential inputs! export PARTITION_INPUT=0 @@ -68,6 +71,13 @@ build WATERS_DEMO $SUFFIX build RELEASE_DEMO $SUFFIX build COPTER_DEMO $SUFFIX +# Stateful -> presumably bad for us +## keep rng states +export IGNORE_INTERNAL_STATE=0 +export PARTITION_INPUT=0 +export IGNORE_INTERRUPTS=0 IGNORE_BYTES=0 SUFFIX="_seq_stateful_full" +build POLYCOPTER_DEMO $SUFFIX + # stateless + dataflow export PARTITION_INPUT=0 export IGNORE_INTERNAL_STATE=1 @@ -76,6 +86,13 @@ export SPECIAL_CFLAGS="-DCOPTER_DATAFLOW=1" build POLYCOPTER_DEMO $SUFFIX unset SPECIAL_CFLAGS +export PARTITION_INPUT=0 +export IGNORE_INTERNAL_STATE=1 +export IGNORE_INTERRUPTS=1 IGNORE_BYTES=0 SUFFIX="_seq_dataflow_bytes" +export SPECIAL_CFLAGS="-DCOPTER_DATAFLOW=1" +build POLYCOPTER_DEMO $SUFFIX +unset SPECIAL_CFLAGS + # stateless + dataflow export PARTITION_INPUT=1 export IGNORE_INTERNAL_STATE=1 @@ -83,3 +100,12 @@ export IGNORE_INTERRUPTS=0 IGNORE_BYTES=0 SUFFIX="_par_dataflow_full" export SPECIAL_CFLAGS="-DCOPTER_DATAFLOW=1" build POLYCOPTER_DEMO $SUFFIX unset SPECIAL_CFLAGS + + +# special waters with no synchronization +export PARTITION_INPUT=0 +export IGNORE_INTERNAL_STATE=1 +export IGNORE_INTERRUPTS=0 IGNORE_BYTES=0 SUFFIX="_seq_unsync_full" +export SPECIAL_CFLAGS="-DWATERS_UNSYNCHRONIZED=1" +build WATERS_DEMO $SUFFIX +unset SPECIAL_CFLAGS diff --git a/fuzzers/FRET/benchmark/plot_all_benchmarks.sh b/fuzzers/FRET/benchmark/plot_all_benchmarks.sh new file mode 100644 index 0000000000..ab43b4b323 --- /dev/null +++ b/fuzzers/FRET/benchmark/plot_all_benchmarks.sh @@ -0,0 +1,8 @@ +#!/bin/sh +if [[ -n "$1" ]]; then + TARGET="$1" +else + TARGET=$BENCHDIR +fi +number_cruncher/target/debug/number_cruncher -i $TARGET/timedump -o $TARGET/bench.sqlite +Rscript plot_sqlite.r bench_$TARGET.sqlite $TARGET diff --git a/fuzzers/FRET/benchmark/plot_all_traces.sh b/fuzzers/FRET/benchmark/plot_all_traces.sh index e9426f07a2..c9509989f1 100644 --- a/fuzzers/FRET/benchmark/plot_all_traces.sh +++ b/fuzzers/FRET/benchmark/plot_all_traces.sh @@ -22,7 +22,7 @@ do # if [ ! -f "$P.html" ]; then # ~/code/FRET/state2gantt/driver.sh "$T" # fi -done < <(find $BENCHDIR/timedump -maxdepth 2 -type 'f' -iregex '.*\.case') +done < <(find $BENCHDIR/timedump -maxdepth 2 -type 'f' -iregex '.*[0-9]+\.case') echo "${PLOTS[@]}" -snakemake -c 6 --rerun-incomplete --keep-incomplete "${PLOTS[@]}" +snakemake -c 20 --rerun-incomplete --keep-incomplete "${PLOTS[@]}" diff --git a/fuzzers/FRET/benchmark/plot_sqlite.r b/fuzzers/FRET/benchmark/plot_sqlite.r index 3664359564..88b1c629ca 100644 --- a/fuzzers/FRET/benchmark/plot_sqlite.r +++ b/fuzzers/FRET/benchmark/plot_sqlite.r @@ -5,8 +5,23 @@ library("DBI") args = commandArgs(trailingOnly=TRUE) KNOWN_WCRT <- list( - watersc14_par_full=242454, - watersc14_seq_full=242454 + waters_seq_bytes=209667, # via INSERT_WC + waters_seq_int=213646, # via INSERT_WC + manual interrupt + waters_seq_full=213646,# via INSERT_WC + manual interrupt + polycopter_seq_dataflow_full=273807, # via INSERT_WC + manual interrupt + polycopter_seq_dataflow_int=273807, # via INSERT_WC + manual interrupt + release_seq_int=624552, # via INSERT_WC + manual interrupt + release_seq_full=624552 # via INSERT_WC + manual interrupt + ) + +STATIC_WCRT <- list( + #waters_seq_bytes=, + waters_seq_int=270789 + #waters_seq_full=, + #polycopter_seq_dataflow_full=, # via INSERT_WC + manual interrupt + #polycopter_seq_dataflow_int=, # via INSERT_WC + manual interrupt + #release_seq_int=, # via INSERT_WC + manual interrupt + #release_seq_full= # via INSERT_WC + manual interrupt ) # Read the first command line argument as an sqlite file diff --git a/fuzzers/FRET/benchmark/target_symbols.csv b/fuzzers/FRET/benchmark/target_symbols.csv index 3728ccd939..5c788dd5dc 100644 --- a/fuzzers/FRET/benchmark/target_symbols.csv +++ b/fuzzers/FRET/benchmark/target_symbols.csv @@ -44,4 +44,5 @@ polycopter_seq_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 polycopter_par_dataflow_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 polycopter_seq_dataflow_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000 watersc14_par_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000 -watersc14_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000 \ No newline at end of file +watersc14_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000 +waters_seq_unsync_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C13,0#1000 \ No newline at end of file diff --git a/fuzzers/FRET/tests/times.py b/fuzzers/FRET/tests/times.py index bc0d94e9c5..6d1c92c502 100755 --- a/fuzzers/FRET/tests/times.py +++ b/fuzzers/FRET/tests/times.py @@ -13,11 +13,20 @@ except ValueError: QEMU_SHIFT=5 ISNS_PER_US=10**3 / (2**QEMU_SHIFT) -print("Time span") -print("ISNS -> µs", f"{number / ISNS_PER_US:.2f} us") -print("µs -> ISNS", f"{number * ISNS_PER_US:.2f}") - int_offset=53430 -print("Interrupt offset") -print("ISNS -> µs", f"{((number + int_offset) / ISNS_PER_US):.2f} us") -print("µs -> ISNS", f"{((number * ISNS_PER_US)-int_offset):.2f}") + +if len(sys.argv) == 2: + print("Time span") + print("ISNS -> µs", f"{number / ISNS_PER_US:.2f} us") + print("µs -> ISNS", f"{number * ISNS_PER_US:.2f}") + print("Interrupt offset") + print("ISNS -> µs", f"{((number + int_offset) / ISNS_PER_US):.2f} us") + print("µs -> ISNS", f"{((number * ISNS_PER_US)-int_offset):.2f}") +elif len(sys.argv) > 2: + for i in range(1, len(sys.argv)): + try: + number = float(sys.argv[i]) + except ValueError: + print(f"The argument {i} must be a number.") + sys.exit(1) + print(f"{((number + int_offset) / (ISNS_PER_US*1000)):.2f}")