config updates

fuzzers/FRET/benchmark/Snakefile

@@ -4,7 +4,7 @@ envvars:
     "BENCHDIR"
 def_flags="--release --no-default-features --features std,snapshot_fast,restarting,do_hash_notify_state,do_hash_notify_value,fuzz_int,trace_job_response_times"
 benchdir=os.environ["BENCHDIR"]
-RUNTIME=1
+RUNTIME=(3600*24)
 
 rule copy_kernel:
     input:
@@ -247,16 +247,31 @@ rule quicktest:
     params:
         benchdir=benchdir
     input:
-        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg', 'random'], target=['polycopter'], variant=['_seq_dataflow_full', '_par_dataflow_full'], num=range(0,int( 1 ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg', 'random'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 1 ))),
 
-rule minimal_set:
+rule set128:
     params:
         benchdir=benchdir
     input:
-        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 5 ))),
-        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['stgwoet'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 4 ))),
-        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['waters'], variant=['_seq_full', '_seq_int', '_seq_bytes'], num=range(0,int( 5 ))),
-        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stg'], target=['release'], variant=['_seq_full', '_seq_int'], num=range(0,int( 5 ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['release'], variant=['_seq_full', '_seq_int'], num=range(0,int( 13 ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['release'], variant=['_seq_full', '_seq_int'], num=range(0,int( 3 ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 13 ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['polycopter'], variant=['_seq_dataflow_full'], num=range(0,int( 3 ))),
+
+rule set64:
+    params:
+        benchdir=benchdir
+    input:
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 10 ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 2 ))),
+
+
+rule set48:
+    params:
+        benchdir=benchdir
+    input:
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['waters'], variant=['_seq_int', '_seq_bytes'], num=range(0,int( 7 ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_int', '_seq_bytes'], num=range(0,int( 3 ))),
 
 
 rule all_bins:

fuzzers/FRET/benchmark/build_all_demos.sh

@@ -2,6 +2,9 @@ build () {
     make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC clean && make -C ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC $1=1 IGNORE_INTERRUPTS=$IGNORE_INTERRUPTS IGNORE_BYTES=$IGNORE_BYTES IGNORE_INTERNAL_STATE=$IGNORE_INTERNAL_STATE
     cp ../../../../FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC/build/RTOSDemo.axf build/$(echo $1 | cut -d_ -f1 | tr '[:upper:]' '[:lower:]')$2.elf 
 }
+# INSERT_WC=1
+
+mkdir -p build
 
 # Sequential inputs!
 export PARTITION_INPUT=0
@@ -68,6 +71,13 @@ build WATERS_DEMO $SUFFIX
 build RELEASE_DEMO $SUFFIX
 build COPTER_DEMO $SUFFIX
 
+# Stateful -> presumably bad for us
+## keep rng states
+export IGNORE_INTERNAL_STATE=0
+export PARTITION_INPUT=0
+export IGNORE_INTERRUPTS=0 IGNORE_BYTES=0 SUFFIX="_seq_stateful_full"
+build POLYCOPTER_DEMO $SUFFIX
+
 # stateless + dataflow
 export PARTITION_INPUT=0
 export IGNORE_INTERNAL_STATE=1
@@ -76,6 +86,13 @@ export SPECIAL_CFLAGS="-DCOPTER_DATAFLOW=1"
 build POLYCOPTER_DEMO $SUFFIX
 unset SPECIAL_CFLAGS
 
+export PARTITION_INPUT=0
+export IGNORE_INTERNAL_STATE=1
+export IGNORE_INTERRUPTS=1 IGNORE_BYTES=0 SUFFIX="_seq_dataflow_bytes"
+export SPECIAL_CFLAGS="-DCOPTER_DATAFLOW=1"
+build POLYCOPTER_DEMO $SUFFIX
+unset SPECIAL_CFLAGS
+
 # stateless + dataflow
 export PARTITION_INPUT=1
 export IGNORE_INTERNAL_STATE=1
@@ -83,3 +100,12 @@ export IGNORE_INTERRUPTS=0 IGNORE_BYTES=0 SUFFIX="_par_dataflow_full"
 export SPECIAL_CFLAGS="-DCOPTER_DATAFLOW=1"
 build POLYCOPTER_DEMO $SUFFIX
 unset SPECIAL_CFLAGS
+
+
+# special waters with no synchronization
+export PARTITION_INPUT=0
+export IGNORE_INTERNAL_STATE=1
+export IGNORE_INTERRUPTS=0 IGNORE_BYTES=0 SUFFIX="_seq_unsync_full"
+export SPECIAL_CFLAGS="-DWATERS_UNSYNCHRONIZED=1"
+build WATERS_DEMO $SUFFIX
+unset SPECIAL_CFLAGS

fuzzers/FRET/benchmark/plot_all_benchmarks.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+if [[ -n "$1" ]]; then
+  TARGET="$1"
+else
+  TARGET=$BENCHDIR
+fi
+number_cruncher/target/debug/number_cruncher -i $TARGET/timedump -o $TARGET/bench.sqlite
+Rscript plot_sqlite.r bench_$TARGET.sqlite $TARGET

fuzzers/FRET/benchmark/plot_all_traces.sh

@@ -22,7 +22,7 @@ do
     # if [ ! -f "$P.html" ]; then
     #     ~/code/FRET/state2gantt/driver.sh "$T"
     # fi
-done < <(find $BENCHDIR/timedump -maxdepth 2 -type 'f' -iregex '.*\.case')
+done < <(find $BENCHDIR/timedump -maxdepth 2 -type 'f' -iregex '.*[0-9]+\.case')
 
 echo "${PLOTS[@]}"
-snakemake -c 6 --rerun-incomplete --keep-incomplete "${PLOTS[@]}"
+snakemake -c 20 --rerun-incomplete --keep-incomplete "${PLOTS[@]}"

fuzzers/FRET/benchmark/plot_sqlite.r

@@ -5,8 +5,23 @@ library("DBI")
 args = commandArgs(trailingOnly=TRUE)
 
 KNOWN_WCRT <- list(
-  watersc14_par_full=242454,
-  watersc14_seq_full=242454
+  waters_seq_bytes=209667, # via INSERT_WC
+  waters_seq_int=213646, # via INSERT_WC + manual interrupt
+  waters_seq_full=213646,# via INSERT_WC + manual interrupt
+  polycopter_seq_dataflow_full=273807, # via INSERT_WC + manual interrupt
+  polycopter_seq_dataflow_int=273807, # via INSERT_WC + manual interrupt
+  release_seq_int=624552, # via INSERT_WC + manual interrupt
+  release_seq_full=624552 # via INSERT_WC + manual interrupt
+  )
+
+STATIC_WCRT <- list(
+  #waters_seq_bytes=,
+  waters_seq_int=270789
+  #waters_seq_full=,
+  #polycopter_seq_dataflow_full=, # via INSERT_WC + manual interrupt
+  #polycopter_seq_dataflow_int=, # via INSERT_WC + manual interrupt
+  #release_seq_int=, # via INSERT_WC + manual interrupt
+  #release_seq_full= # via INSERT_WC + manual interrupt
   )
 
 # Read the first command line argument as an sqlite file

fuzzers/FRET/benchmark/target_symbols.csv

@@ -45,3 +45,4 @@ polycopter_par_dataflow_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#2
 polycopter_seq_dataflow_full,main_osek,FUZZ_INPUT,4096,trigger_Qemu_break,FC,0#20000
 watersc14_par_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000
 watersc14_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000
+waters_seq_unsync_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C13,0#1000

fuzzers/FRET/tests/times.py

@@ -13,11 +13,20 @@ except ValueError:
 
 QEMU_SHIFT=5
 ISNS_PER_US=10**3 / (2**QEMU_SHIFT)
-print("Time span")
-print("ISNS -> µs", f"{number / ISNS_PER_US:.2f} us")
-print("µs -> ISNS", f"{number * ISNS_PER_US:.2f}")
-
 int_offset=53430
-print("Interrupt offset")
-print("ISNS -> µs", f"{((number + int_offset) / ISNS_PER_US):.2f} us")
-print("µs -> ISNS", f"{((number * ISNS_PER_US)-int_offset):.2f}")
+
+if len(sys.argv) == 2:
+    print("Time span")
+    print("ISNS -> µs", f"{number / ISNS_PER_US:.2f} us")
+    print("µs -> ISNS", f"{number * ISNS_PER_US:.2f}")
+    print("Interrupt offset")
+    print("ISNS -> µs", f"{((number + int_offset) / ISNS_PER_US):.2f} us")
+    print("µs -> ISNS", f"{((number * ISNS_PER_US)-int_offset):.2f}")
+elif len(sys.argv) > 2:
+    for i in range(1, len(sys.argv)):
+        try:
+            number = float(sys.argv[i])
+        except ValueError:
+            print(f"The argument {i} must be a number.")
+            sys.exit(1)
+        print(f"{((number + int_offset) / (ISNS_PER_US*1000)):.2f}")