benchmark targets

fuzzers/FRET/benchmark/Snakefile

@@ -5,6 +5,8 @@ envvars:
 def_flags="--release --no-default-features --features std,snapshot_fast,restarting,do_hash_notify_state,do_hash_notify_value,fuzz_int,trace_job_response_times,observe_systemstate_unordered"
 benchdir=os.environ["BENCHDIR"]
 RUNTIME=(3600*24)
+TARGET_SET=['feedgeneration100', 'stgwoet', 'frafl']
+TARGET_NUMBER=10
 
 rule copy_kernel:
     input:
@@ -264,7 +266,7 @@ rule set128:
         benchdir=benchdir
     input:
         # waters full
-        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 10 ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 5 ))),
         expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['frafl'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 10 ))),
         expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 5 ))),
         # release full
@@ -293,6 +295,45 @@ rule set64:
         expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['waters'], variant=['_seq_int', '_seq_bytes'], num=range(0,int( 8 ))),
         expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_int', '_seq_bytes'], num=range(0,int( 5 ))),
 
+rule eval_bytes:
+    params:
+        benchdir=benchdir
+    input:
+        # waters bytes
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['waters'], variant=['_seq_bytes'], num=range(0,int( TARGET_NUMBER ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_bytes'], num=range(0,int( 1 ))),
+        # polycopter full
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['polycopter'], variant=['_seq_full'], num=range(0,int( TARGET_NUMBER ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['polycopter'], variant=['_seq_full'], num=range(0,int( 1 ))),
+
+rule eval_int:
+    params:
+        benchdir=benchdir
+    input:
+        # waters int
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['waters'], variant=['_seq_int'], num=range(0,int( TARGET_NUMBER ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_int'], num=range(0,int( 1 ))),
+        # release int
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['release'], variant=['_seq_int'], num=range(0,int( TARGET_NUMBER ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['release'], variant=['_seq_int'], num=range(0,int( 1 ))),
+
+rule eval_full:
+    params:
+        benchdir=benchdir
+    input:
+        # waters full
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['waters'], variant=['_seq_full'], num=range(0,int( TARGET_NUMBER ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_full'], num=range(0,int( 1 ))),
+        # release full
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['release'], variant=['_seq_full'], num=range(0,int( TARGET_NUMBER ))),
+        expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['release'], variant=['_seq_full'], num=range(0,int( 1 ))),
+
+rule waters_multi:
+    params:
+        benchdir=benchdir
+    input:
+        expand("{benchdir}/timedump/{fuzzer}/{target}{chain}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['waters'], chain=['Ic11','Ic12','Ic13','Ic14','Ic21','Ic22','Ic23','Ic32','Ic33'], variant=['_seq_full'], num=range(0,int( 3 ))), # 'Ic31'
+
 
 rule all_bins:
     params:

fuzzers/FRET/benchmark/target_symbols.csv

@@ -53,7 +53,7 @@ watersIc12_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C12,0#1000
 watersIc13_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C13,0#1000
 watersIc14_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000
 watersIc21_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C21,0#1000
-watersc22_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C22,0#1000
+watersIc22_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C22,0#1000
 watersIc23_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C23,0#1000
 watersIc31_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C31,0#1000
 watersIc32_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C32,0#1000