From cf8f57f3d0c47456d79e15da33f706bcd3139d82 Mon Sep 17 00:00:00 2001 From: Alwin Berger Date: Mon, 11 Aug 2025 12:10:16 +0200 Subject: [PATCH] benchmark targets --- fuzzers/FRET/benchmark/Snakefile | 45 ++++++++++++++++++++++- fuzzers/FRET/benchmark/target_symbols.csv | 4 +- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/fuzzers/FRET/benchmark/Snakefile b/fuzzers/FRET/benchmark/Snakefile index 46d533b59c..eb8220d15c 100644 --- a/fuzzers/FRET/benchmark/Snakefile +++ b/fuzzers/FRET/benchmark/Snakefile @@ -5,6 +5,8 @@ envvars: def_flags="--release --no-default-features --features std,snapshot_fast,restarting,do_hash_notify_state,do_hash_notify_value,fuzz_int,trace_job_response_times,observe_systemstate_unordered" benchdir=os.environ["BENCHDIR"] RUNTIME=(3600*24) +TARGET_SET=['feedgeneration100', 'stgwoet', 'frafl'] +TARGET_NUMBER=10 rule copy_kernel: input: @@ -264,7 +266,7 @@ rule set128: benchdir=benchdir input: # waters full - expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 10 ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 5 ))), expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['frafl'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 10 ))), expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_full', '_seq_unsync_full'], num=range(0,int( 5 ))), # release full @@ -293,6 +295,45 @@ rule set64: expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['waters'], variant=['_seq_int', '_seq_bytes'], num=range(0,int( 8 ))), expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_int', '_seq_bytes'], num=range(0,int( 5 ))), +rule eval_bytes: + params: + benchdir=benchdir + input: + # waters bytes + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['waters'], variant=['_seq_bytes'], num=range(0,int( TARGET_NUMBER ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_bytes'], num=range(0,int( 1 ))), + # polycopter full + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['polycopter'], variant=['_seq_full'], num=range(0,int( TARGET_NUMBER ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['polycopter'], variant=['_seq_full'], num=range(0,int( 1 ))), + +rule eval_int: + params: + benchdir=benchdir + input: + # waters int + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['waters'], variant=['_seq_int'], num=range(0,int( TARGET_NUMBER ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_int'], num=range(0,int( 1 ))), + # release int + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['release'], variant=['_seq_int'], num=range(0,int( TARGET_NUMBER ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['release'], variant=['_seq_int'], num=range(0,int( 1 ))), + +rule eval_full: + params: + benchdir=benchdir + input: + # waters full + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['waters'], variant=['_seq_full'], num=range(0,int( TARGET_NUMBER ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['waters'], variant=['_seq_full'], num=range(0,int( 1 ))), + # release full + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=TARGET_SET, target=['release'], variant=['_seq_full'], num=range(0,int( TARGET_NUMBER ))), + expand("{benchdir}/timedump/{fuzzer}/{target}{variant}#{num}.time", benchdir=benchdir, fuzzer=['random'], target=['release'], variant=['_seq_full'], num=range(0,int( 1 ))), + +rule waters_multi: + params: + benchdir=benchdir + input: + expand("{benchdir}/timedump/{fuzzer}/{target}{chain}{variant}#{num}.time", benchdir=benchdir, fuzzer=['feedgeneration100', 'stgwoet', 'frafl'], target=['waters'], chain=['Ic11','Ic12','Ic13','Ic14','Ic21','Ic22','Ic23','Ic32','Ic33'], variant=['_seq_full'], num=range(0,int( 3 ))), # 'Ic31' + rule all_bins: params: @@ -306,4 +347,4 @@ rule clean: rule full_clean: shell: - "rm -rf {benchdir}/bins & rm -rf {benchdir}/timedump" \ No newline at end of file + "rm -rf {benchdir}/bins & rm -rf {benchdir}/timedump" diff --git a/fuzzers/FRET/benchmark/target_symbols.csv b/fuzzers/FRET/benchmark/target_symbols.csv index 7f22c2dfd8..2d9008503f 100644 --- a/fuzzers/FRET/benchmark/target_symbols.csv +++ b/fuzzers/FRET/benchmark/target_symbols.csv @@ -53,8 +53,8 @@ watersIc12_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C12,0#1000 watersIc13_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C13,0#1000 watersIc14_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C14,0#1000 watersIc21_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C21,0#1000 -watersc22_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C22,0#1000 +watersIc22_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C22,0#1000 watersIc23_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C23,0#1000 watersIc31_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C31,0#1000 watersIc32_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C32,0#1000 -watersIc33_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C33,0#1000 \ No newline at end of file +watersIc33_seq_full,main_waters,FUZZ_INPUT,4096,trigger_Qemu_break,C33,0#1000