From d1c685ccf9d98f3b33c1af1c5dcb8bddbf75ab7c Mon Sep 17 00:00:00 2001
From: Alwin Berger <alwin.berger@tu-dortmund.de>
Date: Sun, 29 May 2022 00:45:17 +0200
Subject: [PATCH] prepare for benchmarking

new features for mutators
fuzzer corpus in memory for benchmarking
showmap single file input
---
 fuzzers/wcet_qemu_sys/Cargo.toml         |  4 +++
 fuzzers/wcet_qemu_sys/Makefile           | 33 +++++++++++++++---------
 fuzzers/wcet_qemu_sys/src/bin/fuzzer.rs  | 28 +++++++++++++++++---
 fuzzers/wcet_qemu_sys/src/bin/showmap.rs |  4 +--
 4 files changed, 51 insertions(+), 18 deletions(-)

diff --git a/fuzzers/wcet_qemu_sys/Cargo.toml b/fuzzers/wcet_qemu_sys/Cargo.toml
index a200fd110e..769772a60f 100644
--- a/fuzzers/wcet_qemu_sys/Cargo.toml
+++ b/fuzzers/wcet_qemu_sys/Cargo.toml
@@ -27,7 +27,11 @@ obj_trace = []
 obj_edges = []
 obj_ticks = []
 
+muta_input = [ "sched_graph" ]
 muta_snip = [ "sched_graph" ]
+muta_suffix = [ "sched_graph" ]
+
+benchmark = []	# don't save corpus to disk, easy parallelizable
 
 [profile.release]
 debug = true
diff --git a/fuzzers/wcet_qemu_sys/Makefile b/fuzzers/wcet_qemu_sys/Makefile
index a6120ca39e..86057c2a23 100644
--- a/fuzzers/wcet_qemu_sys/Makefile
+++ b/fuzzers/wcet_qemu_sys/Makefile
@@ -1,50 +1,59 @@
 BENCHDIR = target/bench
 TARGET = tmr.axf
-EXAMPLE = "\xff\05\x84\x84\x84\x84\x84\x84\x84\x84"
-TARGET_TRACE = tmp/target_trace.ron
-TARGET_EDGES = tmp/target_edges.ron
+TARGET_TRACE = $(BENCHDIR)/traces/tmr_worst.ron
+TARGET_EDGES = $(BENCHDIR)/edges/tmr_worst.ron
 
 $(BENCHDIR)/bin:
 	mkdir -p $@
 
 $(BENCHDIR)/bin/fuzz_vanilla: $(BENCHDIR)/bin
-	cargo build --features feed_afl,sched_queue
+	cargo build --features benchmark,feed_afl,sched_mapmax
 	cp target/debug/fuzzer $@
 
 $(BENCHDIR)/bin/fuzz_state: $(BENCHDIR)/bin
-	cargo build --features feed_state,sched_state
+	cargo build --features benchmark,feed_state,sched_state
 	cp target/debug/fuzzer $@
 
 $(BENCHDIR)/bin/fuzz_graph: $(BENCHDIR)/bin
-	cargo build --features feed_graph,sched_graph
+	cargo build --features benchmark,feed_graph,sched_graph
 	cp target/debug/fuzzer $@
 
 $(BENCHDIR)/bin/fuzz_graph_snip: $(BENCHDIR)/bin
-	cargo build --features feed_graph,sched_graph,muta_snip
+	cargo build --features benchmark,feed_graph,sched_graph,muta_snip,muta_input,muta_suffix
 	cp target/debug/fuzzer $@
 
 $(BENCHDIR)/bin/fuzz_graph_afl: $(BENCHDIR)/bin
-	cargo build --features feed_graph,sched_graph,feed_afl
+	cargo build --features benchmark,feed_graph,sched_graph,feed_afl
 	cp target/debug/fuzzer $@
 
 $(BENCHDIR)/bin/fuzz_graph_all: $(BENCHDIR)/bin
-	cargo build --features feed_graph,sched_graph,feed_afl,muta_snip
+	cargo build --features benchmark,feed_graph,sched_graph,feed_afl,muta_snip,muta_input,muta_suffix
 	cp target/debug/fuzzer $@
 
 binaries: $(BENCHDIR)/bin/fuzz_vanilla $(BENCHDIR)/bin/fuzz_state $(BENCHDIR)/bin/fuzz_graph $(BENCHDIR)/bin/fuzz_graph_snip $(BENCHDIR)/bin/fuzz_graph_afl $(BENCHDIR)/bin/fuzz_graph_all
 
 # variants: vanilla, state, graph, graph_snip, graph_afl, graph_all
 $(BENCHDIR)/bench_%.log: $(BENCHDIR)/bin/fuzz_% $(TARGET_TRACE)
-	for i in {1..5}; do ./fuzzer_bench.sh $< $(TARGET) --libafl-traces $(TARGET_TRACE) > $@_$$i; done
+	for i in {1..1}; do ./fuzzer_bench.sh $< $(TARGET) --libafl-traces $(TARGET_TRACE) --libafl-exectimes $@.exec_$$i > $@_$$i; done
 	for i in $@_*; do tail -n 1 $$i >> $@; done
 
 benchmarks: target/bench/bench_vanilla.log target/bench/bench_state.log target/bench/bench_graph.log target/bench/bench_graph_snip.log target/bench/bench_graph_afl.log target/bench/bench_graph_all.log
 
-all: binaries
+all: binaries benchmarks
 
 clean_bench:
 	rm -rf $(BENCHDIR)/bench_*
 
 clean:
 	rm -rf target/bench
-	
\ No newline at end of file
+	
+%.case: %_inputs
+	mkdir -p $(BENCHDIR)/traces $(BENCHDIR)/edges
+	for i in $</*.case; do \
+		CASE=$$(basename -s.case $$i); echo $$CASE; \
+		PROG=$$(basename -s.case $@); \
+		./showmap.sh $(TARGET) --libafl-single $$i \
+			--libafl-edges $(BENCHDIR)/edges/$$PROG\_$$CASE.ron \
+			--libafl-traces $(BENCHDIR)/traces/$$PROG\_$$CASE.ron | \
+			grep "Qemu Ticks:"; \
+		done
\ No newline at end of file
diff --git a/fuzzers/wcet_qemu_sys/src/bin/fuzzer.rs b/fuzzers/wcet_qemu_sys/src/bin/fuzzer.rs
index 468892eb31..23bdeff435 100644
--- a/fuzzers/wcet_qemu_sys/src/bin/fuzzer.rs
+++ b/fuzzers/wcet_qemu_sys/src/bin/fuzzer.rs
@@ -1,5 +1,8 @@
 //! A singlethreaded QEMU fuzzer that can auto-restart.
 
+use libafl::corpus::InMemoryCorpus;
+use wcet_qemu_sys::sysstate::graph::RandGraphSuffixMutator;
+use wcet_qemu_sys::sysstate::graph::RandInputSnippetMutator;
 use wcet_qemu_sys::worst::DummyFeedback;
 use wcet_qemu_sys::worst::ExecTimeCollectorFeedback;
 use wcet_qemu_sys::worst::EXEC_TIME_COLLECTION;
@@ -412,6 +415,21 @@ fn fuzz(
     let objective = feedback_or!(ExecTimeReachedFeedback::new(targettime));
 
     // create a State from scratch
+    #[cfg(feature = "benchmark")]
+    let mut state = state.unwrap_or_else(||{
+        StdState::new(
+            // RNG
+            StdRand::with_seed(current_nanos()),
+            // Corpus that will be evolved
+            InMemoryCorpus::new(),
+            // Corpus in which we store solutions (crashes in this example),
+            InMemoryCorpus::new(),
+            // States of the feedbacks.
+            // They are the data related to the feedbacks that you want to persist in the State.
+            tuple_list!(feedback_state,clock::MaxIcountMetadata::default(),sysstate_feedback_state),
+        )
+    });
+    #[cfg(not(feature = "benchmark"))]
     let mut state = state.unwrap_or_else(||{
         StdState::new(
             // RNG
@@ -432,11 +450,13 @@ fn fuzz(
     // Setup a randomic Input2State stage
     // let i2s = StdMutationalStage::new(StdScheduledMutator::new(tuple_list!(I2SRandReplace::new())));
 
+    let mutator_list = havoc_mutations();
+    #[cfg(feature = "muta_input")]
+    let mutator_list = mutator_list.merge(tuple_list!(RandInputSnippetMutator::new()));
+    #[cfg(feature = "muta_suffix")]
+    let mutator_list = mutator_list.merge(tuple_list!(RandGraphSuffixMutator::new()));
     #[cfg(feature = "muta_snip")]
-    let mutator_list = havoc_mutations().merge(tokens_mutations())
-        .merge(tuple_list!(RandGraphSnippetMutator::new()));
-    #[cfg(not(feature = "muta_snip"))]
-    let mutator_list = havoc_mutations().merge(tokens_mutations());
+    let mutator_list = mutator_list.merge(tuple_list!(RandGraphSnippetMutator::new()));
     // Setup a MOPT mutator
     let mutator = StdMOptMutator::new(&mut state, mutator_list,5)?;
 
diff --git a/fuzzers/wcet_qemu_sys/src/bin/showmap.rs b/fuzzers/wcet_qemu_sys/src/bin/showmap.rs
index 7ae8a95d47..b79308050d 100644
--- a/fuzzers/wcet_qemu_sys/src/bin/showmap.rs
+++ b/fuzzers/wcet_qemu_sys/src/bin/showmap.rs
@@ -154,7 +154,7 @@ pub fn main() {
             std::io::stdin().read_to_end(&mut buf).expect("Could not read Stdin");
             Left(buf)
         } else {
-            Left(s.as_bytes().to_owned())
+            Left(fs::read(s).expect("Input file for --libafl-single can not be read"))
         },
         None => {
             let in_dir = PathBuf::from(res.value_of("in").unwrap().to_string());
@@ -353,7 +353,7 @@ fn fuzz(
         }
 
         unsafe {
-            libafl_int_offset = 347780+int_tick.unwrap_or(0);
+            // libafl_int_offset = 347780+int_tick.unwrap_or(0);
             // INTR_OFFSET = int_tick;
             emu.write_mem(test_length_ptr,&(len as u32).to_le_bytes());
             emu.write_mem(input_addr,buf);