Execute each benchmark as individual program

- Client now produces one binary per benchmark

.gitignore

@@ -1,3 +1,5 @@
 /target
 *.log
-/nyx_data
+/nyx_data
+/KVM-Nyx-fork
+/benchmarks

Cargo.toml

@@ -11,4 +11,7 @@ edition = "2024"
 libnyx = { git = "https://git.cs.tu-dortmund.de/david.venhoff/libnyx-fork" }
 pt-dump-decoder = { path = "pt-dump-decoder" }
 csv = "1.3.1"
-clap = { version = "4.5.47", features = ["derive"] }
+clap.workspace = true
+
+[workspace.dependencies]
+clap = { version = "4.5.47", features = ["derive"] }

benchmark_all.sh

@@ -0,0 +1,16 @@
+cargo build --workspace --release
+
+echo "Disabling turbo..."
+echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
+
+for file in client/src/bin/*.rs; do
+    bin_name=$(basename "$file" .rs)
+    bin_path=./target/release/${bin_name}
+
+    echo "Benchmarking $bin_name..."
+    sudo -E nice -n -20 taskset -c 0 ./target/release/qemu-nyx-runner "$bin_path"
+done
+
+echo 0 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
+
+echo "Done!"

client/build.rs

@@ -70,11 +70,19 @@ fn main() {
         benchmark_functions.push(function_name);
     }
 
-    write!(generated_code, "pub unsafe fn benchmark_all() {{ ").unwrap();
+    writeln!(generated_code, "#[allow(non_camel_case_types)] pub enum Benchmark {{").unwrap();
-    for function_name in benchmark_functions {
+    for benchmark_name in &benchmark_functions {
-        write!(generated_code, "unsafe {{ {function_name}(); }}\n").unwrap();
+        writeln!(generated_code, "    {benchmark_name},").unwrap();
     }
-    write!(generated_code, "}}").unwrap();
+    writeln!(generated_code, "}}").unwrap();
+
+    writeln!(generated_code, "pub unsafe fn run(benchmark: Benchmark) {{").unwrap();
+    writeln!(generated_code, "match benchmark {{").unwrap();
+    for benchmark_name in benchmark_functions {
+        writeln!(generated_code, "    Benchmark::{benchmark_name} => unsafe {{ {benchmark_name}() }},").unwrap();
+    }
+    writeln!(generated_code, "}}").unwrap();
+    writeln!(generated_code, "}}").unwrap();
 
     fs::write(&format!("{out_dir}/libembench-sys.rs"), generated_code).unwrap();
 }

client/src/bin/client_combined.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_sglib_combined);
+}

client/src/bin/client_crc_32.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_crc32);
+}

client/src/bin/client_edn.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_edn);
+}

client/src/bin/client_huffbench.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_huffbench);
+}

client/src/bin/client_matmult-int.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_matmult_int);
+}

client/src/bin/client_md5.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_md5sum);
+}

client/src/bin/client_minver.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_minver);
+}

client/src/bin/client_mont64.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_aha_mont64);
+}

client/src/bin/client_nbody.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_nbody);
+}

client/src/bin/client_nettle-aes.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_nettle_aes);
+}

client/src/bin/client_nettle-sha256.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_nettle_sha256);
+}

client/src/bin/client_nsichneu.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_nsichneu);
+}

client/src/bin/client_picojpeg.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_picojpeg);
+}

client/src/bin/client_primecount.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_primecount);
+}

client/src/bin/client_slre.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_slre);
+}

client/src/bin/client_st.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_st);
+}

client/src/bin/client_statemate.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_statemate);
+}

client/src/bin/client_tarfind.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_tarfind);
+}

client/src/bin/client_ud.rs

@@ -0,0 +1,5 @@
+use client::{program, Benchmark};
+
+fn main() {
+    program(Benchmark::benchmark_ud);
+}

client/src/lib.rs

@@ -1,16 +1,23 @@
+use std::arch::asm;
+
 mod embench_sys {
     include!(concat!(env!("OUT_DIR"), "/libembench-sys.rs"));
 }
 
-pub fn program() {
+pub use embench_sys::Benchmark;
+
+fn ptwrite(val: u32) {
     unsafe {
-        embench_sys::benchmark_all();
+        asm!("ptwrite {0:e}", in(reg) val);
     }
 }
 
-/// This method is used to benchmark without nyx
+pub fn program(benchmark: embench_sys::Benchmark) {
-pub fn benchmark_once() -> std::time::Duration {
     let start = std::time::Instant::now();
-    program();
+    ptwrite(42);
-    start.elapsed()
+    unsafe {
+        embench_sys::run(benchmark);
+    }
+    ptwrite(43);
+    println!("{}", start.elapsed().as_secs_f64());
 }

client/src/main.rs

@@ -1,14 +0,0 @@
-use std::arch::asm;
-
-fn ptwrite(val: u32) {
-    unsafe {
-        asm!("ptwrite {0:e}", in(reg) val);
-    }
-}
-
-/// main is only executed from nyx
-fn main() {
-    ptwrite(42);
-    client::program();
-    ptwrite(43);
-}

src/benchmark_baseline.rs

@@ -1,17 +1,24 @@
 use crate::benchmark::{Benchmark, ToCsv};
 use csv::Writer;
 use std::io::Write;
+use std::path::Path;
+use std::process::Command;
 use std::time::Duration;
 
-pub struct Baseline;
+pub struct Baseline<'a>(pub &'a Path);
 
-impl Benchmark for Baseline {
+impl Benchmark for Baseline<'_> {
     const TITLE: &'static str = "baseline";
 
     type BenchmarkResult = Duration;
 
     fn execute_once(&mut self) -> Self::BenchmarkResult {
-        client::benchmark_once()
+        let output = Command::new(self.0).output().expect("Command should succeed");
+        let output = String::from_utf8_lossy(&output.stdout);
+        let Ok(duration) = output.trim().parse() else {
+            panic!("Invalid output: {}", output);
+        };
+        Duration::from_secs_f64(duration)
     }
 }
 

src/benchmark_nyx_pt.rs

@@ -36,6 +36,7 @@ impl NyxRunner {
 
     fn execute(&mut self) {
         let result = self.0.exec();
+        // nyx_print_aux_buffer(&mut self.0 as *mut _);
         assert!(
             matches!(result, NyxReturnValue::Normal),
             "Unexpected return value: {result:?}"

src/main.rs

@@ -1,35 +1,46 @@
 //! Translated from libnyx/test.c
 
 mod benchmark;
+mod benchmark_baseline;
 mod benchmark_nyx_pt;
 mod nyx;
 
 use crate::benchmark::{Benchmark, ToCsv};
-use crate::benchmark_nyx_pt::NyxRunner;
+use crate::benchmark_baseline::Baseline;
-use std::error::Error;
-use std::path::PathBuf;
-use std::time::{Duration, Instant};
 use clap::Parser;
+use std::error::Error;
+use std::path::{Path, PathBuf};
+use std::time::{Duration, Instant};
+use crate::benchmark_nyx_pt::NyxRunner;
 
 #[derive(Debug, Parser)]
-#[command(about="Tool to execute a program with nyx and trace it with intel pt")]
+#[command(about = "Tool to execute a program with nyx and trace it with intel pt")]
 struct Args {
     /// Path to the client binary to execute
-    client: PathBuf
+    client: PathBuf,
+    /// Directory where the benchmark files get stored
+    #[arg(short = 'o', long = "output", default_value = "benchmarks")]
+    output_dir: PathBuf,
 }
 
 fn main() -> Result<(), Box<dyn Error>> {
     let args = Args::parse();
     let nyx_runner = NyxRunner::setup(&args.client)?;
-    benchmark(nyx_runner)?;
+    let binary_name = args.client.file_name().unwrap().to_str().unwrap();
+    benchmark(binary_name, &args.output_dir, nyx_runner)?;
+    benchmark(binary_name, &args.output_dir, Baseline(&args.client))?;
 
     Ok(())
 }
 
-fn benchmark<B: Benchmark>(mut benchmark: B) -> Result<(), Box<dyn Error>> {
+fn benchmark<B: Benchmark>(
+    binary_name: &str,
+    output_dir: &Path,
+    mut benchmark: B,
+) -> Result<(), Box<dyn Error>> {
     warmup(&mut benchmark);
     let results = benchmark_loop(&mut benchmark);
-    write_results::<B>(&results)?;
+    write_results::<B>(binary_name, output_dir, &results)?;
 
     Ok(())
 }
@@ -48,16 +59,21 @@ fn warmup(benchmark: &mut impl Benchmark) {
 }
 
 fn benchmark_loop<B: Benchmark>(benchmark: &mut B) -> Vec<B::BenchmarkResult> {
-    let n = 50;
+    let n = 500;
     println!("Perform {n} iterations...");
     (0..n)
         .map(|_| std::hint::black_box(benchmark.execute_once()))
         .collect::<Vec<_>>()
 }
 
-fn write_results<B: Benchmark>(results: &[B::BenchmarkResult]) -> Result<(), Box<dyn Error>> {
+fn write_results<B: Benchmark>(
-    let file = std::fs::File::create(format!("benchmark_{}", B::TITLE))
+    binary_name: &str,
-        .expect("Should be able to create file");
+    output_dir: &Path,
+    results: &[B::BenchmarkResult],
+) -> Result<(), Box<dyn Error>> {
+    std::fs::create_dir_all(output_dir)?;
+    let filename = format!("benchmark_{binary_name}_{}", B::TITLE);
+    let file = std::fs::File::create(output_dir.join(filename))?;
     let mut writer = csv::WriterBuilder::new().from_writer(file);
     writer.write_record(B::BenchmarkResult::HEADERS)?;
     for result in results {