From b44a0a5ef3d9a2028e56342c47d6c38978dd1bf4 Mon Sep 17 00:00:00 2001 From: David Venhoff Date: Mon, 8 Sep 2025 16:35:04 +0200 Subject: [PATCH] Execute each benchmark as individual program - Client now produces one binary per benchmark --- .gitignore | 4 ++- Cargo.toml | 5 ++- benchmark_all.sh | 16 ++++++++++ client/build.rs | 16 +++++++--- client/src/bin/client_combined.rs | 5 +++ client/src/bin/client_crc_32.rs | 5 +++ client/src/bin/client_edn.rs | 5 +++ client/src/bin/client_huffbench.rs | 5 +++ client/src/bin/client_matmult-int.rs | 5 +++ client/src/bin/client_md5.rs | 5 +++ client/src/bin/client_minver.rs | 5 +++ client/src/bin/client_mont64.rs | 5 +++ client/src/bin/client_nbody.rs | 5 +++ client/src/bin/client_nettle-aes.rs | 5 +++ client/src/bin/client_nettle-sha256.rs | 5 +++ client/src/bin/client_nsichneu.rs | 5 +++ client/src/bin/client_picojpeg.rs | 5 +++ client/src/bin/client_primecount.rs | 5 +++ client/src/bin/client_slre.rs | 5 +++ client/src/bin/client_st.rs | 5 +++ client/src/bin/client_statemate.rs | 5 +++ client/src/bin/client_tarfind.rs | 5 +++ client/src/bin/client_ud.rs | 5 +++ client/src/lib.rs | 19 ++++++++---- client/src/main.rs | 14 --------- src/benchmark_baseline.rs | 13 ++++++-- src/benchmark_nyx_pt.rs | 1 + src/main.rs | 42 ++++++++++++++++++-------- 28 files changed, 183 insertions(+), 42 deletions(-) create mode 100755 benchmark_all.sh create mode 100644 client/src/bin/client_combined.rs create mode 100644 client/src/bin/client_crc_32.rs create mode 100644 client/src/bin/client_edn.rs create mode 100644 client/src/bin/client_huffbench.rs create mode 100644 client/src/bin/client_matmult-int.rs create mode 100644 client/src/bin/client_md5.rs create mode 100644 client/src/bin/client_minver.rs create mode 100644 client/src/bin/client_mont64.rs create mode 100644 client/src/bin/client_nbody.rs create mode 100644 client/src/bin/client_nettle-aes.rs create mode 100644 client/src/bin/client_nettle-sha256.rs create mode 100644 client/src/bin/client_nsichneu.rs create mode 100644 client/src/bin/client_picojpeg.rs create mode 100644 client/src/bin/client_primecount.rs create mode 100644 client/src/bin/client_slre.rs create mode 100644 client/src/bin/client_st.rs create mode 100644 client/src/bin/client_statemate.rs create mode 100644 client/src/bin/client_tarfind.rs create mode 100644 client/src/bin/client_ud.rs delete mode 100644 client/src/main.rs diff --git a/.gitignore b/.gitignore index 609584b..bfcaf27 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /target *.log -/nyx_data \ No newline at end of file +/nyx_data +/KVM-Nyx-fork +/benchmarks \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 4f4065f..f33f712 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,7 @@ edition = "2024" libnyx = { git = "https://git.cs.tu-dortmund.de/david.venhoff/libnyx-fork" } pt-dump-decoder = { path = "pt-dump-decoder" } csv = "1.3.1" -clap = { version = "4.5.47", features = ["derive"] } +clap.workspace = true + +[workspace.dependencies] +clap = { version = "4.5.47", features = ["derive"] } \ No newline at end of file diff --git a/benchmark_all.sh b/benchmark_all.sh new file mode 100755 index 0000000..bceedf8 --- /dev/null +++ b/benchmark_all.sh @@ -0,0 +1,16 @@ +cargo build --workspace --release + +echo "Disabling turbo..." +echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo + +for file in client/src/bin/*.rs; do + bin_name=$(basename "$file" .rs) + bin_path=./target/release/${bin_name} + + echo "Benchmarking $bin_name..." + sudo -E nice -n -20 taskset -c 0 ./target/release/qemu-nyx-runner "$bin_path" +done + +echo 0 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo + +echo "Done!" \ No newline at end of file diff --git a/client/build.rs b/client/build.rs index 8e3dc66..57eba5c 100644 --- a/client/build.rs +++ b/client/build.rs @@ -70,11 +70,19 @@ fn main() { benchmark_functions.push(function_name); } - write!(generated_code, "pub unsafe fn benchmark_all() {{ ").unwrap(); - for function_name in benchmark_functions { - write!(generated_code, "unsafe {{ {function_name}(); }}\n").unwrap(); + writeln!(generated_code, "#[allow(non_camel_case_types)] pub enum Benchmark {{").unwrap(); + for benchmark_name in &benchmark_functions { + writeln!(generated_code, " {benchmark_name},").unwrap(); } - write!(generated_code, "}}").unwrap(); + writeln!(generated_code, "}}").unwrap(); + + writeln!(generated_code, "pub unsafe fn run(benchmark: Benchmark) {{").unwrap(); + writeln!(generated_code, "match benchmark {{").unwrap(); + for benchmark_name in benchmark_functions { + writeln!(generated_code, " Benchmark::{benchmark_name} => unsafe {{ {benchmark_name}() }},").unwrap(); + } + writeln!(generated_code, "}}").unwrap(); + writeln!(generated_code, "}}").unwrap(); fs::write(&format!("{out_dir}/libembench-sys.rs"), generated_code).unwrap(); } diff --git a/client/src/bin/client_combined.rs b/client/src/bin/client_combined.rs new file mode 100644 index 0000000..fc6ead0 --- /dev/null +++ b/client/src/bin/client_combined.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_sglib_combined); +} diff --git a/client/src/bin/client_crc_32.rs b/client/src/bin/client_crc_32.rs new file mode 100644 index 0000000..94fb8ae --- /dev/null +++ b/client/src/bin/client_crc_32.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_crc32); +} diff --git a/client/src/bin/client_edn.rs b/client/src/bin/client_edn.rs new file mode 100644 index 0000000..c7e80dd --- /dev/null +++ b/client/src/bin/client_edn.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_edn); +} diff --git a/client/src/bin/client_huffbench.rs b/client/src/bin/client_huffbench.rs new file mode 100644 index 0000000..e74cc9b --- /dev/null +++ b/client/src/bin/client_huffbench.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_huffbench); +} diff --git a/client/src/bin/client_matmult-int.rs b/client/src/bin/client_matmult-int.rs new file mode 100644 index 0000000..beaca31 --- /dev/null +++ b/client/src/bin/client_matmult-int.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_matmult_int); +} diff --git a/client/src/bin/client_md5.rs b/client/src/bin/client_md5.rs new file mode 100644 index 0000000..fa00008 --- /dev/null +++ b/client/src/bin/client_md5.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_md5sum); +} diff --git a/client/src/bin/client_minver.rs b/client/src/bin/client_minver.rs new file mode 100644 index 0000000..458ae40 --- /dev/null +++ b/client/src/bin/client_minver.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_minver); +} diff --git a/client/src/bin/client_mont64.rs b/client/src/bin/client_mont64.rs new file mode 100644 index 0000000..9baf9f0 --- /dev/null +++ b/client/src/bin/client_mont64.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_aha_mont64); +} diff --git a/client/src/bin/client_nbody.rs b/client/src/bin/client_nbody.rs new file mode 100644 index 0000000..99ad6a7 --- /dev/null +++ b/client/src/bin/client_nbody.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_nbody); +} diff --git a/client/src/bin/client_nettle-aes.rs b/client/src/bin/client_nettle-aes.rs new file mode 100644 index 0000000..f262e67 --- /dev/null +++ b/client/src/bin/client_nettle-aes.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_nettle_aes); +} diff --git a/client/src/bin/client_nettle-sha256.rs b/client/src/bin/client_nettle-sha256.rs new file mode 100644 index 0000000..212b7a0 --- /dev/null +++ b/client/src/bin/client_nettle-sha256.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_nettle_sha256); +} diff --git a/client/src/bin/client_nsichneu.rs b/client/src/bin/client_nsichneu.rs new file mode 100644 index 0000000..8ace950 --- /dev/null +++ b/client/src/bin/client_nsichneu.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_nsichneu); +} diff --git a/client/src/bin/client_picojpeg.rs b/client/src/bin/client_picojpeg.rs new file mode 100644 index 0000000..9e86dd3 --- /dev/null +++ b/client/src/bin/client_picojpeg.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_picojpeg); +} diff --git a/client/src/bin/client_primecount.rs b/client/src/bin/client_primecount.rs new file mode 100644 index 0000000..a39ac04 --- /dev/null +++ b/client/src/bin/client_primecount.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_primecount); +} diff --git a/client/src/bin/client_slre.rs b/client/src/bin/client_slre.rs new file mode 100644 index 0000000..a867835 --- /dev/null +++ b/client/src/bin/client_slre.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_slre); +} diff --git a/client/src/bin/client_st.rs b/client/src/bin/client_st.rs new file mode 100644 index 0000000..7fc50b2 --- /dev/null +++ b/client/src/bin/client_st.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_st); +} diff --git a/client/src/bin/client_statemate.rs b/client/src/bin/client_statemate.rs new file mode 100644 index 0000000..5046015 --- /dev/null +++ b/client/src/bin/client_statemate.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_statemate); +} diff --git a/client/src/bin/client_tarfind.rs b/client/src/bin/client_tarfind.rs new file mode 100644 index 0000000..01781d1 --- /dev/null +++ b/client/src/bin/client_tarfind.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_tarfind); +} diff --git a/client/src/bin/client_ud.rs b/client/src/bin/client_ud.rs new file mode 100644 index 0000000..b62dc77 --- /dev/null +++ b/client/src/bin/client_ud.rs @@ -0,0 +1,5 @@ +use client::{program, Benchmark}; + +fn main() { + program(Benchmark::benchmark_ud); +} diff --git a/client/src/lib.rs b/client/src/lib.rs index 965328a..b311178 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,16 +1,23 @@ +use std::arch::asm; + mod embench_sys { include!(concat!(env!("OUT_DIR"), "/libembench-sys.rs")); } -pub fn program() { +pub use embench_sys::Benchmark; + +fn ptwrite(val: u32) { unsafe { - embench_sys::benchmark_all(); + asm!("ptwrite {0:e}", in(reg) val); } } -/// This method is used to benchmark without nyx -pub fn benchmark_once() -> std::time::Duration { +pub fn program(benchmark: embench_sys::Benchmark) { let start = std::time::Instant::now(); - program(); - start.elapsed() + ptwrite(42); + unsafe { + embench_sys::run(benchmark); + } + ptwrite(43); + println!("{}", start.elapsed().as_secs_f64()); } diff --git a/client/src/main.rs b/client/src/main.rs deleted file mode 100644 index 676fe1b..0000000 --- a/client/src/main.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::arch::asm; - -fn ptwrite(val: u32) { - unsafe { - asm!("ptwrite {0:e}", in(reg) val); - } -} - -/// main is only executed from nyx -fn main() { - ptwrite(42); - client::program(); - ptwrite(43); -} diff --git a/src/benchmark_baseline.rs b/src/benchmark_baseline.rs index 8bf2d0c..8ef9dd2 100644 --- a/src/benchmark_baseline.rs +++ b/src/benchmark_baseline.rs @@ -1,17 +1,24 @@ use crate::benchmark::{Benchmark, ToCsv}; use csv::Writer; use std::io::Write; +use std::path::Path; +use std::process::Command; use std::time::Duration; -pub struct Baseline; +pub struct Baseline<'a>(pub &'a Path); -impl Benchmark for Baseline { +impl Benchmark for Baseline<'_> { const TITLE: &'static str = "baseline"; type BenchmarkResult = Duration; fn execute_once(&mut self) -> Self::BenchmarkResult { - client::benchmark_once() + let output = Command::new(self.0).output().expect("Command should succeed"); + let output = String::from_utf8_lossy(&output.stdout); + let Ok(duration) = output.trim().parse() else { + panic!("Invalid output: {}", output); + }; + Duration::from_secs_f64(duration) } } diff --git a/src/benchmark_nyx_pt.rs b/src/benchmark_nyx_pt.rs index e70a3f6..b4e5426 100644 --- a/src/benchmark_nyx_pt.rs +++ b/src/benchmark_nyx_pt.rs @@ -36,6 +36,7 @@ impl NyxRunner { fn execute(&mut self) { let result = self.0.exec(); + // nyx_print_aux_buffer(&mut self.0 as *mut _); assert!( matches!(result, NyxReturnValue::Normal), "Unexpected return value: {result:?}" diff --git a/src/main.rs b/src/main.rs index 4b87ef2..53f19da 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,35 +1,46 @@ //! Translated from libnyx/test.c mod benchmark; +mod benchmark_baseline; mod benchmark_nyx_pt; mod nyx; use crate::benchmark::{Benchmark, ToCsv}; -use crate::benchmark_nyx_pt::NyxRunner; -use std::error::Error; -use std::path::PathBuf; -use std::time::{Duration, Instant}; +use crate::benchmark_baseline::Baseline; use clap::Parser; +use std::error::Error; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; +use crate::benchmark_nyx_pt::NyxRunner; #[derive(Debug, Parser)] -#[command(about="Tool to execute a program with nyx and trace it with intel pt")] +#[command(about = "Tool to execute a program with nyx and trace it with intel pt")] struct Args { /// Path to the client binary to execute - client: PathBuf + client: PathBuf, + /// Directory where the benchmark files get stored + #[arg(short = 'o', long = "output", default_value = "benchmarks")] + output_dir: PathBuf, } fn main() -> Result<(), Box> { let args = Args::parse(); let nyx_runner = NyxRunner::setup(&args.client)?; - benchmark(nyx_runner)?; + let binary_name = args.client.file_name().unwrap().to_str().unwrap(); + benchmark(binary_name, &args.output_dir, nyx_runner)?; + benchmark(binary_name, &args.output_dir, Baseline(&args.client))?; Ok(()) } -fn benchmark(mut benchmark: B) -> Result<(), Box> { +fn benchmark( + binary_name: &str, + output_dir: &Path, + mut benchmark: B, +) -> Result<(), Box> { warmup(&mut benchmark); let results = benchmark_loop(&mut benchmark); - write_results::(&results)?; + write_results::(binary_name, output_dir, &results)?; Ok(()) } @@ -48,16 +59,21 @@ fn warmup(benchmark: &mut impl Benchmark) { } fn benchmark_loop(benchmark: &mut B) -> Vec { - let n = 50; + let n = 500; println!("Perform {n} iterations..."); (0..n) .map(|_| std::hint::black_box(benchmark.execute_once())) .collect::>() } -fn write_results(results: &[B::BenchmarkResult]) -> Result<(), Box> { - let file = std::fs::File::create(format!("benchmark_{}", B::TITLE)) - .expect("Should be able to create file"); +fn write_results( + binary_name: &str, + output_dir: &Path, + results: &[B::BenchmarkResult], +) -> Result<(), Box> { + std::fs::create_dir_all(output_dir)?; + let filename = format!("benchmark_{binary_name}_{}", B::TITLE); + let file = std::fs::File::create(output_dir.join(filename))?; let mut writer = csv::WriterBuilder::new().from_writer(file); writer.write_record(B::BenchmarkResult::HEADERS)?; for result in results {