From c2e432c21953026c2e55cdb79e64d9f5dd9c4b0d Mon Sep 17 00:00:00 2001 From: David Venhoff Date: Fri, 5 Sep 2025 15:00:47 +0200 Subject: [PATCH] Benchmark nyx and baseline --- Cargo.lock | 27 +++++++++++-- Cargo.toml | 4 +- client/Cargo.toml | 6 --- client/src/lib.rs | 20 +++++++++ client/src/main.rs | 9 +---- pt-dump-decoder/Cargo.toml | 2 +- pt-dump-decoder/src/lib.rs | 17 +++++--- src/benchmark.rs | 15 +++++++ src/benchmark_baseline.rs | 24 +++++++++++ src/benchmark_nyx_pt.rs | 71 ++++++++++++++++++++++++++++++++ src/main.rs | 83 ++++++++++++++++++++++---------------- 11 files changed, 219 insertions(+), 59 deletions(-) create mode 100644 client/src/lib.rs create mode 100644 src/benchmark.rs create mode 100644 src/benchmark_baseline.rs create mode 100644 src/benchmark_nyx_pt.rs diff --git a/Cargo.lock b/Cargo.lock index 85b1fc2..a30835e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,10 +207,6 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "client" version = "0.1.0" -dependencies = [ - "cc", - "libc", -] [[package]] name = "cmake" @@ -254,6 +250,27 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +dependencies = [ + "memchr", +] + [[package]] name = "derivative" version = "2.2.0" @@ -646,6 +663,8 @@ dependencies = [ name = "qemu-nyx-runner" version = "0.1.0" dependencies = [ + "client", + "csv", "libnyx", "pt-dump-decoder", ] diff --git a/Cargo.toml b/Cargo.toml index ffea091..7812b6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,5 +8,7 @@ version = "0.1.0" edition = "2024" [dependencies] +client = { path = "client" } libnyx = { git = "https://git.cs.tu-dortmund.de/david.venhoff/libnyx-fork" } -pt-dump-decoder = { path = "pt-dump-decoder" } \ No newline at end of file +pt-dump-decoder = { path = "pt-dump-decoder" } +csv = "1.3.1" \ No newline at end of file diff --git a/client/Cargo.toml b/client/Cargo.toml index 55c1f78..325f3d4 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -2,9 +2,3 @@ name = "client" version = "0.1.0" edition = "2024" - -[dependencies] -libc = "*" - -[build-dependencies] -cc = "1.0" \ No newline at end of file diff --git a/client/src/lib.rs b/client/src/lib.rs new file mode 100644 index 0000000..78636c0 --- /dev/null +++ b/client/src/lib.rs @@ -0,0 +1,20 @@ +pub fn collect_primes(c: u64) { + for i in 2..c { + std::hint::black_box(is_prime(i)); + } +} + +fn is_prime(number: u64) -> bool { + !(2..(number / 2)).any(|it| number % it == 0) +} + +pub fn program() { + collect_primes(std::hint::black_box(300_000)); +} + +/// This method is used to benchmark without nyx +pub fn benchmark_once() -> std::time::Duration { + let start = std::time::Instant::now(); + program(); + start.elapsed() +} diff --git a/client/src/main.rs b/client/src/main.rs index 1696a16..676fe1b 100644 --- a/client/src/main.rs +++ b/client/src/main.rs @@ -6,14 +6,9 @@ fn ptwrite(val: u32) { } } +/// main is only executed from nyx fn main() { - let start = std::time::Instant::now(); ptwrite(42); - loop { - // ptwrite(69); - if start.elapsed().as_secs() >= 2 { - break; - } - } + client::program(); ptwrite(43); } diff --git a/pt-dump-decoder/Cargo.toml b/pt-dump-decoder/Cargo.toml index 9443a4a..ac0b162 100644 --- a/pt-dump-decoder/Cargo.toml +++ b/pt-dump-decoder/Cargo.toml @@ -9,4 +9,4 @@ edition = "2024" [dependencies] libipt = { version = "0.4.0", features = ["libipt_master"] } memmap2 = "0.9.7" -raw-cpuid = "11.5.0" +raw-cpuid = "11.5.0" \ No newline at end of file diff --git a/pt-dump-decoder/src/lib.rs b/pt-dump-decoder/src/lib.rs index d99375f..33ad391 100644 --- a/pt-dump-decoder/src/lib.rs +++ b/pt-dump-decoder/src/lib.rs @@ -1,16 +1,17 @@ use libipt::enc_dec_builder::EncoderDecoderBuilder; use libipt::packet::{Packet, PacketDecoder}; use memmap2::Mmap; +use raw_cpuid::{CpuId, cpuid}; use std::collections::HashMap; use std::error::Error; use std::fs::File; use std::path::Path; use std::time::Duration; -use raw_cpuid::{cpuid, CpuId}; #[derive(Debug)] pub struct AnalyzeData { pub packets: u64, + pub lost_cyc: u64, pub time_tsc: Duration, pub time_cyc: Duration, pub time_main: Duration, @@ -37,6 +38,7 @@ pub fn analyze_dump(path: impl AsRef) -> Result) -> Result { // Ignore cycle packets when there was no cbr packet yet. if current_cbr == 0 { - println!("Ignoring {} cycles, because cbr is still unknown!", cyc.value()); + lost_cyc += 1; continue; } let f_core = f_bus * current_cbr as f64; @@ -89,15 +91,20 @@ pub fn analyze_dump(path: impl AsRef) -> Result println!("Got error: {error:?}"), } } - dbg!(seen_cbrs); + // dbg!(seen_cbrs); let duration_total = segments.iter().map(|(_, duration)| *duration).sum(); let main_duration = duration_in_mode(&segments, Scope::Main); - let f_tsc = CpuId::new().get_tsc_info().unwrap().tsc_frequency().unwrap(); + let f_tsc = CpuId::new() + .get_tsc_info() + .unwrap() + .tsc_frequency() + .unwrap(); let total_seconds_tsc = (last_tsc - first_tsc.unwrap()) as f64 / f_tsc as f64; Ok(AnalyzeData { packets: total, + lost_cyc, time_cyc: duration_total, time_main: main_duration, time_tsc: Duration::from_secs_f64(total_seconds_tsc), @@ -106,7 +113,7 @@ pub fn analyze_dump(path: impl AsRef) -> Result Duration { segments - .into_iter() + .iter() .filter(|(it, _)| *it == scope) .map(|(_, duration)| *duration) .sum::() diff --git a/src/benchmark.rs b/src/benchmark.rs new file mode 100644 index 0000000..d32501b --- /dev/null +++ b/src/benchmark.rs @@ -0,0 +1,15 @@ +use std::io::Write; + +pub trait Benchmark { + const TITLE: &'static str; + + type BenchmarkResult: ToCsv; + + fn execute_once(&mut self) -> Self::BenchmarkResult; +} + +pub trait ToCsv { + const HEADERS: &'static [&'static str]; + + fn write(&self, writer: &mut csv::Writer) -> csv::Result<()>; +} diff --git a/src/benchmark_baseline.rs b/src/benchmark_baseline.rs new file mode 100644 index 0000000..8bf2d0c --- /dev/null +++ b/src/benchmark_baseline.rs @@ -0,0 +1,24 @@ +use crate::benchmark::{Benchmark, ToCsv}; +use csv::Writer; +use std::io::Write; +use std::time::Duration; + +pub struct Baseline; + +impl Benchmark for Baseline { + const TITLE: &'static str = "baseline"; + + type BenchmarkResult = Duration; + + fn execute_once(&mut self) -> Self::BenchmarkResult { + client::benchmark_once() + } +} + +impl ToCsv for Duration { + const HEADERS: &'static [&'static str] = &["time"]; + + fn write(&self, writer: &mut Writer) -> csv::Result<()> { + writer.write_record([self.as_secs_f64().to_string()]) + } +} diff --git a/src/benchmark_nyx_pt.rs b/src/benchmark_nyx_pt.rs new file mode 100644 index 0000000..a15efc9 --- /dev/null +++ b/src/benchmark_nyx_pt.rs @@ -0,0 +1,71 @@ +use crate::benchmark::{Benchmark, ToCsv}; +use csv::Writer; +use libnyx::{NyxConfig, NyxProcess, NyxProcessRole, NyxReturnValue}; +use pt_dump_decoder::{AnalyzeData, analyze_dump}; +use std::error::Error; +use std::io::Write; + +const WORKDIR_PATH: &str = "/tmp/wdir"; +// The sharedir path gets set by the build script +const SHAREDIR_PATH: &str = env!("NYX_SHAREDIR"); + +pub struct NyxRunner(NyxProcess); + +impl NyxRunner { + pub fn setup() -> Result> { + let _ = std::fs::remove_dir_all(WORKDIR_PATH); + + let mut nyx_config = NyxConfig::load(SHAREDIR_PATH)?; + nyx_config.set_workdir_path(WORKDIR_PATH.to_string()); + nyx_config.set_input_buffer_size(0x2000); + nyx_config.set_nyx_debug_log_path("qemu_nyx.log".to_string()); + + nyx_config.set_process_role(NyxProcessRole::StandAlone); + nyx_config.print(); + + let mut nyx_runner = NyxProcess::new(&mut nyx_config, 0)?; + + nyx_runner.option_set_trace_mode(true); + nyx_runner.option_set_timeout(10, 0); + nyx_runner.option_apply(); + + Ok(NyxRunner(nyx_runner)) + } + + fn execute(&mut self) { + let result = self.0.exec(); + assert!( + matches!(result, NyxReturnValue::Normal), + "Unexpected return value: {result:?}" + ); + } +} + +impl Benchmark for NyxRunner { + const TITLE: &'static str = "nyx_and_pt"; + + type BenchmarkResult = AnalyzeData; + + fn execute_once(&mut self) -> AnalyzeData { + self.execute(); + let dump_path = format!("{WORKDIR_PATH}/pt_trace_dump_0"); + analyze_dump(dump_path).expect("Should be able to analyze the data") + } +} + +impl ToCsv for AnalyzeData { + const HEADERS: &'static [&'static str] = &["time_main", "time_tsc"]; + + fn write(&self, writer: &mut Writer) -> csv::Result<()> { + writer.write_record([ + self.time_main.as_secs_f64().to_string(), + self.time_tsc.as_secs_f64().to_string(), + ]) + } +} + +impl Drop for NyxRunner { + fn drop(&mut self) { + self.0.shutdown(); + } +} diff --git a/src/main.rs b/src/main.rs index f36f390..bad9465 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,47 +1,60 @@ //! Translated from libnyx/test.c -use libnyx::ffi::nyx_print_aux_buffer; -use libnyx::{NyxConfig, NyxProcess, NyxProcessRole, NyxReturnValue}; -use pt_dump_decoder::analyze_dump; -use std::error::Error; +mod benchmark; +mod benchmark_baseline; +mod benchmark_nyx_pt; -const WORKDIR_PATH: &str = "/tmp/wdir"; -// The sharedir path gets set by the build script -const SHAREDIR_PATH: &str = env!("NYX_SHAREDIR"); +use crate::benchmark::{Benchmark, ToCsv}; +use crate::benchmark_baseline::Baseline; +use crate::benchmark_nyx_pt::NyxRunner; +use std::error::Error; +use std::time::{Duration, Instant}; fn main() -> Result<(), Box> { - let _ = std::fs::remove_dir_all(WORKDIR_PATH); - run_client()?; - println!("Analyzing pt data..."); + let nyx_runner = NyxRunner::setup()?; + benchmark(nyx_runner)?; + benchmark(Baseline)?; - let dump_path = format!("{WORKDIR_PATH}/pt_trace_dump_0"); - let result = analyze_dump(dump_path)?; - println!("{result:?}"); Ok(()) } -fn run_client() -> Result<(), Box> { - let mut nyx_config = NyxConfig::load(SHAREDIR_PATH)?; - nyx_config.set_workdir_path(WORKDIR_PATH.to_string()); - nyx_config.set_input_buffer_size(0x2000); - nyx_config.set_nyx_debug_log_path("qemu_nyx.log".to_string()); +fn benchmark(mut benchmark: B) -> Result<(), Box> { + warmup(&mut benchmark); + let results = benchmark_loop(&mut benchmark); + write_results::(&results)?; + + Ok(()) +} + +/// Warm up to make sure caches are initialized, etc. +fn warmup(benchmark: &mut impl Benchmark) { + let warmup_duration = Duration::from_secs(60); + println!("Warming up for {warmup_duration:?}"); + let mut iterations = 0; + let start = Instant::now(); + while start.elapsed() < warmup_duration { + benchmark.execute_once(); + iterations += 1; + } + println!("Warmed up for {iterations} iterations"); +} + +fn benchmark_loop(benchmark: &mut B) -> Vec { + let n = 50; + println!("Perform {n} iterations..."); + (0..n) + .map(|_| std::hint::black_box(benchmark.execute_once())) + .collect::>() +} + +fn write_results(results: &[B::BenchmarkResult]) -> Result<(), Box> { + let file = std::fs::File::create(format!("benchmark_{}", B::TITLE)) + .expect("Should be able to create file"); + let mut writer = csv::WriterBuilder::new().from_writer(file); + writer.write_record(B::BenchmarkResult::HEADERS)?; + for result in results { + result.write(&mut writer)?; + } - nyx_config.set_process_role(NyxProcessRole::StandAlone); - nyx_config.print(); - - let mut nyx_runner = NyxProcess::new(&mut nyx_config, 0)?; - - nyx_runner.option_set_trace_mode(true); - nyx_runner.option_set_timeout(10, 0); - nyx_runner.option_apply(); - - let result = nyx_runner.exec(); - assert!( - matches!(result, NyxReturnValue::Normal), - "Unexpected return value {result:?}" - ); - nyx_print_aux_buffer(&mut nyx_runner as *mut _); - - nyx_runner.shutdown(); Ok(()) }