Benchmark nyx and baseline

Cargo.lock

@@ -207,10 +207,6 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
 [[package]]
 name = "client"
 version = "0.1.0"
-dependencies = [
- "cc",
- "libc",
-]
 
 [[package]]
 name = "cmake"
@@ -254,6 +250,27 @@ version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e"
 
+[[package]]
+name = "csv"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "derivative"
 version = "2.2.0"
@@ -646,6 +663,8 @@ dependencies = [
 name = "qemu-nyx-runner"
 version = "0.1.0"
 dependencies = [
+ "client",
+ "csv",
  "libnyx",
  "pt-dump-decoder",
 ]

Cargo.toml

@@ -8,5 +8,7 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+client = { path = "client" }
 libnyx = { git = "https://git.cs.tu-dortmund.de/david.venhoff/libnyx-fork" }
-pt-dump-decoder = { path = "pt-dump-decoder" }
+pt-dump-decoder = { path = "pt-dump-decoder" }
+csv = "1.3.1"

client/Cargo.toml

@@ -2,9 +2,3 @@
 name = "client"
 version = "0.1.0"
 edition = "2024"
-
-[dependencies]
-libc = "*"
-
-[build-dependencies]
-cc = "1.0"

client/src/lib.rs

@@ -0,0 +1,20 @@
+pub fn collect_primes(c: u64) {
+    for i in 2..c {
+        std::hint::black_box(is_prime(i));
+    }
+}
+
+fn is_prime(number: u64) -> bool {
+    !(2..(number / 2)).any(|it| number % it == 0)
+}
+
+pub fn program() {
+    collect_primes(std::hint::black_box(300_000));
+}
+
+/// This method is used to benchmark without nyx
+pub fn benchmark_once() -> std::time::Duration {
+    let start = std::time::Instant::now();
+    program();
+    start.elapsed()
+}

client/src/main.rs

@@ -6,14 +6,9 @@ fn ptwrite(val: u32) {
     }
 }
 
+/// main is only executed from nyx
 fn main() {
-    let start = std::time::Instant::now();
     ptwrite(42);
-    loop {
+    client::program();
-        // ptwrite(69);
-        if start.elapsed().as_secs() >= 2 {
-            break;
-        }
-    }
     ptwrite(43);
 }

pt-dump-decoder/Cargo.toml

@@ -9,4 +9,4 @@ edition = "2024"
 [dependencies]
 libipt = { version = "0.4.0", features = ["libipt_master"] }
 memmap2 = "0.9.7"
-raw-cpuid = "11.5.0"
+raw-cpuid = "11.5.0"

pt-dump-decoder/src/lib.rs

@@ -1,16 +1,17 @@
 use libipt::enc_dec_builder::EncoderDecoderBuilder;
 use libipt::packet::{Packet, PacketDecoder};
 use memmap2::Mmap;
+use raw_cpuid::{CpuId, cpuid};
 use std::collections::HashMap;
 use std::error::Error;
 use std::fs::File;
 use std::path::Path;
 use std::time::Duration;
-use raw_cpuid::{cpuid, CpuId};
 
 #[derive(Debug)]
 pub struct AnalyzeData {
     pub packets: u64,
+    pub lost_cyc: u64,
     pub time_tsc: Duration,
     pub time_cyc: Duration,
     pub time_main: Duration,
@@ -37,6 +38,7 @@ pub fn analyze_dump(path: impl AsRef<Path>) -> Result<AnalyzeData, Box<dyn Error
 
     let mut segments = vec![(Scope::PreRun, Duration::ZERO)];
     let mut total = 0u64;
+    let mut lost_cyc = 0;
 
     // cpuid(0x16):ecx returns the bus frequency in mHz (See volume 2, chapter 1.3 CPUID in the intel manual)
     // Combined with the core to bus ration we can calculate the core frequency
@@ -54,7 +56,7 @@ pub fn analyze_dump(path: impl AsRef<Path>) -> Result<AnalyzeData, Box<dyn Error
             Ok(Packet::Cyc(cyc)) => {
                 // Ignore cycle packets when there was no cbr packet yet.
                 if current_cbr == 0 {
-                    println!("Ignoring {} cycles, because cbr is still unknown!", cyc.value());
+                    lost_cyc += 1;
                     continue;
                 }
                 let f_core = f_bus * current_cbr as f64;
@@ -89,15 +91,20 @@ pub fn analyze_dump(path: impl AsRef<Path>) -> Result<AnalyzeData, Box<dyn Error
             Err(error) => println!("Got error: {error:?}"),
         }
     }
-    dbg!(seen_cbrs);
+    // dbg!(seen_cbrs);
     let duration_total = segments.iter().map(|(_, duration)| *duration).sum();
     let main_duration = duration_in_mode(&segments, Scope::Main);
 
-    let f_tsc = CpuId::new().get_tsc_info().unwrap().tsc_frequency().unwrap();
+    let f_tsc = CpuId::new()
+        .get_tsc_info()
+        .unwrap()
+        .tsc_frequency()
+        .unwrap();
     let total_seconds_tsc = (last_tsc - first_tsc.unwrap()) as f64 / f_tsc as f64;
 
     Ok(AnalyzeData {
         packets: total,
+        lost_cyc,
         time_cyc: duration_total,
         time_main: main_duration,
         time_tsc: Duration::from_secs_f64(total_seconds_tsc),
@@ -106,7 +113,7 @@ pub fn analyze_dump(path: impl AsRef<Path>) -> Result<AnalyzeData, Box<dyn Error
 
 fn duration_in_mode(segments: &[(Scope, Duration)], scope: Scope) -> Duration {
     segments
-        .into_iter()
+        .iter()
         .filter(|(it, _)| *it == scope)
         .map(|(_, duration)| *duration)
         .sum::<Duration>()

src/benchmark.rs

@@ -0,0 +1,15 @@
+use std::io::Write;
+
+pub trait Benchmark {
+    const TITLE: &'static str;
+
+    type BenchmarkResult: ToCsv;
+
+    fn execute_once(&mut self) -> Self::BenchmarkResult;
+}
+
+pub trait ToCsv {
+    const HEADERS: &'static [&'static str];
+
+    fn write(&self, writer: &mut csv::Writer<impl Write>) -> csv::Result<()>;
+}

src/benchmark_baseline.rs

@@ -0,0 +1,24 @@
+use crate::benchmark::{Benchmark, ToCsv};
+use csv::Writer;
+use std::io::Write;
+use std::time::Duration;
+
+pub struct Baseline;
+
+impl Benchmark for Baseline {
+    const TITLE: &'static str = "baseline";
+
+    type BenchmarkResult = Duration;
+
+    fn execute_once(&mut self) -> Self::BenchmarkResult {
+        client::benchmark_once()
+    }
+}
+
+impl ToCsv for Duration {
+    const HEADERS: &'static [&'static str] = &["time"];
+
+    fn write(&self, writer: &mut Writer<impl Write>) -> csv::Result<()> {
+        writer.write_record([self.as_secs_f64().to_string()])
+    }
+}

src/benchmark_nyx_pt.rs

@@ -0,0 +1,71 @@
+use crate::benchmark::{Benchmark, ToCsv};
+use csv::Writer;
+use libnyx::{NyxConfig, NyxProcess, NyxProcessRole, NyxReturnValue};
+use pt_dump_decoder::{AnalyzeData, analyze_dump};
+use std::error::Error;
+use std::io::Write;
+
+const WORKDIR_PATH: &str = "/tmp/wdir";
+// The sharedir path gets set by the build script
+const SHAREDIR_PATH: &str = env!("NYX_SHAREDIR");
+
+pub struct NyxRunner(NyxProcess);
+
+impl NyxRunner {
+    pub fn setup() -> Result<Self, Box<dyn Error>> {
+        let _ = std::fs::remove_dir_all(WORKDIR_PATH);
+
+        let mut nyx_config = NyxConfig::load(SHAREDIR_PATH)?;
+        nyx_config.set_workdir_path(WORKDIR_PATH.to_string());
+        nyx_config.set_input_buffer_size(0x2000);
+        nyx_config.set_nyx_debug_log_path("qemu_nyx.log".to_string());
+
+        nyx_config.set_process_role(NyxProcessRole::StandAlone);
+        nyx_config.print();
+
+        let mut nyx_runner = NyxProcess::new(&mut nyx_config, 0)?;
+
+        nyx_runner.option_set_trace_mode(true);
+        nyx_runner.option_set_timeout(10, 0);
+        nyx_runner.option_apply();
+
+        Ok(NyxRunner(nyx_runner))
+    }
+
+    fn execute(&mut self) {
+        let result = self.0.exec();
+        assert!(
+            matches!(result, NyxReturnValue::Normal),
+            "Unexpected return value: {result:?}"
+        );
+    }
+}
+
+impl Benchmark for NyxRunner {
+    const TITLE: &'static str = "nyx_and_pt";
+
+    type BenchmarkResult = AnalyzeData;
+
+    fn execute_once(&mut self) -> AnalyzeData {
+        self.execute();
+        let dump_path = format!("{WORKDIR_PATH}/pt_trace_dump_0");
+        analyze_dump(dump_path).expect("Should be able to analyze the data")
+    }
+}
+
+impl ToCsv for AnalyzeData {
+    const HEADERS: &'static [&'static str] = &["time_main", "time_tsc"];
+
+    fn write(&self, writer: &mut Writer<impl Write>) -> csv::Result<()> {
+        writer.write_record([
+            self.time_main.as_secs_f64().to_string(),
+            self.time_tsc.as_secs_f64().to_string(),
+        ])
+    }
+}
+
+impl Drop for NyxRunner {
+    fn drop(&mut self) {
+        self.0.shutdown();
+    }
+}

src/main.rs

@@ -1,47 +1,60 @@
 //! Translated from libnyx/test.c
 
-use libnyx::ffi::nyx_print_aux_buffer;
+mod benchmark;
-use libnyx::{NyxConfig, NyxProcess, NyxProcessRole, NyxReturnValue};
+mod benchmark_baseline;
-use pt_dump_decoder::analyze_dump;
+mod benchmark_nyx_pt;
-use std::error::Error;
 
-const WORKDIR_PATH: &str = "/tmp/wdir";
+use crate::benchmark::{Benchmark, ToCsv};
-// The sharedir path gets set by the build script
+use crate::benchmark_baseline::Baseline;
-const SHAREDIR_PATH: &str = env!("NYX_SHAREDIR");
+use crate::benchmark_nyx_pt::NyxRunner;
+use std::error::Error;
+use std::time::{Duration, Instant};
 
 fn main() -> Result<(), Box<dyn Error>> {
-    let _ = std::fs::remove_dir_all(WORKDIR_PATH);
+    let nyx_runner = NyxRunner::setup()?;
-    run_client()?;
+    benchmark(nyx_runner)?;
-    println!("Analyzing pt data...");
+    benchmark(Baseline)?;
 
-    let dump_path = format!("{WORKDIR_PATH}/pt_trace_dump_0");
-    let result = analyze_dump(dump_path)?;
-    println!("{result:?}");
     Ok(())
 }
 
-fn run_client() -> Result<(), Box<dyn Error>> {
+fn benchmark<B: Benchmark>(mut benchmark: B) -> Result<(), Box<dyn Error>> {
-    let mut nyx_config = NyxConfig::load(SHAREDIR_PATH)?;
+    warmup(&mut benchmark);
-    nyx_config.set_workdir_path(WORKDIR_PATH.to_string());
+    let results = benchmark_loop(&mut benchmark);
-    nyx_config.set_input_buffer_size(0x2000);
+    write_results::<B>(&results)?;
-    nyx_config.set_nyx_debug_log_path("qemu_nyx.log".to_string());
+
+    Ok(())
+}
+
+/// Warm up to make sure caches are initialized, etc.
+fn warmup(benchmark: &mut impl Benchmark) {
+    let warmup_duration = Duration::from_secs(60);
+    println!("Warming up for {warmup_duration:?}");
+    let mut iterations = 0;
+    let start = Instant::now();
+    while start.elapsed() < warmup_duration {
+        benchmark.execute_once();
+        iterations += 1;
+    }
+    println!("Warmed up for {iterations} iterations");
+}
+
+fn benchmark_loop<B: Benchmark>(benchmark: &mut B) -> Vec<B::BenchmarkResult> {
+    let n = 50;
+    println!("Perform {n} iterations...");
+    (0..n)
+        .map(|_| std::hint::black_box(benchmark.execute_once()))
+        .collect::<Vec<_>>()
+}
+
+fn write_results<B: Benchmark>(results: &[B::BenchmarkResult]) -> Result<(), Box<dyn Error>> {
+    let file = std::fs::File::create(format!("benchmark_{}", B::TITLE))
+        .expect("Should be able to create file");
+    let mut writer = csv::WriterBuilder::new().from_writer(file);
+    writer.write_record(B::BenchmarkResult::HEADERS)?;
+    for result in results {
+        result.write(&mut writer)?;
+    }
 
-    nyx_config.set_process_role(NyxProcessRole::StandAlone);
-    nyx_config.print();
-
-    let mut nyx_runner = NyxProcess::new(&mut nyx_config, 0)?;
-
-    nyx_runner.option_set_trace_mode(true);
-    nyx_runner.option_set_timeout(10, 0);
-    nyx_runner.option_apply();
-
-    let result = nyx_runner.exec();
-    assert!(
-        matches!(result, NyxReturnValue::Normal),
-        "Unexpected return value {result:?}"
-    );
-    nyx_print_aux_buffer(&mut nyx_runner as *mut _);
-
-    nyx_runner.shutdown();
     Ok(())
 }