libafl_nyx: Allow custom input buffer size to be passed to NyxHelper (#1960)

* add option to specify input buffer size

* fix typo

* use `libafl::Error` as default error type

* derive `TypedBuilder` for `NyxSettings`

* update nyx_libxml2_standalone

* update nyx_libxml2_parallel

* update nyx_libxml2_standalone

* update nyx_libxml2_standalone

* update nyx_libxml2_parallel

fuzzers/nyx_libxml2_parallel/src/main.rs

@@ -19,7 +19,7 @@ use libafl_bolts::{
     shmem::{ShMemProvider, StdShMemProvider},
     tuples::tuple_list,
 };
-use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper};
+use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper, settings::NyxSettings};
 
 fn main() {
     let shmem_provider = StdShMemProvider::new().expect("Failed to init shared memory");
@@ -32,21 +32,17 @@ fn main() {
 
     // region: fuzzer start function
     let mut run_client = |state: Option<_>, mut restarting_mgr, core_id: CoreId| {
-        // nyx shared dir, created by nyx-fuzz/packer/packer/nyx_packer.py
-        let share_dir = Path::new("/tmp/nyx_libxml2/");
-        let cpu_id = core_id.0.try_into().unwrap();
-        let parallel_mode = true;
         // nyx stuff
-        let mut helper = NyxHelper::new(
+        let settings = NyxSettings::builder()
-            share_dir,
+            .cpu_id(0)
-            cpu_id,
+            .snap_mode(true)
-            true,
+            .parallel_mode(true)
-            parallel_mode,
+            .parent_cpu_id(Some(parent_cpu_id.0 as u32))
-            Some(parent_cpu_id.0.try_into().unwrap()),
+            .build();
-        )
+        let helper = NyxHelper::new("/tmp/nyx_libxml2/", settings).unwrap();
-        .unwrap();
+        let observer = unsafe {
-        let observer =
+            StdMapObserver::from_mut_ptr("trace", helper.bitmap_buffer, helper.bitmap_size)
-            unsafe { StdMapObserver::from_mut_ptr("trace", helper.trace_bits, helper.map_size) };
+        };
 
         let input = BytesInput::new(b"22".to_vec());
         let rand = StdRand::new();
@@ -60,7 +56,7 @@ fn main() {
         let mut feedback = MaxMapFeedback::new(&observer);
         let mut objective = CrashFeedback::new();
         let scheduler = RandScheduler::new();
-        let mut executor = NyxExecutor::new(&mut helper, tuple_list!(observer)).unwrap();
+        let mut executor = NyxExecutor::new(helper, tuple_list!(observer));
 
         // If not restarting, create a State from scratch
         let mut state = state.unwrap_or_else(|| {

fuzzers/nyx_libxml2_standalone/src/main.rs

@@ -1,4 +1,4 @@
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
 use libafl::{
     corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus, Testcase},
@@ -17,17 +17,19 @@ use libafl_bolts::{
     rands::{RandomSeed, StdRand},
     tuples::tuple_list,
 };
-use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper};
+use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper, settings::NyxSettings};
 
 fn main() {
-    let share_dir = Path::new("/tmp/nyx_libxml2/");
-    let cpu_id = 0;
-    let parallel_mode = false;
-
     // nyx stuff
-    let mut helper = NyxHelper::new(share_dir, cpu_id, true, parallel_mode, None).unwrap();
+    let settings = NyxSettings::builder()
+        .cpu_id(0)
+        .snap_mode(true)
+        .parallel_mode(false)
+        .parent_cpu_id(None)
+        .build();
+    let helper = NyxHelper::new("/tmp/nyx_libxml2/", settings).unwrap();
     let observer =
-        unsafe { StdMapObserver::from_mut_ptr("trace", helper.trace_bits, helper.map_size) };
+        unsafe { StdMapObserver::from_mut_ptr("trace", helper.bitmap_buffer, helper.bitmap_size) };
 
     let input = BytesInput::new(b"22".to_vec());
     let rand = StdRand::new();
@@ -50,7 +52,7 @@ fn main() {
     let monitor = TuiMonitor::new(ui);
 
     let mut mgr = SimpleEventManager::new(monitor);
-    let mut executor = NyxExecutor::new(&mut helper, tuple_list!(observer)).unwrap();
+    let mut executor = NyxExecutor::new(helper, tuple_list!(observer));
     let mutator = StdScheduledMutator::new(havoc_mutations());
     let mut stages = tuple_list!(StdMutationalStage::new(mutator));
 

libafl_nyx/Cargo.toml

@@ -18,3 +18,5 @@ libnyx = {git = "https://github.com/nyx-fuzz/libnyx.git",rev = "acaf7f6"}
 libafl = { path = "../libafl", version = "0.11.2", features = ["std", "libafl_derive", "frida_cli" ]}
 libafl_bolts = { path = "../libafl_bolts", version = "0.11.2", features = ["std", "libafl_derive", "frida_cli" ]}
 libafl_targets = { path = "../libafl_targets", version = "0.11.2", features = ["std", "sancov_cmplog"] }
+
+typed-builder = "0.18.1"

libafl_nyx/src/executor.rs

@@ -1,4 +1,4 @@
-use std::{fmt::Debug, marker::PhantomData};
+use std::marker::PhantomData;
 
 use libafl::{
     executors::{Executor, ExitKind, HasObservers},
@@ -13,31 +13,23 @@ use libnyx::NyxReturnValue;
 use crate::helper::NyxHelper;
 
 /// executor for nyx standalone mode
-pub struct NyxExecutor<'a, S, OT> {
+pub struct NyxExecutor<S, OT> {
     /// implement nyx function
-    pub helper: &'a mut NyxHelper,
+    pub helper: NyxHelper,
     /// observers
     observers: OT,
     /// phantom data to keep generic type <I,S>
     phantom: PhantomData<S>,
 }
 
-impl<'a, S, OT> Debug for NyxExecutor<'a, S, OT> {
+impl<S, OT> UsesState for NyxExecutor<S, OT>
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("NyxInprocessExecutor")
-            .field("helper", &self.helper)
-            .finish()
-    }
-}
-
-impl<'a, S, OT> UsesState for NyxExecutor<'a, S, OT>
 where
     S: State,
 {
     type State = S;
 }
 
-impl<'a, S, OT> UsesObservers for NyxExecutor<'a, S, OT>
+impl<S, OT> UsesObservers for NyxExecutor<S, OT>
 where
     OT: ObserversTuple<S>,
     S: State,
@@ -45,7 +37,7 @@ where
     type Observers = OT;
 }
 
-impl<'a, EM, S, Z, OT> Executor<EM, Z> for NyxExecutor<'a, S, OT>
+impl<EM, S, Z, OT> Executor<EM, Z> for NyxExecutor<S, OT>
 where
     EM: UsesState<State = S>,
     S: State + HasExecutions,
@@ -60,56 +52,59 @@ where
         input: &Self::Input,
     ) -> Result<ExitKind, Error> {
         *state.executions_mut() += 1;
-        let input_owned = input.target_bytes();
+
-        let input = input_owned.as_slice();
+        let bytes = input.target_bytes();
-        self.helper.nyx_process.set_input(
+        let buffer = bytes.as_slice();
-            input,
+        let size = u32::try_from(buffer.len())
-            input
+            .map_err(|_| Error::unsupported("Inputs larger than 4GB are not supported"))?;
-                .len()
+
-                .try_into()
+        self.helper.nyx_process.set_input(buffer, size);
-                .expect("Inputs larger than 4GB not supported"),
-        );
 
         // exec will take care of trace_bits, so no need to reset
-        let ret_val = self.helper.nyx_process.exec();
+        match self.helper.nyx_process.exec() {
-        match ret_val {
             NyxReturnValue::Normal => Ok(ExitKind::Ok),
             NyxReturnValue::Crash | NyxReturnValue::Asan => Ok(ExitKind::Crash),
             NyxReturnValue::Timeout => Ok(ExitKind::Timeout),
-            NyxReturnValue::InvalidWriteToPayload => Err(libafl::Error::illegal_state(
+            NyxReturnValue::InvalidWriteToPayload => {
+                self.helper.nyx_process.shutdown();
+                Err(Error::illegal_state(
                     "FixMe: Nyx InvalidWriteToPayload handler is missing",
-            )),
+                ))
-            NyxReturnValue::Error => Err(libafl::Error::illegal_state(
+            }
-                "Error: Nyx runtime error has occurred...",
+            NyxReturnValue::Error => {
-            )),
+                self.helper.nyx_process.shutdown();
+                Err(Error::illegal_state("Nyx runtime error has occurred"))
+            }
             NyxReturnValue::IoError => {
-                // todo! *stop_soon_p = 0
+                self.helper.nyx_process.shutdown();
-                Err(libafl::Error::unknown("Error: QEMU-nyx died..."))
+                Err(Error::unknown("QEMU-nyx died"))
             }
             NyxReturnValue::Abort => {
                 self.helper.nyx_process.shutdown();
-                Err(libafl::Error::shutting_down())
+                Err(Error::shutting_down())
             }
         }
     }
 }
 
-impl<'a, S, OT> NyxExecutor<'a, S, OT> {
+impl<S, OT> NyxExecutor<S, OT> {
-    pub fn new(helper: &'a mut NyxHelper, observers: OT) -> Result<Self, Error> {
+    pub fn new(helper: NyxHelper, observers: OT) -> Self {
-        Ok(Self {
+        Self {
             helper,
             observers,
             phantom: PhantomData,
-        })
+        }
     }
 
     /// convert `trace_bits` ptr into real trace map
     pub fn trace_bits(self) -> &'static mut [u8] {
-        unsafe { std::slice::from_raw_parts_mut(self.helper.trace_bits, self.helper.real_map_size) }
+        unsafe {
+            std::slice::from_raw_parts_mut(self.helper.bitmap_buffer, self.helper.bitmap_size)
+        }
     }
 }
 
-impl<'a, S, OT> HasObservers for NyxExecutor<'a, S, OT>
+impl<S, OT> HasObservers for NyxExecutor<S, OT>
 where
     S: State,
     OT: ObserversTuple<S>,

libafl_nyx/src/helper.rs

@@ -1,25 +1,18 @@
 /// [`NyxHelper`] is used to wrap `NyxProcess`
-use std::{
+use std::{fmt::Debug, path::Path};
-    fmt::{self, Debug},
-    path::Path,
-    time::Duration,
-};
 
 use libafl::Error;
-use libnyx::{NyxProcess, NyxReturnValue};
+use libnyx::NyxProcess;
+
+use crate::settings::NyxSettings;
 
-const INIT_TIMEOUT: Duration = Duration::new(2, 0);
 pub struct NyxHelper {
     pub nyx_process: NyxProcess,
-    /// real size of `trace_bits`
+
-    pub real_map_size: usize,
+    pub bitmap_size: usize,
-    // real size of the trace_bits
+    pub bitmap_buffer: *mut u8,
-    pub map_size: usize,
-    /// shared memory with instruction bitmaps
-    pub trace_bits: *mut u8,
 }
 
-const MAX_FILE: u32 = 1024 * 1024;
 #[derive(Clone, Copy, Debug)]
 pub enum NyxProcessType {
     /// stand alone mode
@@ -29,125 +22,77 @@ pub enum NyxProcessType {
     /// parallel mode's child, consume snapshot and execute
     CHILD,
 }
+
 impl NyxHelper {
-    /// Create [`NyxProcess`] and do basic settings
+    /// Create [`NyxProcess`] and do basic settings. It will convert the
-    /// It will convert instance to parent or child using `parent_cpu_id` when set`parallel_mode`
+    /// instance to a parent or child using `parent_cpu_id` when
-    /// will fail if initial connection takes more than 2 seconds
+    /// `parallel_mode` is set.
-    pub fn new(
+    pub fn new<P>(share_dir: P, settings: NyxSettings) -> Result<Self, Error>
-        target_dir: &Path,
+    where
-        cpu_id: u32,
+        P: AsRef<Path>,
-        snap_mode: bool,
+    {
-        parallel_mode: bool,
+        let work_dir = share_dir.as_ref().join("workdir");
-        parent_cpu_id: Option<u32>,
+        let share_dir_str = share_dir.as_ref().to_str().ok_or(Error::illegal_argument(
-    ) -> Result<Self, Error> {
+            "`share_dir` contains invalid UTF-8",
-        NyxHelper::with_initial_timeout(
+        ))?;
-            target_dir,
+        let work_dir_str = work_dir
-            cpu_id,
+            .to_str()
-            snap_mode,
+            .ok_or(Error::illegal_argument("`work_dir` contains invalid UTF-8"))?;
-            parallel_mode,
+
-            parent_cpu_id,
+        let nyx_process_type = match (settings.parallel_mode, settings.parent_cpu_id) {
-            INIT_TIMEOUT,
+            (false, _) => NyxProcessType::ALONE,
-        )
+            (true, Some(parent_cpu_id)) if settings.cpu_id == parent_cpu_id => {
-    }
-    /// Create [`NyxProcess`] and do basic settings
-    /// It will convert instance to parent or child using `parent_cpu_id` when set`parallel_mode`
-    /// will fail if initial connection takes more than `initial_timeout` seconds
-    pub fn with_initial_timeout(
-        target_dir: &Path,
-        cpu_id: u32,
-        snap_mode: bool,
-        parallel_mode: bool,
-        parent_cpu_id: Option<u32>,
-        initial_timeout: Duration,
-    ) -> Result<Self, Error> {
-        let Some(sharedir) = target_dir.to_str() else {
-            return Err(Error::illegal_argument("can't convert sharedir to str"));
-        };
-        let work_dir = target_dir.join("workdir");
-        let work_dir = work_dir.to_str().expect("unable to convert workdir to str");
-        let nyx_type = if parallel_mode {
-            let Some(parent_cpu_id) = parent_cpu_id else {
-                return Err(Error::illegal_argument(
-                    "please set parent_cpu_id in nyx parallel mode",
-                ));
-            };
-            if cpu_id == parent_cpu_id {
                 NyxProcessType::PARENT
-            } else {
-                NyxProcessType::CHILD
             }
-        } else {
+            (true, Some(_)) => NyxProcessType::CHILD,
-            NyxProcessType::ALONE
-        };
 
-        let nyx_process = match nyx_type {
+            (true, _) => {
-            NyxProcessType::ALONE => NyxProcess::new(sharedir, work_dir, cpu_id, MAX_FILE, true),
+                return Err(Error::illegal_argument(
-            NyxProcessType::PARENT => {
+                    "`parent_cpu_id` is required in nyx parallel mode",
-                NyxProcess::new_parent(sharedir, work_dir, cpu_id, MAX_FILE, true)
+                ))
             }
-            NyxProcessType::CHILD => NyxProcess::new_child(sharedir, work_dir, cpu_id, cpu_id),
         };
+        let mut nyx_process = (match nyx_process_type {
+            NyxProcessType::ALONE => NyxProcess::new(
+                share_dir_str,
+                work_dir_str,
+                settings.cpu_id,
+                settings.input_buffer_size,
+                /* input_buffer_write_protection= */ true,
+            ),
+            NyxProcessType::PARENT => NyxProcess::new_parent(
+                share_dir_str,
+                work_dir_str,
+                settings.cpu_id,
+                settings.input_buffer_size,
+                /* input_buffer_write_protection= */ true,
+            ),
+            NyxProcessType::CHILD => NyxProcess::new_child(
+                share_dir_str,
+                work_dir_str,
+                settings.cpu_id,
+                /* worker_id= */ settings.cpu_id,
+            ),
+        })
+        .map_err(Error::illegal_argument)?;
 
-        let mut nyx_process =
+        nyx_process.option_set_reload_mode(settings.snap_mode);
-            nyx_process.map_err(|msg: String| -> Error { Error::illegal_argument(msg) })?;
+        nyx_process.option_set_timeout(settings.timeout_secs, settings.timeout_micro_secs);
-
-        let real_map_size = nyx_process.bitmap_buffer_size();
-        let map_size = ((real_map_size + 63) >> 6) << 6;
-        let trace_bits = nyx_process.bitmap_buffer_mut().as_mut_ptr();
-        nyx_process.option_set_reload_mode(snap_mode);
         nyx_process.option_apply();
 
-        // default timeout for initial dry-run
+        let bitmap_size = nyx_process.bitmap_buffer_size();
-        let sec = initial_timeout
+        let bitmap_buffer = nyx_process.bitmap_buffer_mut().as_mut_ptr();
-            .as_secs()
-            .try_into()
-            .map_err(|_| -> Error { Error::illegal_argument("can't cast time's sec to u8") })?;
 
-        let micro_sec: u32 = initial_timeout.subsec_micros();
-        nyx_process.option_set_timeout(sec, micro_sec);
-        nyx_process.option_apply();
-
-        // dry run to check if qemu is spawned
-        nyx_process.set_input(b"INIT", 4);
-        match nyx_process.exec() {
-            NyxReturnValue::Error => {
-                nyx_process.shutdown();
-                let msg = "Error: Nyx runtime error has occurred...";
-                return Err(Error::illegal_state(msg));
-            }
-            NyxReturnValue::IoError => {
-                let msg = "Error: QEMU-nyx died...";
-                return Err(Error::illegal_state(msg));
-            }
-            NyxReturnValue::Abort => {
-                nyx_process.shutdown();
-                let msg = "Error: Nyx abort occurred...";
-                return Err(Error::illegal_state(msg));
-            }
-            _ => {}
-        }
         Ok(Self {
             nyx_process,
-            real_map_size,
+            bitmap_size,
-            map_size,
+            bitmap_buffer,
-            trace_bits,
         })
     }
 
-    /// Set a timeout for Nyx
+    /// Set a timeout for Nyx.
-    pub fn set_timeout(&mut self, time: Duration) {
+    pub fn set_timeout(&mut self, secs: u8, micro_secs: u32) {
-        let sec: u8 = time
+        self.nyx_process.option_set_timeout(secs, micro_secs);
-            .as_secs()
-            .try_into()
-            .expect("can't cast time's sec to u8");
-        let micro_sec: u32 = time.subsec_micros();
-        self.nyx_process.option_set_timeout(sec, micro_sec);
         self.nyx_process.option_apply();
     }
 }
-
-impl Debug for NyxHelper {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("NyxInprocessHelper").finish()
-    }
-}

libafl_nyx/src/lib.rs

@@ -3,3 +3,5 @@
 pub mod executor;
 #[cfg(target_os = "linux")]
 pub mod helper;
+#[cfg(target_os = "linux")]
+pub mod settings;

libafl_nyx/src/settings.rs

@@ -0,0 +1,23 @@
+use typed_builder::TypedBuilder;
+
+const DEFAULT_INPUT_BUFFER_SIZE: u32 = 1024 * 1024;
+const DEFAULT_TIMEOUT_SECS: u8 = 2;
+const DEFAULT_TIMEOUT_MICRO_SECS: u32 = 0;
+
+#[derive(Debug, Clone, Copy, TypedBuilder)]
+pub struct NyxSettings {
+    pub cpu_id: u32,
+    pub parent_cpu_id: Option<u32>,
+
+    pub snap_mode: bool,
+    pub parallel_mode: bool,
+
+    #[builder(default = DEFAULT_INPUT_BUFFER_SIZE)]
+    pub input_buffer_size: u32,
+
+    #[builder(default = DEFAULT_TIMEOUT_SECS)]
+    pub timeout_secs: u8,
+
+    #[builder(default = DEFAULT_TIMEOUT_MICRO_SECS)]
+    pub timeout_micro_secs: u32,
+}