Soft recovery from crashes in libafl qemu usermode (#3073)

* soft recovery from crashes in qemu

* regen bindings for clippy

* configurable crash behaviour

Cargo.toml

@@ -103,6 +103,7 @@ log = "0.4.22"
 meminterval = "0.4.1"
 mimalloc = { version = "0.1.43", default-features = false }
 nix = { version = "0.29.0", default-features = false }
+num-derive = { version = "0.4.2", default-features = false }
 num_enum = { version = "0.7.3", default-features = false }
 num-traits = { version = "0.2.19", default-features = false }
 paste = "1.0.15"

fuzzers/binary_only/fuzzbench_qemu/src/fuzzer.rs

@@ -53,7 +53,7 @@ use libafl_qemu::{
         edges::StdEdgeCoverageModule,
     },
     Emulator, GuestReg, MmapPerms, QemuExecutor, QemuExitError, QemuExitReason, QemuShutdownCause,
-    Regs,
+    Regs, TargetSignalHandling,
 };
 use libafl_targets::{edges_map_mut_ptr, EDGES_MAP_ALLOCATED_SIZE, MAX_EDGES_FOUND};
 #[cfg(unix)]
@@ -194,6 +194,10 @@ fn fuzz(
         .modules(modules)
         .build()?;
 
+    // return to harness instead of crashing the process.
+    // greatly speeds up crash recovery.
+    emulator.set_target_crash_handling(&TargetSignalHandling::RaiseSignal);
+
     let qemu = emulator.qemu();
 
     let mut elf_buffer = Vec::new();
@@ -359,13 +363,15 @@ fn fuzz(
                 qemu.write_reg(Regs::Rip, test_one_input_ptr).unwrap();
                 qemu.write_reg(Regs::Rsp, stack_ptr).unwrap();
 
-                match qemu.run() {
+                let qemu_ret = qemu.run();
+
+                match qemu_ret {
                     Ok(QemuExitReason::Breakpoint(_)) => {}
-                    Ok(QemuExitReason::End(QemuShutdownCause::HostSignal(signal))) => {
-                        signal.handle();
-                    }
+                    Ok(QemuExitReason::Crash) => return ExitKind::Crash,
+                    Ok(QemuExitReason::Timeout) => return ExitKind::Timeout,
+
                     Err(QemuExitError::UnexpectedExit) => return ExitKind::Crash,
-                    _ => panic!("Unexpected QEMU exit."),
+                    _ => panic!("Unexpected QEMU exit: {qemu_ret:?}"),
                 }
             }
 

libafl_qemu/Cargo.toml

@@ -109,7 +109,7 @@ hashbrown = { workspace = true, default-features = true, features = [
   "serde",
 ] } # A faster hashmap, nostd compatible
 num-traits = { workspace = true, default-features = true }
-num-derive = "0.4.2"
+num-derive = { workspace = true }
 num_enum = { workspace = true, default-features = true }
 goblin = "0.9.2"
 libc = { workspace = true }
@@ -133,6 +133,7 @@ bytes-utils = "0.1.4"
 typed-builder = { workspace = true }
 memmap2 = "0.9.5"
 getset = "0.1.3"
+
 # Document all features of this crate (for `cargo doc`)
 document-features = { workspace = true, optional = true }
 

libafl_qemu/libafl_qemu_build/src/build.rs

@@ -11,7 +11,7 @@ use crate::cargo_add_rpath;
 
 pub const QEMU_URL: &str = "https://github.com/AFLplusplus/qemu-libafl-bridge";
 pub const QEMU_DIRNAME: &str = "qemu-libafl-bridge";
-pub const QEMU_REVISION: &str = "3c60ef9b83107a160021075b485831edecb5a1c3";
+pub const QEMU_REVISION: &str = "fea68856b9410ca6f0076a6bf9ccc4b4b11aa09c";
 
 pub struct BuildResult {
     pub qemu_path: PathBuf,

libafl_qemu/runtime/libafl_qemu_stub_bindings.rs

@@ -1,5 +1,5 @@
 /* 1.87.0-nightly */
-/* qemu git hash: 06c738f64a4a92d5fc8184c9b5a9fe9340f4a63f */
+/* qemu git hash: fea68856b9410ca6f0076a6bf9ccc4b4b11aa09c */
 /* automatically generated by rust-bindgen 0.71.1 */
 
 #[repr(C)]

libafl_qemu/runtime/nyx_stub_bindings.rs

@@ -1,5 +1,5 @@
 /* 1.87.0-nightly */
-/* qemu git hash: 06c738f64a4a92d5fc8184c9b5a9fe9340f4a63f */
+/* qemu git hash: fea68856b9410ca6f0076a6bf9ccc4b4b11aa09c */
 /* automatically generated by rust-bindgen 0.71.1 */
 
 #[repr(C)]

libafl_qemu/src/emu/builder.rs

@@ -117,6 +117,7 @@ where
         }
     }
 }
+
 impl<C, CM, ED, ET, QP, I, S, SM> EmulatorBuilder<C, CM, ED, ET, QP, I, S, SM>
 where
     I: Unpin,

libafl_qemu/src/emu/drivers/mod.rs

@@ -265,6 +265,9 @@ where
                 }
                 _ => panic!("Unhandled QEMU shutdown cause: {shutdown_cause:?}."),
             },
+            EmulatorExitResult::Crash => {
+                return Ok(Some(EmulatorDriverResult::EndOfRun(ExitKind::Crash)));
+            }
             EmulatorExitResult::Timeout => {
                 return Ok(Some(EmulatorDriverResult::EndOfRun(ExitKind::Timeout)));
             }

libafl_qemu/src/emu/drivers/nyx.rs

@@ -188,6 +188,9 @@ where
                 }
                 _ => panic!("Unhandled QEMU shutdown cause: {shutdown_cause:?}."),
             },
+            EmulatorExitResult::Crash => {
+                return Ok(Some(EmulatorDriverResult::EndOfRun(ExitKind::Crash)));
+            }
             EmulatorExitResult::Timeout => {
                 return Ok(Some(EmulatorDriverResult::EndOfRun(ExitKind::Timeout)));
             }

libafl_qemu/src/emu/mod.rs

@@ -57,6 +57,7 @@ pub enum EmulatorExitResult<C> {
     QemuExit(QemuShutdownCause), // QEMU ended for some reason.
     Breakpoint(Breakpoint<C>),   // Breakpoint triggered. Contains the address of the trigger.
     CustomInsn(CustomInsn<C>), // Synchronous backdoor: The guest triggered a backdoor and should return to LibAFL.
+    Crash,                     // Crash
     Timeout,                   // Timeout
 }
 
@@ -75,6 +76,9 @@ where
             EmulatorExitResult::CustomInsn(sync_exit) => {
                 write!(f, "{sync_exit:?}")
             }
+            EmulatorExitResult::Crash => {
+                write!(f, "Crash")
+            }
             EmulatorExitResult::Timeout => {
                 write!(f, "Timeout")
             }
@@ -219,6 +223,9 @@ where
             EmulatorExitResult::CustomInsn(sync_exit) => {
                 write!(f, "Sync exit: {sync_exit:?}")
             }
+            EmulatorExitResult::Crash => {
+                write!(f, "Crash")
+            }
             EmulatorExitResult::Timeout => {
                 write!(f, "Timeout")
             }
@@ -455,6 +462,7 @@ where
                 QemuExitReason::End(qemu_shutdown_cause) => {
                     EmulatorExitResult::QemuExit(qemu_shutdown_cause)
                 }
+                QemuExitReason::Crash => EmulatorExitResult::Crash,
                 QemuExitReason::Timeout => EmulatorExitResult::Timeout,
                 QemuExitReason::Breakpoint(bp_addr) => {
                     let bp = self

libafl_qemu/src/emu/usermode.rs

@@ -1,6 +1,6 @@
 use libafl_qemu_sys::{GuestAddr, MmapPerms, VerifyAccess};
 
-use crate::{Emulator, GuestMaps, NopSnapshotManager};
+use crate::{Emulator, GuestMaps, NopSnapshotManager, TargetSignalHandling};
 
 pub type StdSnapshotManager = NopSnapshotManager;
 
@@ -88,4 +88,12 @@ impl<C, CM, ED, ET, I, S, SM> Emulator<C, CM, ED, ET, I, S, SM> {
     pub fn unmap(&self, addr: GuestAddr, size: usize) -> Result<(), String> {
         self.qemu.unmap(addr, size)
     }
+
+    /// Set how QEMU should handle crashing signals:
+    /// Check [`TargetSignalHandling`] documentation for more details.
+    pub fn set_target_crash_handling(&self, handling: &TargetSignalHandling) {
+        unsafe {
+            self.qemu.set_target_crash_handling(handling);
+        }
+    }
 }

libafl_qemu/src/executor.rs

@@ -45,6 +45,9 @@ use crate::{EmulatorModules, Qemu, QemuSignalContext, run_target_crash_hooks};
 type EmulatorInProcessExecutor<'a, C, CM, ED, EM, ET, H, I, OT, S, SM, Z> =
     StatefulInProcessExecutor<'a, EM, Emulator<C, CM, ED, ET, I, S, SM>, H, I, OT, S, Z>;
 
+#[cfg(feature = "systemmode")]
+pub(crate) static BREAK_ON_TMOUT: AtomicBool = AtomicBool::new(false);
+
 pub struct QemuExecutor<'a, C, CM, ED, EM, ET, H, I, OT, S, SM, Z> {
     inner: EmulatorInProcessExecutor<'a, C, CM, ED, EM, ET, H, I, OT, S, SM, Z>,
     first_exec: bool,
@@ -99,7 +102,7 @@ pub unsafe fn inproc_qemu_crash_handler<E, EM, ET, I, OF, S, Z>(
                         qemu.run_signal_handler(signal.into(), info, puc);
                     }
 
-                    // if we are there, we can safely to execution
+                    // if we are there, we can safely resume from the signal handler.
                     return;
                 }
                 QemuSignalContext::InQemuSignalHandlerHost => {
@@ -120,7 +123,7 @@ pub unsafe fn inproc_qemu_crash_handler<E, EM, ET, I, OF, S, Z>(
                     // run qemu hooks then report the crash.
 
                     log::debug!(
-                        "QEMU Target signal received that should be handled by host. Most likely a target crash."
+                        "QEMU Target signal received that should be handled by host. It is a target crash."
                     );
 
                     log::debug!("Running crash hooks.");
@@ -157,9 +160,6 @@ pub unsafe fn inproc_qemu_crash_handler<E, EM, ET, I, OF, S, Z>(
     }
 }
 
-#[cfg(feature = "systemmode")]
-pub(crate) static BREAK_ON_TMOUT: AtomicBool = AtomicBool::new(false);
-
 /// # Safety
 /// Can call through the `unix_signal_handler::inproc_timeout_handler`.
 /// Calling this method multiple times concurrently can lead to race conditions.

libafl_qemu/src/qemu/mod.rs

@@ -101,6 +101,9 @@ pub enum QemuExitReason {
     /// Synchronous exit: The guest triggered a backdoor and should return to `LibAFL`.
     SyncExit,
 
+    // Target crash, and it has been requested to be handled by the harness.
+    Crash,
+
     /// Timeout, and it has been requested to be handled by the harness.
     Timeout,
 }
@@ -184,6 +187,7 @@ impl Display for QemuExitReason {
             QemuExitReason::End(shutdown_cause) => write!(f, "End: {shutdown_cause:?}"),
             QemuExitReason::Breakpoint(bp) => write!(f, "Breakpoint: {bp}"),
             QemuExitReason::SyncExit => write!(f, "Sync Exit"),
+            QemuExitReason::Crash => write!(f, "Crash"),
             QemuExitReason::Timeout => write!(f, "Timeout"),
         }
     }
@@ -711,6 +715,8 @@ impl Qemu {
                 #[cfg(feature = "systemmode")]
                 libafl_qemu_sys::libafl_exit_reason_kind_TIMEOUT => QemuExitReason::Timeout,
 
+                libafl_qemu_sys::libafl_exit_reason_kind_CRASH => QemuExitReason::Crash,
+
                 _ => return Err(QemuExitError::UnknownKind),
             })
         }

libafl_qemu/src/qemu/usermode.rs

@@ -16,11 +16,38 @@ use pyo3::{IntoPyObject, Py, PyRef, PyRefMut, Python, pyclass, pymethods};
 
 use crate::{CPU, Qemu, qemu::QEMU_IS_RUNNING};
 
+/// Choose how QEMU target signals should be handled.
+/// It's main use is to describe how crashes and timeouts should be treated.
+pub enum TargetSignalHandling {
+    /// Return to harness with the associated exit request on target crashing or timeout signal.
+    /// The snapshot mechanism should make sure to recover correctly from the crash.
+    /// For instance, snapshots do not take into account side effects related to file descriptors.
+    /// If it could have an impact in case of a crash, prefer the other policy.
+    ///
+    /// *Warning*: this policy should be used with [`SnapshotModule`]. It can be used without
+    /// snapshotting, but it is up to the user to make sure the recovery is possible without
+    /// corrupting the target.
+    ReturnToHarness,
+    /// Propagate target signal to host (following QEMU target to host signal translation) by
+    /// raising the proper signal.
+    /// This the safe policy, since the target is completely reset.
+    /// However, it could make the fuzzer much slower if many crashes are triggered during the
+    /// fuzzing campaign.
+    RaiseSignal,
+}
+
 pub struct QemuMappingsViewer<'a> {
     qemu: &'a Qemu,
     mappings: Vec<MapInfo>,
 }
 
+impl Default for TargetSignalHandling {
+    /// Historically, `LibAFL` QEMU raises the target signal to the host.
+    fn default() -> Self {
+        TargetSignalHandling::RaiseSignal
+    }
+}
+
 impl<'a> QemuMappingsViewer<'a> {
     /// Capture the memory mappings of Qemu at the moment when we create this object
     /// Thus if qemu make updates to the mappings, they won't be reflected to this object.
@@ -411,6 +438,24 @@ impl Qemu {
             libc::raise(signal.into());
         }
     }
+
+    /// Set the target crash handling policy according to [`TargetSignalHandling`]'s documentation.
+    ///
+    /// # Safety
+    ///
+    /// It has an important impact on how crashes are handled by QEMU on target crashing signals.
+    /// Please make sure to read the documentation of [`TargetSignalHandling`] before touching
+    /// this.
+    pub unsafe fn set_target_crash_handling(&self, handling: &TargetSignalHandling) {
+        match handling {
+            TargetSignalHandling::ReturnToHarness => unsafe {
+                libafl_qemu_sys::libafl_set_return_on_crash(true);
+            },
+            TargetSignalHandling::RaiseSignal => unsafe {
+                libafl_qemu_sys::libafl_set_return_on_crash(false);
+            },
+        }
+    }
 }
 
 #[cfg(feature = "python")]