Fast device+mem QEMU snapshots (#930)

* Simple fast root snapshots

* clippy

* epd

* mips

fuzzers/qemu_systemmode/Cargo.toml

@@ -9,9 +9,10 @@ default = ["std"]
 std = []
 
 [profile.release]
-lto = true
+incremental = true
-codegen-units = 1
 debug = true
+lto = "fat"
+codegen-units = 1
 
 [dependencies]
 libafl = { path = "../../libafl/" }

fuzzers/qemu_systemmode/src/fuzzer.rs

@@ -28,7 +28,6 @@ use libafl::{
     stages::StdMutationalStage,
     state::{HasCorpus, StdState},
     Error,
-    //prelude::{SimpleMonitor, SimpleEventManager},
 };
 use libafl_qemu::{
     edges, edges::QemuEdgeCoverageHelper, elf::EasyElf, emu::Emulator, GuestPhysAddr, QemuExecutor,
@@ -88,20 +87,15 @@ pub fn fuzz() {
         }
         emu.remove_breakpoint(main_addr);
 
-        // emu.save_snapshot("start", true);
-
         emu.set_breakpoint(breakpoint); // BREAKPOINT
 
-        //use libafl_qemu::IntoEnumIterator;
+        // let saved_cpu_states: Vec<_> = (0..emu.num_cpus())
-        // Save the GPRs
+        //     .map(|i| emu.cpu_from_index(i).save_state())
-        //let mut saved_regs: Vec<u64> = vec![];
+        //     .collect();
-        //for r in Regs::iter() {
-        //    saved_regs.push(emu.cpu_from_index(0).read_reg(r).unwrap());
-        //}
 
-        let mut saved_cpu_states: Vec<_> = (0..emu.num_cpus())
+        // emu.save_snapshot("start", true);
-            .map(|i| emu.cpu_from_index(i).save_state())
+
-            .collect();
+        let snap = emu.create_fast_snapshot(true);
 
         // The wrapped harness function, calling out to the LLVM-style harness
         let mut harness = |input: &BytesInput| {
@@ -114,10 +108,6 @@ pub fn fuzz() {
                     // len = MAX_INPUT_SIZE;
                 }
 
-                //for (r, v) in saved_regs.iter().enumerate() {
-                //    emu.cpu_from_index(0).write_reg(r as i32, *v).unwrap();
-                //}
-
                 emu.write_phys_mem(input_addr, buf);
 
                 emu.run();
@@ -126,19 +116,24 @@ pub fn fuzz() {
                 let mut pcs = (0..emu.num_cpus())
                     .map(|i| emu.cpu_from_index(i))
                     .map(|cpu| -> Result<u32, String> { cpu.read_reg(Regs::Pc) });
-                let ret = match pcs
+                let _ret = match pcs
                     .find(|pc| (breakpoint..breakpoint + 5).contains(pc.as_ref().unwrap_or(&0)))
                 {
                     Some(_) => ExitKind::Ok,
                     None => ExitKind::Crash,
                 };
 
-                for (i, s) in saved_cpu_states.iter().enumerate() {
+                // OPTION 1: restore only the CPU state (registers et. al)
-                    emu.cpu_from_index(i).restore_state(s);
+                // for (i, s) in saved_cpu_states.iter().enumerate() {
-                }
+                //     emu.cpu_from_index(i).restore_state(s);
+                // }
 
+                // OPTION 2: restore a slow vanilla QEMU snapshot
                 // emu.load_snapshot("start", true);
 
+                // OPTION 3: restore a fast devices+mem snapshot
+                emu.restore_fast_snapshot(snap);
+
                 ret
             }
         };
@@ -161,7 +156,7 @@ pub fn fuzz() {
         );
 
         // A feedback to choose if an input is a solution or not
-        let mut objective = CrashFeedback::new(); //feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new());
+        let mut objective = feedback_or_fast!(CrashFeedback::new(), TimeoutFeedback::new());
 
         // If not restarting, create a State from scratch
         let mut state = state.unwrap_or_else(|| {

libafl_qemu/Cargo.toml

@@ -21,6 +21,8 @@ x86_64 = ["libafl_qemu_sys/x86_64"] # build qemu for x86_64 (default)
 i386 = ["libafl_qemu_sys/i386"] # build qemu for i386
 arm = ["libafl_qemu_sys/arm"] # build qemu for arm
 aarch64 = ["libafl_qemu_sys/aarch64"] # build qemu for aarch64
+mips = ["libafl_qemu_sys/mips"] # build qemu for mips (el, use with the 'be' feature of mips be)
+
 be = ["libafl_qemu_sys/be"]
 
 usermode = ["libafl_qemu_sys/usermode"]

libafl_qemu/build_linux.rs

@@ -26,6 +26,8 @@ pub fn build() {
         "aarch64".to_string()
     } else if cfg!(feature = "i386") {
         "i386".to_string()
+    } else if cfg!(feature = "mips") {
+        "mips".to_string()
     } else {
         env::var("CPU_TARGET").unwrap_or_else(|_| {
             "x86_64".to_string()

libafl_qemu/libafl_qemu_build/src/bindings.rs

@@ -45,6 +45,8 @@ const WRAPPER_HEADER: &str = r#"
 #include "sysemu/tcg.h"
 #include "sysemu/replay.h"
 
+#include "libafl_extras/syx-snapshot/syx-snapshot.h"
+
 #endif
 
 #include "exec/cpu-common.h"
@@ -105,6 +107,10 @@ pub fn generate(
         .allowlist_function("tlb_plugin_lookup")
         .allowlist_function("qemu_plugin_hwaddr_phys_addr")
         .allowlist_function("qemu_plugin_get_hwaddr")
+        .allowlist_function("syx_snapshot_init")
+        .allowlist_function("syx_snapshot_create")
+        .allowlist_function("syx_snapshot_root_restore")
+        .allowlist_function("syx_snapshot_dirty_list_add")
         .blocklist_function("main_loop_wait") // bindgen issue #1313
         .parse_callbacks(Box::new(bindgen::CargoCallbacks));
 

libafl_qemu/libafl_qemu_build/src/build.rs

@@ -8,7 +8,7 @@ use which::which;
 
 const QEMU_URL: &str = "https://github.com/AFLplusplus/qemu-libafl-bridge";
 const QEMU_DIRNAME: &str = "qemu-libafl-bridge";
-const QEMU_REVISION: &str = "9707dd2d211221367915d5da21fe8693d6842eaf";
+const QEMU_REVISION: &str = "e5424c34d223c2b638af6e4c9eef039db8b69dd4";
 
 fn build_dep_check(tools: &[&str]) {
     for tool in tools {
@@ -39,6 +39,10 @@ pub fn build(
         cpu_target += "eb";
     }
 
+    if !is_big_endian && cpu_target == "mips" && !cfg!(feature = "clippy") {
+        cpu_target += "el";
+    }
+
     let custum_qemu_dir = env::var_os("CUSTOM_QEMU_DIR").map(|x| x.to_string_lossy().to_string());
     let custum_qemu_no_build = env::var("CUSTOM_QEMU_NO_BUILD").is_ok();
     let custum_qemu_no_configure = env::var("CUSTOM_QEMU_NO_CONFIGURE").is_ok();

libafl_qemu/libafl_qemu_build/src/main.rs

@@ -5,5 +5,5 @@ use libafl_qemu_build::build_with_bindings;
 // RUST_BACKTRACE=1 OUT_DIR=/tmp/foo/a/b/c cargo run
 fn main() {
     let bfile = PathBuf::from("generated_qemu_bindings.rs");
-    build_with_bindings("arm", false, true, None, &bfile);
+    build_with_bindings("arm", false, false, None, &bfile);
 }

libafl_qemu/libafl_qemu_sys/Cargo.toml

@@ -17,6 +17,8 @@ x86_64 = [] # build qemu for x86_64 (default)
 i386 = [] # build qemu for i386
 arm = [] # build qemu for arm
 aarch64 = [] # build qemu for aarch64
+mips = [] # build qemu for mips (el, use with the 'be' feature of mips be)
+
 be = []
 
 usermode = []

libafl_qemu/libafl_qemu_sys/build_linux.rs

@@ -38,9 +38,9 @@ pub fn build() {
 
     // Make sure we have at most one architecutre feature set
     // Else, we default to `x86_64` - having a default makes CI easier :)
-    assert_unique_feature!("arm", "aarch64", "i386", "i86_64");
+    assert_unique_feature!("arm", "aarch64", "i386", "i86_64", "mips");
 
-    // Make sure that we don't have BE set for any architecture other than arm
+    // Make sure that we don't have BE set for any architecture other than arm and mips
     // Sure aarch64 may support BE, but its not in common usage and we don't
     // need it yet and so haven't tested it
     assert_unique_feature!("be", "aarch64", "i386", "i86_64");
@@ -53,6 +53,8 @@ pub fn build() {
         "aarch64".to_string()
     } else if cfg!(feature = "i386") {
         "i386".to_string()
+    } else if cfg!(feature = "mips") {
+        "mips".to_string()
     } else {
         env::var("CPU_TARGET").unwrap_or_else(|_| {
             println!(

libafl_qemu/src/blocks.rs

@@ -43,36 +43,7 @@ pub fn pc2basicblock(pc: GuestAddr, emu: &Emulator) -> Result<Vec<Instruction>,
     let mut iaddr = pc;
     let mut block = Vec::<Instruction>::new();
 
-    #[cfg(cpu_target = "x86_64")]
+    let cs = crate::capstone().detail(true).build().unwrap();
-    let cs = Capstone::new()
-        .x86()
-        .mode(capstone::arch::x86::ArchMode::Mode64)
-        .detail(true)
-        .build()
-        .unwrap();
-
-    #[cfg(cpu_target = "i386")]
-    let cs = Capstone::new()
-        .x86()
-        .mode(capstone::arch::x86::ArchMode::Mode32)
-        .detail(true)
-        .build()
-        .unwrap();
-
-    #[cfg(cpu_target = "arm")]
-    let cs = Capstone::new()
-        .arm()
-        .mode(capstone::arch::arm::ArchMode::Arm)
-        .detail(true)
-        .build()
-        .unwrap();
-    #[cfg(cpu_target = "aarch64")]
-    let cs = Capstone::new()
-        .arm64()
-        .mode(capstone::arch::arm64::ArchMode::Arm)
-        .detail(true)
-        .build()
-        .unwrap();
 
     'disasm: while let Ok(insns) = cs.disasm_count(code, iaddr.into(), 1) {
         if insns.is_empty() {

libafl_qemu/src/emu.rs

@@ -24,6 +24,9 @@ pub type GuestPhysAddr = libafl_qemu_sys::hwaddr;
 
 pub type GuestHwAddrInfo = libafl_qemu_sys::qemu_plugin_hwaddr;
 
+#[cfg(emulation_mode = "systemmode")]
+pub type FastSnapshot = *mut libafl_qemu_sys::syx_snapshot_t;
+
 #[repr(transparent)]
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct MemAccessInfo {
@@ -279,7 +282,6 @@ extern "C" {
     fn qemu_cleanup();
 
     fn libafl_save_qemu_snapshot(name: *const u8, sync: bool);
-    #[allow(unused)]
     fn libafl_load_qemu_snapshot(name: *const u8, sync: bool);
 }
 
@@ -462,7 +464,7 @@ impl Drop for GuestMaps {
 
 #[repr(C)]
 #[derive(Clone, Copy, PartialEq, Eq)]
-pub(crate) struct FatPtr(*const c_void, *const c_void);
+pub(crate) struct FatPtr(pub *const c_void, pub *const c_void);
 
 static mut GDB_COMMANDS: Vec<FatPtr> = vec![];
 
@@ -708,6 +710,7 @@ impl Emulator {
                     envp.as_ptr() as *const *const u8,
                 );
                 libc::atexit(qemu_cleanup_atexit);
+                libafl_qemu_sys::syx_snapshot_init();
             }
             EMULATOR_IS_INITIALIZED = true;
         }
@@ -1061,6 +1064,17 @@ impl Emulator {
         unsafe { libafl_load_qemu_snapshot(s.as_ptr() as *const _, sync) };
     }
 
+    #[cfg(emulation_mode = "systemmode")]
+    #[must_use]
+    pub fn create_fast_snapshot(&self, track: bool) -> FastSnapshot {
+        unsafe { libafl_qemu_sys::syx_snapshot_create(track) }
+    }
+
+    #[cfg(emulation_mode = "systemmode")]
+    pub fn restore_fast_snapshot(&self, snapshot: FastSnapshot) {
+        unsafe { libafl_qemu_sys::syx_snapshot_root_restore(snapshot) }
+    }
+
     #[cfg(emulation_mode = "usermode")]
     pub fn set_pre_syscall_hook(
         &self,

libafl_qemu/src/lib.rs

@@ -44,6 +44,11 @@ pub mod x86_64;
 #[cfg(cpu_target = "x86_64")]
 pub use x86_64::*;
 
+#[cfg(cpu_target = "mips")]
+pub mod mips;
+#[cfg(cpu_target = "mips")]
+pub use mips::*;
+
 pub mod elf;
 
 pub mod helper;
@@ -53,20 +58,27 @@ pub use hooks::*;
 
 pub mod edges;
 pub use edges::QemuEdgeCoverageHelper;
+
+#[cfg(not(cpu_target = "mips"))]
 pub mod cmplog;
-pub mod drcov;
+#[cfg(not(cpu_target = "mips"))]
 pub use cmplog::QemuCmpLogHelper;
+
 #[cfg(emulation_mode = "usermode")]
 pub mod snapshot;
 #[cfg(emulation_mode = "usermode")]
 pub use snapshot::QemuSnapshotHelper;
+
 #[cfg(emulation_mode = "usermode")]
 pub mod asan;
 #[cfg(emulation_mode = "usermode")]
 pub use asan::{init_with_asan, QemuAsanHelper};
 
 pub mod blocks;
+
+#[cfg(not(cpu_target = "mips"))]
 pub mod calls;
+pub mod drcov;
 
 pub mod executor;
 pub use executor::QemuExecutor;

libafl_qemu/src/mips.rs

@@ -0,0 +1,66 @@
+use num_enum::{IntoPrimitive, TryFromPrimitive};
+#[cfg(feature = "python")]
+use pyo3::prelude::*;
+pub use strum_macros::EnumIter;
+pub use syscall_numbers::mips::*;
+
+/// Registers for the ARM instruction set.
+#[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter)]
+#[repr(i32)]
+pub enum Regs {
+    R0 = 0,
+    R1 = 1,
+    R2 = 2,
+    R3 = 3,
+    R4 = 4,
+    R5 = 5,
+    R6 = 6,
+    R7 = 7,
+    R8 = 8,
+    R9 = 9,
+    R10 = 10,
+    R11 = 11,
+    R12 = 12,
+    R13 = 13,
+    R14 = 14,
+    R15 = 15,
+    R16 = 16,
+    R17 = 17,
+    R18 = 18,
+    R19 = 19,
+    R20 = 20,
+    R21 = 21,
+    R22 = 22,
+    R23 = 23,
+    R24 = 24,
+    R25 = 25,
+    R26 = 26,
+    R27 = 27,
+    R28 = 28,
+    R29 = 29,
+    R30 = 30,
+    R31 = 31,
+}
+
+/// alias registers
+#[allow(non_upper_case_globals)]
+impl Regs {
+    pub const Zero: Regs = Regs::R0;
+    pub const Gp: Regs = Regs::R28;
+    pub const Sp: Regs = Regs::R29;
+    pub const Fp: Regs = Regs::R30;
+    pub const Ra: Regs = Regs::R31;
+}
+
+#[cfg(feature = "python")]
+impl IntoPy<PyObject> for Regs {
+    fn into_py(self, py: Python) -> PyObject {
+        let n: i32 = self.into();
+        n.into_py(py)
+    }
+}
+
+/// Return an MIPS ArchCapstoneBuilder
+pub fn capstone() -> capstone::arch::mips::ArchCapstoneBuilder {
+    capstone::Capstone::new().mips()
+}

libafl_qemu/src/snapshot.rs

@@ -8,6 +8,14 @@ use libafl::{inputs::UsesInput, state::HasMetadata};
 use meminterval::{Interval, IntervalTree};
 use thread_local::ThreadLocal;
 
+#[cfg(any(cpu_target = "arm", cpu_target = "i386", cpu_target = "mips"))]
+use crate::SYS_fstatat64;
+#[cfg(not(cpu_target = "arm"))]
+use crate::SYS_mmap;
+#[cfg(any(cpu_target = "arm", cpu_target = "mips"))]
+use crate::SYS_mmap2;
+#[cfg(not(any(cpu_target = "arm", cpu_target = "mips", cpu_target = "i386")))]
+use crate::SYS_newfstatat;
 use crate::{
     emu::{Emulator, MmapPerms, SyscallHookResult},
     helper::{QemuHelper, QemuHelperTuple},
@@ -15,12 +23,6 @@ use crate::{
     GuestAddr, SYS_fstat, SYS_fstatfs, SYS_futex, SYS_getrandom, SYS_mprotect, SYS_mremap,
     SYS_munmap, SYS_pread64, SYS_read, SYS_readlinkat, SYS_statfs,
 };
-#[cfg(cpu_target = "i386")]
-use crate::{SYS_fstatat64, SYS_mmap};
-#[cfg(cpu_target = "arm")]
-use crate::{SYS_fstatat64, SYS_mmap2};
-#[cfg(not(any(cpu_target = "arm", cpu_target = "i386")))]
-use crate::{SYS_mmap, SYS_newfstatat};
 
 // TODO use the functions provided by Emulator
 pub const SNAPSHOT_PAGE_SIZE: usize = 4096;
@@ -301,6 +303,10 @@ impl QemuSnapshotHelper {
     }
 
     pub fn add_mapped(&mut self, start: GuestAddr, mut size: usize, perms: Option<MmapPerms>) {
+        if size == 0 {
+            return;
+        }
+
         let total_size = {
             if size % SNAPSHOT_PAGE_SIZE != 0 {
                 size = size + (SNAPSHOT_PAGE_SIZE - size % SNAPSHOT_PAGE_SIZE);
@@ -632,14 +638,14 @@ where
             let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
             h.access(a0 as GuestAddr, a3 as usize);
         }
-        #[cfg(not(any(cpu_target = "arm", cpu_target = "i386")))]
+        #[cfg(not(any(cpu_target = "arm", cpu_target = "i386", cpu_target = "mips")))]
         SYS_newfstatat => {
             if a2 != 0 {
                 let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
                 h.access(a2 as GuestAddr, 4096); // stat is not greater than a page
             }
         }
-        #[cfg(any(cpu_target = "arm", cpu_target = "i386"))]
+        #[cfg(any(cpu_target = "arm", cpu_target = "mips", cpu_target = "i386"))]
         SYS_fstatat64 => {
             if a2 != 0 {
                 let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
@@ -664,27 +670,12 @@ where
 
             // TODO handle huge pages
 
-            #[cfg(cpu_target = "arm")]
+            #[cfg(any(cpu_target = "arm", cpu_target = "mips"))]
             if i64::from(sys_num) == SYS_mmap2 {
                 if let Ok(prot) = MmapPerms::try_from(a2 as i32) {
                     let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
                     h.add_mapped(result as GuestAddr, a1 as usize, Some(prot));
                 }
-            } else if i64::from(sys_num) == SYS_mremap {
-                let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
-                h.remove_mapped(a0 as GuestAddr, a1 as usize);
-                h.add_mapped(result as GuestAddr, a2 as usize, None);
-                // TODO get the old permissions from the removed mapping
-            } else if i64::from(sys_num) == SYS_mprotect {
-                if let Ok(prot) = MmapPerms::try_from(a2 as i32) {
-                    let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
-                    h.add_mapped(a0 as GuestAddr, a1 as usize, Some(prot));
-                }
-            } else if i64::from(sys_num) == SYS_munmap {
-                let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
-                if !h.accurate_unmap && !h.is_unmap_allowed(a0 as GuestAddr, a1 as usize) {
-                    h.remove_mapped(a0 as GuestAddr, a1 as usize);
-                }
             }
 
             #[cfg(not(cpu_target = "arm"))]
@@ -693,7 +684,9 @@ where
                     let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
                     h.add_mapped(result as GuestAddr, a1 as usize, Some(prot));
                 }
-            } else if i64::from(sys_num) == SYS_mremap {
+            }
+
+            if i64::from(sys_num) == SYS_mremap {
                 let h = hooks.match_helper_mut::<QemuSnapshotHelper>().unwrap();
                 h.remove_mapped(a0 as GuestAddr, a1 as usize);
                 h.add_mapped(result as GuestAddr, a2 as usize, None);