LibAFL_QEMU: Fix incorrect handling of brk syscall when shrinking the heap (#2776)

* added libafl_get_initial_brk API to properly handle brk growing and shrinking

* cargo fmt

* updated qemu revision

---------

Co-authored-by: Dominik Maier <domenukk@gmail.com>

libafl_qemu/libafl_qemu_build/src/build.rs

@@ -11,7 +11,7 @@ use crate::cargo_add_rpath;
 
 pub const QEMU_URL: &str = "https://github.com/AFLplusplus/qemu-libafl-bridge";
 pub const QEMU_DIRNAME: &str = "qemu-libafl-bridge";
-pub const QEMU_REVISION: &str = "b01a0bc334cf11bfc5e8f121d9520ef7f47dbcd1";
+pub const QEMU_REVISION: &str = "06bf8facec33548840413fba1b20858f58e38e2d";
 
 #[allow(clippy::module_name_repetitions)]
 pub struct BuildResult {

libafl_qemu/libafl_qemu_sys/src/bindings/x86_64_stub_bindings.rs

@@ -5705,6 +5705,9 @@ extern "C" {
 extern "C" {
     pub fn libafl_get_brk() -> u64;
 }
+extern "C" {
+    pub fn libafl_get_initial_brk() -> u64;
+}
 extern "C" {
     pub fn libafl_set_brk(new_brk: u64) -> u64;
 }

libafl_qemu/src/emu/usermode.rs

@@ -50,6 +50,11 @@ where
         self.qemu.get_brk()
     }
 
+    #[must_use]
+    pub fn get_initial_brk(&self) -> GuestAddr {
+        self.qemu.get_initial_brk()
+    }
+
     pub fn set_brk(&self, brk: GuestAddr) {
         self.qemu.set_brk(brk);
     }

libafl_qemu/src/modules/usermode/snapshot.rs

@@ -89,6 +89,7 @@ pub struct SnapshotModule {
     pub maps: MappingInfo,
     pub new_maps: Mutex<MappingInfo>,
     pub pages: HashMap<GuestAddr, SnapshotPageInfo>,
+    pub initial_brk: GuestAddr,
     pub brk: GuestAddr,
     pub mmap_start: GuestAddr,
     pub mmap_limit: usize,
@@ -120,6 +121,7 @@ impl SnapshotModule {
             maps: MappingInfo::default(),
             new_maps: Mutex::new(MappingInfo::default()),
             pages: HashMap::default(),
+            initial_brk: 0,
             brk: 0,
             mmap_start: 0,
             mmap_limit: 0,
@@ -137,6 +139,7 @@ impl SnapshotModule {
             maps: MappingInfo::default(),
             new_maps: Mutex::new(MappingInfo::default()),
             pages: HashMap::default(),
+            initial_brk: 0,
             brk: 0,
             mmap_start: 0,
             mmap_limit: 0,
@@ -154,6 +157,7 @@ impl SnapshotModule {
             maps: MappingInfo::default(),
             new_maps: Mutex::new(MappingInfo::default()),
             pages: HashMap::default(),
+            initial_brk: 0,
             brk: 0,
             mmap_start: 0,
             mmap_limit,
@@ -191,6 +195,7 @@ impl SnapshotModule {
     pub fn snapshot(&mut self, qemu: Qemu) {
         log::info!("Start snapshot");
         self.brk = qemu.get_brk();
+        self.initial_brk = qemu.get_initial_brk();
         self.mmap_start = qemu.get_mmap_start();
         self.pages.clear();
         for map in qemu.mappings() {
@@ -843,12 +848,16 @@ where
         }
         SYS_brk => {
             let h = emulator_modules.get_mut::<SnapshotModule>().unwrap();
-            if h.brk != result && result != 0 {
+            if h.brk != result && result != 0 && result > h.initial_brk {
-                /* brk has changed. we change mapping from the snapshotted brk address to the new target_brk
+                /* brk has changed, and it doesn't shrink below initial_brk. We change mapping from the snapshotted initial brk address to the new target_brk
                  * If no brk mapping has been made until now, change_mapped won't change anything and just create a new mapping.
                  * It is safe to assume RW perms here
                  */
-                h.change_mapped(h.brk, (result - h.brk) as usize, Some(MmapPerms::ReadWrite));
+                h.change_mapped(
+                    h.initial_brk,
+                    (result - h.initial_brk) as usize,
+                    Some(MmapPerms::ReadWrite),
+                );
             }
         }
         // mmap syscalls

libafl_qemu/src/qemu/usermode.rs

@@ -4,10 +4,10 @@ use std::{
 };
 
 use libafl_qemu_sys::{
-    exec_path, free_self_maps, guest_base, libafl_force_dfl, libafl_get_brk, libafl_load_addr,
+    exec_path, free_self_maps, guest_base, libafl_force_dfl, libafl_get_brk,
-    libafl_maps_first, libafl_maps_next, libafl_qemu_run, libafl_set_brk, mmap_next_start,
+    libafl_get_initial_brk, libafl_load_addr, libafl_maps_first, libafl_maps_next, libafl_qemu_run,
-    pageflags_get_root, read_self_maps, GuestAddr, GuestUsize, IntervalTreeNode, IntervalTreeRoot,
+    libafl_set_brk, mmap_next_start, pageflags_get_root, read_self_maps, GuestAddr, GuestUsize,
-    MapInfo, MmapPerms, VerifyAccess,
+    IntervalTreeNode, IntervalTreeRoot, MapInfo, MmapPerms, VerifyAccess,
 };
 use libc::{c_int, c_uchar, strlen};
 #[cfg(feature = "python")]
@@ -177,6 +177,11 @@ impl Qemu {
         unsafe { libafl_get_brk() as GuestAddr }
     }
 
+    #[must_use]
+    pub fn get_initial_brk(&self) -> GuestAddr {
+        unsafe { libafl_get_initial_brk() as GuestAddr }
+    }
+
     pub fn set_brk(&self, brk: GuestAddr) {
         unsafe { libafl_set_brk(brk.into()) };
     }