Forkserver: 1. Add mem barrier 2. Don't send the initial 4 bytes message when it uses dynamic map option only (#1073)

* fix

* Real fix

* a

libafl/src/executors/forkserver.rs

@@ -4,6 +4,7 @@ use alloc::{borrow::ToOwned, string::ToString, vec::Vec};
 use core::{
     fmt::{self, Debug, Formatter},
     marker::PhantomData,
+    sync::atomic::{compiler_fence, Ordering},
     time::Duration,
 };
 use std::{
@@ -733,14 +734,37 @@ impl<'a, SP> ForkserverExecutorBuilder<'a, SP> {
             return Err(Error::unknown("Failed to start a forkserver".to_string()));
         }
         log::info!("All right - fork server is up.");
+
+        if status & FS_OPT_MAPSIZE == FS_OPT_MAPSIZE {
+            let mut map_size = fs_opt_get_mapsize(status);
+            // When 0, we assume that map_size was filled by the user or const
+            /* TODO autofill map size from the observer
+
+            if map_size > 0 {
+                self.map_size = Some(map_size as usize);
+            }
+            */
+
+            self.real_map_size = map_size;
+            if map_size % 64 != 0 {
+                map_size = ((map_size + 63) >> 6) << 6;
+            }
+
+            // TODO set AFL_MAP_SIZE
+            assert!(self.map_size.is_none() || map_size as usize <= self.map_size.unwrap());
+
+            log::info!("Target MAP SIZE = {:#x}", self.real_map_size);
+            self.map_size = Some(map_size as usize);
+        }
+
+        // Only with SHMEM or AUTODICT we can send send_status back or it breaks!
         // If forkserver is responding, we then check if there's any option enabled.
         // We'll send 4-bytes message back to the forkserver to tell which features to use
         // The forkserver is listening to our response if either shmem fuzzing is enabled or auto dict is enabled
         // <https://github.com/AFLplusplus/AFLplusplus/blob/147654f8715d237fe45c1657c87b2fe36c4db22a/instrumentation/afl-compiler-rt.o.c#L1026>
         if status & FS_OPT_ENABLED == FS_OPT_ENABLED
             && (status & FS_OPT_SHDMEM_FUZZ == FS_OPT_SHDMEM_FUZZ
-                || status & FS_OPT_AUTODICT == FS_OPT_AUTODICT
+                || status & FS_OPT_AUTODICT == FS_OPT_AUTODICT)
-                || status & FS_OPT_MAPSIZE == FS_OPT_MAPSIZE)
         {
             let mut send_status = FS_OPT_ENABLED;
 
@@ -750,28 +774,6 @@ impl<'a, SP> ForkserverExecutorBuilder<'a, SP> {
                 self.uses_shmem_testcase = true;
             }
 
-            if status & FS_OPT_MAPSIZE == FS_OPT_MAPSIZE {
-                let mut map_size = fs_opt_get_mapsize(status);
-                // When 0, we assume that map_size was filled by the user or const
-                /* TODO autofill map size from the observer
-
-                if map_size > 0 {
-                    self.map_size = Some(map_size as usize);
-                }
-                */
-
-                self.real_map_size = map_size;
-                if map_size % 64 != 0 {
-                    map_size = ((map_size + 63) >> 6) << 6;
-                }
-
-                // TODO set AFL_MAP_SIZE
-                assert!(self.map_size.is_none() || map_size as usize <= self.map_size.unwrap());
-
-                log::info!("Target MAP SIZE = {:#x}", self.real_map_size);
-                self.map_size = Some(map_size as usize);
-            }
-
             let send_len = forkserver.write_ctl(send_status)?;
             if send_len != 4 {
                 return Err(Error::unknown("Writing to forkserver failed.".to_string()));
@@ -1046,6 +1048,9 @@ where
             self.input_file.write_buf(input.target_bytes().as_slice())?;
         }
 
+        // Don't tell the forkserver to spawn a new process before clearing the cov map
+        compiler_fence(Ordering::SeqCst);
+
         let send_len = self
             .forkserver
             .write_ctl(self.forkserver().last_run_timed_out())?;
@@ -1098,6 +1103,9 @@ where
 
         self.forkserver.set_child_pid(Pid::from_raw(0));
 
+        // Clear the observer map after the execution is finished
+        compiler_fence(Ordering::SeqCst);
+
         Ok(exit_kind)
     }
 }

libafl/src/observers/mod.rs

@@ -29,7 +29,6 @@ use alloc::{
     vec::Vec,
 };
 use core::{fmt::Debug, time::Duration};
-
 #[cfg(feature = "std")]
 use std::time::Instant;
 
@@ -38,7 +37,6 @@ pub use value::*;
 
 #[cfg(feature = "no_std")]
 use crate::bolts::current_time;
-
 use crate::{
     bolts::{
         ownedref::OwnedMutPtr,