Set rlimit to inifinity for core dumps if AFL_DEBUG=1 (#2643)

* forkserver: set rlimit to inifinity for core dumps if AFL_DEBUG=1

* move coredump rlimit to a separate func

* update docs

libafl/src/executors/forkserver.rs

@@ -26,6 +26,7 @@ use libafl_bolts::{
     tuples::{Handle, Handled, MatchNameRef, Prepend, RefIndexable},
     AsSlice, AsSliceMut, Truncate,
 };
+use libc::RLIM_INFINITY;
 use nix::{
     sys::{
         select::{pselect, FdSet},
@@ -134,6 +135,8 @@ pub trait ConfigTarget {
     fn setsid(&mut self) -> &mut Self;
     /// Sets a mem limit
     fn setlimit(&mut self, memlimit: u64) -> &mut Self;
+    /// enables core dumps (rlimit = infinity)
+    fn set_coredump(&mut self, enable: bool) -> &mut Self;
     /// Sets the stdin
     fn setstdin(&mut self, fd: RawFd, use_stdin: bool) -> &mut Self;
     /// Sets the AFL forkserver pipes
@@ -220,19 +223,27 @@ impl ConfigTarget for Command {
                 rlim_cur: memlimit,
                 rlim_max: memlimit,
             };
-            let r0 = libc::rlimit {
-                rlim_cur: 0,
-                rlim_max: 0,
-            };
-
             #[cfg(target_os = "openbsd")]
-            let mut ret = unsafe { libc::setrlimit(libc::RLIMIT_RSS, &r) };
+            let ret = unsafe { libc::setrlimit(libc::RLIMIT_RSS, &r) };
             #[cfg(not(target_os = "openbsd"))]
-            let mut ret = unsafe { libc::setrlimit(libc::RLIMIT_AS, &r) };
+            let ret = unsafe { libc::setrlimit(libc::RLIMIT_AS, &r) };
             if ret < 0 {
                 return Err(io::Error::last_os_error());
             }
-            ret = unsafe { libc::setrlimit(libc::RLIMIT_CORE, &r0) };
+            Ok(())
+        };
+        // # Safety
+        // This calls our non-shady function from above.
+        unsafe { self.pre_exec(func) }
+    }
+
+    fn set_coredump(&mut self, enable: bool) -> &mut Self {
+        let func = move || {
+            let r0 = libc::rlimit {
+                rlim_cur: if enable { RLIM_INFINITY } else { 0 },
+                rlim_max: if enable { RLIM_INFINITY } else { 0 },
+            };
+            let ret = unsafe { libc::setrlimit(libc::RLIMIT_CORE, &r0) };
             if ret < 0 {
                 return Err(io::Error::last_os_error());
             }
@@ -367,6 +378,15 @@ impl Forkserver {
             return Err(Error::unknown("__AFL_SHM_ID not set. It is necessary to set this env, otherwise the forkserver cannot communicate with the fuzzer".to_string()));
         }
 
+        let afl_debug = if let Ok(afl_debug) = env::var("AFL_DEBUG") {
+            if afl_debug != "1" && afl_debug != "0" {
+                return Err(Error::illegal_argument("AFL_DEBUG must be either 1 or 0"));
+            }
+            afl_debug == "1"
+        } else {
+            false
+        };
+
         let mut st_pipe = Pipe::new().unwrap();
         let mut ctl_pipe = Pipe::new().unwrap();
 
@@ -409,6 +429,7 @@ impl Forkserver {
             .env("LD_BIND_NOW", "1")
             .envs(envs)
             .setlimit(memlimit)
+            .set_coredump(afl_debug)
             .setsid()
             .setstdin(input_filefd, use_stdin)
             .setpipe(