libafl_libfuzzer fixes for port and fd allocation (#1525)

* better port and fd handling

* fix multitude of CI failures

libafl/src/monitors/tui/mod.rs

@@ -3,8 +3,8 @@
 use alloc::{boxed::Box, string::ToString};
 use std::{
     collections::VecDeque,
-    fmt::Write,
+    fmt::Write as _,
-    io::{self, BufRead},
+    io::{self, BufRead, Write},
     panic,
     string::String,
     sync::{Arc, RwLock},
@@ -434,7 +434,33 @@ impl TuiMonitor {
     #[must_use]
     pub fn with_time(tui_ui: TuiUI, start_time: Duration) -> Self {
         let context = Arc::new(RwLock::new(TuiContext::new(start_time)));
-        run_tui_thread(context.clone(), Duration::from_millis(250), tui_ui);
+
+        enable_raw_mode().unwrap();
+        #[cfg(unix)]
+        {
+            use std::{
+                fs::File,
+                os::fd::{AsRawFd, FromRawFd},
+            };
+
+            let stdout = unsafe { libc::dup(io::stdout().as_raw_fd()) };
+            let stdout = unsafe { File::from_raw_fd(stdout) };
+            run_tui_thread(
+                context.clone(),
+                Duration::from_millis(250),
+                tui_ui,
+                move || stdout.try_clone().unwrap(),
+            );
+        }
+        #[cfg(not(unix))]
+        {
+            run_tui_thread(
+                context.clone(),
+                Duration::from_millis(250),
+                tui_ui,
+                io::stdout,
+            );
+        }
         Self {
             context,
             start_time,
@@ -528,11 +554,15 @@ impl TuiMonitor {
     }
 }
 
-fn run_tui_thread(context: Arc<RwLock<TuiContext>>, tick_rate: Duration, tui_ui: TuiUI) {
+fn run_tui_thread<W: Write + Send + Sync + 'static>(
+    context: Arc<RwLock<TuiContext>>,
+    tick_rate: Duration,
+    tui_ui: TuiUI,
+    stdout_provider: impl Send + Sync + 'static + Fn() -> W,
+) {
     thread::spawn(move || -> io::Result<()> {
         // setup terminal
-        let mut stdout = io::stdout();
+        let mut stdout = stdout_provider();
-        enable_raw_mode()?;
         execute!(stdout, EnterAlternateScreen, EnableMouseCapture)?;
 
         let backend = CrosstermBackend::new(stdout);
@@ -546,9 +576,10 @@ fn run_tui_thread(context: Arc<RwLock<TuiContext>>, tick_rate: Duration, tui_ui:
         // Catching panics when the main thread dies
         let old_hook = panic::take_hook();
         panic::set_hook(Box::new(move |panic_info| {
+            let mut stdout = stdout_provider();
             disable_raw_mode().unwrap();
             execute!(
-                io::stdout(),
+                stdout,
                 LeaveAlternateScreen,
                 DisableMouseCapture,
                 Show,

libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml

@@ -30,9 +30,9 @@ path = "src/lib.rs"
 crate-type = ["staticlib", "rlib"]
 
 [dependencies]
-libafl = { version = "0.11", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "errors_backtrace", "regex", "serdeany_autoreg", "tui_monitor"] }
+libafl = { path = "../../libafl", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "regex", "errors_backtrace", "serdeany_autoreg", "tui_monitor"] }
-libafl_bolts = { version = "0.11", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "errors_backtrace"] }
+libafl_bolts = { path = "../../libafl_bolts", default-features = false, features = ["std", "derive", "llmp_compression", "rand_trait", "serdeany_autoreg", "errors_backtrace"] }
-libafl_targets = { version = "0.11", features = ["sancov_8bit", "sancov_cmplog", "libfuzzer", "libfuzzer_oom", "libfuzzer_define_run_driver", "sanitizers_flags"] }
+libafl_targets = { path = "../../libafl_targets", features = ["sancov_8bit", "sancov_cmplog", "libfuzzer", "libfuzzer_oom", "libfuzzer_define_run_driver", "sanitizers_flags"] }
 
 ahash = { version = "0.8.3", default-features = false }
 libc = "0.2.139"
@@ -46,10 +46,12 @@ serde = { version = "1.0", default-features = false, features = ["alloc", "deriv
 bytecount = "0.6.3"
 
 # for identifying if we can grimoire-ify
-utf8-chars = "2.0.3"
+utf8-chars = "3.0.1"
+
+env_logger = "0.10"
 
 [build-dependencies]
-bindgen = "0.65.1"
+bindgen = "0.68.1"
 cc = { version = "1.0", features = ["parallel"] }
 
 [workspace]

libafl_libfuzzer/libafl_libfuzzer_runtime/src/fuzz.rs

@@ -1,15 +1,13 @@
 use core::ffi::c_int;
+#[cfg(unix)]
+use std::io::Write;
 use std::{
     fmt::Debug,
     fs::File,
     net::TcpListener,
+    str::FromStr,
     time::{SystemTime, UNIX_EPOCH},
 };
-#[cfg(unix)]
-use std::{
-    io::Write,
-    os::fd::{AsRawFd, FromRawFd, IntoRawFd},
-};
 
 use libafl::{
     corpus::Corpus,
@@ -37,6 +35,31 @@ use libafl_bolts::{
 
 use crate::{feedbacks::LibfuzzerCrashCauseMetadata, fuzz_with, options::LibfuzzerOptions};
 
+fn destroy_output_fds(options: &LibfuzzerOptions) {
+    #[cfg(unix)]
+    {
+        use std::os::fd::AsRawFd;
+
+        if options.tui() {
+            let file_null = File::open("/dev/null").unwrap();
+            unsafe {
+                libc::dup2(file_null.as_raw_fd(), 1);
+                libc::dup2(file_null.as_raw_fd(), 2);
+            }
+        } else if options.close_fd_mask() != 0 {
+            let file_null = File::open("/dev/null").unwrap();
+            unsafe {
+                if options.close_fd_mask() & 1 != 0 {
+                    libc::dup2(file_null.as_raw_fd(), 1);
+                }
+                if options.close_fd_mask() & 2 != 0 {
+                    libc::dup2(file_null.as_raw_fd(), 2);
+                }
+            }
+        }
+    }
+}
+
 fn do_fuzz<F, ST, E, S, EM>(
     options: &LibfuzzerOptions,
     fuzzer: &mut F,
@@ -93,6 +116,7 @@ fn fuzz_single_forking<M>(
 where
     M: Monitor + Debug,
 {
+    destroy_output_fds(options);
     fuzz_with!(options, harness, do_fuzz, |fuzz_single| {
         let (state, mgr): (
             Option<StdState<_, _, _, _>>,
@@ -109,24 +133,13 @@ where
                 }
             },
         };
-        #[cfg(unix)]
-        {
-            if options.close_fd_mask() != 0 {
-                let file_null = File::open("/dev/null")?;
-                unsafe {
-                    if options.close_fd_mask() & 1 != 0 {
-                        libc::dup2(file_null.as_raw_fd(), 1);
-                    }
-                    if options.close_fd_mask() & 2 != 0 {
-                        libc::dup2(file_null.as_raw_fd(), 2);
-                    }
-                }
-            }
-        }
         crate::start_fuzzing_single(fuzz_single, state, mgr)
     })
 }
 
+/// Communicate the selected port to subprocesses
+const PORT_PROVIDER_VAR: &str = "_LIBAFL_LIBFUZZER_FORK_PORT";
+
 fn fuzz_many_forking<M>(
     options: &LibfuzzerOptions,
     harness: &extern "C" fn(*const u8, usize) -> c_int,
@@ -137,12 +150,19 @@ fn fuzz_many_forking<M>(
 where
     M: Monitor + Clone + Debug,
 {
+    destroy_output_fds(options);
+    let broker_port = std::env::var(PORT_PROVIDER_VAR)
+        .map_err(Error::from)
+        .and_then(|s| u16::from_str(&s).map_err(Error::from))
+        .or_else(|_| {
+            TcpListener::bind("127.0.0.1:0").map(|sock| {
+                let port = sock.local_addr().unwrap().port();
+                std::env::set_var(PORT_PROVIDER_VAR, port.to_string());
+                port
+            })
+        })?;
     fuzz_with!(options, harness, do_fuzz, |mut run_client| {
         let cores = Cores::from((0..forks).collect::<Vec<_>>());
-        let broker_port = TcpListener::bind("127.0.0.1:0")?
-            .local_addr()
-            .unwrap()
-            .port();
 
         match Launcher::builder()
             .shmem_provider(shmem_provider)
@@ -164,6 +184,26 @@ where
     })
 }
 
+fn create_monitor_closure() -> impl Fn(String) + Clone {
+    #[cfg(unix)]
+    let stderr_fd =
+        std::os::fd::RawFd::from_str(&std::env::var(crate::STDERR_FD_VAR).unwrap()).unwrap(); // set in main
+    move |s| {
+        #[cfg(unix)]
+        {
+            use std::os::fd::FromRawFd;
+
+            // unfortunate requirement to meet Clone... thankfully, this does not
+            // generate effectively any overhead (no allocations, calls get merged)
+            let mut stderr = unsafe { File::from_raw_fd(stderr_fd) };
+            writeln!(stderr, "{s}").expect("Could not write to stderr???");
+            std::mem::forget(stderr); // do not close the descriptor!
+        }
+        #[cfg(not(unix))]
+        eprintln!("{s}");
+    }
+}
+
 pub fn fuzz(
     options: &LibfuzzerOptions,
     harness: &extern "C" fn(*const u8, usize) -> c_int,
@@ -174,37 +214,14 @@ pub fn fuzz(
             let monitor = TuiMonitor::new(TuiUI::new(options.fuzzer_name().to_string(), true));
             fuzz_many_forking(options, harness, shmem_provider, forks, monitor)
         } else if forks == 1 {
-            #[cfg(unix)]
-            let mut stderr = unsafe {
-                let new_fd = libc::dup(std::io::stderr().as_raw_fd());
-                File::from_raw_fd(new_fd)
-            };
             let monitor = MultiMonitor::with_time(
-                move |s| {
+                create_monitor_closure(),
-                    #[cfg(unix)]
-                    writeln!(stderr, "{s}").expect("Could not write to stderr???");
-                    #[cfg(not(unix))]
-                    eprintln!("{s}");
-                },
                 SystemTime::now().duration_since(UNIX_EPOCH).unwrap(),
             );
             fuzz_single_forking(options, harness, shmem_provider, monitor)
         } else {
-            #[cfg(unix)]
-            let stderr_fd = unsafe { libc::dup(std::io::stderr().as_raw_fd()) };
             let monitor = MultiMonitor::with_time(
-                move |s| {
+                create_monitor_closure(),
-                    #[cfg(unix)]
-                    {
-                        // unfortunate requirement to meet Clone... thankfully, this does not
-                        // generate effectively any overhead (no allocations, calls get merged)
-                        let mut stderr = unsafe { File::from_raw_fd(stderr_fd) };
-                        writeln!(stderr, "{s}").expect("Could not write to stderr???");
-                        let _ = stderr.into_raw_fd(); // discard the file without closing
-                    }
-                    #[cfg(not(unix))]
-                    eprintln!("{s}");
-                },
                 SystemTime::now().duration_since(UNIX_EPOCH).unwrap(),
             );
             fuzz_many_forking(options, harness, shmem_provider, forks, monitor)
@@ -216,8 +233,9 @@ pub fn fuzz(
         let monitor = TuiMonitor::new(TuiUI::new(options.fuzzer_name().to_string(), true));
         fuzz_many_forking(options, harness, shmem_provider, 1, monitor)
     } else {
+        destroy_output_fds(options);
         fuzz_with!(options, harness, do_fuzz, |fuzz_single| {
-            let mgr = SimpleEventManager::new(SimpleMonitor::new(|s| eprintln!("{s}")));
+            let mgr = SimpleEventManager::new(SimpleMonitor::new(create_monitor_closure()));
             crate::start_fuzzing_single(fuzz_single, None, mgr)
         })
     }

libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs

@@ -71,7 +71,9 @@
 #![allow(clippy::borrow_deref_ref)]
 
 use core::ffi::{c_char, c_int, CStr};
+use std::{fs::File, io::stderr, os::fd::RawFd};
 
+use env_logger::Target;
 use libafl::{
     inputs::{BytesInput, HasTargetBytes, Input},
     Error,
@@ -207,7 +209,7 @@ macro_rules! fuzz_with {
             let backtrace_observer = BacktraceObserver::new(
                 "BacktraceObserver",
                 unsafe { &mut BACKTRACE },
-                libafl::observers::HarnessType::InProcess,
+                if $options.forks().is_some() || $options.tui() { libafl::observers::HarnessType::Child } else { libafl::observers::HarnessType::InProcess }
             );
 
             // New maximization map feedback linked to the edges observer
@@ -528,6 +530,9 @@ extern "C" {
     fn libafl_targets_libfuzzer_init(argc: *mut c_int, argv: *mut *mut *const c_char) -> i32;
 }
 
+/// Communicate the stderr duplicated fd to subprocesses
+pub const STDERR_FD_VAR: &str = "_LIBAFL_LIBFUZZER_STDERR_FD";
+
 /// A method to start the fuzzer at a later point in time from a library.
 /// To quote the `libfuzzer` docs:
 /// > when it’s ready to start fuzzing, it can call `LLVMFuzzerRunDriver`, passing in the program arguments and a callback. This callback is invoked just like `LLVMFuzzerTestOneInput`, and has the same signature.
@@ -547,6 +552,29 @@ pub unsafe extern "C" fn LLVMFuzzerRunDriver(
         .as_ref()
         .expect("Illegal harness provided to libafl.");
 
+    // early duplicate the stderr fd so we can close it later for the target
+    #[cfg(unix)]
+    {
+        use std::{
+            os::fd::{AsRawFd, FromRawFd},
+            str::FromStr,
+        };
+
+        let stderr_fd = std::env::var(STDERR_FD_VAR)
+            .map_err(Error::from)
+            .and_then(|s| RawFd::from_str(&s).map_err(Error::from))
+            .unwrap_or_else(|_| {
+                let stderr = libc::dup(stderr().as_raw_fd());
+                std::env::set_var(STDERR_FD_VAR, stderr.to_string());
+                stderr
+            });
+        let stderr = File::from_raw_fd(stderr_fd);
+        env_logger::builder()
+            .parse_default_env()
+            .target(Target::Pipe(Box::new(stderr)))
+            .init();
+    }
+
     // it appears that no one, not even libfuzzer, uses this return value
     // https://github.com/llvm/llvm-project/blob/llvmorg-15.0.7/compiler-rt/lib/fuzzer/FuzzerDriver.cpp#L648
     libafl_targets_libfuzzer_init(argc, argv);