fixup: don't download unicode categories data (#1732)

* fixup: don't download unicode categories data

* whoops, document

* fmt

* ci

---------

Co-authored-by: toka <tokazerkje@outlook.com>

libafl/build.rs

@@ -5,10 +5,6 @@ use std::error::Error;
 fn main() -> Result<(), Box<dyn Error>> {
     println!("cargo:rerun-if-changed=build.rs");
     println!("cargo:rustc-cfg=nightly");
-    #[cfg(feature = "unicode")]
-    {
-        // build_unicode_property_map()?;
-    }
     Ok(())
 }
 
@@ -20,52 +16,5 @@ fn main() -> Result<(), Box<dyn Error>> {
         cfg!(all(not(docrs), not(feature = "nautilus"))),
         "The 'nautilus' feature of libafl requires a nightly compiler"
     );
-    #[cfg(feature = "unicode")]
-    {
-        // build_unicode_property_map()?;
-    }
-    Ok(())
-}
-
-#[cfg(feature = "unicode")]
-fn build_unicode_property_map() -> Result<(), Box<dyn Error>> {
-    use std::{
-        env,
-        fs::File,
-        io::{BufWriter, Write},
-        path::PathBuf,
-        process::{Command, Stdio},
-    };
-
-    let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
-    let ucd_dir = out_dir.join("ucd-dir");
-    let generated_file = out_dir.join("unicode_categories.rs");
-
-    std::fs::create_dir_all(&ucd_dir)?;
-
-    let zip_path = ucd_dir.join("ucd.zip");
-    let mut ucd_file = BufWriter::new(File::create(&zip_path)?);
-    for chunk in reqwest::blocking::get("https://www.unicode.org/Public/zipped/latest/UCD.zip")?
-        .bytes()?
-        .chunks(1 << 12)
-    {
-        ucd_file.write_all(chunk)?;
-    }
-    ucd_file.flush()?;
-    drop(ucd_file);
-
-    let mut zip_file = zip::ZipArchive::new(File::open(&zip_path)?)?;
-    zip_file.extract(&ucd_dir)?;
-    drop(zip_file);
-
-    std::fs::remove_file(zip_path)?;
-
-    let status = Command::new("ucd-generate")
-        .arg("general-category")
-        .arg(ucd_dir.as_os_str())
-        .stdout(Stdio::from(File::create(generated_file)?))
-        .status()?;
-    assert!(status.success());
-
     Ok(())
 }

libafl/src/mutators/scheduled.rs

@@ -92,7 +92,7 @@ where
     fn schedule(&self, state: &mut S, input: &I) -> MutationId;
 
     /// New default implementation for mutate.
-    /// Implementations must forward mutate() to this method
+    /// Implementations must forward `mutate()` to this method
     fn scheduled_mutate(
         &mut self,
         state: &mut S,

libafl/src/mutators/string.rs

@@ -19,6 +19,12 @@ use crate::{
     state::{HasCorpus, HasMaxSize, HasMetadata, HasRand},
 };
 
+/// Unicode category data, as used by string analysis and mutators.
+#[allow(unused)]
+#[allow(missing_docs)]
+#[allow(clippy::redundant_static_lifetimes)]
+pub mod unicode_categories;
+
 /// Input which contains the context necessary to perform unicode mutations
 pub type UnicodeInput = (BytesInput, StringIdentificationMetadata);
 
@@ -267,15 +273,6 @@ fn rand_replace_range<S: HasRand + HasMaxSize, F: Fn(&mut S) -> char>(
     MutationResult::Mutated
 }
 
-/// Unicode category data, as used by string analysis and mutators.
-pub mod unicode_categories {
-    #![allow(unused)]
-    #![allow(missing_docs)]
-    #![allow(clippy::redundant_static_lifetimes)]
-
-    include!(concat!(env!("OUT_DIR"), "/unicode_categories.rs"));
-}
-
 /// Mutator which randomly replaces a randomly selected range of bytes with bytes that preserve the
 /// range's category
 #[derive(Debug, Default)]

libafl/src/observers/map.rs

@@ -105,7 +105,7 @@ pub trait MapObserver: HasLen + Named + Serialize + serde::de::DeserializeOwned
     /// Compute the hash of the map
     fn hash(&self) -> u64;
 
-    /// Get the initial value for reset()
+    /// Get the initial value for `reset()`
     fn initial(&self) -> Self::Entry;
 
     /// Reset the map

libafl/src/stages/colorization.rs

@@ -1,4 +1,4 @@
-//! The colorization stage from colorization() in afl++
+//! The colorization stage from `colorization()` in afl++
 use alloc::{
     collections::binary_heap::BinaryHeap,
     string::{String, ToString},