diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 489dd9ab5f..919587da2d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -22,9 +22,13 @@ jobs: toolchain: nightly - uses: Swatinem/rust-cache@v1 - name: install mdbook - run: cargo install mdbook + uses: baptiste0928/cargo-install@v1.3.0 + with: + crate: mdbook - name: install linkcheck - run: cargo install mdbook-linkcheck + uses: baptiste0928/cargo-install@v1.3.0 + with: + crate: mdbook-linkcheck - uses: actions/checkout@v2 - name: Build libafl debug run: cargo build -p libafl @@ -51,8 +55,10 @@ jobs: - uses: Swatinem/rust-cache@v1 - name: set mold linker as default linker uses: rui314/setup-mold@v1 - - name: Install deps - run: sudo apt-get install -y llvm llvm-dev clang ninja-build clang-format-13 shellcheck gcc-arm-linux-gnueabi g++-arm-linux-gnueabi + - name: Install and cache deps + uses: awalsh128/cache-apt-pkgs-action@v1.1.0 + with: + packages: llvm llvm-dev clang ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi - name: get clang version run: command -v llvm-config && clang -v - name: Install cargo-hack @@ -60,6 +66,27 @@ jobs: - name: Add nightly rustfmt and clippy run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - uses: actions/checkout@v2 + + # ---- format check ---- + # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately + - name: Check pcguard edges + run: cargo check --features=sancov_pcguard_edges + - name: Format + run: cargo fmt -- --check + - name: Run clang-format style check for C/C++ programs. + run: clang-format-13 -n -Werror --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c' | grep -v 'QEMU-Nyx') + - name: run shellcheck + run: shellcheck ./scripts/*.sh + - name: Run clippy + run: ./scripts/clippy.sh + + # ---- doc check ---- + - name: Build Docs + run: cargo doc + - name: Test Docs + run: cargo +nightly test --doc --all-features + + # ---- build and feature check ---- - name: Run a normal build run: cargo build --verbose # cargo-hack tests/checks each crate in the workspace @@ -68,26 +95,9 @@ jobs: # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs - name: Check each feature # Skipping python as it has to be built with the `maturin` tool - run: cargo hack check --feature-powerset --depth=2 --exclude-features=agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386 --no-dev-deps - # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately - - name: Check pcguard edges - run: cargo check --features=sancov_pcguard_edges + run: cargo hack check --feature-powerset --depth=2 --exclude-features=agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386 --no-dev-deps - name: Build examples run: cargo build --examples --verbose - - uses: actions/checkout@v2 - - name: Format - run: cargo fmt -- --check - - name: Run clang-format style check for C/C++ programs. - run: clang-format-13 -n -Werror --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c') - - name: run shellcheck - run: shellcheck ./scripts/*.sh - - uses: actions/checkout@v2 - - name: Build Docs - run: cargo doc - - name: Test Docs - run: cargo +nightly test --doc --all-features - - name: Run clippy - run: ./scripts/clippy.sh ubuntu-concolic: runs-on: ubuntu-latest @@ -122,9 +132,6 @@ jobs: run: cd ./bindings/pylibafl && maturin build fuzzers: - env: - CC: ccache clang # use ccache in default - CXX: ccache clang # use ccache in default strategy: matrix: os: [ubuntu-latest, macos-latest] @@ -137,11 +144,6 @@ jobs: - name: set mold linker as default linker if: runner.os == 'Linux' # mold only support linux until now uses: rui314/setup-mold@v1 - - name: add ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ github.job }}-${{ matrix.os }} - max-size: 2000M - name: enable mult-thread for `make` run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)" - uses: Swatinem/rust-cache@v1 @@ -149,19 +151,25 @@ jobs: run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - uses: lyricwulf/abc@v1 with: - linux: llvm llvm-dev clang nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi + # todo: remove afl++-clang when nyx support samcov_pcguard + linux: llvm llvm-dev clang nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libgtk-3-dev afl++-clang pax-utils # update bash for macos to support `declare -A` command` macos: llvm libpng nasm coreutils z3 bash + - name: pip install + run: python3 -m pip install msgpack jinja2 - name: install cargo-make - run: cargo install --force cargo-make + uses: baptiste0928/cargo-install@v1.3.0 + with: + crate: cargo-make - uses: actions/checkout@v2 + with: + submodules: true # recursively checkout submodules - name: Build and run example fuzzers if: runner.os == 'Linux' run: ./scripts/test_all_fuzzers.sh - name: Build and run example fuzzers if: runner.os == 'macOS' # use bash v4 run: /usr/local/bin/bash ./scripts/test_all_fuzzers.sh - - run: ccache --show-stats nostd-build: runs-on: ubuntu-latest @@ -229,7 +237,7 @@ jobs: - name: Add nightly rustfmt and clippy run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - name: Install deps - run: brew install z3 + run: brew install z3 gtk+3 - uses: actions/checkout@v2 - name: MacOS Build run: cargo build --verbose diff --git a/.gitignore b/.gitignore index f43a82e582..c79b27f4ef 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ __pycache__ *.lafl_lock *atomic_file_testfile* +**/libxml2 +**/corpus_discovered +**/libxml2-*.tar.gz \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 47d2bed34e..6f398f40b2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "libafl_concolic/symcc_runtime/symcc"] path = libafl_concolic/symcc_runtime/symcc url = https://github.com/AFLplusplus/symcc.git +[submodule "libafl_nyx/QEMU-Nyx"] + path = libafl_nyx/QEMU-Nyx + url = https://github.com/nyx-fuzz/QEMU-Nyx.git +[submodule "libafl_nyx/packer"] + path = libafl_nyx/packer + url = https://github.com/syheliel/packer.git diff --git a/Cargo.toml b/Cargo.toml index aeb96ea7b4..25915ea8b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "libafl_frida", "libafl_qemu", "libafl_sugar", + "libafl_nyx", "libafl_concolic/symcc_runtime", "libafl_concolic/symcc_libafl", "libafl_concolic/test/dump_constraints", diff --git a/Dockerfile b/Dockerfile index 27764ddd20..75aa3e707f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,6 +63,9 @@ COPY scripts/dummy.rs libafl_concolic/symcc_runtime/src/lib.rs COPY libafl_concolic/symcc_libafl/Cargo.toml libafl_concolic/symcc_libafl/ COPY scripts/dummy.rs libafl_concolic/symcc_libafl/src/lib.rs +COPY libafl_nyx/Cargo.toml libafl_nyx/build.rs libafl_nyx/ +COPY scripts/dummy.rs libafl_nyx/src/lib.rs + COPY utils utils RUN cargo build && cargo build --release @@ -94,6 +97,8 @@ RUN touch libafl_frida/src/lib.rs COPY libafl_concolic/symcc_libafl libafl_concolic/symcc_libafl COPY libafl_concolic/symcc_runtime libafl_concolic/symcc_runtime COPY libafl_concolic/test libafl_concolic/test +COPY libafl_nyx/src libafl_nyx/src +RUN touch libafl_nyx/src/lib.rs RUN cargo build && cargo build --release # Copy fuzzers over diff --git a/docs/src/core_concepts/launcher.md b/docs/src/core_concepts/launcher.md new file mode 100644 index 0000000000..12e57be380 --- /dev/null +++ b/docs/src/core_concepts/launcher.md @@ -0,0 +1,10 @@ +# Launcher + +Launcher is used to launch multiple fuzzer instances in parallel in one click. On `Unix` systems, Launcher will use `fork` if the `fork` feature is enabled. Else, it will start subsequent nodes with the same command line, and will set special `env` variables accordingly. + +To use launcher, first you need to write an anonymous function `let mut run_client = |state: Option<_>, mut mgr, _core_id|{}`, which uses three parameters to create individual fuzzer. Then you can specify the `shmem_provider`,`broker_port`,`monitor`,`cores` and other stuff through `Launcher::builder()`: +1. To connect multiple nodes together via TCP, you can use the `remote_broker_addr`. this requires the `llmp_bind_public` compile-time feature for `LibAFL`. +2. To use multiple launchers for individual configurations, you can set `spawn_broker` to `false` on all but one. +3. Launcher will not select the cores automatically, so you need to specify the `cores` that you want. + +For more examples, you can check out `qemu_launcher` and `libfuzzer_libpng_launcher` in `./fuzzers/`. \ No newline at end of file diff --git a/fuzzers/nyx_libxml2_parallel/Cargo.toml b/fuzzers/nyx_libxml2_parallel/Cargo.toml new file mode 100644 index 0000000000..7136d6ab1e --- /dev/null +++ b/fuzzers/nyx_libxml2_parallel/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "nyx_libxml2_parallel" +version = "0.1.0" +edition = "2021" +default-run = "nyx_libxml2_parallel" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libafl = {path = "../../libafl"} +libafl_cc = {path = "../../libafl_cc"} +libafl_nyx = {path = "../../libafl_nyx"} + +[profile.release] +codegen-units = 1 +opt-level = 3 diff --git a/fuzzers/nyx_libxml2_parallel/Makefile.toml b/fuzzers/nyx_libxml2_parallel/Makefile.toml new file mode 100644 index 0000000000..ee7fc262c6 --- /dev/null +++ b/fuzzers/nyx_libxml2_parallel/Makefile.toml @@ -0,0 +1,57 @@ + +# Variables +[env] +FUZZER_NAME='nyx_libxml2_parallel' +PROJECT_DIR = { script = ["pwd"] } + +[config] +skip_core_tasks = true # skip `cargo test` to avoid error + +[tasks.unsupported] +script_runner="@shell" +script=''' +echo "Cargo-make not integrated yet on this platform" +''' + +[tasks.build] +dependencies = [ "libxml2" ] + +[tasks.libxml2] +linux_alias = "libxml2_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.libxml2_unix] +# condition = { files_not_exist = ["./libxml2"]} +script_runner="@shell" +script=''' +./setup_libxml2.sh +''' + +# Run the fuzzer +[tasks.run] +linux_alias = "run_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.run_unix] +script_runner = "@shell" +script=''' +cargo run +''' +dependencies = [ "libxml2" ] + +# Clean up +[tasks.clean] +linux_alias = "clean_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.clean_unix] +# Disable default `clean` definition +clear = true +script_runner="@shell" +script=''' +make -C ./libxml2 clean +cargo clean +''' diff --git a/fuzzers/nyx_libxml2_parallel/README.md b/fuzzers/nyx_libxml2_parallel/README.md new file mode 100644 index 0000000000..5a7bdc49a3 --- /dev/null +++ b/fuzzers/nyx_libxml2_parallel/README.md @@ -0,0 +1,15 @@ +this example shows to use `libafl_nyx` to fuzz `libxml2` + +# requirement +the following command will: +1. run `cargo build --release` to generate `libafl_cc`,`libafl_cxx` +2. download and extract `libxml2` +3. instruct `libxml2` using `libafl_cc` and `libafl_cxx` +4. prepare nyx shared dir and config file at `/tmp/nyx_libxml2` +5. open kvm support +``` +./setup_libxml2.sh +``` + +# run the fuzzer +use `cargo make run` to run the fuzzer. If you have setup all environment, you can use `cargo run` directly. diff --git a/fuzzers/nyx_libxml2_parallel/setup_libxml2.sh b/fuzzers/nyx_libxml2_parallel/setup_libxml2.sh new file mode 100755 index 0000000000..9100a0e7fc --- /dev/null +++ b/fuzzers/nyx_libxml2_parallel/setup_libxml2.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# cargo build --release +# PWD=$(pwd) +# export CC="$PWD/target/release/libafl_cc" +# export CXX="$PWD/target/release/libafl_cxx" + +export CC=afl-clang-fast +export CXX=afl-clang-fast++ +curl -C - https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.14/libxml2-v2.9.14.tar.gz --output libxml2-v2.9.14.tar.gz +tar -xf ./libxml2-v2.9.14.tar.gz --transform s/libxml2-v2.9.14/libxml2/ || exit +cd ./libxml2/ || exit +./autogen.sh --enable-shared=no || exit +make -j || exit +cd - || exit +python3 "../../libafl_nyx/packer/packer/nyx_packer.py" \ + ./libxml2/xmllint \ + /tmp/nyx_libxml2 \ + afl \ + instrumentation \ + -args "/tmp/input" \ + -file "/tmp/input" \ + --fast_reload_mode \ + --purge || exit + +python3 ../../libafl_nyx/packer/packer/nyx_config_gen.py /tmp/nyx_libxml2/ Kernel || exit +sudo modprobe -r kvm-intel # or kvm-amd for AMD +sudo modprobe -r kvm +sudo modprobe kvm enable_vmware_backdoor=y +sudo modprobe kvm-intel diff --git a/fuzzers/nyx_libxml2_parallel/src/bin/libafl_cc.rs b/fuzzers/nyx_libxml2_parallel/src/bin/libafl_cc.rs new file mode 100644 index 0000000000..018314e9b7 --- /dev/null +++ b/fuzzers/nyx_libxml2_parallel/src/bin/libafl_cc.rs @@ -0,0 +1,34 @@ +use libafl_cc::{ClangWrapper, CompilerWrapper}; +use std::env; + +pub fn main() { + let args: Vec = env::args().collect(); + if args.len() > 1 { + let mut dir = env::current_exe().unwrap(); + let wrapper_name = dir.file_name().unwrap().to_str().unwrap(); + + let is_cpp = match wrapper_name[wrapper_name.len()-2..].to_lowercase().as_str() { + "cc" => false, + "++" | "pp" | "xx" => true, + _ => panic!("Could not figure out if c or c++ warpper was called. Expected {:?} to end with c or cxx", dir), + }; + + dir.pop(); + + let mut cc = ClangWrapper::new(); + if let Some(code) = cc + .cpp(is_cpp) + // silence the compiler wrapper output, needed for some configure scripts. + .silence(true) + .parse_args(&args) + .expect("Failed to parse the command line") + .add_arg("-fsanitize-coverage=trace-pc-guard") + .run() + .expect("Failed to run the wrapped compiler") + { + std::process::exit(code); + } + } else { + panic!("LibAFL CC: No Arguments given"); + } +} diff --git a/fuzzers/nyx_libxml2_parallel/src/bin/libafl_cxx.rs b/fuzzers/nyx_libxml2_parallel/src/bin/libafl_cxx.rs new file mode 100644 index 0000000000..dabd22971a --- /dev/null +++ b/fuzzers/nyx_libxml2_parallel/src/bin/libafl_cxx.rs @@ -0,0 +1,5 @@ +pub mod libafl_cc; + +fn main() { + libafl_cc::main(); +} diff --git a/fuzzers/nyx_libxml2_parallel/src/main.rs b/fuzzers/nyx_libxml2_parallel/src/main.rs new file mode 100644 index 0000000000..b2f7732167 --- /dev/null +++ b/fuzzers/nyx_libxml2_parallel/src/main.rs @@ -0,0 +1,104 @@ +use std::path::{Path, PathBuf}; + +use libafl::{ + bolts::{ + core_affinity::Cores, + launcher::Launcher, + rands::{RandomSeed, StdRand}, + shmem::{ShMemProvider, StdShMemProvider}, + tuples::tuple_list, + }, + corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus, Testcase}, + events::EventConfig, + feedbacks::{CrashFeedback, MaxMapFeedback}, + inputs::BytesInput, + monitors::MultiMonitor, + mutators::{havoc_mutations, StdScheduledMutator}, + observers::StdMapObserver, + schedulers::RandScheduler, + stages::StdMutationalStage, + state::StdState, + Error, Fuzzer, StdFuzzer, +}; +use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper}; + +fn main() { + let shmem_provider = StdShMemProvider::new().expect("Failed to init shared memory"); + let broker_port = 7777; + + let monitor = MultiMonitor::new(|s| println!("{}", s)); + + let cores = Cores::all().expect("unable to get all core id"); + let parent_cpu_id = cores + .ids + .first() + .expect("unable to get first core id") + .clone(); + + // region: fuzzer start function + let mut run_client = |state: Option<_>, mut restarting_mgr, _core_id: usize| { + // nyx shared dir, created by nyx-fuzz/packer/packer/nyx_packer.py + let share_dir = Path::new("/tmp/nyx_libxml2/"); + let cpu_id = _core_id as u32; + let parallel_mode = true; + // nyx stuff + let mut helper = NyxHelper::new( + share_dir, + cpu_id, + true, + parallel_mode, + Some(parent_cpu_id.id as u32), + ) + .unwrap(); + let trace_bits = + unsafe { std::slice::from_raw_parts_mut(helper.trace_bits, helper.map_size) }; + let observer = StdMapObserver::new("trace", trace_bits); + + let input = BytesInput::new(b"22".to_vec()); + let rand = StdRand::new(); + let mut corpus = CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(); + corpus + .add(Testcase::new(input)) + .expect("error in adding corpus"); + let solutions = OnDiskCorpus::::new(PathBuf::from("./crashes")).unwrap(); + + // libafl stuff + let mut feedback = MaxMapFeedback::new(&observer); + let mut objective = CrashFeedback::new(); + let scheduler = RandScheduler::new(); + let mut executor = NyxExecutor::new(&mut helper, tuple_list!(observer)).unwrap(); + + // If not restarting, create a State from scratch + let mut state = state.unwrap_or_else(|| { + StdState::new(rand, corpus, solutions, &mut feedback, &mut objective).unwrap() + }); + + println!("We're a client, let's fuzz :)"); + let mutator = StdScheduledMutator::new(havoc_mutations()); + let mut stages = tuple_list!(StdMutationalStage::new(mutator)); + + // A fuzzer with feedbacks and a corpus scheduler + let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); + + fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut restarting_mgr)?; + Ok(()) + }; + + match Launcher::builder() + .shmem_provider(shmem_provider) + .configuration(EventConfig::from_name("default")) + .monitor(monitor) + .run_client(&mut run_client) + .cores(&cores) + .broker_port(broker_port) + // .stdout_file(Some("/dev/null")) + .build() + .launch() + { + Ok(()) => (), + Err(Error::ShuttingDown) => println!("Fuzzing stopped by user. Good bye."), + Err(err) => panic!("Failed to run launcher: {:?}", err), + } + + // endregion +} diff --git a/fuzzers/nyx_libxml2_standalone/Cargo.toml b/fuzzers/nyx_libxml2_standalone/Cargo.toml new file mode 100644 index 0000000000..1a861a8f7a --- /dev/null +++ b/fuzzers/nyx_libxml2_standalone/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "nyx_libxml2_standalone" +version = "0.1.0" +edition = "2021" +default-run = "nyx_libxml2_standalone" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libafl = {path = "../../libafl"} +libafl_cc = {path = "../../libafl_cc"} +libafl_nyx = {path = "../../libafl_nyx"} + +[profile.release] +codegen-units = 1 +opt-level = 3 diff --git a/fuzzers/nyx_libxml2_standalone/Makefile.toml b/fuzzers/nyx_libxml2_standalone/Makefile.toml new file mode 100644 index 0000000000..ee79b6703c --- /dev/null +++ b/fuzzers/nyx_libxml2_standalone/Makefile.toml @@ -0,0 +1,57 @@ + +# Variables +[env] +FUZZER_NAME='nyx_libxml2_standalone' +PROJECT_DIR = { script = ["pwd"] } + +[config] +skip_core_tasks = true # skip `cargo test` to avoid error + +[tasks.unsupported] +script_runner="@shell" +script=''' +echo "Cargo-make not integrated yet on this platform" +''' + +[tasks.build] +dependencies = [ "libxml2" ] + +[tasks.libxml2] +linux_alias = "libxml2_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.libxml2_unix] +# condition = { files_not_exist = ["./libxml2"]} +script_runner="@shell" +script=''' +./setup_libxml2.sh +''' + +# Run the fuzzer +[tasks.run] +linux_alias = "run_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.run_unix] +script_runner = "@shell" +script=''' +cargo run +''' +dependencies = [ "libxml2" ] + +# Clean up +[tasks.clean] +linux_alias = "clean_unix" +mac_alias = "unsupported" +windows_alias = "unsupported" + +[tasks.clean_unix] +# Disable default `clean` definition +clear = true +script_runner="@shell" +script=''' +make -C ./libxml2 clean +cargo clean +''' diff --git a/fuzzers/nyx_libxml2_standalone/README.md b/fuzzers/nyx_libxml2_standalone/README.md new file mode 100644 index 0000000000..5a7bdc49a3 --- /dev/null +++ b/fuzzers/nyx_libxml2_standalone/README.md @@ -0,0 +1,15 @@ +this example shows to use `libafl_nyx` to fuzz `libxml2` + +# requirement +the following command will: +1. run `cargo build --release` to generate `libafl_cc`,`libafl_cxx` +2. download and extract `libxml2` +3. instruct `libxml2` using `libafl_cc` and `libafl_cxx` +4. prepare nyx shared dir and config file at `/tmp/nyx_libxml2` +5. open kvm support +``` +./setup_libxml2.sh +``` + +# run the fuzzer +use `cargo make run` to run the fuzzer. If you have setup all environment, you can use `cargo run` directly. diff --git a/fuzzers/nyx_libxml2_standalone/setup_libxml2.sh b/fuzzers/nyx_libxml2_standalone/setup_libxml2.sh new file mode 100755 index 0000000000..9100a0e7fc --- /dev/null +++ b/fuzzers/nyx_libxml2_standalone/setup_libxml2.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# cargo build --release +# PWD=$(pwd) +# export CC="$PWD/target/release/libafl_cc" +# export CXX="$PWD/target/release/libafl_cxx" + +export CC=afl-clang-fast +export CXX=afl-clang-fast++ +curl -C - https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.14/libxml2-v2.9.14.tar.gz --output libxml2-v2.9.14.tar.gz +tar -xf ./libxml2-v2.9.14.tar.gz --transform s/libxml2-v2.9.14/libxml2/ || exit +cd ./libxml2/ || exit +./autogen.sh --enable-shared=no || exit +make -j || exit +cd - || exit +python3 "../../libafl_nyx/packer/packer/nyx_packer.py" \ + ./libxml2/xmllint \ + /tmp/nyx_libxml2 \ + afl \ + instrumentation \ + -args "/tmp/input" \ + -file "/tmp/input" \ + --fast_reload_mode \ + --purge || exit + +python3 ../../libafl_nyx/packer/packer/nyx_config_gen.py /tmp/nyx_libxml2/ Kernel || exit +sudo modprobe -r kvm-intel # or kvm-amd for AMD +sudo modprobe -r kvm +sudo modprobe kvm enable_vmware_backdoor=y +sudo modprobe kvm-intel diff --git a/fuzzers/nyx_libxml2_standalone/src/bin/libafl_cc.rs b/fuzzers/nyx_libxml2_standalone/src/bin/libafl_cc.rs new file mode 100644 index 0000000000..018314e9b7 --- /dev/null +++ b/fuzzers/nyx_libxml2_standalone/src/bin/libafl_cc.rs @@ -0,0 +1,34 @@ +use libafl_cc::{ClangWrapper, CompilerWrapper}; +use std::env; + +pub fn main() { + let args: Vec = env::args().collect(); + if args.len() > 1 { + let mut dir = env::current_exe().unwrap(); + let wrapper_name = dir.file_name().unwrap().to_str().unwrap(); + + let is_cpp = match wrapper_name[wrapper_name.len()-2..].to_lowercase().as_str() { + "cc" => false, + "++" | "pp" | "xx" => true, + _ => panic!("Could not figure out if c or c++ warpper was called. Expected {:?} to end with c or cxx", dir), + }; + + dir.pop(); + + let mut cc = ClangWrapper::new(); + if let Some(code) = cc + .cpp(is_cpp) + // silence the compiler wrapper output, needed for some configure scripts. + .silence(true) + .parse_args(&args) + .expect("Failed to parse the command line") + .add_arg("-fsanitize-coverage=trace-pc-guard") + .run() + .expect("Failed to run the wrapped compiler") + { + std::process::exit(code); + } + } else { + panic!("LibAFL CC: No Arguments given"); + } +} diff --git a/fuzzers/nyx_libxml2_standalone/src/bin/libafl_cxx.rs b/fuzzers/nyx_libxml2_standalone/src/bin/libafl_cxx.rs new file mode 100644 index 0000000000..dabd22971a --- /dev/null +++ b/fuzzers/nyx_libxml2_standalone/src/bin/libafl_cxx.rs @@ -0,0 +1,5 @@ +pub mod libafl_cc; + +fn main() { + libafl_cc::main(); +} diff --git a/fuzzers/nyx_libxml2_standalone/src/main.rs b/fuzzers/nyx_libxml2_standalone/src/main.rs new file mode 100644 index 0000000000..21022096bf --- /dev/null +++ b/fuzzers/nyx_libxml2_standalone/src/main.rs @@ -0,0 +1,60 @@ +use std::path::{Path, PathBuf}; + +use libafl::{ + bolts::{ + rands::{RandomSeed, StdRand}, + tuples::tuple_list, + }, + corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus, Testcase}, + events::SimpleEventManager, + feedbacks::{CrashFeedback, MaxMapFeedback}, + inputs::BytesInput, + monitors::tui::TuiMonitor, + mutators::{havoc_mutations, StdScheduledMutator}, + observers::StdMapObserver, + schedulers::RandScheduler, + stages::StdMutationalStage, + state::StdState, + Fuzzer, StdFuzzer, +}; +use libafl_nyx::{executor::NyxExecutor, helper::NyxHelper}; + +fn main() { + let share_dir = Path::new("/tmp/nyx_libxml2/"); + let cpu_id = 0; + let parallel_mode = false; + + // nyx stuff + let mut helper = NyxHelper::new(share_dir, cpu_id, true, parallel_mode, None).unwrap(); + let trace_bits = unsafe { std::slice::from_raw_parts_mut(helper.trace_bits, helper.map_size) }; + let observer = StdMapObserver::new("trace", trace_bits); + + let input = BytesInput::new(b"22".to_vec()); + let rand = StdRand::new(); + let mut corpus = CachedOnDiskCorpus::new(PathBuf::from("./corpus_discovered"), 64).unwrap(); + corpus + .add(Testcase::new(input)) + .expect("error in adding corpus"); + let solutions = OnDiskCorpus::new(PathBuf::from("./crashes")).unwrap(); + + // libafl stuff + let mut feedback = MaxMapFeedback::new(&observer); + let mut objective = CrashFeedback::new(); + let mut state = StdState::new(rand, corpus, solutions, &mut feedback, &mut objective).unwrap(); + let scheduler = RandScheduler::new(); + let mut fuzzer = StdFuzzer::new(scheduler, feedback, objective); + + // switch monitor if you want + // let monitor = SimpleMonitor::new(|x|-> () {println!("{}",x)}); + let monitor = TuiMonitor::new("test_fuzz".to_string(), true); + + let mut mgr: SimpleEventManager = SimpleEventManager::new(monitor); + let mut executor = NyxExecutor::new(&mut helper, tuple_list!(observer)).unwrap(); + let mutator = StdScheduledMutator::new(havoc_mutations()); + let mut stages = tuple_list!(StdMutationalStage::new(mutator)); + + // start fuzz + fuzzer + .fuzz_loop(&mut stages, &mut executor, &mut state, &mut mgr) + .expect("error when fuzz"); +} diff --git a/libafl_nyx/.clang-format b/libafl_nyx/.clang-format new file mode 100644 index 0000000000..15f9b6fc9f --- /dev/null +++ b/libafl_nyx/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: Never \ No newline at end of file diff --git a/libafl_nyx/Cargo.toml b/libafl_nyx/Cargo.toml new file mode 100644 index 0000000000..71b7ae9f7f --- /dev/null +++ b/libafl_nyx/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "libafl_nyx" +version = "0.1.0" +edition = "2021" +authors = ["syheliel "] +description = "libafl using nyx, only avaliable on linux" +documentation = "https://docs.rs/libafl_nyx" +repository = "https://github.com/AFLplusplus/LibAFL/" +readme = "../README.md" +license = "MIT OR Apache-2.0" +keywords = ["fuzzing", "testing", "security"] +categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[target.'cfg(target_os = "linux")'.dependencies] +libnyx = {git = "https://github.com/nyx-fuzz/libnyx.git",rev = "acaf7f6"} +libafl = { path = "../libafl", version = "0.8.0", features = ["std", "libafl_derive", "frida_cli" ]} +libafl_targets = { path = "../libafl_targets", version = "0.8.0", features = ["std", "sancov_cmplog"] } diff --git a/libafl_nyx/QEMU-Nyx b/libafl_nyx/QEMU-Nyx new file mode 160000 index 0000000000..4df041cd8e --- /dev/null +++ b/libafl_nyx/QEMU-Nyx @@ -0,0 +1 @@ +Subproject commit 4df041cd8e886e5dee07a58bb3508860918ef908 diff --git a/libafl_nyx/README.md b/libafl_nyx/README.md new file mode 100644 index 0000000000..58e5e224c8 --- /dev/null +++ b/libafl_nyx/README.md @@ -0,0 +1,5 @@ +`libafl_nyx` is the `libafl`'s front-end for nyx fuzzer. This crate provides both the standalone mode and parallel mode: +- In standalone mode, no VM snapshot is serialized and stored in the working directory. That might be useful if you really want to run the fuzzer with only one process (meaning one VM). +- In parallel mode, the first fuzzer process (parent) has to create the VM snapshot while all other child processes will wait for the snapshot files to appear in the working directory. + +In order to use this crate, you need to specify the shared directory and mode in `NyxHelper`, then use `NyxExecutor`. For more details, please see `./fuzzers/nyx_libxml2_standalone` and `./fuzzers/nyx_libxml2_parallel`. \ No newline at end of file diff --git a/libafl_nyx/build.rs b/libafl_nyx/build.rs new file mode 100644 index 0000000000..78867d3e57 --- /dev/null +++ b/libafl_nyx/build.rs @@ -0,0 +1,18 @@ +use std::process::Command; + +fn main() { + if cfg!(target_os = "linux") && cfg!(target_arch = "x86_64") { + println!("cargo:rerun-if-changed=build.rs"); + // let output = Command::new("./build_nyx_support.sh").output().expect("can't run ./build_nyx_support.sh"); + let status = Command::new("./build_nyx_support.sh") + .status() + .expect("can't run ./build_nyx_support.sh"); + if status.success() { + println!("success to run ./build_nyx_support.sh"); + } else { + panic!("fail to run ./build_nyx_support.sh"); + } + } else { + println!("cargo:warning=NYX node is only avaliable on x64 Linux"); + } +} diff --git a/libafl_nyx/build_nyx_support.sh b/libafl_nyx/build_nyx_support.sh new file mode 100755 index 0000000000..bb3f00be4e --- /dev/null +++ b/libafl_nyx/build_nyx_support.sh @@ -0,0 +1,39 @@ +#!/bin/bash +echo "=================================================" +echo " Nyx build script" +echo "=================================================" +echo + + +echo "[*] Making sure all Nyx is checked out" + +git status 1>/dev/null 2>/dev/null + +git submodule init || exit 1 +echo "[*] initializing QEMU-Nyx submodule" +git submodule update ./QEMU-Nyx 2>/dev/null # ignore errors +echo "[*] initializing packer submodule" +git submodule update ./packer 2>/dev/null # ignore errors + + +test -e packer/.git || { echo "[-] packer not checked out, please install git or check your internet connection." ; exit 1 ; } +test -e QEMU-Nyx/.git || { echo "[-] QEMU-Nyx not checked out, please install git or check your internet connection." ; exit 1 ; } + +echo "[*] checking packer init.cpio.gz ..." +if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then + cd packer/linux_initramfs/ || return + sh pack.sh || exit 1 + cd ../../ +fi + + +echo "[*] Checking QEMU-Nyx ..." +if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then + cd QEMU-Nyx/ || return + ./compile_qemu_nyx.sh static || exit 1 + cd .. +fi + +echo "[+] All done for nyx_mode, enjoy!" + +exit 0 diff --git a/libafl_nyx/packer b/libafl_nyx/packer new file mode 160000 index 0000000000..86b159bafc --- /dev/null +++ b/libafl_nyx/packer @@ -0,0 +1 @@ +Subproject commit 86b159bafc0b2ba8feeaa8761a45b6201d34084f diff --git a/libafl_nyx/src/executor.rs b/libafl_nyx/src/executor.rs new file mode 100644 index 0000000000..538a778b55 --- /dev/null +++ b/libafl_nyx/src/executor.rs @@ -0,0 +1,96 @@ +use std::{fmt::Debug, marker::PhantomData}; + +use libafl::{ + executors::{Executor, ExitKind, HasObservers}, + inputs::{HasBytesVec, Input}, + observers::ObserversTuple, + Error, +}; +use libnyx::NyxReturnValue; + +use crate::helper::NyxHelper; + +/// executor for nyx standalone mode +pub struct NyxExecutor<'a, I, S, OT> { + /// implement nyx function + pub helper: &'a mut NyxHelper, + /// observers + observers: OT, + /// phantom data to keep generic type + phantom: PhantomData<(I, S)>, +} + +impl<'a, I, S, OT> Debug for NyxExecutor<'a, I, S, OT> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NyxInprocessExecutor") + .field("helper", &self.helper) + .finish() + } +} + +impl<'a, EM, I, S, Z, OT> Executor for NyxExecutor<'a, I, S, OT> +where + I: Input + HasBytesVec, +{ + fn run_target( + &mut self, + _fuzzer: &mut Z, + _state: &mut S, + _mgr: &mut EM, + input: &I, + ) -> Result { + let input = input.bytes(); + self.helper.nyx_process.set_input(input, input.len() as u32); + + // exec will take care of trace_bits, so no need to reset + let ret_val = self.helper.nyx_process.exec(); + match ret_val { + NyxReturnValue::Normal => Ok(ExitKind::Ok), + NyxReturnValue::Crash | NyxReturnValue::Asan => Ok(ExitKind::Crash), + NyxReturnValue::Timeout => Ok(ExitKind::Timeout), + NyxReturnValue::InvalidWriteToPayload => Err(libafl::Error::illegal_state( + "FixMe: Nyx InvalidWriteToPayload handler is missing", + )), + NyxReturnValue::Error => Err(libafl::Error::illegal_state( + "Error: Nyx runtime error has occured...", + )), + NyxReturnValue::IoError => { + // todo! *stop_soon_p = 0 + Err(libafl::Error::unknown("Error: QEMU-nyx died...")) + } + NyxReturnValue::Abort => { + self.helper.nyx_process.shutdown(); + Err(libafl::Error::shutting_down()) + } + } + } +} + +impl<'a, I, S, OT> NyxExecutor<'a, I, S, OT> { + pub fn new(helper: &'a mut NyxHelper, observers: OT) -> Result { + Ok(Self { + helper, + observers, + phantom: PhantomData, + }) + } + + /// convert `trace_bits` ptr into real trace map + pub fn trace_bits(self) -> &'static mut [u8] { + unsafe { std::slice::from_raw_parts_mut(self.helper.trace_bits, self.helper.real_map_size) } + } +} + +impl<'a, I, S, OT> HasObservers for NyxExecutor<'a, I, S, OT> +where + I: Input, + OT: ObserversTuple, +{ + fn observers(&self) -> &OT { + &self.observers + } + + fn observers_mut(&mut self) -> &mut OT { + &mut self.observers + } +} diff --git a/libafl_nyx/src/helper.rs b/libafl_nyx/src/helper.rs new file mode 100644 index 0000000000..6a27cc61c8 --- /dev/null +++ b/libafl_nyx/src/helper.rs @@ -0,0 +1,157 @@ +/// [`NyxHelper`] is used to wrap `NyxProcess` +use std::{ + fmt::{self, Debug}, + path::Path, + time::Duration, +}; + +use libafl::Error; +use libnyx::{NyxProcess, NyxReturnValue}; + +const INIT_TIMEOUT: Duration = Duration::new(2, 0); +pub struct NyxHelper { + pub nyx_process: NyxProcess, + /// real size of trace_bits + pub real_map_size: usize, + // real size of the trace_bits + pub map_size: usize, + /// shared memory with instruction bitmaps + pub trace_bits: *mut u8, +} + +const MAX_FILE: u32 = 1024 * 1024; +#[derive(Clone, Copy, Debug)] +pub enum NyxProcessType { + /// stand alone mode + ALONE, + /// parallel mode's parent, used to create snapshot + PARENT, + /// parallel mode's child, consume snapshot and execute + CHILD, +} +impl NyxHelper { + /// create `NyxProcess` and do basic settings + /// It will convert instance to parent or child using `parent_cpu_id` when set`parallel_mode` + /// will fail if initial connection takes more than 2 seconds + pub fn new( + target_dir: &Path, + cpu_id: u32, + snap_mode: bool, + parallel_mode: bool, + parent_cpu_id: Option, + ) -> Result { + NyxHelper::new_with_initial_timeout( + target_dir, + cpu_id, + snap_mode, + parallel_mode, + parent_cpu_id, + INIT_TIMEOUT, + ) + } + /// create `NyxProcess` and do basic settings + /// It will convert instance to parent or child using `parent_cpu_id` when set`parallel_mode` + /// will fail if initial connection takes more than `initial_timeout` seconds + pub fn new_with_initial_timeout( + target_dir: &Path, + cpu_id: u32, + snap_mode: bool, + parallel_mode: bool, + parent_cpu_id: Option, + initial_timeout: Duration, + ) -> Result { + let sharedir = match target_dir.to_str() { + Some(x) => x, + None => return Err(Error::illegal_argument("can't convert sharedir to str")), + }; + let work_dir = target_dir.join("workdir"); + let work_dir = work_dir.to_str().expect("unable to convert workdir to str"); + let nyx_type = if parallel_mode { + let parent_cpu_id = match parent_cpu_id { + None => { + return Err(Error::illegal_argument( + "please set parent_cpu_id in nyx parallel mode", + )) + } + Some(x) => x, + }; + if cpu_id == parent_cpu_id { + NyxProcessType::PARENT + } else { + NyxProcessType::CHILD + } + } else { + NyxProcessType::ALONE + }; + + let nyx_process = match nyx_type { + NyxProcessType::ALONE => NyxProcess::new(sharedir, work_dir, cpu_id, MAX_FILE, true), + NyxProcessType::PARENT => { + NyxProcess::new_parent(sharedir, work_dir, cpu_id, MAX_FILE, true) + } + NyxProcessType::CHILD => NyxProcess::new_child(sharedir, work_dir, cpu_id, cpu_id), + }; + + let mut nyx_process = + nyx_process.map_err(|msg: String| -> Error { Error::illegal_argument(msg) })?; + + let real_map_size = nyx_process.bitmap_buffer_size(); + let map_size = ((real_map_size + 63) >> 6) << 6; + let trace_bits = nyx_process.bitmap_buffer_mut().as_mut_ptr(); + nyx_process.option_set_reload_mode(snap_mode); + nyx_process.option_apply(); + + // default timeout for initial dry-run + let sec = initial_timeout + .as_secs() + .try_into() + .map_err(|_| -> Error { Error::illegal_argument("can't cast time's sec to u8") })?; + + let micro_sec: u32 = initial_timeout.subsec_micros(); + nyx_process.option_set_timeout(sec, micro_sec); + nyx_process.option_apply(); + + // dry run to check if qemu is spawned + nyx_process.set_input(b"INIT", 4); + match nyx_process.exec() { + NyxReturnValue::Error => { + nyx_process.shutdown(); + let msg = "Error: Nyx runtime error has occured..."; + return Err(Error::illegal_state(msg)); + } + NyxReturnValue::IoError => { + let msg = "Error: QEMU-nyx died..."; + return Err(Error::illegal_state(msg)); + } + NyxReturnValue::Abort => { + nyx_process.shutdown(); + let msg = "Error: Nyx abort occured..."; + return Err(Error::illegal_state(msg)); + } + _ => {} + } + Ok(Self { + nyx_process, + real_map_size, + map_size, + trace_bits, + }) + } + + /// set timeout + pub fn set_timeout(mut self, time: Duration) { + let sec: u8 = time + .as_secs() + .try_into() + .expect("can't cast time's sec to u8"); + let micro_sec: u32 = time.subsec_micros(); + self.nyx_process.option_set_timeout(sec, micro_sec); + self.nyx_process.option_apply(); + } +} + +impl Debug for NyxHelper { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NyxInprocessHelper").finish() + } +} diff --git a/libafl_nyx/src/lib.rs b/libafl_nyx/src/lib.rs new file mode 100644 index 0000000000..83b65bdfc6 --- /dev/null +++ b/libafl_nyx/src/lib.rs @@ -0,0 +1,4 @@ +#[cfg(target_os = "linux")] +pub mod executor; +#[cfg(target_os = "linux")] +pub mod helper; diff --git a/scripts/test_all_fuzzers.sh b/scripts/test_all_fuzzers.sh index c077e541ca..5680af5010 100755 --- a/scripts/test_all_fuzzers.sh +++ b/scripts/test_all_fuzzers.sh @@ -28,6 +28,26 @@ declare -A time_record || (echo "declare -A not avaliable, please update your ba # shellcheck disable=SC2116 for fuzzer in $(echo "$fuzzers" "$backtrace_fuzzers"); do + # for nyx examples + if [[ $fuzzer == *"nyx_"* ]]; then + + # only test on linux + if [[ $(uname -s) == "Linux" ]]; then + cd "$fuzzer" || exit 1 + if [ "$1" != "--no-fmt" ]; then + echo "[*] Checking fmt for $fuzzer" + cargo fmt --all -- --check || exit 1 + echo "[*] Running clippy for $fuzzer" + cargo clippy || exit 1 + else + echo "[+] Skipping fmt and clippy for $fuzzer (--no-fmt specified)" + fi + cargo make build + cd - || exit + fi + continue + fi + cd "$fuzzer" || exit 1 start=$(date +%s) # Clippy checks @@ -40,7 +60,7 @@ do else echo "[+] Skipping fmt and clippy for $fuzzer (--no-fmt specified)" fi - + if [ -e ./Makefile.toml ]; then echo "[*] Testing $fuzzer" cargo make test || exit 1