From cedcee01c09ed03cfd10ba4b7ef1a1e2a1839fdf Mon Sep 17 00:00:00 2001 From: Mrmaxmeier <3913977+Mrmaxmeier@users.noreply.github.com> Date: Thu, 20 Oct 2022 21:38:58 +0200 Subject: [PATCH] CI: Build fuzzers with shared cargo target dir (#845) * build fuzzers with shared cargo target dir * Make external build scripts aware of CARGO_TARGET_DIR * fix libmozjpeg fuzzer with shared target dir * fix cargo-make default value for CARGO_TARGET_DIR * avoid ./ in cargo-make for windows compat * CI: cargo-hack's --feature-powerset is too powerful * fuzzer_concolic: support CARGO_TARGET_DIR * ci: install z3 to avoid building from source * ci: update actions * ci: test nightly features with nightly rust * test_all_fuzzers: try pruning more compilation artifacts * ci: fix nightly feature check * ci: apply rust-cache action after checkout (d'oh) The rust-cache action populates the checkout directory, which is promply deleted by the checkout action during checkout.. whoops! --- .github/workflows/build_and_test.yml | 57 ++++++++++--------- fuzzers/frida_libpng/Makefile.toml | 5 +- fuzzers/fuzzbench/Makefile.toml | 5 +- fuzzers/fuzzbench_text/Makefile.toml | 5 +- fuzzers/fuzzbench_weighted/Makefile.toml | 5 +- fuzzers/libfuzzer_libmozjpeg/Makefile.toml | 11 ++-- fuzzers/libfuzzer_libpng/Makefile.toml | 15 ++--- .../libfuzzer_libpng_accounting/Makefile.toml | 11 ++-- fuzzers/libfuzzer_libpng_cmin/Makefile.toml | 15 ++--- fuzzers/libfuzzer_libpng_ctx/Makefile.toml | 11 ++-- .../libfuzzer_libpng_launcher/Makefile.toml | 11 ++-- fuzzers/libfuzzer_stb_image/Makefile.toml | 7 ++- .../Makefile.toml | 3 +- .../fuzzer/build.rs | 14 +++-- .../libfuzzer_stb_image_sugar/Makefile.toml | 7 ++- scripts/test_all_fuzzers.sh | 23 ++++++-- 16 files changed, 119 insertions(+), 86 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 240ed0d173..ad56997968 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -20,7 +20,6 @@ jobs: with: profile: minimal toolchain: nightly - - uses: Swatinem/rust-cache@v1 - name: install mdbook uses: baptiste0928/cargo-install@v1.3.0 with: @@ -29,7 +28,8 @@ jobs: uses: baptiste0928/cargo-install@v1.3.0 with: crate: mdbook-linkcheck - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 - name: Build libafl debug run: cargo build -p libafl - name: Build the book @@ -54,7 +54,6 @@ jobs: with: profile: minimal toolchain: stable - - uses: Swatinem/rust-cache@v1 - name: set mold linker as default linker uses: rui314/setup-mold@v1 - name: Install and cache deps @@ -67,7 +66,8 @@ jobs: run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin - name: Add nightly rustfmt and clippy run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 # ---- format check ---- # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately @@ -91,13 +91,14 @@ jobs: # ---- build and feature check ---- - name: Run a normal build run: cargo build --verbose - # cargo-hack tests/checks each crate in the workspace - #- name: Run tests - # run: cargo hack test --all-features # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs - name: Check each feature - # Skipping python as it has to be built with the `maturin` tool - run: cargo hack check --feature-powerset --depth=2 --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be --no-dev-deps + # Skipping `python` as it has to be built with the `maturin` tool + # `agpl`, `nautilus` require nightly + # `sancov_pcguard_edges` is tested seperately + run: cargo hack check --each-feature --exclude-features=agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be + - name: Check nightly features + run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus - name: Build examples run: cargo build --examples --verbose @@ -108,8 +109,8 @@ jobs: with: profile: minimal toolchain: stable - - uses: Swatinem/rust-cache@v1 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 - name: Install smoke test deps run: sudo ./libafl_concolic/test/smoke_test_ubuntu_deps.sh - name: Run smoke test @@ -122,14 +123,14 @@ jobs: with: profile: minimal toolchain: stable - - uses: Swatinem/rust-cache@v1 - name: set mold linker as default linker uses: rui314/setup-mold@v1 - name: Install deps run: sudo apt-get install -y llvm llvm-dev clang ninja-build python3-dev python3-pip python3-venv - name: Install maturin run: python3 -m pip install maturin - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 - name: Run a maturin build run: cd ./bindings/pylibafl && maturin build @@ -148,7 +149,6 @@ jobs: uses: rui314/setup-mold@v1 - name: enable mult-thread for `make` run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)" - - uses: Swatinem/rust-cache@v1 - name: Add nightly rustfmt and clippy run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - name: Add no_std toolchain @@ -156,7 +156,7 @@ jobs: - uses: lyricwulf/abc@v1 with: # todo: remove afl++-clang when nyx support samcov_pcguard - linux: llvm llvm-dev clang nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libgtk-3-dev afl++-clang pax-utils + linux: llvm llvm-dev clang nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libgtk-3-dev afl++-clang pax-utils z3 libz3-dev # update bash for macos to support `declare -A` command` macos: llvm libpng nasm coreutils z3 bash - name: pip install @@ -165,13 +165,14 @@ jobs: uses: baptiste0928/cargo-install@v1.3.0 with: crate: cargo-make - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true # recursively checkout submodules - - name: Build and run example fuzzers + - uses: Swatinem/rust-cache@v2 + - name: Build and run example fuzzers (Linux) if: runner.os == 'Linux' run: ./scripts/test_all_fuzzers.sh - - name: Build and run example fuzzers + - name: Build and run example fuzzers (macOS) if: runner.os == 'macOS' # use bash v4 run: /usr/local/bin/bash ./scripts/test_all_fuzzers.sh @@ -182,10 +183,10 @@ jobs: with: profile: minimal toolchain: nightly - - uses: Swatinem/rust-cache@v1 - name: Add nightly rustfmt and clippy run: rustup toolchain install nightly && rustup target add --toolchain nightly aarch64-unknown-none && rustup component add --toolchain nightly rust-src && rustup target add thumbv6m-none-eabi - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 - name: Build aarch64-unknown-none run: cd ./fuzzers/baby_no_std && cargo +nightly build -Zbuild-std=core,alloc --target aarch64-unknown-none -v --release && cd ../.. - name: run x86_64 until panic! @@ -198,7 +199,7 @@ jobs: build-docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build docker run: docker build -t libafl . @@ -209,8 +210,8 @@ jobs: with: profile: minimal toolchain: stable - - uses: Swatinem/rust-cache@v1 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 - name: Windows Build run: cargo build --verbose - name: Run clippy @@ -236,12 +237,12 @@ jobs: with: profile: minimal toolchain: stable - - uses: Swatinem/rust-cache@v1 - name: Add nightly rustfmt and clippy run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade - name: Install deps run: brew install z3 gtk+3 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 - name: MacOS Build run: cargo build --verbose - name: Run clippy @@ -258,7 +259,6 @@ jobs: with: profile: minimal toolchain: stable - - uses: Swatinem/rust-cache@v1 - uses: nttld/setup-ndk@v1 with: ndk-version: r21e @@ -268,7 +268,8 @@ jobs: run: rustup target add aarch64-linux-android - name: install cargo ndk run: cargo install cargo-ndk - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 - name: Build iOS run: cargo build --target aarch64-apple-ios - name: Build Android @@ -287,7 +288,7 @@ jobs: runs-on: macos-12 name: Simple build in FreeBSD steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Test in FreeBSD id: test uses: vmactions/freebsd-vm@v0 diff --git a/fuzzers/frida_libpng/Makefile.toml b/fuzzers/frida_libpng/Makefile.toml index 087961191a..5dd57af624 100644 --- a/fuzzers/frida_libpng/Makefile.toml +++ b/fuzzers/frida_libpng/Makefile.toml @@ -1,7 +1,8 @@ # Variables [env] +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } FUZZER_NAME={ source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = "frida_fuzzer", mapping = {"linux" = "frida_fuzzer", "macos" = "frida_fuzzer", "windows" = "frida_fuzzer.exe"} } -FUZZER = './target/release/${FUZZER_NAME}' +FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}' [tasks.unsupported] script_runner="@shell" @@ -63,7 +64,7 @@ cl /c harness_win.cpp && link harness_win.obj /dll script_runner="@shell" script=''' cargo build --release -cp ./target/release/${FUZZER_NAME} . +cp ${CARGO_TARGET_DIR}/release/${FUZZER_NAME} . ''' # Run the fuzzer diff --git a/fuzzers/fuzzbench/Makefile.toml b/fuzzers/fuzzbench/Makefile.toml index be7dca4516..14c6b92387 100644 --- a/fuzzers/fuzzbench/Makefile.toml +++ b/fuzzers/fuzzbench/Makefile.toml @@ -1,4 +1,5 @@ [env] +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } FUZZER_NAME="fuzzer" PROJECT_DIR = { script = ["pwd"] } @@ -34,7 +35,7 @@ mac_alias = "fuzz_o_unix" windows_alias = "unsupported" [tasks.fuzz_o_unix] -command = "target/release/libafl_cc" +command = "${CARGO_TARGET_DIR}/release/libafl_cc" args = ["--libafl-no-link", "-O3", "-c", "fuzz.c", "-o", "fuzz.o"] dependencies = ["cc", "cxx"] @@ -45,7 +46,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["--libafl", "fuzz.o", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = ["cc", "cxx", "fuzz_o"] diff --git a/fuzzers/fuzzbench_text/Makefile.toml b/fuzzers/fuzzbench_text/Makefile.toml index 5ec845b5f4..b3abf57a89 100644 --- a/fuzzers/fuzzbench_text/Makefile.toml +++ b/fuzzers/fuzzbench_text/Makefile.toml @@ -1,4 +1,5 @@ [env] +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } FUZZER_NAME="fuzzer" PROJECT_DIR = { script = ["pwd"] } @@ -34,7 +35,7 @@ mac_alias = "fuzz_o_unix" windows_alias = "unsupported" [tasks.fuzz_o_unix] -command = "target/release/libafl_cc" +command = "${CARGO_TARGET_DIR}/release/libafl_cc" args = ["--libafl-no-link", "-O3", "-c", "fuzz.c", "-o", "fuzz.o"] dependencies = ["cc", "cxx"] @@ -45,7 +46,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["--libafl", "fuzz.o", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = ["cc", "cxx", "fuzz_o"] diff --git a/fuzzers/fuzzbench_weighted/Makefile.toml b/fuzzers/fuzzbench_weighted/Makefile.toml index be7dca4516..14c6b92387 100644 --- a/fuzzers/fuzzbench_weighted/Makefile.toml +++ b/fuzzers/fuzzbench_weighted/Makefile.toml @@ -1,4 +1,5 @@ [env] +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } FUZZER_NAME="fuzzer" PROJECT_DIR = { script = ["pwd"] } @@ -34,7 +35,7 @@ mac_alias = "fuzz_o_unix" windows_alias = "unsupported" [tasks.fuzz_o_unix] -command = "target/release/libafl_cc" +command = "${CARGO_TARGET_DIR}/release/libafl_cc" args = ["--libafl-no-link", "-O3", "-c", "fuzz.c", "-o", "fuzz.o"] dependencies = ["cc", "cxx"] @@ -45,7 +46,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["--libafl", "fuzz.o", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = ["cc", "cxx", "fuzz_o"] diff --git a/fuzzers/libfuzzer_libmozjpeg/Makefile.toml b/fuzzers/libfuzzer_libmozjpeg/Makefile.toml index 103bbc13ad..7eb98ff1df 100644 --- a/fuzzers/libfuzzer_libmozjpeg/Makefile.toml +++ b/fuzzers/libfuzzer_libmozjpeg/Makefile.toml @@ -1,9 +1,10 @@ # Variables [env] FUZZER_NAME='fuzzer_mozjpeg' -LIBAFL_CC = './target/release/libafl_cc' -LIBAFL_CXX = './target/release/libafl_cxx' -FUZZER = './target/release/${FUZZER_NAME}' +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } +LIBAFL_CC = '${CARGO_TARGET_DIR}/release/libafl_cc' +LIBAFL_CXX = '${CARGO_TARGET_DIR}/release/libafl_cxx' +FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}' PROJECT_DIR = { script = ["pwd"] } [tasks.unsupported] @@ -53,7 +54,7 @@ windows_alias = "unsupported" [tasks.lib_unix] script=''' -cd mozjpeg-4.0.3 && cmake . -DENABLE_SHARED=false -DPNG_SUPPORTED=false -DCMAKE_C_COMPILER="${PROJECT_DIR}/${LIBAFL_CC}" -DCMAKE_CXX_COMPILER="${PROJECT_DIR}/${LIBAFL_CXX}" -G "Unix Makefiles" +cd mozjpeg-4.0.3 && cmake . -DENABLE_SHARED=false -DPNG_SUPPORTED=false -DCMAKE_C_COMPILER="${LIBAFL_CC}" -DCMAKE_CXX_COMPILER="${LIBAFL_CXX}" -G "Unix Makefiles" cd "${PROJECT_DIR}" make -C mozjpeg-4.0.3 ''' @@ -67,7 +68,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/mozjpeg-4.0.3/libjpeg.a", "${PROJECT_DIR}/mozjpeg-4.0.3/libturbojpeg.a", "-I", "${PROJECT_DIR}/mozjpeg-4.0.3/", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = [ "lib", "cxx", "cc" ] diff --git a/fuzzers/libfuzzer_libpng/Makefile.toml b/fuzzers/libfuzzer_libpng/Makefile.toml index f95c932bf1..b3eec10d76 100644 --- a/fuzzers/libfuzzer_libpng/Makefile.toml +++ b/fuzzers/libfuzzer_libpng/Makefile.toml @@ -1,9 +1,10 @@ # Variables [env] FUZZER_NAME='fuzzer_libpng' -LIBAFL_CC = './target/release/libafl_cc' -LIBAFL_CXX = './target/release/libafl_cxx' -FUZZER = './target/release/${FUZZER_NAME}' +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } +LIBAFL_CC = '${CARGO_TARGET_DIR}/release/libafl_cc' +LIBAFL_CXX = '${CARGO_TARGET_DIR}/release/libafl_cxx' +FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}' PROJECT_DIR = { script = ["pwd"] } [tasks.unsupported] @@ -74,7 +75,7 @@ script_runner="@shell" script=''' cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes cd "${PROJECT_DIR}" -make -C libpng-1.6.37 CC="${PROJECT_DIR}/target/release/libafl_cc" CXX="${PROJECT_DIR}/target/release/libafl_cxx" +make -C libpng-1.6.37 CC="${CARGO_TARGET_DIR}/release/libafl_cc" CXX="${CARGO_TARGET_DIR}/release/libafl_cxx" ''' dependencies = [ "libpng", "cxx", "cc" ] @@ -89,7 +90,7 @@ script_runner="@shell" script=''' cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes cd "${PROJECT_DIR}" -make -C libpng-1.6.37 CC="${PROJECT_DIR}/target/release/libafl_cc" CXX="${PROJECT_DIR}/target/release/libafl_cxx" +make -C libpng-1.6.37 CC="${CARGO_TARGET_DIR}/release/libafl_cc" CXX="${CARGO_TARGET_DIR}/release/libafl_cxx" ''' dependencies = [ "libpng", "crash_cxx", "crash_cc" ] @@ -100,7 +101,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/libpng-1.6.37/.libs/libpng16.a", "-I", "${PROJECT_DIR}/libpng-1.6.37/", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = [ "lib", "cxx", "cc" ] @@ -111,7 +112,7 @@ mac_alias = "fuzzer_crash_unix" windows_alias = "unsupported" [tasks.fuzzer_crash_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/libpng-1.6.37/.libs/libpng16.a", "-I", "${PROJECT_DIR}/libpng-1.6.37/", "-o", "${FUZZER_NAME}_crash", "-lm", "-lz"] dependencies = [ "crash_lib", "crash_cxx", "crash_cc" ] diff --git a/fuzzers/libfuzzer_libpng_accounting/Makefile.toml b/fuzzers/libfuzzer_libpng_accounting/Makefile.toml index 23442ff862..63d129ef16 100644 --- a/fuzzers/libfuzzer_libpng_accounting/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_accounting/Makefile.toml @@ -1,9 +1,10 @@ # Variables [env] +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } FUZZER_NAME='fuzzer_libpng_accounting' -LIBAFL_CC = './target/release/libafl_cc' -LIBAFL_CXX = './target/release/libafl_cxx' -FUZZER = './target/release/${FUZZER_NAME}' +LIBAFL_CC = '${CARGO_TARGET_DIR}/release/libafl_cc' +LIBAFL_CXX = '${CARGO_TARGET_DIR}/release/libafl_cxx' +FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}' PROJECT_DIR = { script = ["pwd"] } [tasks.unsupported] @@ -56,7 +57,7 @@ script_runner="@shell" script=''' cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes cd "${PROJECT_DIR}" -make -C libpng-1.6.37 CC="${PROJECT_DIR}/target/release/libafl_cc" CXX="${PROJECT_DIR}/target/release/libafl_cxx" +make -C libpng-1.6.37 CC="${CARGO_TARGET_DIR}/release/libafl_cc" CXX="${CARGO_TARGET_DIR}/release/libafl_cxx" ''' dependencies = [ "libpng", "cxx", "cc" ] @@ -68,7 +69,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/libpng-1.6.37/.libs/libpng16.a", "-I", "${PROJECT_DIR}/libpng-1.6.37/", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = [ "lib", "cxx", "cc" ] diff --git a/fuzzers/libfuzzer_libpng_cmin/Makefile.toml b/fuzzers/libfuzzer_libpng_cmin/Makefile.toml index 3d57308674..81d9b046d8 100644 --- a/fuzzers/libfuzzer_libpng_cmin/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_cmin/Makefile.toml @@ -1,9 +1,10 @@ # Variables [env] FUZZER_NAME='fuzzer_libpng' -LIBAFL_CC = './target/release/libafl_cc' -LIBAFL_CXX = './target/release/libafl_cxx' -FUZZER = './target/release/${FUZZER_NAME}' +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } +LIBAFL_CC = '${CARGO_TARGET_DIR}/release/libafl_cc' +LIBAFL_CXX = '${CARGO_TARGET_DIR}/release/libafl_cxx' +FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}' PROJECT_DIR = { script = ["pwd"] } [tasks.unsupported] @@ -74,7 +75,7 @@ script_runner="@shell" script=''' cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes cd "${PROJECT_DIR}" -make -C libpng-1.6.37 CC="${PROJECT_DIR}/target/release/libafl_cc" CXX="${PROJECT_DIR}/target/release/libafl_cxx" +make -C libpng-1.6.37 CC="${CARGO_TARGET_DIR}/release/libafl_cc" CXX="${CARGO_TARGET_DIR}/release/libafl_cxx" ''' dependencies = [ "libpng", "cxx", "cc" ] @@ -89,7 +90,7 @@ script_runner="@shell" script=''' cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes cd "${PROJECT_DIR}" -make -C libpng-1.6.37 CC="${PROJECT_DIR}/target/release/libafl_cc" CXX="${PROJECT_DIR}/target/release/libafl_cxx" +make -C libpng-1.6.37 CC="${CARGO_TARGET_DIR}/release/libafl_cc" CXX="${CARGO_TARGET_DIR}/release/libafl_cxx" ''' dependencies = [ "libpng", "crash_cxx", "crash_cc" ] @@ -100,7 +101,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/libpng-1.6.37/.libs/libpng16.a", "-I", "${PROJECT_DIR}/libpng-1.6.37/", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = [ "lib", "cxx", "cc" ] @@ -111,7 +112,7 @@ mac_alias = "fuzzer_crash_unix" windows_alias = "unsupported" [tasks.fuzzer_crash_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/libpng-1.6.37/.libs/libpng16.a", "-I", "${PROJECT_DIR}/libpng-1.6.37/", "-o", "${FUZZER_NAME}_crash", "-lm", "-lz"] dependencies = [ "crash_lib", "crash_cxx", "crash_cc" ] diff --git a/fuzzers/libfuzzer_libpng_ctx/Makefile.toml b/fuzzers/libfuzzer_libpng_ctx/Makefile.toml index 2dc10feaf8..60a2781efe 100644 --- a/fuzzers/libfuzzer_libpng_ctx/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_ctx/Makefile.toml @@ -1,9 +1,10 @@ # Variables [env] FUZZER_NAME='fuzzer_libpng_ctx' -LIBAFL_CC = './target/release/libafl_cc' -LIBAFL_CXX = './target/release/libafl_cxx' -FUZZER = './target/release/${FUZZER_NAME}' +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } +LIBAFL_CC = '${CARGO_TARGET_DIR}/release/libafl_cc' +LIBAFL_CXX = '${CARGO_TARGET_DIR}/release/libafl_cxx' +FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}' PROJECT_DIR = { script = ["pwd"] } [tasks.unsupported] @@ -56,7 +57,7 @@ script_runner="@shell" script=''' cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes cd "${PROJECT_DIR}" -make -C libpng-1.6.37 CC="${PROJECT_DIR}/target/release/libafl_cc" CXX="${PROJECT_DIR}/target/release/libafl_cxx" +make -C libpng-1.6.37 CC="${CARGO_TARGET_DIR}/release/libafl_cc" CXX="${CARGO_TARGET_DIR}/release/libafl_cxx" ''' dependencies = [ "libpng", "cxx", "cc" ] @@ -68,7 +69,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/libpng-1.6.37/.libs/libpng16.a", "-I", "${PROJECT_DIR}/libpng-1.6.37/", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = [ "lib", "cxx", "cc" ] diff --git a/fuzzers/libfuzzer_libpng_launcher/Makefile.toml b/fuzzers/libfuzzer_libpng_launcher/Makefile.toml index edb1150554..4f0ca2db28 100644 --- a/fuzzers/libfuzzer_libpng_launcher/Makefile.toml +++ b/fuzzers/libfuzzer_libpng_launcher/Makefile.toml @@ -1,9 +1,10 @@ # Variables [env] FUZZER_NAME='fuzzer_libpng_launcher' -LIBAFL_CC = './target/release/libafl_cc' -LIBAFL_CXX = './target/release/libafl_cxx' -FUZZER = './target/release/${FUZZER_NAME}' +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } +LIBAFL_CC = '${CARGO_TARGET_DIR}/release/libafl_cc' +LIBAFL_CXX = '${CARGO_TARGET_DIR}/release/libafl_cxx' +FUZZER = '${CARGO_TARGET_DIR}/release/${FUZZER_NAME}' PROJECT_DIR = { script = ["pwd"] } [tasks.unsupported] @@ -56,7 +57,7 @@ script_runner="@shell" script=''' cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes cd "${PROJECT_DIR}" -make -C libpng-1.6.37 CC="${PROJECT_DIR}/target/release/libafl_cc" CXX="${PROJECT_DIR}/target/release/libafl_cxx" +make -C libpng-1.6.37 CC="${CARGO_TARGET_DIR}/release/libafl_cc" CXX="${CARGO_TARGET_DIR}/release/libafl_cxx" ''' dependencies = [ "libpng", "cxx", "cc" ] @@ -68,7 +69,7 @@ mac_alias = "fuzzer_unix" windows_alias = "unsupported" [tasks.fuzzer_unix] -command = "target/release/libafl_cxx" +command = "${CARGO_TARGET_DIR}/release/libafl_cxx" args = ["${PROJECT_DIR}/harness.cc", "${PROJECT_DIR}/libpng-1.6.37/.libs/libpng16.a", "-I", "${PROJECT_DIR}/libpng-1.6.37/", "-o", "${FUZZER_NAME}", "-lm", "-lz"] dependencies = [ "lib", "cxx", "cc" ] diff --git a/fuzzers/libfuzzer_stb_image/Makefile.toml b/fuzzers/libfuzzer_stb_image/Makefile.toml index bd6c4a027f..90d84d1c04 100644 --- a/fuzzers/libfuzzer_stb_image/Makefile.toml +++ b/fuzzers/libfuzzer_stb_image/Makefile.toml @@ -1,9 +1,10 @@ # Variables [env] FUZZER_NAME='libfuzzer_stb_image' -LIBAFL_CC = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = './target/release/libafl_cc', mapping = {"windows" = '.\\target\\release\\libafl_cc.exe'} } -LIBAFL_CXX = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = './target/release/libafl_cxx', mapping = {"windows" = '.\\target\\release\\libafl_cxx.exe'} } -FUZZER = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = './target/release/libfuzzer_stb_image', mapping = {"windows" = '.\\target\\release\\libfuzzer_stb_image.exe'} } +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } +LIBAFL_CC = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = '${CARGO_TARGET_DIR}/release/libafl_cc', mapping = {"windows" = '.\\target\\release\\libafl_cc.exe'} } +LIBAFL_CXX = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = '${CARGO_TARGET_DIR}/release/libafl_cxx', mapping = {"windows" = '.\\target\\release\\libafl_cxx.exe'} } +FUZZER = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = '${CARGO_TARGET_DIR}/release/libfuzzer_stb_image', mapping = {"windows" = '.\\target\\release\\libfuzzer_stb_image.exe'} } # Compilers [tasks.cxx] diff --git a/fuzzers/libfuzzer_stb_image_concolic/Makefile.toml b/fuzzers/libfuzzer_stb_image_concolic/Makefile.toml index a595a325b0..9564f6f71f 100644 --- a/fuzzers/libfuzzer_stb_image_concolic/Makefile.toml +++ b/fuzzers/libfuzzer_stb_image_concolic/Makefile.toml @@ -1,5 +1,6 @@ # Variables [env] +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } FUZZER_NAME='libfuzzer_stb_image_concolic' # Compilers @@ -9,7 +10,7 @@ mac_alias = "unsupported" windows_alias = "unsupported" [tasks.runtime_unix] -condition = { files_not_exist = ["runtime/target/libSymRuntime.so"] } +condition = { files_not_exist = ["${CARGO_TARGET_DIR}/libSymRuntime.so"] } script_runner = "@shell" script = ''' cd runtime diff --git a/fuzzers/libfuzzer_stb_image_concolic/fuzzer/build.rs b/fuzzers/libfuzzer_stb_image_concolic/fuzzer/build.rs index 32077e5ca8..847aa8f2b7 100644 --- a/fuzzers/libfuzzer_stb_image_concolic/fuzzer/build.rs +++ b/fuzzers/libfuzzer_stb_image_concolic/fuzzer/build.rs @@ -49,11 +49,15 @@ fn main() { let symcc_dir = clone_and_build_symcc(&out_path); - let runtime_dir = std::env::current_dir() - .unwrap() - .join("..") - .join("runtime") - .join("target") + let runtime_dir = std::env::var("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| { + std::env::current_dir() + .unwrap() + .join("..") + .join("runtime") + .join("target") + }) .join(std::env::var("PROFILE").unwrap()); if !runtime_dir.join("libSymRuntime.so").exists() { diff --git a/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml b/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml index 09edf62e1d..7aa09a0116 100644 --- a/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml +++ b/fuzzers/libfuzzer_stb_image_sugar/Makefile.toml @@ -1,9 +1,10 @@ # Variables [env] FUZZER_NAME='libfuzzer_stb_image_sugar' -LIBAFL_CC = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = './target/release/libafl_cc', mapping = {"windows" = '.\\target\\release\\libafl_cc.exe'} } -LIBAFL_CXX = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = './target/release/libafl_cxx', mapping = {"windows" = '.\\target\\release\\libafl_cxx.exe'} } -FUZZER = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = './target/release/libfuzzer_stb_image_sugar', mapping = {"windows" = '.\\target\\release\\libfuzzer_stb_image_sugar.exe'} } +CARGO_TARGET_DIR = { value = "target", condition = { env_not_set = ["CARGO_TARGET_DIR"] } } +LIBAFL_CC = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = '${CARGO_TARGET_DIR}/release/libafl_cc', mapping = {"windows" = '.\\target\\release\\libafl_cc.exe'} } +LIBAFL_CXX = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = '${CARGO_TARGET_DIR}/release/libafl_cxx', mapping = {"windows" = '.\\target\\release\\libafl_cxx.exe'} } +FUZZER = { source = "${CARGO_MAKE_RUST_TARGET_OS}", default_value = '${CARGO_TARGET_DIR}/release/libfuzzer_stb_image_sugar', mapping = {"windows" = '.\\target\\release\\libfuzzer_stb_image_sugar.exe'} } # Compilers [tasks.cxx] diff --git a/scripts/test_all_fuzzers.sh b/scripts/test_all_fuzzers.sh index 5680af5010..cba530e14e 100755 --- a/scripts/test_all_fuzzers.sh +++ b/scripts/test_all_fuzzers.sh @@ -5,11 +5,14 @@ cd "$SCRIPT_DIR/.." || exit 1 # TODO: This should be rewritten in rust, a Makefile, or some platform-independent language -fuzzers=$(find ./fuzzers -maxdepth 1 -type d) -backtrace_fuzzers=$(find ./fuzzers/backtrace_baby_fuzzers -maxdepth 1 -type d) +fuzzers=$(find ./fuzzers -mindepth 1 -maxdepth 1 -type d) +backtrace_fuzzers=$(find ./fuzzers/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d) libafl=$(pwd) +export CARGO_TARGET_DIR="$libafl/target" +mkdir -p "$CARGO_TARGET_DIR" + git submodule init && git submodule update # override default profile settings for speed @@ -17,7 +20,10 @@ git submodule init && git submodule update for profile in DEV RELEASE; # loop for all profiles do export CARGO_PROFILE_"$profile"_OPT_LEVEL=z # optimize for size - export CARGO_PROFILE_"$profile"_SPLIT_DEBUGINFO=unpacked # minimize debug info + # runs into shared target dir bug: + # [pid 351769] openat(AT_FDCWD, "LibAFL/target/release/deps/libc-dbff77a14da5d893.libc.5deb7d4a-cgu.0.rcgu.dwo", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + # error: failed to build archive: No such file or directory + # export CARGO_PROFILE_"$profile"_SPLIT_DEBUGINFO=unpacked # minimize debug info # export CARGO_PROFILE_"$profile"_PANIC=abort export CARGO_PROFILE_"$profile"_INCREMENTAL=true done @@ -72,8 +78,17 @@ do fi end=$(date +%s) time_record[$fuzzer]=$((end-start)) + du -sh "$CARGO_TARGET_DIR" # Save disk space - cargo clean + cargo clean -p "$(basename "$fuzzer")" + cargo clean --release -p "$(basename "$fuzzer")" 2> /dev/null + # Leaving these in the cache results in lots of duplicate build artefacts + # (many different feature flag combinations, ...), so let's prune them. + for clean_pkgid in libafl libafl_targets libafl_sugar; do + cargo clean -p "$clean_pkgid" 2> /dev/null + cargo clean --release -p "$clean_pkgid" 2> /dev/null + done + du -sh "$CARGO_TARGET_DIR" cd "$libafl" || exit 1 echo "" done