From 82f11c22ad0f8d7610c20ae4a271a8f5ae645306 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Fri, 19 Jul 2024 22:32:50 +0200 Subject: [PATCH] Various CI improvements + fix wrong format check in CI (#2417) use of LLVM 19 by default LLVM version should be much easier to switch from now on a lot of code repetition has been deleted removed llvm-related github action fix format check other small things --------- Co-authored-by: Toka --- .github/workflows/build_and_test.yml | 98 +++++-------------- .../fuzzer-tester-prepare/action.yml | 42 ++------ .../qemu-fuzzer-tester-prepare/action.yml | 24 +---- .github/workflows/ubuntu-prepare/action.yml | 39 ++++++-- Cargo.toml | 1 + fuzzers/others/libafl-fuzz/src/scheduler.rs | 13 ++- fuzzers/others/libafl-fuzz/test/test-cmplog.c | 7 -- fuzzers/others/libafl-fuzz/test/test-instr.c | 14 --- libafl_cc/build.rs | 4 +- libafl_qemu/src/modules/calls.rs | 21 ++-- libafl_qemu/src/modules/usermode/asan.rs | 20 +--- .../src/modules/usermode/asan_guest.rs | 4 +- libafl_qemu/src/modules/usermode/drcov.rs | 6 +- .../src/modules/usermode/injections.rs | 4 +- libafl_qemu/src/modules/usermode/snapshot.rs | 64 +++--------- scripts/fmt_all.sh | 4 +- utils/desyscall/build.rs | 4 +- utils/libafl_fmt/src/main.rs | 29 +++--- 18 files changed, 134 insertions(+), 264 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9b2db7a050..879b752068 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -10,6 +10,7 @@ on: env: CARGO_TERM_COLOR: always CARGO_NET_GIT_FETCH_WITH_CLI: true + MAIN_LLVM_VERSION: 19 concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -22,17 +23,12 @@ jobs: os: [ ubuntu-latest, windows-latest, macOS-latest ] runs-on: ${{ matrix.os }} steps: + - uses: actions/checkout@v3 + - if: runner.os == 'Linux' + uses: ./.github/workflows/ubuntu-prepare - name: Install mimetype if: runner.os == 'Linux' - run: sudo apt-get install libfile-mimeinfo-perl - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - - name: Install mimetype - if: runner.os == 'Linux' - run: sudo apt-get install libfile-mimeinfo-perl + run: sudo apt-get install -y libfile-mimeinfo-perl - name: install mdbook uses: baptiste0928/cargo-install@v1.3.0 with: @@ -41,7 +37,6 @@ jobs: uses: baptiste0928/cargo-install@v1.3.0 with: crate: mdbook-linkcheck - - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 with: { shared-key: "ubuntu" } if: runner.os == 'Linux' @@ -126,26 +121,12 @@ jobs: ubuntu: runs-on: ubuntu-22.04 steps: - - name: Remove Dotnet & Haskell - run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Remove existing clang and LLVM - run: sudo apt purge llvm* clang* lld* lldb* opt* - - name: Install and cache deps - run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev - - name: Add nightly clippy - run: rustup toolchain install nightly --component clippy --component miri --allow-downgrade - uses: actions/checkout@v3 + - uses: ./.github/workflows/ubuntu-prepare - uses: Swatinem/rust-cache@v2 with: { shared-key: "ubuntu" } - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v2 - with: - directory: ${{ runner.temp }}/llvm - version: 17 + - name: Remove Dotnet & Haskell + run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately - name: Check pcguard edges run: cargo check --features=sancov_pcguard_edges @@ -160,25 +141,14 @@ jobs: ubuntu-clippy: runs-on: ubuntu-22.04 steps: - - name: Remove Dotnet & Haskell - run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - - name: Install and cache deps - run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev - - name: Add nightly clippy - run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly - uses: actions/checkout@v3 + - uses: ./.github/workflows/ubuntu-prepare - uses: Swatinem/rust-cache@v2 with: { shared-key: "ubuntu" } - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v2 - with: - directory: ${{ runner.temp }}/llvm - version: 17 + - name: Remove Dotnet & Haskell + run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc + - name: Add nightly clippy + run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly - name: Run clippy run: ./scripts/clippy.sh # --- test embedding the libafl_libfuzzer_runtime library @@ -223,23 +193,14 @@ jobs: python-bindings: runs-on: ubuntu-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Remove existing clang and LLVM - run: sudo apt purge llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v2 - with: - directory: ${{ runner.temp }}/llvm - version: 17 + - uses: actions/checkout@v3 + - uses: ./.github/workflows/ubuntu-prepare + - uses: Swatinem/rust-cache@v2 + with: { shared-key: "ubuntu" } - name: Install deps - run: sudo apt-get install -y ninja-build python3-dev python3-pip python3-venv libz3-dev + run: sudo apt-get install -y lsb-release wget software-properties-common gnupg ninja-build python3-dev python3-pip python3-venv libz3-dev - name: Install maturin run: python3 -m pip install maturin - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - name: Run a maturin build run: export LLVM_CONFIG=llvm-config-16 && cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh - name: Run python test @@ -248,20 +209,13 @@ jobs: cargo-fmt: runs-on: ubuntu-latest steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - components: rustfmt - uses: actions/checkout@v3 - - name: Remove existing clang and LLVM - run: sudo apt purge llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v2 - with: - directory: ${{ runner.temp }}/llvm - version: 17 + - uses: ./.github/workflows/ubuntu-prepare + - uses: Swatinem/rust-cache@v2 + with: { shared-key: "ubuntu" } + - name: Add rustfmt nightly + shell: bash + run: rustup component add --toolchain nightly-x86_64-unknown-linux-gnu rustfmt - name: Format Check run: ./scripts/fmt_all.sh check @@ -337,7 +291,7 @@ jobs: # Stb - ./fuzzers/stb/libfuzzer_stb_image_sugar - ./fuzzers/stb/libfuzzer_stb_image - - ./fuzzers/stb/libfuzzer_stb_image_concolic + # - ./fuzzers/stb/libfuzzer_stb_image_concolic # Others - ./fuzzers/others/cargo_fuzz @@ -374,6 +328,7 @@ jobs: with: filters: | qemu: + - '.github/**' - 'libafl/**' - 'libafl_bolts/**' - 'libafl_targets/**' @@ -383,6 +338,7 @@ jobs: fuzzers-qemu: needs: - common + - changes if: ${{ needs.changes.outputs.qemu == 'true' }} strategy: matrix: diff --git a/.github/workflows/fuzzer-tester-prepare/action.yml b/.github/workflows/fuzzer-tester-prepare/action.yml index f5acdf7cdd..db53cc3cad 100644 --- a/.github/workflows/fuzzer-tester-prepare/action.yml +++ b/.github/workflows/fuzzer-tester-prepare/action.yml @@ -7,42 +7,24 @@ runs: with: submodules: true fetch-depth: 0 + - uses: ./.github/workflows/ubuntu-prepare - uses: Swatinem/rust-cache@v2 with: { shared-key: "${{ runner.os }}-shared-fuzzer-cache" } - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - name: Add stable clippy + - name: Install fuzzers deps shell: bash - run: rustup toolchain install stable --component clippy --allow-downgrade - - name: Add nightly clippy - shell: bash - run: rustup toolchain install nightly --component clippy --allow-downgrade - - name: Add no_std toolchain - shell: bash - run: rustup toolchain install nightly-x86_64-unknown-linux-gnu ; rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu - - name: Add wasm target - shell: bash - run: rustup target add wasm32-unknown-unknown - - name: Remove obsolete llvm (Linux) - if: runner.os == 'Linux' - shell: bash - run: sudo apt purge -y llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v2 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - - name: Install deps - shell: bash - run: sudo apt update && sudo apt install -y nasm nlohmann-json3-dev ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev + run: sudo apt-get update && sudo apt-get install -y nasm nlohmann-json3-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils - name: pip install shell: bash run: python3 -m pip install msgpack jinja2 find_libpython - name: enable mult-thread for `make` shell: bash run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)" + - name: Add no_std toolchain + shell: bash + run: rustup toolchain install nightly-x86_64-unknown-linux-gnu ; rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - name: Add wasm target + shell: bash + run: rustup target add wasm32-unknown-unknown - name: install cargo-make uses: baptiste0928/cargo-install@v1.3.0 with: @@ -58,8 +40,4 @@ runs: - name: install chrome uses: browser-actions/setup-chrome@v1 with: - chrome-version: stable - - name: Symlink Headers - if: runner.os == 'Linux' - shell: bash - run: sudo ln -s /usr/include/asm-generic /usr/include/asm + chrome-version: stable \ No newline at end of file diff --git a/.github/workflows/qemu-fuzzer-tester-prepare/action.yml b/.github/workflows/qemu-fuzzer-tester-prepare/action.yml index 23a1af3a2a..c3016aac03 100644 --- a/.github/workflows/qemu-fuzzer-tester-prepare/action.yml +++ b/.github/workflows/qemu-fuzzer-tester-prepare/action.yml @@ -7,30 +7,16 @@ runs: with: submodules: true fetch-depth: 0 - - name: Install deps + - name: Install QEMU deps shell: bash - run: apt update && apt install -y nasm ninja-build libc6-dev libgtk-3-dev pax-utils libz3-dev wget qemu-utils libsqlite3-dev gcc-arm-none-eabi sudo gcc g++ build-essential gcc-arm-linux-gnueabi g++-arm-linux-gnueabi + run: apt-get update && apt-get install -y qemu-utils sudo + - uses: ./.github/workflows/ubuntu-prepare - uses: Swatinem/rust-cache@v2 with: { shared-key: "${{ runner.os }}-shared-fuzzer-cache" } - uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: stable - - name: Add stable clippy - shell: bash - run: rustup toolchain install stable --component clippy --allow-downgrade - - name: Add nightly clippy - shell: bash - run: rustup toolchain install nightly --component clippy --allow-downgrade - - name: Remove obsolete llvm (Linux) - if: runner.os == 'Linux' - shell: bash - run: sudo apt purge -y llvm* clang* - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v2 - with: - directory: ${{ runner.temp }}/llvm - version: 17 - name: pip install shell: bash run: python3 -m pip install msgpack jinja2 find_libpython @@ -41,7 +27,3 @@ runs: uses: baptiste0928/cargo-install@v1.3.0 with: crate: cargo-make - - name: Symlink Headers - if: runner.os == 'Linux' - shell: bash - run: sudo ln -s /usr/include/asm-generic /usr/include/asm diff --git a/.github/workflows/ubuntu-prepare/action.yml b/.github/workflows/ubuntu-prepare/action.yml index 9883b892f1..0ed874617a 100644 --- a/.github/workflows/ubuntu-prepare/action.yml +++ b/.github/workflows/ubuntu-prepare/action.yml @@ -3,25 +3,44 @@ description: Sets up the Rust environment for the CI workflow runs: using: composite steps: + - name: Install and cache deps + shell: bash + run: sudo apt-get update && sudo apt-get install -y curl lsb-release wget software-properties-common gnupg ninja-build shellcheck pax-utils nasm libsqlite3-dev libc6-dev libgtk-3-dev gcc g++ gcc-arm-none-eabi gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev build-essential - uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: stable - components: llvm-tools + - name: Add stable clippy + shell: bash + run: rustup toolchain install stable --component clippy --allow-downgrade + - name: Add nightly clippy + shell: bash + run: rustup toolchain install nightly --component clippy --allow-downgrade - name: Remove existing clang and LLVM shell: bash - run: sudo apt purge llvm* clang* - - name: Install and cache deps - shell: bash - run: sudo apt update && sudo apt install ninja-build clang-format shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev + run: sudo apt-get purge -y *llvm* *clang* lld* lldb* opt* - name: Install cargo-hack shell: bash run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin - name: Add nightly shell: bash run: rustup toolchain install nightly --allow-downgrade - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v2 - with: - directory: ${{ runner.temp }}/llvm - version: 17 \ No newline at end of file + - name: Default to nightly + shell: bash + run: rustup default nightly + - name: Add LLVM in sources list + shell: bash + run: | + sudo tee -a /etc/apt/sources.list.d/llvm.list << EOF + deb https://apt.llvm.org/jammy/ llvm-toolchain-jammy main + deb-src https://apt.llvm.org/jammy/ llvm-toolchain-jammy main + EOF + - name: Add LLVM archive signature + shell: bash + run: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + - name: Install LLVM and Clang (Version ${{ env.MAIN_LLVM_VERSION }}) + shell: bash + run: sudo apt-get update && sudo apt-get install -y llvm llvm-dev clang libclang-cpp-dev llvm-${{ env.MAIN_LLVM_VERSION }} clang-${{ env.MAIN_LLVM_VERSION }} llvm-${{ env.MAIN_LLVM_VERSION }}-dev libclang-cpp${{ env.MAIN_LLVM_VERSION }} libclang-cpp${{ env.MAIN_LLVM_VERSION }}-dev + - name: Symlink Headers + shell: bash + run: sudo ln -s /usr/include/asm-generic /usr/include/asm \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 5919d55ce1..a5e8c97c65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ exclude = [ "libafl_tinyinst", "libafl_qemu/libafl_qemu_build", "libafl_qemu/libafl_qemu_sys", + "libafl_concolic/test/symcc/util/symcc_fuzzing_helper", ] [workspace.package] diff --git a/fuzzers/others/libafl-fuzz/src/scheduler.rs b/fuzzers/others/libafl-fuzz/src/scheduler.rs index 0b5477abb1..674241b441 100644 --- a/fuzzers/others/libafl-fuzz/src/scheduler.rs +++ b/fuzzers/others/libafl-fuzz/src/scheduler.rs @@ -4,7 +4,9 @@ use libafl::{ corpus::{CorpusId, HasTestcase, Testcase}, inputs::UsesInput, observers::{CanTrack, ObserversTuple}, - schedulers::{HasQueueCycles, MinimizerScheduler, RemovableScheduler, Scheduler, TestcaseScore}, + schedulers::{ + HasQueueCycles, MinimizerScheduler, RemovableScheduler, Scheduler, TestcaseScore, + }, state::{HasCorpus, HasRand, State, UsesState}, Error, HasMetadata, }; @@ -12,7 +14,10 @@ use libafl_bolts::{serdeany::SerdeAny, AsIter, HasRefCnt}; pub enum SupportedSchedulers { Queue(Q, PhantomData<(S, Q, CS, F, M, O)>), - Weighted(MinimizerScheduler, PhantomData<(S, Q, CS, F, M, O)>), + Weighted( + MinimizerScheduler, + PhantomData<(S, Q, CS, F, M, O)>, + ), } impl UsesState for SupportedSchedulers @@ -63,7 +68,7 @@ where CS: Scheduler, M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, O: CanTrack, - F: TestcaseScore + F: TestcaseScore, { fn on_add(&mut self, state: &mut Self::State, id: CorpusId) -> Result<(), Error> { match self { @@ -113,7 +118,7 @@ where CS: Scheduler + HasQueueCycles, O: CanTrack, M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, - F: TestcaseScore + F: TestcaseScore, { fn queue_cycles(&self) -> u64 { match self { diff --git a/fuzzers/others/libafl-fuzz/test/test-cmplog.c b/fuzzers/others/libafl-fuzz/test/test-cmplog.c index 0c91b21c45..60e981a44c 100644 --- a/fuzzers/others/libafl-fuzz/test/test-cmplog.c +++ b/fuzzers/others/libafl-fuzz/test/test-cmplog.c @@ -7,32 +7,25 @@ #include int LLVMFuzzerTestOneInput(const uint8_t *buf, size_t i) { - if (i < 15) return -1; if (buf[0] != 'A') return 0; int *icmp = (int *)(buf + 1); if (*icmp != 0x69694141) return 0; if (memcmp(buf + 5, "1234EF", 6) == 0) abort(); return 0; - } #ifdef __AFL_COMPILER int main(int argc, char *argv[]) { - unsigned char buf[1024]; ssize_t i; while (__AFL_LOOP(1000)) { - i = read(0, (char *)buf, sizeof(buf) - 1); if (i > 0) buf[i] = 0; LLVMFuzzerTestOneInput(buf, i); - } return 0; - } #endif - diff --git a/fuzzers/others/libafl-fuzz/test/test-instr.c b/fuzzers/others/libafl-fuzz/test/test-instr.c index 285528932c..a71baf4c19 100644 --- a/fuzzers/others/libafl-fuzz/test/test-instr.c +++ b/fuzzers/others/libafl-fuzz/test/test-instr.c @@ -23,38 +23,28 @@ #endif int main(int argc, char **argv) { - int fd = 0, cnt; char buff[8]; char *buf = buff; // we support command line parameter and stdin if (argc == 2) { - buf = argv[1]; } else { - if (argc >= 3 && strcmp(argv[1], "-f") == 0) { - if ((fd = open(argv[2], O_RDONLY)) < 0) { - fprintf(stderr, "Error: unable to open %s\n", argv[2]); exit(-1); - } - } if ((cnt = read(fd, buf, sizeof(buf) - 1)) < 1) { - printf("Hum?\n"); return 1; - } buf[cnt] = 0; - } if (getenv("AFL_DEBUG")) fprintf(stderr, "test-instr: %s\n", buf); @@ -62,7 +52,6 @@ int main(int argc, char **argv) { // we support three input cases (plus a 4th if stdin is used but there is no // input) switch (buf[0]) { - case '0': printf("Looks like a zero to me!\n"); break; @@ -74,10 +63,7 @@ int main(int argc, char **argv) { default: printf("Neither one or zero? How quaint!\n"); break; - } return 0; - } - diff --git a/libafl_cc/build.rs b/libafl_cc/build.rs index 9b7a3972b5..5154d593d6 100644 --- a/libafl_cc/build.rs +++ b/libafl_cc/build.rs @@ -107,12 +107,12 @@ fn find_llvm_config() -> Result { fn exec_llvm_config(args: &[&str]) -> String { let llvm_config = find_llvm_config().expect("Unexpected error"); - match Command::new(llvm_config).args(args).output() { + match Command::new(&llvm_config).args(args).output() { Ok(output) => String::from_utf8(output.stdout) .expect("Unexpected llvm-config output") .trim() .to_string(), - Err(e) => panic!("Could not execute llvm-config: {e}"), + Err(e) => panic!("Could not execute {llvm_config}: {e}"), } } diff --git a/libafl_qemu/src/modules/calls.rs b/libafl_qemu/src/modules/calls.rs index a412abcbed..f870af154c 100644 --- a/libafl_qemu/src/modules/calls.rs +++ b/libafl_qemu/src/modules/calls.rs @@ -265,10 +265,7 @@ where .as_mut() .unwrap() .on_ret_all(emulator_modules, state, pc, ret_addr); - emulator_modules - .get_mut::() - .unwrap() - .collectors = collectors; + emulator_modules.get_mut::().unwrap().collectors = collectors; } fn gen_blocks_calls( @@ -360,12 +357,11 @@ where let call_cb = Box::new( move |emulator_modules: &mut EmulatorModules, state: Option<&mut S>, pc| { // eprintln!("CALL @ 0x{:#x}", pc + call_len); - let mut collectors = - if let Some(h) = emulator_modules.get_mut::() { - h.collectors.take() - } else { - return; - }; + let mut collectors = if let Some(h) = emulator_modules.get_mut::() { + h.collectors.take() + } else { + return; + }; if collectors.is_none() { return; // TODO fix this, it can be None on races ret } @@ -373,10 +369,7 @@ where .as_mut() .unwrap() .on_call_all(emulator_modules, state, pc, call_len); - emulator_modules - .get_mut::() - .unwrap() - .collectors = collectors; + emulator_modules.get_mut::().unwrap().collectors = collectors; }, ); emulator_modules.instruction_closure(call_addr, call_cb, false); diff --git a/libafl_qemu/src/modules/usermode/asan.rs b/libafl_qemu/src/modules/usermode/asan.rs index b48854a380..0917b52ad4 100644 --- a/libafl_qemu/src/modules/usermode/asan.rs +++ b/libafl_qemu/src/modules/usermode/asan.rs @@ -1204,9 +1204,7 @@ pub fn trace_write1_asan_snapshot( let h = emulator_modules.get_mut::().unwrap(); h.write_1(qemu, id as GuestAddr, addr); } - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(addr, 1); } @@ -1224,9 +1222,7 @@ pub fn trace_write2_asan_snapshot( let h = emulator_modules.get_mut::().unwrap(); h.write_2(qemu, id as GuestAddr, addr); } - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(addr, 2); } @@ -1244,9 +1240,7 @@ pub fn trace_write4_asan_snapshot( let h = emulator_modules.get_mut::().unwrap(); h.write_4(qemu, id as GuestAddr, addr); } - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(addr, 4); } @@ -1264,9 +1258,7 @@ pub fn trace_write8_asan_snapshot( let h = emulator_modules.get_mut::().unwrap(); h.write_8(qemu, id as GuestAddr, addr); } - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(addr, 8); } @@ -1285,9 +1277,7 @@ pub fn trace_write_n_asan_snapshot( let h = emulator_modules.get_mut::().unwrap(); h.read_n(qemu, id as GuestAddr, addr, size); } - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(addr, size); } diff --git a/libafl_qemu/src/modules/usermode/asan_guest.rs b/libafl_qemu/src/modules/usermode/asan_guest.rs index 65c314a6ed..cadb1a9bc9 100644 --- a/libafl_qemu/src/modules/usermode/asan_guest.rs +++ b/libafl_qemu/src/modules/usermode/asan_guest.rs @@ -218,9 +218,7 @@ where S: Unpin + UsesInput, ET: EmulatorModuleTuple, { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); if !h.must_instrument(pc) { return None; } diff --git a/libafl_qemu/src/modules/usermode/drcov.rs b/libafl_qemu/src/modules/usermode/drcov.rs index ad0e4a71bc..f57adb6bec 100644 --- a/libafl_qemu/src/modules/usermode/drcov.rs +++ b/libafl_qemu/src/modules/usermode/drcov.rs @@ -286,11 +286,7 @@ pub fn exec_trace_block( ET: EmulatorModuleTuple, S: Unpin + UsesInput + HasMetadata, { - if emulator_modules - .get::() - .unwrap() - .full_trace - { + if emulator_modules.get::().unwrap().full_trace { DRCOV_IDS.lock().unwrap().as_mut().unwrap().push(id); } } diff --git a/libafl_qemu/src/modules/usermode/injections.rs b/libafl_qemu/src/modules/usermode/injections.rs index fcbda512fe..67e4ac253e 100644 --- a/libafl_qemu/src/modules/usermode/injections.rs +++ b/libafl_qemu/src/modules/usermode/injections.rs @@ -353,9 +353,7 @@ where log::trace!("syscall_hook {syscall} {SYS_execve}"); debug_assert!(i32::try_from(SYS_execve).is_ok()); if syscall == SYS_execve as i32 { - let _module = emulator_modules - .get_mut::() - .unwrap(); + let _module = emulator_modules.get_mut::().unwrap(); if x0 > 0 && x1 > 0 { let c_array = x1 as *const *const c_char; let cmd = unsafe { diff --git a/libafl_qemu/src/modules/usermode/snapshot.rs b/libafl_qemu/src/modules/usermode/snapshot.rs index 66f3dc6b46..7424766829 100644 --- a/libafl_qemu/src/modules/usermode/snapshot.rs +++ b/libafl_qemu/src/modules/usermode/snapshot.rs @@ -710,9 +710,7 @@ pub fn trace_write_snapshot( S: Unpin + UsesInput, ET: EmulatorModuleTuple, { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(addr, SIZE); } @@ -726,9 +724,7 @@ pub fn trace_write_n_snapshot( S: Unpin + UsesInput, ET: EmulatorModuleTuple, { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(addr, size); } @@ -752,9 +748,7 @@ where ET: EmulatorModuleTuple, { if i64::from(sys_num) == SYS_munmap { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); if !h.is_unmap_allowed(a0 as GuestAddr, a1 as usize) { return SyscallHookResult::new(Some(0)); } @@ -785,21 +779,15 @@ where // NOT A COMPLETE LIST OF MEMORY EFFECTS match i64::from(sys_num) { SYS_read | SYS_pread64 => { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(a1, a2 as usize); } SYS_readlinkat => { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(a2, a3 as usize); } SYS_futex => { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(a0, a3 as usize); } #[cfg(not(any( @@ -810,37 +798,27 @@ where )))] SYS_newfstatat => { if a2 != 0 { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(a2, 4096); // stat is not greater than a page } } #[cfg(any(cpu_target = "arm", cpu_target = "mips", cpu_target = "i386"))] SYS_fstatat64 => { if a2 != 0 { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(a2, 4096); // stat is not greater than a page } } SYS_statfs | SYS_fstatfs | SYS_fstat => { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(a1, 4096); // stat is not greater than a page } SYS_getrandom => { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.access(a0, a1 as usize); } SYS_brk => { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); if h.brk != result && result != 0 { /* brk has changed. we change mapping from the snapshotted brk address to the new target_brk * If no brk mapping has been made until now, change_mapped won't change anything and just create a new mapping. @@ -862,9 +840,7 @@ where #[cfg(any(cpu_target = "arm", cpu_target = "mips"))] if sys_const == SYS_mmap2 { if let Ok(prot) = MmapPerms::try_from(a2 as i32) { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.add_mapped(result, a1 as usize, Some(prot)); } } @@ -872,31 +848,23 @@ where #[cfg(not(cpu_target = "arm"))] if sys_const == SYS_mmap { if let Ok(prot) = MmapPerms::try_from(a2 as i32) { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.add_mapped(result, a1 as usize, Some(prot)); } } if sys_const == SYS_mremap { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); // TODO get the old permissions from the removed mapping h.remove_mapped(a0, a1 as usize); h.add_mapped(result, a2 as usize, None); } else if sys_const == SYS_mprotect { if let Ok(prot) = MmapPerms::try_from(a2 as i32) { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); h.change_mapped(a0, a1 as usize, Some(prot)); } } else if sys_const == SYS_munmap { - let h = emulator_modules - .get_mut::() - .unwrap(); + let h = emulator_modules.get_mut::().unwrap(); if !h.accurate_unmap && !h.is_unmap_allowed(a0, a1 as usize) { h.remove_mapped(a0, a1 as usize); } diff --git a/scripts/fmt_all.sh b/scripts/fmt_all.sh index c311443700..0c3529308e 100755 --- a/scripts/fmt_all.sh +++ b/scripts/fmt_all.sh @@ -4,9 +4,9 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" LIBAFL_DIR=$(realpath "$SCRIPT_DIR/..") if [ "$1" = "check" ]; then - cargo run --manifest-path "$LIBAFL_DIR/utils/libafl_fmt/Cargo.toml" --release -- -c --verbose + cargo run --manifest-path "$LIBAFL_DIR/utils/libafl_fmt/Cargo.toml" --release -- -c --verbose || exit 1 else - cargo run --manifest-path "$LIBAFL_DIR/utils/libafl_fmt/Cargo.toml" --release -- --verbose + cargo run --manifest-path "$LIBAFL_DIR/utils/libafl_fmt/Cargo.toml" --release -- --verbose || exit 1 fi if command -v black > /dev/null; then diff --git a/utils/desyscall/build.rs b/utils/desyscall/build.rs index 26c2a18712..2c4cd7dc0d 100644 --- a/utils/desyscall/build.rs +++ b/utils/desyscall/build.rs @@ -14,8 +14,8 @@ fn main() { println!("cargo:rerun-if-changed=src/syscalls.c"); // Enforce clang for its -fsanitize-coverage support. - std::env::set_var("CC", "clang"); - std::env::set_var("CXX", "clang++"); + env::set_var("CC", "clang"); + env::set_var("CXX", "clang++"); cc::Build::new().file("src/syscalls.c").compile("syscalls"); println!("cargo:rerun-if-changed=src/syscalls.c"); diff --git a/utils/libafl_fmt/src/main.rs b/utils/libafl_fmt/src/main.rs index 60d835705a..c304319750 100644 --- a/utils/libafl_fmt/src/main.rs +++ b/utils/libafl_fmt/src/main.rs @@ -77,6 +77,8 @@ use tokio::{process::Command, task::JoinSet}; use walkdir::{DirEntry, WalkDir}; use which::which; +const REF_LLVM_VERSION: u32 = 18; + async fn run_cargo_fmt(path: PathBuf, is_check: bool, verbose: bool) -> io::Result<()> { // Make sure we parse the correct file assert_eq!(path.file_name().unwrap().to_str().unwrap(), "Cargo.toml"); @@ -102,10 +104,12 @@ async fn run_cargo_fmt(path: PathBuf, is_check: bool, verbose: bool) -> io::Resu let res = fmt_command.output().await?; if !res.status.success() { - println!("{}", from_utf8(&res.stderr).unwrap()); + let stdout = from_utf8(&res.stdout).unwrap(); + let stderr = from_utf8(&res.stderr).unwrap(); return Err(io::Error::new( ErrorKind::Other, - format!("Cargo fmt failed. Run cargo fmt for {path:#?}"), + format!( + "Cargo fmt failed. Run cargo fmt for {path:#?}.\nstdout: {stdout}\nstderr: {stderr}"), )); } @@ -114,13 +118,13 @@ async fn run_cargo_fmt(path: PathBuf, is_check: bool, verbose: bool) -> io::Resu async fn run_clang_fmt( path: PathBuf, - clang: &str, + clang: String, is_check: bool, verbose: bool, ) -> io::Result<()> { let task_str = if is_check { "Checking" } else { "Formatting" }; - let mut fmt_command = Command::new(clang); + let mut fmt_command = Command::new(&clang); fmt_command .arg("-i") @@ -143,11 +147,12 @@ async fn run_clang_fmt( if res.status.success() { Ok(()) } else { - let stderr = from_utf8(&res.stderr).unwrap().to_string(); + let stdout = from_utf8(&res.stdout).unwrap(); + let stderr = from_utf8(&res.stderr).unwrap(); println!("{stderr}"); Err(io::Error::new( ErrorKind::Other, - format!("{clang} failed: {stderr}"), + format!("{clang} failed.\nstdout:{stdout}\nstderr:{stderr}"), )) } } @@ -216,13 +221,15 @@ async fn main() -> io::Result<()> { tokio_joinset.spawn(run_cargo_fmt(project, cli.check, cli.verbose)); } - let (clang, warning) = if which("clang-format-17").is_ok() { + let ref_clang_format = format!("clang-format-{REF_LLVM_VERSION}"); + + let (clang, warning) = if which(ref_clang_format.clone()).is_ok() { // can't use 18 for ci. - (Some("clang-format-17"), None) + (Some(ref_clang_format), None) } else if which("clang-format").is_ok() { ( - Some("clang-format"), - Some("using clang-format, could provide a different result from clang-format-18"), + Some("clang-format".to_string()), + Some("using clang-format, could provide a different result from clang-format-17"), ) } else { ( @@ -242,7 +249,7 @@ async fn main() -> io::Result<()> { .collect(); for c_file in c_files_to_fmt { - tokio_joinset.spawn(run_clang_fmt(c_file, clang, cli.check, cli.verbose)); + tokio_joinset.spawn(run_clang_fmt(c_file, clang.clone(), cli.check, cli.verbose)); } }