CI: A few updates and fixes (#2023)

* CI: Don't run cargo clean in order to preserve build caches

* CI: run yaml formatter >:)

* CI: try sharing build caches between similar jobs

* CI: it looks like the `ucd-generate` tool is not used anymore

* CI: Sneak in merge queue support :)

* CI: split out `cargo fmt --check` job

* libafl_qemu injections: be more resilient about mapping paths

.github/workflows/build_and_test.yml

@@ -2,10 +2,11 @@ name: build and test
 
 on:
   push:
-    branches: [ main, 'pr/**' ]
+    branches: [main, "pr/**"]
   pull_request:
     branches: [main]
   workflow_dispatch:
+  merge_group:
 env:
   CARGO_TERM_COLOR: always
 concurrency:
@@ -40,6 +41,10 @@ jobs:
           crate: mdbook-linkcheck
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
+        if: runner.os == 'Linux'
+      - uses: Swatinem/rust-cache@v2
+        if: runner.os != 'Linux'
       - name: Check for binary blobs
         if: runner.os == 'Linux'
         run: ./scripts/check_for_blobs.sh
@@ -75,6 +80,7 @@ jobs:
           toolchain: stable
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "llvm-tester" }
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@v1
         with:
@@ -82,7 +88,6 @@ jobs:
       - name: Build and test with llvm-${{ matrix.llvm-version }}
         run: pwd && ls & cd libafl_cc && cargo build --release
 
-
   ubuntu-doc:
     runs-on: ubuntu-22.04
     needs: ubuntu
@@ -99,10 +104,9 @@ jobs:
         run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
       - name: Add nightly rustfmt and clippy
         run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
-    - name: Install ucd-generate
-      run: cargo install -f ucd-generate
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@v1
         with:
@@ -130,10 +134,9 @@ jobs:
         run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
       - name: Add nightly rustfmt and clippy
         run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
-    - name: Install ucd-generate
-      run: cargo install -f ucd-generate
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@v1
         with:
@@ -158,10 +161,9 @@ jobs:
         run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
       - name: Add nightly rustfmt and clippy
         run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
-    - name: Install ucd-generate
-      run: cargo install -f ucd-generate
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@v1
         with:
@@ -171,10 +173,6 @@ jobs:
       # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately
       - name: Check pcguard edges
         run: cargo check --features=sancov_pcguard_edges
-    - name: Format
-      run: cargo fmt -- --check
-    - name: Cleanup
-      run: cargo clean
       - name: Run clang-format style check for C/C++ programs.
         run: clang-format -n -Werror --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c' | grep -v 'QEMU-Nyx')
       - name: run shellcheck
@@ -202,6 +200,7 @@ jobs:
         run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@v1
         with:
@@ -210,10 +209,6 @@ jobs:
       - name: Run clippy
         run: ./scripts/clippy.sh
 
-    # Clean up files to save up disk space
-    - name: Cleanup
-      run: cargo clean
-
     # --- test embedding the libafl_libfuzzer_runtime library
     # Fix me plz
     # - name: Test Build libafl_libfuzzer with embed
@@ -249,12 +244,11 @@ jobs:
         run: sudo apt update && sudo apt install ninja-build clang-format shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
       - name: Install cargo-hack
         run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
-    - name: Install ucd-generate
-      run: cargo install -f ucd-generate
       - name: Add nightly
         run: rustup toolchain install nightly --allow-downgrade
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@v1
         with:
@@ -280,6 +274,7 @@ jobs:
           toolchain: stable
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
       - name: Install smoke test deps
         run: sudo ./libafl_concolic/test/smoke_test_ubuntu_deps.sh
       - name: Run smoke test
@@ -310,6 +305,19 @@ jobs:
       - name: Run python test
         run: . ./bindings/pylibafl/.env/bin/activate # && cd ./fuzzers/python_qemu/ && python3 fuzzer.py 2>&1 | grep "Bye"
 
+  cargo-fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+      - uses: actions/checkout@v3
+      - name: Format Check
+        run: cargo fmt -- --check
+
   fuzzers:
     needs: ubuntu
     strategy:
@@ -482,8 +490,6 @@ jobs:
           toolchain: stable
       - name: Add nightly rustfmt and clippy
         run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade && rustup default nightly
-    - name: Install ucd-generate
-      run: cargo install -f ucd-generate
       - name: Install deps
         run: brew install z3 gtk+3
       - name: Install cxxbridge
@@ -551,7 +557,6 @@ jobs:
             freebsd-version
             . "$HOME/.cargo/env"
             rustup toolchain install nightly
-          cargo install -f ucd-generate
             export LLVM_CONFIG=/usr/local/bin/llvm-config16
             pwd
             ls -lah

.github/workflows/fuzzer-tester-prepare/action.yml

@@ -8,8 +8,7 @@ runs:
         submodules: true
         fetch-depth: 0
     - uses: Swatinem/rust-cache@v2
-      with:
-        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      with: { shared-key: "${{ runner.os }}-shared-fuzzer-cache" }
     - uses: actions-rs/toolchain@v1
       with:
         profile: minimal
@@ -23,9 +22,6 @@ runs:
     - name: Add wasm target
       shell: bash
       run: rustup target add wasm32-unknown-unknown
-    - name: Install ucd-generate
-      shell: bash
-      run: cargo install -f ucd-generate
     - name: Remove obsolete llvm (Linux)
       if: runner.os == 'Linux'
       shell: bash

libafl_libfuzzer/build.rs

@@ -78,7 +78,7 @@ fn main() {
 
     assert!(
         command.status().map_or(false, |s| s.success()),
-        "Couldn't build runtime crate! Did you remember to use nightly? (`rustup default nightly` to install) Or, did you remember to install ucd-generate? (`cargo install ucd-generate` to install)"
+        "Couldn't build runtime crate! Did you remember to use nightly? (`rustup default nightly` to install)"
     );
 
     let mut archive_path = custom_lib_dir.join(std::env::var_os("TARGET").unwrap());

libafl_qemu/src/injections.rs

@@ -272,7 +272,8 @@ where
 
         for region in qemu.mappings() {
             if let Some(path) = region.path().map(ToOwned::to_owned) {
-                if !path.is_empty() {
+                // skip [heap], [vdso] and friends
+                if !path.is_empty() && !path.starts_with('[') {
                     LibInfo::add_unique(
                         &mut libs,
                         LibInfo {