Improve CI (#3258)

* Improve CI

* Fix markup errors

* Remove unnecessary matrix parameters

* Rename matrix jobs to tidy up the UI

* Allow the docker cache to be updated

* Fix cache name

* Share some caches

* Rename tools

.github/workflows/build_and_test.yml

@@ -254,12 +254,12 @@ jobs:
         run: ./scripts/check_tested_fuzzers.sh
 
   fuzzers:
+    name: 🚀 ${{ matrix.fuzzer }}
     needs:
       - fuzzers-preflight
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-24.04 ]
         fuzzer:
           # Baby
           - ./fuzzers/baby/baby_fuzzer_swap_differential
@@ -338,26 +338,59 @@ jobs:
           - ./fuzzers/fuzz_anything/baby_no_std
           - ./fuzzers/fuzz_anything/baby_fuzzer_wasm
 
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+        # Get the name of the fuzzer so that we can use it as the key for a cache
-      - uses: ./.github/workflows/fuzzer-tester-prepare
+        # of the built artefacts. The key cannot have any special characters.
+      - name: Get fuzzer name
+        id: fuzzer_name
+        run: |
+          fname=$(basename "${{ matrix.fuzzer }}")
+          echo "fuzzer_name=$fname" >> $GITHUB_OUTPUT
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Prepare
+        uses: ./.github/workflows/fuzzer-tester-prepare
         with:
           fuzzer-name: ${{ matrix.fuzzer }}
-      - name: Build and run example fuzzers (Linux)
+      - name: Configure Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # We will have these guys all share a cache (since they should share
+          # and awful lot of their dependencies). That way we won't use up
+          # as much space.
+          shared-key: fuzzers-x86_64
+          cache-all-crates: true
+          # The code is built in the fuzzers own directory, not in the target
+          # directory in the root of the workspace
+          workspaces: |
+            ${{ matrix.fuzzer }}
+      - name: Test
         if: runner.os == 'Linux'
         shell: bash
-        run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} cargo run --manifest-path ./utils/ci_runner/Cargo.toml -- ${{ matrix.fuzzer }}
+        run: |
+          RUN_ON_CI=1 \
+          LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} \
+          cargo \
+            run \
+            --manifest-path ./utils/ci_runner/Cargo.toml \
+            -- \
+            ${{ matrix.fuzzer }}
 
-  changes:
+  # This job checks whether any changes have been made to the QEMU code to avoid
+  # rebuilding and testing the QEMU related fuzzers unnecessarily as they are 
+  # more expensive to build
+  qemu-changes:
     runs-on: ubuntu-24.04
     permissions:
       pull-requests: read
     outputs:
       qemu: ${{ steps.filter.outputs.qemu }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
-      - uses: dorny/paths-filter@v3
+        uses: actions/checkout@v4
+      - name: Filter
+        uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
@@ -368,6 +401,7 @@ jobs:
               - 'libafl_targets/**'
               - 'libafl_qemu/**'
               - 'fuzzers/**/*qemu*/**'
+
   fuzzer-unicorn:
     runs-on: ubuntu-24.04
     needs:
@@ -390,53 +424,69 @@ jobs:
         shell: bash
         run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} cargo run --manifest-path ./utils/ci_runner/Cargo.toml -- ${{ matrix.fuzzer }}
 
-  fuzzers-qemu-user:
+  # Job to build and test the QEMU fuzzers
+  qemu:
+    name: 🚀 ${{ matrix.fuzzer }}
     needs:
-      - changes
+      - fuzzers-preflight # Check that all the fuzzers listed for testing or explicitly ignored
-    if: ${{ needs.changes.outputs.qemu == 'true' }}
+      - qemu-changes # Only build if the QEMU code has changed
+    if: ${{ needs.qemu-changes.outputs.qemu == 'true' }}
     strategy:
       matrix:
-        os: [ubuntu-24.04]
         fuzzer:
-          # Binary only
+            - fuzzers/binary_only/qemu_cmin
-          - ./fuzzers/binary_only/qemu_cmin
+            - fuzzers/binary_only/qemu_tmin
-          - ./fuzzers/binary_only/qemu_tmin
+            - fuzzers/binary_only/qemu_coverage
-          - ./fuzzers/binary_only/qemu_coverage
+            - fuzzers/binary_only/qemu_launcher
-          - ./fuzzers/binary_only/qemu_launcher
+            - fuzzers/full_system/qemu_baremetal
-        arch:
+            # - fuzzers/full_system/qemu_linux_kernel
-          # unless somebody pays us for the servers.
+            # - fuzzers/full_system/qemu_linux_process
-          # - aarch64
+    runs-on: ubuntu-24.04
-          # - arm
-          # - i386
-          # - ppc
-          - x86_64
-
-    runs-on: [ self-hosted, qemu ]
     container: registry.gitlab.com/qemu-project/qemu/qemu/ubuntu2204:latest
     steps:
-      - uses: actions/checkout@v4
+        # Get the name of the fuzzer so that we can use it as the key for a cache
-      - uses: ./.github/workflows/qemu-fuzzer-tester-prepare
+        # of the built artefacts. The key cannot have any special characters.
-      - name: Build and run example QEMU fuzzers (Linux)
+      - name: Get fuzzer name
-        if: runner.os == 'Linux'
+        id: fuzzer_name
+        run: |
+          fname=$(basename "${{ matrix.fuzzer }}")
+          echo "fuzzer_name=$fname" >> $GITHUB_OUTPUT
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Prepare
+        uses: ./.github/workflows/qemu-fuzzer-tester-prepare
+      - name: Configure Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # We will have each of these fuzzers have it's own cache since these
+          # are some of the heaviest fuzzers to build.
+          shared-key: qemu-${{ steps.fuzzer_name.outputs.fuzzer_name }}-x86_64
+          cache-all-crates: true
+          # The code is built in the fuzzers own directory, not in the target
+          # directory in the root of the workspace
+          workspaces: |
+            ${{ matrix.fuzzer }}
+      - name: Test
         shell: bash
-        run: unset RUSTC && ARCH=${{ matrix.arch }} RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} cargo run --manifest-path ./utils/ci_runner/Cargo.toml --  ${{ matrix.fuzzer }}
+        run: |
+          unset RUSTC && \
+          ARCH=x86_64 \
+          RUN_ON_CI=1 \
+          LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} \
+          cargo \
+            run \
+            --manifest-path ./utils/ci_runner/Cargo.toml \
+            --  \
+            ${{ matrix.fuzzer }}
 
   utils:
+    name: 🔧 ${{ matrix.util }}
     strategy:
       matrix:
-        os: [ubuntu-24.04]
+        util:
-        fuzzer:
-          # Binary only
           - ./utils/gdb_qemu
-        arch:
-          # unless somebody pays us for the servers.
-          # - aarch64
-          # - arm
-          # - i386
-          # - ppc
-          - x86_64
 
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/workflows/ubuntu-prepare
@@ -448,69 +498,67 @@ jobs:
       - name: Build and run utils (Linux)
         if: runner.os == 'Linux'
         shell: bash
-        run: just -d ${{ matrix.fuzzer }} --justfile ${{ matrix.fuzzer }}/Justfile test
+        run: just -d ${{ matrix.util }} --justfile ${{ matrix.util }}/Justfile test
 
-  librasan-build:
+  # Build and test librasan. The setup is common, so we use a matrix build and
-      runs-on: ubuntu-24.04
+  # use the task name at the end to select which target to build
-      needs:
+  librasan:
-        - changes
+    name: 🔧 librasan - ${{ matrix.task }}
-      if: ${{ needs.changes.outputs.qemu == 'true' }}
-      steps:
-        - uses: actions/checkout@v4
-        - uses: ./.github/workflows/librasan-prepare
-        - name: Build
-          if: runner.os == 'Linux'
-          shell: bash
-          run: |
-            RUN_ON_CI=1 \
-            RUSTC_BOOTSTRAP=1 \
-            LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} \
-            just \
-              -f ./libafl_qemu/librasan/Justfile \
-              build_everything_dev \
-              build_x86_64_release
-
-  librasan-test:
-      runs-on: ubuntu-24.04
-      needs:
-        - changes
-      if: ${{ needs.changes.outputs.qemu == 'true' }}
-      steps:
-        - uses: actions/checkout@v4
-        - uses: ./.github/workflows/librasan-prepare
-        - name: Build
-          if: runner.os == 'Linux'
-          shell: bash
-          run: |
-            RUN_ON_CI=1 \
-            RUSTC_BOOTSTRAP=1 \
-            LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} \
-            just \
-              -f ./libafl_qemu/librasan/Justfile \
-              test_everything
-
-  fuzzers-qemu-system:
-    needs:
-      - changes
-    if: ${{ needs.changes.outputs.qemu == 'true' }}
     strategy:
       matrix:
-        os: [ubuntu-24.04]
+        task:
-        fuzzer:
+          - build
-          # Full-system
+          - test
-          - ./fuzzers/full_system/qemu_baremetal
+      # If one task fails, then don't abandon the other
-          # - ./fuzzers/full_system/qemu_linux_kernel
+      fail-fast: false
-          # - ./fuzzers/full_system/qemu_linux_process
+    # Only build this if the QEMU code changes for now. Though agnostic, nobody
-
+    # else has integrated librasan with other fuzzers for now.
-    runs-on: [ self-hosted, qemu ]
+    needs:
-    container: registry.gitlab.com/qemu-project/qemu/qemu/ubuntu2204:latest
+        - qemu-changes
+    if: ${{ needs.qemu-changes.outputs.qemu == 'true' }}
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
-      - uses: ./.github/workflows/qemu-fuzzer-tester-prepare
+        uses: actions/checkout@v4
-      - name: Build and run example QEMU fuzzers (Linux)
+        with:
-        if: runner.os == 'Linux'
+          submodules: true
+          fetch-depth: 0
+      - name: Prepare
+        uses: ./.github/workflows/librasan-prepare
+      - name: Configure Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # Use the task name to build the cache key. We will have a separate 
+          # cache for each since they are both expensive to build and one builds
+          # for many different architectures.
+          shared-key: librasan-${{ matrix.task }}
+          cache-all-crates: true
+          # Again the artefacts aren't built in the target root directory.
+          workspaces: |
+            libafl_qemu/librasan
+        # Run only for the build task
+      - name: Build
+        if: ${{ matrix.task == 'build' }}
         shell: bash
-        run: unset RUSTC && RUN_ON_CI=1 LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} cargo run --manifest-path ./utils/ci_runner/Cargo.toml --  ${{ matrix.fuzzer }}
+        run: |
+          RUN_ON_CI=1 \
+          RUSTC_BOOTSTRAP=1 \
+          LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} \
+          just \
+            -f ./libafl_qemu/librasan/Justfile \
+            build_everything_dev \
+            build_x86_64_release
+        # Run only for the test task
+      - name: Test
+        if: ${{ matrix.task == 'test' }}
+        shell: bash
+        run: |
+          RUN_ON_CI=1 \
+          RUSTC_BOOTSTRAP=1 \
+          LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} \
+          just \
+            -f ./libafl_qemu/librasan/Justfile \
+            test_everything
 
   nostd-build:
     runs-on: ubuntu-24.04
@@ -555,72 +603,126 @@ jobs:
 
   build-docker:
     runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Build docker
-        run: docker build -t libafl .
 
-  windows-frida-libpng:
+    permissions:
-    runs-on: windows-latest
+      packages: write
-    needs:
+      contents: read
-      - common
+
-    steps:
+    outputs:
-      - uses: actions/checkout@v4
+      image: ghcr.io/aflplusplus/libafl:latest
-      - uses: ./.github/workflows/windows-tester-prepare
+
-      - name: Build fuzzers/binary_only/frida_libpng
+    steps:
-        run: cd fuzzers/binary_only/frida_libpng/ && just test
+      - name: Checkout repo
-
+        uses: actions/checkout@v3
-  windows-libafl-libfuzzer:
+
-    runs-on: windows-latest
+      - name: Check Dockerfile for changes
-    needs:
+        uses: dorny/paths-filter@v3
-      - common
+        id: filter
-    steps:
+        with:
-      - uses: actions/checkout@v4
+          filters: |
-      - uses: ./.github/workflows/windows-tester-prepare
+            changed:
-      - name: Build fuzzers/inprocess/libafl_libfuzzer_windows
+              - 'Dockerfile'
-        run: cd fuzzers/inprocess/libafl_libfuzzer_windows && just test
+              - '.github/workflows/build_and_test.yml'
-
+
-  windows-libfuzzer-stb-image:
+      - name: Restore Docker build cache
-    runs-on: windows-latest
+        if: steps.filter.outputs.changed == 'true'
-    needs:
+        uses: actions/cache@v4
-      - common
+        with:
-    steps:
+          path: /tmp/.buildx-cache
-      - uses: actions/checkout@v4
+          # We want to be able to update the cache if the Dockerfile is changed,
-      - uses: ./.github/workflows/windows-tester-prepare
+          # but still be able to use previous versions if available.
-      - name: Build fuzzers/inprocess/libfuzzer_stb_image
+          key: ${{ runner.os }}-buildx-${{ github.ref }}-${{ hashFiles('Dockerfile') }}
-        run: cd fuzzers/inprocess/libfuzzer_stb_image && cargo build --release
+          restore-keys: |
-
+            ${{ runner.os }}-buildx-${{ github.ref }}-
-  # windows-libfuzzer-asan:
+            ${{ runner.os }}-buildx-
-  #   runs-on: windows-latest
+
-  #   needs:
+      - name: Set up Docker Buildx
-  #     - common
+        if: steps.filter.outputs.changed == 'true'
-  #   steps:
+        uses: docker/setup-buildx-action@v3
-  #     - uses: actions/checkout@v4
+        with:
-  #     - uses: ./.github/workflows/windows-tester-prepare
+          driver: docker-container
-  #     - name: Build fuzzers/inprocess/libfuzzer_windows_asan
+
-  #       run: cd fuzzers/inprocess/libfuzzer_windows_asan && just test
+      - name: Log in to GitHub Container Registry
-
+        if: steps.filter.outputs.changed == 'true' && github.ref == 'refs/heads/main'
-  windows-frida-gdiplus:
+        uses: docker/login-action@v3
-    runs-on: windows-latest
+        with:
-    needs:
+          registry: ghcr.io
-      - common
+          username: ${{ github.actor }}
-    steps:
+          password: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/checkout@v4
+
-      - uses: ./.github/workflows/windows-tester-prepare
+      - name: Build Docker image
-      - name: Build fuzzers/binary_only/frida_windows_gdiplus
+        if: steps.filter.outputs.changed == 'true'
-        run: cd fuzzers/binary_only/frida_windows_gdiplus/ && just test && just test_cmplog
+        uses: docker/build-push-action@v5
-
+        with:
-  windows-tinyinst-simple:
+          context: .
+          file: ./Dockerfile
+          push: ${{ github.ref == 'refs/heads/main' }}
+          load: ${{ github.ref != 'refs/heads/main' }}
+          tags: 
+            ghcr.io/aflplusplus/libafl:latest
+          cache-from: |
+            type=registry,ref=ghcr.io/aflplusplus/libafl:cache
+            type=local,src=/tmp/.buildx-cache
+          cache-to: |
+            ${{ github.ref == 'refs/heads/main' && 'type=registry,ref=ghcr.io/aflplusplus/libafl:cache,mode=max' || 'type=local,dest=/tmp/.buildx-cache' }}
+
+      - name: Save Docker build cache
+        if: steps.filter.outputs.changed == 'true'
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          # We want to be able to update the cache if the Dockerfile is changed,
+          # but still be able to use previous versions if available.
+          key: ${{ runner.os }}-buildx-${{ github.ref }}-${{ hashFiles('Dockerfile') }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-${{ github.ref }}-
+            ${{ runner.os }}-buildx-
+
+  # The windows fuzzers require the same setup, so we will use a matrix build
+  windows:
+    name: 🚀 ${{ matrix.fuzzer }}
+    strategy:
+      matrix:
+        fuzzer:
+            - fuzzers/binary_only/frida_libpng
+            - fuzzers/inprocess/libafl_libfuzzer_windows
+            - fuzzers/inprocess/libfuzzer_stb_image
+            - fuzzers/binary_only/frida_windows_gdiplus
+            - fuzzers/binary_only/tinyinst_simple
+            # - fuzzers/inprocess/libfuzzer_windows_asan
+      fail-fast: false
     runs-on: windows-latest
     needs:
+      - fuzzers-preflight
       - common
     steps:
+        # Get the name of the fuzzer so that we can use it as the key for a cache
+        # of the built artefacts. The key cannot have any special characters.
+      - name: Get fuzzer name
+        id: fuzzer_name
+        shell: pwsh
+        run: |
+          $fname = Split-Path -Leaf "${{ matrix.fuzzer }}"
+          "fuzzer_name=$fname" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Prepare
+        uses: ./.github/workflows/windows-tester-prepare
       - name: install cxx bridge
         run: cargo install cxxbridge-cmd
-      - uses: actions/checkout@v4
+      - name: Configure Cache
-      - uses: ./.github/workflows/windows-tester-prepare
+        uses: Swatinem/rust-cache@v2
-      - name: Build fuzzers/binary_only/tinyinst_simple
+        with:
-        run: cd fuzzers/binary_only/tinyinst_simple/ && just test
+          # There aren't too many of these fuzzers, so lets just give them a 
+          # separate cache each for now.
+          shared-key: fuzzers-windows-${{ steps.fuzzer_name.outputs.fuzzer_name }}-x86_64
+          cache-all-crates: true
+          # The code is built in the fuzzers own directory, not in the target
+          # directory in the root of the workspace
+          workspaces: |
+            ${{ matrix.fuzzer }}
+      - name: Test
+        run: cd ${{ matrix.fuzzer }} && just test
 
   windows-clippy:
     runs-on: windows-latest

fuzzers/binary_only/frida_windows_gdiplus/Justfile

@@ -38,11 +38,14 @@ test_cmplog: build harness_cmplog_test
 
 [windows]
 [script("cmd.exe", "/c")]
-test: build harness
+test_default: build harness
     start "" "{{FUZZER_NAME}}" -H harness.dll -i corpus -o output --libs-to-instrument gdi32.dll --libs-to-instrument gdi32full.dll --libs-to-instrument gdiplus.dll --libs-to-instrument WindowsCodecs.dll --disable-excludes
     ping -n 10 127.0.0.1>NUL && taskkill /im frida_windows_gdiplus.exe /F
     dir /a-d corpus_discovered && (echo Files exist) || (exit /b 1337)
 
+[windows]
+test: test_default test_cmplog
+
 [windows]
 clean:
     make -C libpng-1.6.37 clean

fuzzers/inprocess/libfuzzer_stb_image/Justfile

@@ -7,8 +7,9 @@ PROFILE_DIR := if PROFILE == "release" { "release" } else if PROFILE == "dev" {
 EXTENSION := if os() == "windows" {".exe"} else { "" }
 LIBAFL_CC := PROJECT_DIR / CARGO_TARGET_DIR / PROFILE_DIR / "libafl_cc" + EXTENSION
 LIBAFL_CXX := PROJECT_DIR / CARGO_TARGET_DIR / PROFILE_DIR / "libafl_cxx" + EXTENSION
- 
 FUZZER := PROJECT_DIR / CARGO_TARGET_DIR / PROFILE_DIR / FUZZER_NAME + EXTENSION
+set windows-shell := ['cmd.exe', '/c']
+set unstable
 
 alias cc := cxx
 
@@ -47,6 +48,11 @@ test: fuzzer
         exit 1
     fi
 
+[windows]
+[script("cmd.exe", "/c")]
+test:
+    cargo build --release
+
 clean:
     #!/bin/bash
     rm -f {{FUZZER_NAME}}

libafl_qemu/libafl_qemu_build/src/build.rs

@@ -335,9 +335,9 @@ pub fn build(
             .as_deref()
             .unwrap_or(LIBAFL_QEMU_GIT_REV);
 
-        let qemu_rev = target_dir.join("QEMU_REVISION");
+        let qemu_rev = qemu_path.join("QEMU_REVISION");
-        if qemu_rev.exists()
+        if !qemu_rev.exists()
-            && fs::read_to_string(&qemu_rev).expect("Failed to read QEMU_REVISION") != qemu_git_rev
+            || fs::read_to_string(&qemu_rev).expect("Failed to read QEMU_REVISION") != qemu_git_rev
         {
             drop(fs::remove_dir_all(&qemu_path));
         }

scripts/check_tested_fuzzers.sh

@@ -11,8 +11,10 @@ while read -r fuzzdir; do
     echo "Fuzzer ${fuzzdir} is explicitly ignored"
   fi
 done < <(
-          find ./fuzzers -mindepth 2 -maxdepth 2 -type d
+  {
-          find ./fuzzers/baby/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d
+    find ./fuzzers -mindepth 2 -maxdepth 2 -type d
-        )
+    find ./fuzzers/baby/backtrace_baby_fuzzers -mindepth 1 -maxdepth 1 -type d
+  } | sed 's|^\./||'
+)
 
-exit $ret
+exit $ret