Faster cargo check (#2084)

* speed up cargo check

* fix

* composite

* plural

* wrong name

* go brrrrrrrr

* bash

* aaaaa

* why not even more?

* more composite

* fix

* miri

* test for llvm >= 16

* v2

* v2

* naaaa

* JUST MORE INSTANCES

* aa

* fix

* no bash for windows

* reduce a bit as some guys are delayed

* no deps

.github/workflows/build_and_test.yml

@@ -70,7 +70,7 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        llvm-version: [ "11", "12", "13", "14", "16", "17" ]
+        llvm-version: [ "16", "17" ] # Add 18 when KyleMayes/install-llvm-action enables it
     steps:
       - name: Remove Dotnet & Haskell
         run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
@@ -82,39 +82,31 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with: { shared-key: "llvm-tester" }
       - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v1
+        uses: KyleMayes/install-llvm-action@v2
         with:
           version: "${{matrix.llvm-version}}"
       - name: Build and test with llvm-${{ matrix.llvm-version }}
         run: pwd && ls & cd libafl_cc && cargo build --release
 
-  ubuntu-doc:
+  ubuntu-doc-build:
     runs-on: ubuntu-22.04
     needs: ubuntu
     steps:
-      - name: Remove Dotnet & Haskell
-        run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-      - name: Remove existing clang and LLVM
-        run: sudo apt purge llvm* clang* lld* lldb* opt*
-      - name: Install and cache deps
-        run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
-      - name: Add nightly rustfmt and clippy
-        run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
       - uses: actions/checkout@v3
+      - uses: ./.github/workflows/ubuntu-prepare
       - uses: Swatinem/rust-cache@v2
-        with: { shared-key: "ubuntu" }
-      - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v1
-        with:
-          directory: ${{ runner.temp }}/llvm
-          version: 17
       # ---- doc check ----
       - name: Build Docs
-        run: RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features
+        run: RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features --no-deps
+
+  ubuntu-doc-test:
+    runs-on: ubuntu-22.04
+    needs: ubuntu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/ubuntu-prepare
+      - uses: Swatinem/rust-cache@v2
+      # ---- doc check ----
       - name: Test Docs
         run: RUSTFLAGS="--cfg docsrs" cargo +nightly test --doc --all-features
 
@@ -122,26 +114,11 @@ jobs:
     runs-on: ubuntu-22.04
     needs: ubuntu
     steps:
-      - name: Remove Dotnet & Haskell
-        run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-      - name: Remove existing clang and LLVM
-        run: sudo apt purge llvm* clang* lld* lldb* opt*
-      - name: Install and cache deps
-        run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
-      - name: Add nightly rustfmt and clippy
-        run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
       - uses: actions/checkout@v3
+      - uses: ./.github/workflows/ubuntu-prepare
       - uses: Swatinem/rust-cache@v2
-        with: { shared-key: "ubuntu" }
+      - name: Add nightly rustfmt and clippy
-      - name: Install LLVM and Clang
+        run: rustup toolchain install nightly --component miri --allow-downgrade
-        uses: KyleMayes/install-llvm-action@v1
-        with:
-          directory: ${{ runner.temp }}/llvm
-          version: 17
       # --- miri undefined behavior test --
       - name: Run miri tests
         run: RUST_BACKTRACE=1 MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test
@@ -185,7 +162,6 @@ jobs:
 
   ubuntu-clippy:
     runs-on: ubuntu-22.04
-    needs: ubuntu
     steps:
       - name: Remove Dotnet & Haskell
         run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
@@ -213,56 +189,38 @@ jobs:
     # Fix me plz
     # - name: Test Build libafl_libfuzzer with embed
     #  run: cargo +nightly test --features=embed-runtime --manifest-path libafl_libfuzzer/Cargo.toml
+  ubuntu-check-nightly:
+    runs-on: ubuntu-22.04
+    needs: ubuntu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/ubuntu-prepare
+      - uses: Swatinem/rust-cache@v2
+        with: { shared-key: "ubuntu" }
+      # ---- build and feature check ----
+      # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs
+      - name: Check nightly features
+        run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus
+
 
   ubuntu-check:
     runs-on: ubuntu-22.04
     needs: ubuntu
+    strategy:
+      matrix:
+        instance_idx: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17" ]
     steps:
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          components: llvm-tools
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: false
-          docker-images: true
-          swap-storage: true
-      - name: Remove existing clang and LLVM
-        run: sudo apt purge llvm* clang*
-      - name: Install and cache deps
-        run: sudo apt update && sudo apt install ninja-build clang-format shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
-      - name: Install cargo-hack
-        run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
-      - name: Add nightly
-        run: rustup toolchain install nightly --allow-downgrade
       - uses: actions/checkout@v3
+      - uses: ./.github/workflows/ubuntu-prepare
       - uses: Swatinem/rust-cache@v2
         with: { shared-key: "ubuntu" }
-      - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v1
-        with:
-          directory: ${{ runner.temp }}/llvm
-          version: 17
       # ---- build and feature check ----
       # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs
       - name: Check each feature
         # Skipping `python` as it has to be built with the `maturin` tool
         # `agpl`, `nautilus` require nightly
         # `sancov_pcguard_edges` is tested seperatelyc
-        run: LLVM_CONFIG=llvm-config cargo hack check --workspace --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode,whole_archive --no-dev-deps --exclude libafl_libfuzzer
+        run: python3 ./scripts/parallellize_cargo_check.py ${{ matrix.instance_idx }}
-      - name: Check nightly features
-        run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus
 
   ubuntu-concolic:
     runs-on: ubuntu-latest
@@ -448,37 +406,52 @@ jobs:
       - name: Build docker
         run: docker build -t libafl .
 
-  windows:
+  windows-frida-libpng:
     runs-on: windows-latest
-    steps:
+    needs:
-      - uses: actions-rs/toolchain@v1
+      - common  
-        with:
+    steps:  
-          profile: minimal
-          toolchain: stable
       - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
+      - uses: ./.github/workflows/windows-tester-prepare
-      - name: Windows Build
-        run: cargo build --verbose
-      - name: Build docs
-        run: cargo doc
-      - uses: ilammy/msvc-dev-cmd@v1
-      - name: Set LIBCLANG_PATH
-        run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV
-      - name: install cargo-make
-        run: cargo install --force cargo-make
-      - name: install cxx bridge
-        run: cargo install cxxbridge-cmd
-      - name: Build fuzzers/libfuzzer_stb_image
-        run: cd fuzzers/libfuzzer_stb_image && cargo build --release
       - name: Build fuzzers/frida_libpng
         run: cd fuzzers/frida_libpng/ && cargo make test
+
+  windows-frida-libfuzzer-stb-image:
+    runs-on: windows-latest
+    needs:
+      - common
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/windows-tester-prepare
+      - name: Build fuzzers/libfuzzer_stb_image
+        run: cd fuzzers/libfuzzer_stb_image && cargo build --release
+
+  windows-frida-gdiplus:
+    runs-on: windows-latest
+    needs:
+      - common
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/windows-tester-prepare
       - name: Build fuzzers/frida_gdiplus
         run: cd fuzzers/frida_gdiplus/ && cargo make test && cargo make test_cmplog
+
+  windows-tinyinst-simple:
+    runs-on: windows-latest
+    needs:
+      - common 
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/windows-tester-prepare
+      - name: install cxx bridge
+        run: cargo install cxxbridge-cmd
       - name: Build fuzzers/tinyinst_simple
         run: cd fuzzers/tinyinst_simple/ && cargo make test
 
   windows-clippy:
     runs-on: windows-latest
+    needs:
+      - common    
     steps:
       - uses: actions-rs/toolchain@v1
         with:

.github/workflows/ubuntu-prepare/action.yml

@@ -0,0 +1,27 @@
+name: Setup Rust Environment
+description: Sets up the Rust environment for the CI workflow
+runs:
+  using: composite
+  steps:
+    - uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+        components: llvm-tools
+    - name: Remove existing clang and LLVM
+      shell: bash
+      run: sudo apt purge llvm* clang*
+    - name: Install and cache deps
+      shell: bash
+      run: sudo apt update && sudo apt install ninja-build clang-format shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
+    - name: Install cargo-hack
+      shell: bash
+      run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
+    - name: Add nightly
+      shell: bash
+      run: rustup toolchain install nightly --allow-downgrade
+    - name: Install LLVM and Clang
+      uses: KyleMayes/install-llvm-action@v1
+      with:
+        directory: ${{ runner.temp }}/llvm
+        version: 17

.github/workflows/windows-tester-prepare/action.yml

@@ -0,0 +1,24 @@
+name: Setup Rust Environment
+description: Sets up the Rust environment for the CI workflow
+runs:
+  using: composite
+  steps:
+    - uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+    - uses: actions/checkout@v3
+    - uses: Swatinem/rust-cache@v2
+    - name: Windows Build
+      shell: pwsh
+      run: cargo build --verbose
+    - name: Build docs
+      shell: pwsh
+      run: cargo doc
+    - uses: ilammy/msvc-dev-cmd@v1
+    - name: Set LIBCLANG_PATH
+      shell: pwsh
+      run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV
+    - name: install cargo-make
+      shell: pwsh
+      run: cargo install --force cargo-make

scripts/parallellize_cargo_check.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python3
+import subprocess
+import os
+import sys
+import math
+# Current CI Runner
+ci_instances = 18
+
+if len(sys.argv) != 2:
+    exit(1)
+
+instance_idx = int(sys.argv[1])
+# set llvm config
+os.environ["LLVM_CONFIG"] = "llvm-config"
+command = "cargo hack check --workspace --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode,whole_archive --no-dev-deps --exclude libafl_libfuzzer --print-command-list"
+
+# Run the command and capture the output
+output = subprocess.check_output(command, shell=True, text=True)
+output = output.strip().split('\n')[0:]
+all_task_cnt = len(output) // 2 # by 2 cuz one task has two lines
+task_per_core = math.ceil(all_task_cnt // ci_instances)
+print(task_per_core, "tasks assigned to this instance")
+
+for task in output[instance_idx * 2 * task_per_core: (instance_idx + 1) * 2 * task_per_core]:
+    print("Running ", task)
+    cargo_check = subprocess.check_output(task, shell=True, text=True)