Faster cargo check (#2084)
* speed up cargo check * fix * composite * plural * wrong name * go brrrrrrrr * bash * aaaaa * why not even more? * more composite * fix * miri * test for llvm >= 16 * v2 * v2 * naaaa * JUST MORE INSTANCES * aa * fix * no bash for windows * reduce a bit as some guys are delayed * no deps
This commit is contained in:
Dongjia "toka" Zhang
GitHub
parent
fd364d15fe
commit
2df71e7a36
165
.github/workflows/build_and_test.yml
vendored
165
.github/workflows/build_and_test.yml
vendored
@ -70,7 +70,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
matrix:
|
||||
llvm-version: [ "11", "12", "13", "14", "16", "17" ]
|
||||
llvm-version: [ "16", "17" ] # Add 18 when KyleMayes/install-llvm-action enables it
|
||||
steps:
|
||||
- name: Remove Dotnet & Haskell
|
||||
run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
|
||||
@ -82,39 +82,31 @@ jobs:
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with: { shared-key: "llvm-tester" }
|
||||
- name: Install LLVM and Clang
|
||||
uses: KyleMayes/install-llvm-action@v1
|
||||
uses: KyleMayes/install-llvm-action@v2
|
||||
with:
|
||||
version: "${{matrix.llvm-version}}"
|
||||
- name: Build and test with llvm-${{ matrix.llvm-version }}
|
||||
run: pwd && ls & cd libafl_cc && cargo build --release
|
||||
|
||||
ubuntu-doc:
|
||||
ubuntu-doc-build:
|
||||
runs-on: ubuntu-22.04
|
||||
needs: ubuntu
|
||||
steps:
|
||||
- name: Remove Dotnet & Haskell
|
||||
run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
- name: Remove existing clang and LLVM
|
||||
run: sudo apt purge llvm* clang* lld* lldb* opt*
|
||||
- name: Install and cache deps
|
||||
run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
|
||||
- name: Add nightly rustfmt and clippy
|
||||
run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ./.github/workflows/ubuntu-prepare
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with: { shared-key: "ubuntu" }
|
||||
- name: Install LLVM and Clang
|
||||
uses: KyleMayes/install-llvm-action@v1
|
||||
with:
|
||||
directory: ${{ runner.temp }}/llvm
|
||||
version: 17
|
||||
# ---- doc check ----
|
||||
- name: Build Docs
|
||||
run: RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features
|
||||
run: RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features --no-deps
|
||||
|
||||
ubuntu-doc-test:
|
||||
runs-on: ubuntu-22.04
|
||||
needs: ubuntu
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ./.github/workflows/ubuntu-prepare
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
# ---- doc check ----
|
||||
- name: Test Docs
|
||||
run: RUSTFLAGS="--cfg docsrs" cargo +nightly test --doc --all-features
|
||||
|
||||
@ -122,26 +114,11 @@ jobs:
|
||||
runs-on: ubuntu-22.04
|
||||
needs: ubuntu
|
||||
steps:
|
||||
- name: Remove Dotnet & Haskell
|
||||
run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
- name: Remove existing clang and LLVM
|
||||
run: sudo apt purge llvm* clang* lld* lldb* opt*
|
||||
- name: Install and cache deps
|
||||
run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
|
||||
- name: Add nightly rustfmt and clippy
|
||||
run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ./.github/workflows/ubuntu-prepare
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with: { shared-key: "ubuntu" }
|
||||
- name: Install LLVM and Clang
|
||||
uses: KyleMayes/install-llvm-action@v1
|
||||
with:
|
||||
directory: ${{ runner.temp }}/llvm
|
||||
version: 17
|
||||
- name: Add nightly rustfmt and clippy
|
||||
run: rustup toolchain install nightly --component miri --allow-downgrade
|
||||
# --- miri undefined behavior test --
|
||||
- name: Run miri tests
|
||||
run: RUST_BACKTRACE=1 MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test
|
||||
@ -185,7 +162,6 @@ jobs:
|
||||
|
||||
ubuntu-clippy:
|
||||
runs-on: ubuntu-22.04
|
||||
needs: ubuntu
|
||||
steps:
|
||||
- name: Remove Dotnet & Haskell
|
||||
run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
|
||||
@ -213,56 +189,38 @@ jobs:
|
||||
# Fix me plz
|
||||
# - name: Test Build libafl_libfuzzer with embed
|
||||
# run: cargo +nightly test --features=embed-runtime --manifest-path libafl_libfuzzer/Cargo.toml
|
||||
ubuntu-check-nightly:
|
||||
runs-on: ubuntu-22.04
|
||||
needs: ubuntu
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ./.github/workflows/ubuntu-prepare
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with: { shared-key: "ubuntu" }
|
||||
# ---- build and feature check ----
|
||||
# cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs
|
||||
- name: Check nightly features
|
||||
run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus
|
||||
|
||||
|
||||
ubuntu-check:
|
||||
runs-on: ubuntu-22.04
|
||||
needs: ubuntu
|
||||
strategy:
|
||||
matrix:
|
||||
instance_idx: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17" ]
|
||||
steps:
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
components: llvm-tools
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
# this might remove tools that are actually needed,
|
||||
# if set to "true" but frees about 6 GB
|
||||
tool-cache: false
|
||||
|
||||
# all of these default to true, but feel free to set to
|
||||
# "false" if necessary for your workflow
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Remove existing clang and LLVM
|
||||
run: sudo apt purge llvm* clang*
|
||||
- name: Install and cache deps
|
||||
run: sudo apt update && sudo apt install ninja-build clang-format shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
|
||||
- name: Install cargo-hack
|
||||
run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
|
||||
- name: Add nightly
|
||||
run: rustup toolchain install nightly --allow-downgrade
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ./.github/workflows/ubuntu-prepare
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with: { shared-key: "ubuntu" }
|
||||
- name: Install LLVM and Clang
|
||||
uses: KyleMayes/install-llvm-action@v1
|
||||
with:
|
||||
directory: ${{ runner.temp }}/llvm
|
||||
version: 17
|
||||
# ---- build and feature check ----
|
||||
# cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs
|
||||
- name: Check each feature
|
||||
# Skipping `python` as it has to be built with the `maturin` tool
|
||||
# `agpl`, `nautilus` require nightly
|
||||
# `sancov_pcguard_edges` is tested seperatelyc
|
||||
run: LLVM_CONFIG=llvm-config cargo hack check --workspace --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode,whole_archive --no-dev-deps --exclude libafl_libfuzzer
|
||||
- name: Check nightly features
|
||||
run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus
|
||||
run: python3 ./scripts/parallellize_cargo_check.py ${{ matrix.instance_idx }}
|
||||
|
||||
ubuntu-concolic:
|
||||
runs-on: ubuntu-latest
|
||||
@ -448,37 +406,52 @@ jobs:
|
||||
- name: Build docker
|
||||
run: docker build -t libafl .
|
||||
|
||||
windows:
|
||||
windows-frida-libpng:
|
||||
runs-on: windows-latest
|
||||
needs:
|
||||
- common
|
||||
steps:
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
- uses: actions/checkout@v3
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Windows Build
|
||||
run: cargo build --verbose
|
||||
- name: Build docs
|
||||
run: cargo doc
|
||||
- uses: ilammy/msvc-dev-cmd@v1
|
||||
- name: Set LIBCLANG_PATH
|
||||
run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV
|
||||
- name: install cargo-make
|
||||
run: cargo install --force cargo-make
|
||||
- name: install cxx bridge
|
||||
run: cargo install cxxbridge-cmd
|
||||
- name: Build fuzzers/libfuzzer_stb_image
|
||||
run: cd fuzzers/libfuzzer_stb_image && cargo build --release
|
||||
- uses: ./.github/workflows/windows-tester-prepare
|
||||
- name: Build fuzzers/frida_libpng
|
||||
run: cd fuzzers/frida_libpng/ && cargo make test
|
||||
|
||||
windows-frida-libfuzzer-stb-image:
|
||||
runs-on: windows-latest
|
||||
needs:
|
||||
- common
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ./.github/workflows/windows-tester-prepare
|
||||
- name: Build fuzzers/libfuzzer_stb_image
|
||||
run: cd fuzzers/libfuzzer_stb_image && cargo build --release
|
||||
|
||||
windows-frida-gdiplus:
|
||||
runs-on: windows-latest
|
||||
needs:
|
||||
- common
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ./.github/workflows/windows-tester-prepare
|
||||
- name: Build fuzzers/frida_gdiplus
|
||||
run: cd fuzzers/frida_gdiplus/ && cargo make test && cargo make test_cmplog
|
||||
|
||||
windows-tinyinst-simple:
|
||||
runs-on: windows-latest
|
||||
needs:
|
||||
- common
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: ./.github/workflows/windows-tester-prepare
|
||||
- name: install cxx bridge
|
||||
run: cargo install cxxbridge-cmd
|
||||
- name: Build fuzzers/tinyinst_simple
|
||||
run: cd fuzzers/tinyinst_simple/ && cargo make test
|
||||
|
||||
windows-clippy:
|
||||
runs-on: windows-latest
|
||||
needs:
|
||||
- common
|
||||
steps:
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
|
||||
27
.github/workflows/ubuntu-prepare/action.yml
vendored
Normal file
27
.github/workflows/ubuntu-prepare/action.yml
vendored
Normal file
@ -0,0 +1,27 @@
|
||||
name: Setup Rust Environment
|
||||
description: Sets up the Rust environment for the CI workflow
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
components: llvm-tools
|
||||
- name: Remove existing clang and LLVM
|
||||
shell: bash
|
||||
run: sudo apt purge llvm* clang*
|
||||
- name: Install and cache deps
|
||||
shell: bash
|
||||
run: sudo apt update && sudo apt install ninja-build clang-format shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
|
||||
- name: Install cargo-hack
|
||||
shell: bash
|
||||
run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
|
||||
- name: Add nightly
|
||||
shell: bash
|
||||
run: rustup toolchain install nightly --allow-downgrade
|
||||
- name: Install LLVM and Clang
|
||||
uses: KyleMayes/install-llvm-action@v1
|
||||
with:
|
||||
directory: ${{ runner.temp }}/llvm
|
||||
version: 17
|
||||
24
.github/workflows/windows-tester-prepare/action.yml
vendored
Normal file
24
.github/workflows/windows-tester-prepare/action.yml
vendored
Normal file
@ -0,0 +1,24 @@
|
||||
name: Setup Rust Environment
|
||||
description: Sets up the Rust environment for the CI workflow
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
- uses: actions/checkout@v3
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Windows Build
|
||||
shell: pwsh
|
||||
run: cargo build --verbose
|
||||
- name: Build docs
|
||||
shell: pwsh
|
||||
run: cargo doc
|
||||
- uses: ilammy/msvc-dev-cmd@v1
|
||||
- name: Set LIBCLANG_PATH
|
||||
shell: pwsh
|
||||
run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV
|
||||
- name: install cargo-make
|
||||
shell: pwsh
|
||||
run: cargo install --force cargo-make
|
||||
26
scripts/parallellize_cargo_check.py
Executable file
26
scripts/parallellize_cargo_check.py
Executable file
@ -0,0 +1,26 @@
|
||||
#!/usr/bin/python3
|
||||
import subprocess
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
# Current CI Runner
|
||||
ci_instances = 18
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
exit(1)
|
||||
|
||||
instance_idx = int(sys.argv[1])
|
||||
# set llvm config
|
||||
os.environ["LLVM_CONFIG"] = "llvm-config"
|
||||
command = "cargo hack check --workspace --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode,whole_archive --no-dev-deps --exclude libafl_libfuzzer --print-command-list"
|
||||
|
||||
# Run the command and capture the output
|
||||
output = subprocess.check_output(command, shell=True, text=True)
|
||||
output = output.strip().split('\n')[0:]
|
||||
all_task_cnt = len(output) // 2 # by 2 cuz one task has two lines
|
||||
task_per_core = math.ceil(all_task_cnt // ci_instances)
|
||||
print(task_per_core, "tasks assigned to this instance")
|
||||
|
||||
for task in output[instance_idx * 2 * task_per_core: (instance_idx + 1) * 2 * task_per_core]:
|
||||
print("Running ", task)
|
||||
cargo_check = subprocess.check_output(task, shell=True, text=True)
|
||||
Loading…
x
Reference in New Issue
Block a user