Add header (#1777)

* add * rdm * CHATGPT * CHATDUMB * different action * no * fix * ci * ci * ci * fuck * dir * checkout? * ci * ci * ci * ci * ci * ci * ci * ci * ci * ci * ci * ci * ci * ci * don't use shit * ci * ci * ci
2024-01-05 19:35:36 +01:00 · 2024-01-05 19:35:36 +01:00 · 0a72d8abbc
commit 0a72d8abbc
parent 207e7dbdff
6 changed files with 79 additions and 56 deletions
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@ -57,6 +57,29 @@ jobs:
      - name: Test libafl_targets no_std
        run: cd libafl_targets && cargo test --no-default-features

+  llvm-tester:
+    runs-on: ubuntu-22.04
+    continue-on-error: true
+    strategy:
+      matrix:
+        llvm-version: ["11", "12", "13", "14", "16", "17"]
+    steps:
+      - name: Remove Dotnet & Haskell
+        run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+      - name: Install LLVM and Clang
+        uses: KyleMayes/install-llvm-action@v1
+        with:
+          version: "${{matrix.llvm-version}}"
+      - name: Build and test with llvm-${{ matrix.llvm-version }}
+        run: pwd && ls & cd libafl_cc && cargo build --release
+
+
  ubuntu:
    runs-on: ubuntu-22.04
    steps:
@ -67,20 +90,20 @@ jobs:
        profile: minimal
        toolchain: stable
    - name: Remove existing clang and LLVM
-      run: sudo apt purge llvm* clang*
+      run: sudo apt purge llvm* clang* lld* lldb* opt*
    - name: Install and cache deps
-      uses: awalsh128/cache-apt-pkgs-action@v1.1.0
-      with:
-        packages: llvm-15 llvm-15-dev clang-15 ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
-    - name: get clang version
-      run: command -v llvm-config-15 && clang-15 -v
+      run: sudo apt install ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
    - name: Add nightly rustfmt and clippy
      run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
    - name: Install ucd-generate
      run: cargo install -f ucd-generate
    - uses: actions/checkout@v3
    - uses: Swatinem/rust-cache@v2
-
+    - name: Install LLVM and Clang
+      uses: KyleMayes/install-llvm-action@v1
+      with:
+        directory: ${{ runner.temp }}/llvm
+        version: 16
    # ---- format check ----
    # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately
    - name: Check pcguard edges
@ -118,16 +141,18 @@ jobs:
      with:
        profile: minimal
        toolchain: stable
+
    - name: Install and cache deps
-      uses: awalsh128/cache-apt-pkgs-action@v1.1.0
-      with:
-        packages: llvm-15 clang-15 llvm-15-dev ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
-    - name: get clang version
-      run: command -v llvm-config-15 && clang-15 -v
+      run: sudo apt install ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
    - name: Add nightly rustfmt and clippy
      run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly
    - uses: actions/checkout@v3
    - uses: Swatinem/rust-cache@v2
+    - name: Install LLVM and Clang
+      uses: KyleMayes/install-llvm-action@v1
+      with:
+        directory: ${{ runner.temp }}/llvm
+        version: 16
    - name: Run clippy
      run: ./scripts/clippy.sh

@ -153,11 +178,7 @@ jobs:
    - name: Remove existing clang and LLVM
      run: sudo apt purge llvm* clang*
    - name: Install and cache deps
-      uses: awalsh128/cache-apt-pkgs-action@v1.1.0
-      with:
-        packages: llvm-15 llvm-15-dev clang-15 ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
-    - name: get clang version
-      run: command -v llvm-config-15 && clang-15 -v
+      run: sudo apt install ninja-build clang-format-13 shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
    - name: Install cargo-hack
      run: curl -LsSf https://github.com/taiki-e/cargo-hack/releases/latest/download/cargo-hack-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
    - name: Install ucd-generate
@ -166,14 +187,18 @@ jobs:
      run: rustup toolchain install nightly --allow-downgrade
    - uses: actions/checkout@v3
    - uses: Swatinem/rust-cache@v2
-
+    - name: Install LLVM and Clang
+      uses: KyleMayes/install-llvm-action@v1
+      with:
+        directory: ${{ runner.temp }}/llvm
+        version: 16
    # ---- build and feature check ----
    # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs
    - name: Check each feature
      # Skipping `python` as it has to be built with the `maturin` tool
      # `agpl`, `nautilus` require nightly
-      # `sancov_pcguard_edges` is tested seperately
-      run: LLVM_CONFIG=llvm-config-15 cargo hack check --workspace --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode,whole_archive --no-dev-deps --exclude libafl_libfuzzer
+      # `sancov_pcguard_edges` is tested seperatelyc
+      run: LLVM_CONFIG=llvm-config cargo hack check --workspace --each-feature --clean-per-run --exclude-features=prelude,agpl,nautilus,python,sancov_pcguard_edges,arm,aarch64,i386,be,systemmode,whole_archive --no-dev-deps --exclude libafl_libfuzzer
    - name: Check nightly features
      run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus

@ -200,14 +225,19 @@ jobs:
        toolchain: stable
    - name: Remove existing clang and LLVM
      run: sudo apt purge llvm* clang*
+    - name: Install LLVM and Clang
+      uses: KyleMayes/install-llvm-action@v1
+      with:
+        directory: ${{ runner.temp }}/llvm
+        version: 16
    - name: Install deps
-      run: sudo apt-get install -y llvm-15 llvm-15-dev clang-15 ninja-build python3-dev python3-pip python3-venv libz3-dev
+      run: sudo apt-get install -y ninja-build python3-dev python3-pip python3-venv libz3-dev
    - name: Install maturin
      run: python3 -m pip install maturin
    - uses: actions/checkout@v3
    - uses: Swatinem/rust-cache@v2
    - name: Run a maturin build
-      run: export LLVM_CONFIG=llvm-config-15 && cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh
+      run: export LLVM_CONFIG=llvm-config-16 && cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh
    - name: Run python test
      run: . ./bindings/pylibafl/.env/bin/activate && cd ./fuzzers/baby_fuzzer && python3 baby_fuzzer.py 2>&1 | grep "Bye"

@ -246,13 +276,13 @@ jobs:
      run: cargo install -f ucd-generate
    - name: Remove obsolete llvm (Linux)
      run: sudo apt purge llvm* clang*
-    - uses: lyricwulf/abc@v1
+    - name: Install LLVM and Clang
+      uses: KyleMayes/install-llvm-action@v1
      with:
-        linux: llvm-15 llvm-15-dev clang-15 nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev
-    - name: Set clang version
-      run: sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100
-    - name: Set clang++ version
-      run: sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 100
+        directory: ${{ runner.temp }}/llvm
+        version: 16
+    - name: Install deps
+      run: sudo apt install nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev
    - name: pip install
      run: python3 -m pip install msgpack jinja2 find_libpython
    # Note that nproc needs to have coreutils installed on macOS, so the order of CI commands matters.
@ -281,7 +311,7 @@ jobs:
      run: sudo ln -s /usr/include/asm-generic /usr/include/asm
    - name: Build and run example fuzzers (Linux)
      if: runner.os == 'Linux'
-      run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config-15 ./scripts/test_all_fuzzers.sh
+      run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config ./scripts/test_all_fuzzers.sh

  qemu_fuzzers:
    strategy:
@ -320,17 +350,13 @@ jobs:
    - name: Remove obsolete llvm (Linux)
      if: runner.os == 'Linux'
      run: sudo apt purge llvm* clang*
-    - uses: lyricwulf/abc@v1
+    - name: Install LLVM and Clang
+      uses: KyleMayes/install-llvm-action@v1
      with:
-        linux: llvm-15 llvm-15-dev clang-15 nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev
-        # update bash for macos to support `declare -A` command`
-        macos: llvm@15 libpng nasm coreutils z3 bash wget
-    - name: Set clang version
-      if: runner.os == 'Linux'
-      run: sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100
-    - name: Set clang++ version
-      if: runner.os == 'Linux'
-      run: sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 100
+        directory: ${{ runner.temp }}/llvm
+        version: 16
+    - name: Install deps
+      run: sudo apt install nasm ninja-build gcc-arm-linux-gnueabi g++-arm-linux-gnueabi gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-mipsel-linux-gnu g++-mipsel-linux-gnu gcc-powerpc-linux-gnu g++-powerpc-linux-gnu libc6-dev-i386-cross libc6-dev libc6-dev-i386 lib32gcc-11-dev lib32stdc++-11-dev libgtk-3-dev pax-utils libz3-dev
    - name: pip install
      run: python3 -m pip install msgpack jinja2 find_libpython
    # Note that nproc needs to have coreutils installed on macOS, so the order of CI commands matters.
@ -359,7 +385,7 @@ jobs:
      run: sudo ln -s /usr/include/asm-generic /usr/include/asm
    - name: Build and run example fuzzers (Linux)
      if: runner.os == 'Linux'
-      run: RUN_ON_CI=1 RUN_QEMU_FUZZER=1 LLVM_CONFIG=llvm-config-15 ./scripts/test_all_fuzzers.sh
+      run: RUN_ON_CI=1 RUN_QEMU_FUZZER=1 LLVM_CONFIG=llvm-config ./scripts/test_all_fuzzers.sh


  nostd-build:
--- a/2
+++ b/2
@ -21,7 +21,7 @@ RUN rustup default nightly

 # Install clang 11, common build tools
 RUN apt update && apt install -y build-essential gdb git wget python3-venv ninja-build lsb-release software-properties-common gnupg
-RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16
+RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 15

 # Copy a dummy.rs and Cargo.toml first, so that dependencies are cached
 WORKDIR /libafl
--- a/libafl_cc/README.md
+++ b/libafl_cc/README.md
@ -4,7 +4,7 @@ LibAFL CC provides the functionalities to write compiler wrappers for LibAFL, by

 The online documentation for this crate is available [here](https://docs.rs/crate/libafl_cc/latest).

-Currently, we support LLVM version 11 up to 18, but other versions may work.
+Currently, we support LLVM version 11 up to 17, but other versions may work.
 To install LLVM, use the official [download page](https://releases.llvm.org/download.html).

 The LLVM tools (including clang, clang++) are needed (newer than LLVM 11.0.0 up to LLVM 17.0.0)
--- a/libafl_cc/src/common-llvm.h
+++ b/libafl_cc/src/common-llvm.h
@ -33,6 +33,8 @@ constexpr std::nullopt_t None = std::nullopt;
  #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #endif

+#include "llvm/IR/Function.h"
+
 #define FATAL(...)                          \
  do {                                      \
    fprintf(stderr, "FATAL: " __VA_ARGS__); \
--- a/libafl_targets/src/coverage.c
+++ b/libafl_targets/src/coverage.c
@ -32,7 +32,7 @@ uint8_t *__token_stop = &__stop_libafl_token;

 #endif

-//#if defined(__ANDROID__) || defined(__HAIKU__)
+// #if defined(__ANDROID__) || defined(__HAIKU__)
 MAYBE_THREAD_LOCAL prev_loc_t __afl_prev_loc[NGRAM_SIZE_MAX];
 MAYBE_THREAD_LOCAL prev_loc_t __afl_prev_caller[CTX_MAX_K];
 MAYBE_THREAD_LOCAL uint32_t   __afl_prev_ctx;
--- a/scripts/test_all_fuzzers.sh
+++ b/scripts/test_all_fuzzers.sh
@ -15,17 +15,12 @@ else
    export PROFILE_DIR=debug
 fi

-
-if [[ -z "${RUN_ON_CI}" ]]; then
-    :
+if [[ -z "${RUN_QEMU_FUZZER}" ]]; then
+    fuzzers=$(echo "$fuzzers" | tr ' ' '\n' | grep -v "qemu")
+    backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n' | grep -v "qemu")
 else
-    if [[ -z "${RUN_QEMU_FUZZER}" ]]; then
-        fuzzers=$(echo "$fuzzers" | tr ' ' '\n' | grep -v "qemu")
-        backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n' | grep "qemu")
-    else
-        fuzzers=$(echo "$fuzzers" | tr ' ' '\n' | grep  "qemu")
-        backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n' | grep -v "qemu")
-    fi
+    fuzzers=$(echo "$fuzzers" | tr ' ' '\n' | grep "qemu")
+    backtrace_fuzzers=$(echo "$backtrace_fuzzers" | tr ' ' '\n' | grep "qemu")
 fi

 libafl=$(pwd)