Merge ubuntu and macos fuzzer && Record time for ./test_all_fuzzers.sh (#629)

* Add ccache

* Update codecov.yml

* Update build_and_test.yml

* Update build_and_test.yml

* Update test_all_fuzzers.sh

Add fuzzer timer

* Fix `./test_all_fuzzers.sh` on macos

* Fix CI

* Fix CI

* Update build_and_test.yml

* Fix typo

.github/workflows/build_and_test.yml

@@ -1,4 +1,4 @@
-name: Build and Test
+name: build and test
 
 on:
   push:
@@ -94,23 +94,44 @@ jobs:
       run: sudo ./libafl_concolic/test/smoke_test_ubuntu_deps.sh 
     - name: Run smoke test
       run: ./libafl_concolic/test/smoke_test.sh 
-  ubuntu-fuzzers:
+  fuzzers:
-    runs-on: ubuntu-latest
+    env:
+      CC: ccache clang # use ccache in default
+      CXX: ccache clang # use ccache in default
+    strategy:
+      matrix:
+          os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions-rs/toolchain@v1
       with:
         profile: minimal
         toolchain: stable
+    - name: add ccache 
+      uses: hendrikmuhs/ccache-action@v1.2
+      with:
+        key: ${{ github.job }}-${{ matrix.os }}
+        max-size: 2000M
+    - name: enable mult-thread for `make`
+      run: export MAKEFLAGS="-j$(expr $(nproc) \+ 1)"
     - uses: Swatinem/rust-cache@v1
     - name: Add nightly rustfmt and clippy
       run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade
-    - name: Install deps
+    - uses: lyricwulf/abc@v1
-      run: sudo apt-get install -y llvm llvm-dev clang nasm ninja-build
+      with: 
+        linux: llvm llvm-dev clang nasm ninja-build
+        # update bash for macos to support `declare -A` command`
+        macos: llvm libpng nasm coreutils z3 bash
     - name: install cargo-make
       run: cargo install --force cargo-make
     - uses: actions/checkout@v2
     - name: Build and run example fuzzers
+      if: runner.os == 'Linux'
       run: ./scripts/test_all_fuzzers.sh
+    - name: Build and run example fuzzers
+      if: runner.os == 'macOS' # use bash v4
+      run: /usr/local/bin/bash ./scripts/test_all_fuzzers.sh
+    - run: ccache --show-stats
   nostd-build:
     runs-on: ubuntu-latest
     steps:
@@ -182,25 +203,7 @@ jobs:
       run: ./scripts/shmem_limits_macos.sh
     - name: Run Tests
       run: cargo test
-  macos-fuzzers:
+
-    runs-on: macOS-latest
-    steps:
-    - uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: stable
-    - uses: Swatinem/rust-cache@v1
-    - name: Add nightly rustfmt and clippy
-      run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade
-    - name: Install deps
-      run: brew install llvm libpng nasm coreutils z3 && brew link --force llvm
-    - name: install cargo-make
-      run: cargo install --force cargo-make
-    - uses: actions/checkout@v2
-    - name: Increase map sizes
-      run: ./scripts/shmem_limits_macos.sh
-    - name: Build and run example fuzzers
-      run: ./scripts/test_all_fuzzers.sh
   other_targets:
     runs-on: macOS-latest
     steps:

.github/workflows/codecov.yml

@@ -1,6 +1,6 @@
 on: [push]
 
-name: build
+name: codecov
 
 jobs:
   check:
@@ -32,4 +32,4 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: code-coverage-report
-          path: cobertura.xml
+          path: cobertura.xml

scripts/test_all_fuzzers.sh

@@ -12,9 +12,12 @@ libafl=$(pwd)
 
 git submodule init && git submodule update
 
+# record time of each fuzzer
+declare -A time_record || (echo "declare -A not avaliable, please update your bash version to 4";exit 1)
 for fuzzer in $(echo $fuzzers $backtrace_fuzzers);
 do
     cd $fuzzer
+    start=`date +%s`
     # Clippy checks
     if [ "$1" != "--no-fmt" ]; then
         
@@ -35,9 +38,15 @@ do
         cargo build || exit 1
         echo "[+] Done building $fuzzer"
     fi
-
+    end=`date +%s`
+    time_record[$fuzzer]=$((end-start))
     # Save disk space
     cargo clean
     cd $libafl
     echo ""
 done
+
+# print time for each fuzzer
+for key in ${!time_record[@]}; do
+    echo "dir:"$key" time:"${time_record[$key]};
+done