SYS-OSS/FRET-LibAFL
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

LLVM passes for Windows (#710)

Browse Source 
* libafl_cc fixes for windows

* libafl_cc checks for llvm-config (again)

* libafl_cc clang-format

* libafl_cc fixes for macos

* maintain libafl_cc pass manager selection logic

* libafl_cc rustfmt
This commit is contained in:
Alexandru Geană 2022-08-12 20:25:59 +02:00 committed by GitHub
parent 2504b6dae3
commit c1aafe3e98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 269 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

289
libafl_cc/build.rs
View File

@ -4,7 +4,6 @@ use std::{env, fs::File, io::Write, path::Path, process::Command, str};
#[cfg(target_vendor = "apple")]
use glob::glob;
#[cfg(not(target_vendor = "apple"))]
use which::which;
/// The max version of `LLVM` we're looking for
@ -18,7 +17,7 @@ const LLVM_VERSION_MIN: u32 = 6;
/// Get the extension for a shared object
fn dll_extension<'a>() -> &'a str {
match env::var("CARGO_CFG_TARGET_OS").unwrap().as_str() {
"windwos" => "dll",
"windows" => "dll",
"macos" | "ios" => "dylib",
_ => "so",
}
@ -50,28 +49,44 @@ fn find_llvm_config_brew() -> Result<PathBuf, String> {
}
}
fn find_llvm_config() -> String {
env::var("LLVM_CONFIG").unwrap_or_else(|_| {
// for Ghithub Actions, we check if we find llvm-config in brew.
fn find_llvm_config() -> Result<String, String> {
if let Ok(var) = env::var("LLVM_CONFIG") {
return Ok(var);
}
// for Github Actions, we check if we find llvm-config in brew.
#[cfg(target_vendor = "apple")]
match find_llvm_config_brew() {
Ok(llvm_dir) => llvm_dir.to_str().unwrap().to_string(),
Ok(llvm_dir) => return Ok(llvm_dir.to_str().unwrap().to_string()),
Err(err) => {
println!("cargo:warning={}", err);
// falling back to system llvm-config
"llvm-config".to_string()
}
}
};
#[cfg(not(target_vendor = "apple"))]
for version in (LLVM_VERSION_MIN..=LLVM_VERSION_MAX).rev() {
let llvm_config_name = format!("llvm-config-{}", version);
let llvm_config_name: String = format!("llvm-config-{}", version);
if which(&llvm_config_name).is_ok() {
return llvm_config_name;
return Ok(llvm_config_name);
}
}
#[cfg(not(target_vendor = "apple"))]
"llvm-config".to_string()
})
if which("llvm-config").is_ok() {
return Ok("llvm-config".to_owned());
}
Err("could not find llvm-config".to_owned())
}
fn exec_llvm_config(args: &[&str]) -> String {
let llvm_config = find_llvm_config().expect("Unexpected error");
match Command::new(&llvm_config).args(args).output() {
Ok(output) => String::from_utf8(output.stdout)
.expect("Unexpected llvm-config output")
.trim()
.to_string(),
Err(e) => panic!("Could not execute llvm-config: {}", e),
}
}
/// Use `xcrun` to get the path to the Xcode SDK tools library path, for linking
@ -86,51 +101,132 @@ fn find_macos_sdk_libs() -> String {
)
}
fn find_llvm_version() -> Option<i32> {
let output = exec_llvm_config(&["--version"]);
if let Some(major) = output.split('.').collect::<Vec<&str>>().first() {
if let Ok(res) = major.parse::<i32>() {
return Some(res);
}
}
None
}
fn build_pass(
bindir_path: &Path,
out_dir: &Path,
cxxflags: &Vec<String>,
ldflags: &Vec<&str>,
src_dir: &Path,
src_file: &str,
) {
let dot_offset = src_file.rfind('.').unwrap();
let src_stub = &src_file[..dot_offset];
println!("cargo:rerun-if-changed=src/{}", src_file);
if cfg!(unix) {
assert!(Command::new(bindir_path.join("clang++"))
.arg("-v")
.args(cxxflags)
.arg(src_dir.join(src_file))
.args(ldflags)
.args(&["-o"])
.arg(out_dir.join(format!("{}.{}", src_stub, dll_extension())))
.status()
.unwrap_or_else(|_| panic!("Failed to compile {}", src_file))
.success());
} else if cfg!(windows) {
println!("{:?}", cxxflags);
assert!(Command::new(bindir_path.join("clang-cl"))
.arg("-v")
.args(cxxflags)
.arg(src_dir.join(src_file))
.arg("/link")
.args(ldflags)
.arg(format!(
"/OUT:{}",
out_dir
.join(format!("{}.{}", src_stub, dll_extension()))
.display()
))
.status()
.unwrap_or_else(|_| panic!("Failed to compile {}", src_file))
.success());
}
}
#[allow(clippy::too_many_lines)]
fn main() {
let out_dir = env::var_os("OUT_DIR").unwrap();
let out_dir = Path::new(&out_dir);
let src_dir = Path::new("src");
let dest_path = Path::new(&out_dir).join("clang_constants.rs");
let mut clang_constants_file = File::create(&dest_path).expect("Could not create file");
println!("cargo:rerun-if-env-changed=LLVM_CONFIG");
println!("cargo:rerun-if-env-changed=LIBAFL_EDGES_MAP_SIZE");
println!("cargo:rerun-if-env-changed=LIBAFL_ACCOUNTING_MAP_SIZE");
println!("cargo:rerun-if-changed=src/common-llvm.h");
println!("cargo:rerun-if-changed=build.rs");
let mut custom_flags = vec![];
// test if llvm-config is available and we can compile the passes
if find_llvm_config().is_err() {
println!(
"cargo:warning=Failed to find llvm-config, we will not build LLVM passes. If you need them, set the LLVM_CONFIG environment variable to a recent llvm-config."
);
let dest_path = Path::new(&out_dir).join("clang_constants.rs");
let mut clang_constants_file = File::create(&dest_path).expect("Could not create file");
write!(
clang_constants_file,
"// These constants are autogenerated by build.rs
/// The path to the `clang` executable
pub const CLANG_PATH: &str = \"clang\";
/// The path to the `clang++` executable
pub const CLANGXX_PATH: &str = \"clang++\";
/// The llvm version used to build llvm passes
pub const LIBAFL_CC_LLVM_VERSION: Option<usize> = None;
"
)
.expect("Could not write file");
return;
}
let cxxflags = exec_llvm_config(&["--cxxflags"]);
let mut cxxflags: Vec<String> = cxxflags.split_whitespace().map(String::from).collect();
let edges_map_size: usize = option_env!("LIBAFL_EDGES_MAP_SIZE")
.map_or(Ok(65536), str::parse)
.expect("Could not parse LIBAFL_EDGES_MAP_SIZE");
custom_flags.push(format!("-DLIBAFL_EDGES_MAP_SIZE={}", edges_map_size));
cxxflags.push(format!("-DLIBAFL_EDGES_MAP_SIZE={}", edges_map_size));
let acc_map_size: usize = option_env!("LIBAFL_ACCOUNTING_MAP_SIZE")
.map_or(Ok(65536), str::parse)
.expect("Could not parse LIBAFL_ACCOUNTING_MAP_SIZE");
custom_flags.push(format!("-DLIBAFL_ACCOUNTING_MAP_SIZE={}", acc_map_size));
cxxflags.push(format!("-DLIBAFL_ACCOUNTING_MAP_SIZE={}", acc_map_size));
let llvm_config = find_llvm_config();
// Get LLVM version.
let llvm_version = match llvm_config.split('-').collect::<Vec<&str>>().get(2) {
let llvm_version = match find_llvm_config()
.unwrap()
.split('-')
.collect::<Vec<&str>>()
.get(2)
{
Some(ver) => ver.parse::<usize>().ok(),
None => None,
};
// The approach below causes issues with arguments to optimization passes.
// An example is fuzzers/libfuzzer_libpng_accounting which passes -granularity=FUNC.
// In CI/CD, the new pass manager is not used. For now, maintain the same behavior.
//let llvm_version = find_llvm_version();
if let Some(ver) = llvm_version {
if ver >= 14 {
custom_flags.push("-DUSE_NEW_PM".to_string());
cxxflags.push(String::from("-DUSE_NEW_PM"));
}
}
if let Ok(output) = Command::new(&llvm_config).args(&["--bindir"]).output() {
let llvm_bindir = Path::new(
str::from_utf8(&output.stdout)
.expect("Invalid llvm-config output")
.trim(),
);
let llvm_bindir = exec_llvm_config(&["--bindir"]);
let bindir_path = Path::new(&llvm_bindir);
write!(
clang_constants_file,
"// These constants are autogenerated by build.rs
@ -149,36 +245,51 @@ fn main() {
/// The llvm version used to build llvm passes
pub const LIBAFL_CC_LLVM_VERSION: Option<usize> = {:?};
",
llvm_bindir.join("clang"),
llvm_bindir.join("clang++"),
bindir_path.join("clang"),
bindir_path.join("clang++"),
edges_map_size,
acc_map_size,
llvm_version,
)
.expect("Could not write file");
let output = Command::new(&llvm_config)
.args(&["--cxxflags"])
.output()
.expect("Failed to execute llvm-config");
let cxxflags = str::from_utf8(&output.stdout).expect("Invalid llvm-config output");
let mut cmd = Command::new(&llvm_config);
#[cfg(target_vendor = "apple")]
{
cmd.args(&["--libs"]);
let mut llvm_config_ld = vec![];
if cfg!(target_vendor = "apple") {
llvm_config_ld.push("--libs");
}
if cfg!(windows) {
llvm_config_ld.push("--libs");
llvm_config_ld.push("--system-libs");
}
llvm_config_ld.push("--ldflags");
let output = cmd
.args(&["--ldflags"])
.output()
.expect("Failed to execute llvm-config");
let ldflags = str::from_utf8(&output.stdout).expect("Invalid llvm-config output");
let cxxflags: Vec<&str> = cxxflags.split_whitespace().collect();
let ldflags = exec_llvm_config(&llvm_config_ld);
let mut ldflags: Vec<&str> = ldflags.split_whitespace().collect();
if cfg!(unix) {
cxxflags.push(String::from("-shared"));
cxxflags.push(String::from("-fPIC"));
}
if cfg!(windows) {
cxxflags.push(String::from("-fuse-ld=lld"));
cxxflags.push(String::from("/LD"));
/* clang on windows links against the libcmt.lib runtime
* however, the distributed binaries are compiled against msvcrt.lib
* we need to also use msvcrt.lib instead of libcmt.lib when building the optimization passes
* first, we tell clang-cl (and indirectly link) to ignore libcmt.lib via -nodefaultlib:libcmt
* second, we pass the /MD flag to clang-cl to use the msvcrt.lib runtime instead when generating the object file
*/
ldflags.push("-nodefaultlib:libcmt");
cxxflags.push(String::from("/MD"));
/* the include directories are not always added correctly when running --cxxflags or --includedir on windows
* this is somehow related to where/how llvm was compiled (vm, docker container, host)
* add the option of setting additional flags via the LLVM_CXXFLAGS variable
*/
if let Some(env_cxxflags) = option_env!("LLVM_CXXFLAGS") {
cxxflags.append(&mut env_cxxflags.split_whitespace().map(String::from).collect());
}
}
let sdk_path;
if env::var("CARGO_CFG_TARGET_VENDOR").unwrap().as_str() == "apple" {
// Needed on macos.
@ -191,80 +302,16 @@ fn main() {
ldflags.push(&sdk_path);
};
println!("cargo:rerun-if-changed=src/common-llvm.h");
println!("cargo:rerun-if-changed=src/cmplog-routines-pass.cc");
println!("cargo:rerun-if-changed=src/afl-coverage-pass.cc");
println!("cargo:rerun-if-changed=src/autotokens-pass.cc");
println!("cargo:rerun-if-changed=src/coverage-accounting-pass.cc");
assert!(Command::new(llvm_bindir.join("clang++"))
.args(&cxxflags)
.args(&custom_flags)
.arg(src_dir.join("cmplog-routines-pass.cc"))
.args(&ldflags)
.args(&["-fPIC", "-shared", "-o"])
.arg(out_dir.join(format!("cmplog-routines-pass.{}", dll_extension())))
.status()
.expect("Failed to compile cmplog-routines-pass.cc")
.success());
assert!(Command::new(llvm_bindir.join("clang++"))
.args(&cxxflags)
.args(&custom_flags)
.arg(src_dir.join("afl-coverage-pass.cc"))
.args(&ldflags)
.args(&["-fPIC", "-shared", "-o"])
.arg(out_dir.join(format!("afl-coverage-pass.{}", dll_extension())))
.status()
.expect("Failed to compile afl-coverage-pass.cc")
.success());
assert!(Command::new(llvm_bindir.join("clang++"))
.args(&cxxflags)
.args(&custom_flags)
.arg(src_dir.join("autotokens-pass.cc"))
.args(&ldflags)
.args(&["-fPIC", "-shared", "-o"])
.arg(out_dir.join(format!("autotokens-pass.{}", dll_extension())))
.status()
.expect("Failed to compile autotokens-pass.cc")
.success());
assert!(Command::new(llvm_bindir.join("clang++"))
.args(&cxxflags)
.args(&custom_flags)
.arg(src_dir.join("coverage-accounting-pass.cc"))
.args(&ldflags)
.args(&["-fPIC", "-shared", "-o"])
.arg(out_dir.join(format!("coverage-accounting-pass.{}", dll_extension())))
.status()
.expect("Failed to compile coverage-accounting-pass.cc")
.success());
} else {
write!(
clang_constants_file,
"// These constants are autogenerated by build.rs
/// The path to the `clang` executable
pub const CLANG_PATH: &str = \"clang\";
/// The path to the `clang++` executable
pub const CLANGXX_PATH: &str = \"clang++\";
/// The llvm version used to build llvm passes
pub const LIBAFL_CC_LLVM_VERSION: Option<usize> = None;
"
)
.expect("Could not write file");
println!(
"cargo:warning=Failed to locate the LLVM path using {}, we will not build LLVM passes
(if you need them, set point the LLVM_CONFIG env to a recent llvm-config, or make sure {} is available)",
llvm_config, llvm_config
);
for pass in &[
"cmplog-routines-pass.cc",
"afl-coverage-pass.cc",
"autotokens-pass.cc",
"coverage-accounting-pass.cc",
] {
build_pass(bindir_path, out_dir, &cxxflags, &ldflags, src_dir, pass);
}
cc::Build::new()
.file(src_dir.join("no-link-rt.c"))
.compile("no-link-rt");
println!("cargo:rerun-if-changed=build.rs");
}

13
libafl_cc/src/afl-coverage-pass.cc
View File

@ -31,8 +31,12 @@
#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/time.h>
#ifndef _WIN32
#include <unistd.h>
#include <sys/time.h>
#else
#include <io.h>
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
@ -781,8 +785,13 @@ bool AFLCoverage::runOnModule(Module &M) {
}
if (DumpCFG) {
int fd;
#ifndef _WIN32
if ((fd = open(DumpCFGPath.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0644)) <
0)
#else
if ((fd = _open(DumpCFGPath.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0644)) <
0)
#endif
FATAL("Could not open/create CFG dump file.");
std::string cfg = "";
for (auto record = entry_bb.begin(); record != entry_bb.end(); record++) {

14
libafl_cc/src/autotokens-pass.cc
View File

@ -18,9 +18,13 @@
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#ifndef _WIN32
#include <unistd.h>
#include <sys/time.h>
#else
#include <io.h>
#endif
#include <string.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
@ -213,7 +217,9 @@ void dict2file(int fd, uint8_t *mem, uint32_t len) {
if (write(fd, line, strlen(line)) <= 0) {
FATAL("Could not write to the dictionary file");
}
#ifndef _WIN32
fsync(fd);
#endif
}
#if USE_NEW_PM
@ -242,7 +248,11 @@ bool AutoTokensPass::runOnModule(Module &M) {
}
if (use_file) {
#ifndef _WIN32
if ((fd = open(ptr, O_WRONLY | O_APPEND | O_CREAT | O_DSYNC, 0644)) < 0)
#else
if ((fd = open(ptr, O_WRONLY | O_APPEND | O_CREAT, 0644)) < 0)
#endif
FATAL("Could not open/create %s.", ptr);
}

29
libafl_cc/src/clang.rs
View File

@ -112,7 +112,11 @@ impl CompilerWrapper for ClangWrapper {
self.name = args[0].as_ref().to_string();
// Detect C++ compiler looking at the wrapper name
self.is_cpp = self.is_cpp || self.name.ends_with("++");
self.is_cpp = if cfg!(windows) {
self.is_cpp || self.name.ends_with("++.exe")
} else {
self.is_cpp || self.name.ends_with("++")
};
// Sancov flag
// new_args.push("-fsanitize-coverage=trace-pc-guard".into());
@ -252,21 +256,22 @@ impl CompilerWrapper for ClangWrapper {
where
S: AsRef<str>,
{
if cfg!(target_vendor = "apple") {
//self.add_link_arg("-force_load".into())?;
} else {
self.add_link_arg("-Wl,--whole-archive");
}
self.add_link_arg(
dir.join(format!("{}{}.{}", LIB_PREFIX, name.as_ref(), LIB_EXT))
let lib_file = dir
.join(format!("{}{}.{}", LIB_PREFIX, name.as_ref(), LIB_EXT))
.into_os_string()
.into_string()
.unwrap(),
);
.unwrap();
if cfg!(unix) {
if cfg!(target_vendor = "apple") {
self
self.add_link_arg(lib_file)
} else {
self.add_link_arg("-Wl,-no-whole-archive")
self.add_link_arg("-Wl,--whole-archive")
.add_link_arg(lib_file)
.add_link_arg("-Wl,--no-whole-archive")
}
} else {
self.add_link_arg(format!("-Wl,-wholearchive:{}", lib_file))
}
}

6
libafl_cc/src/cmplog-routines-pass.cc
View File

@ -17,12 +17,14 @@
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#ifndef _WIN32
#include <unistd.h>
#include <sys/time.h>
#endif
#include <list>
#include <string>
#include <fstream>
#include <sys/time.h>
#include "llvm/Config/llvm-config.h"
#if USE_NEW_PM