361 lines
15 KiB
Python
361 lines
15 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import sys
|
||
|
|
import os
|
||
|
|
|
||
|
|
from subprocess import call
|
||
|
|
|
||
|
|
SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
|
||
|
|
PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
|
||
|
|
DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR,
|
||
|
|
os.path.pardir,
|
||
|
|
os.path.pardir,
|
||
|
|
os.path.pardir))
|
||
|
|
|
||
|
|
|
||
|
|
def add(parser, args):
|
||
|
|
import SATestAdd
|
||
|
|
from ProjectMap import ProjectInfo
|
||
|
|
|
||
|
|
if args.source == "git" and (args.origin == "" or args.commit == ""):
|
||
|
|
parser.error(
|
||
|
|
"Please provide both --origin and --commit if source is 'git'")
|
||
|
|
|
||
|
|
if args.source != "git" and (args.origin != "" or args.commit != ""):
|
||
|
|
parser.error("Options --origin and --commit don't make sense when "
|
||
|
|
"source is not 'git'")
|
||
|
|
|
||
|
|
project = ProjectInfo(args.name[0], args.mode, args.source, args.origin,
|
||
|
|
args.commit)
|
||
|
|
|
||
|
|
SATestAdd.add_new_project(project)
|
||
|
|
|
||
|
|
|
||
|
|
def build(parser, args):
|
||
|
|
import SATestBuild
|
||
|
|
|
||
|
|
SATestBuild.VERBOSE = args.verbose
|
||
|
|
|
||
|
|
projects = get_projects(parser, args)
|
||
|
|
tester = SATestBuild.RegressionTester(args.jobs,
|
||
|
|
projects,
|
||
|
|
args.override_compiler,
|
||
|
|
args.extra_analyzer_config,
|
||
|
|
args.regenerate,
|
||
|
|
args.strictness)
|
||
|
|
tests_passed = tester.test_all()
|
||
|
|
|
||
|
|
if not tests_passed:
|
||
|
|
sys.stderr.write("ERROR: Tests failed.\n")
|
||
|
|
sys.exit(42)
|
||
|
|
|
||
|
|
|
||
|
|
def compare(parser, args):
|
||
|
|
import CmpRuns
|
||
|
|
|
||
|
|
choices = [CmpRuns.HistogramType.RELATIVE.value,
|
||
|
|
CmpRuns.HistogramType.LOG_RELATIVE.value,
|
||
|
|
CmpRuns.HistogramType.ABSOLUTE.value]
|
||
|
|
|
||
|
|
if args.histogram is not None and args.histogram not in choices:
|
||
|
|
parser.error("Incorrect histogram type, available choices are {}"
|
||
|
|
.format(choices))
|
||
|
|
|
||
|
|
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
|
||
|
|
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
|
||
|
|
|
||
|
|
CmpRuns.dump_scan_build_results_diff(dir_old, dir_new,
|
||
|
|
show_stats=args.show_stats,
|
||
|
|
stats_only=args.stats_only,
|
||
|
|
histogram=args.histogram,
|
||
|
|
verbose_log=args.verbose_log)
|
||
|
|
|
||
|
|
|
||
|
|
def update(parser, args):
|
||
|
|
import SATestUpdateDiffs
|
||
|
|
from ProjectMap import ProjectMap
|
||
|
|
|
||
|
|
project_map = ProjectMap()
|
||
|
|
for project in project_map.projects:
|
||
|
|
SATestUpdateDiffs.update_reference_results(project, args.git)
|
||
|
|
|
||
|
|
|
||
|
|
def benchmark(parser, args):
|
||
|
|
from SATestBenchmark import Benchmark
|
||
|
|
|
||
|
|
projects = get_projects(parser, args)
|
||
|
|
benchmark = Benchmark(projects, args.iterations, args.output)
|
||
|
|
benchmark.run()
|
||
|
|
|
||
|
|
|
||
|
|
def benchmark_compare(parser, args):
|
||
|
|
import SATestBenchmark
|
||
|
|
SATestBenchmark.compare(args.old, args.new, args.output)
|
||
|
|
|
||
|
|
|
||
|
|
def get_projects(parser, args):
|
||
|
|
from ProjectMap import ProjectMap, Size
|
||
|
|
|
||
|
|
project_map = ProjectMap()
|
||
|
|
projects = project_map.projects
|
||
|
|
|
||
|
|
def filter_projects(projects, predicate, force=False):
|
||
|
|
return [project.with_fields(enabled=(force or project.enabled) and
|
||
|
|
predicate(project))
|
||
|
|
for project in projects]
|
||
|
|
|
||
|
|
if args.projects:
|
||
|
|
projects_arg = args.projects.split(",")
|
||
|
|
available_projects = [project.name
|
||
|
|
for project in projects]
|
||
|
|
|
||
|
|
# validate that given projects are present in the project map file
|
||
|
|
for manual_project in projects_arg:
|
||
|
|
if manual_project not in available_projects:
|
||
|
|
parser.error("Project '{project}' is not found in "
|
||
|
|
"the project map file. Available projects are "
|
||
|
|
"{all}.".format(project=manual_project,
|
||
|
|
all=available_projects))
|
||
|
|
|
||
|
|
projects = filter_projects(projects, lambda project:
|
||
|
|
project.name in projects_arg,
|
||
|
|
force=True)
|
||
|
|
|
||
|
|
try:
|
||
|
|
max_size = Size.from_str(args.max_size)
|
||
|
|
except ValueError as e:
|
||
|
|
parser.error("{}".format(e))
|
||
|
|
|
||
|
|
projects = filter_projects(projects, lambda project:
|
||
|
|
project.size <= max_size)
|
||
|
|
|
||
|
|
return projects
|
||
|
|
|
||
|
|
|
||
|
|
def docker(parser, args):
|
||
|
|
if len(args.rest) > 0:
|
||
|
|
if args.rest[0] != "--":
|
||
|
|
parser.error("REST arguments should start with '--'")
|
||
|
|
args.rest = args.rest[1:]
|
||
|
|
|
||
|
|
if args.build_image:
|
||
|
|
docker_build_image()
|
||
|
|
elif args.shell:
|
||
|
|
docker_shell(args)
|
||
|
|
else:
|
||
|
|
sys.exit(docker_run(args, ' '.join(args.rest)))
|
||
|
|
|
||
|
|
|
||
|
|
def docker_build_image():
|
||
|
|
sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR),
|
||
|
|
shell=True))
|
||
|
|
|
||
|
|
|
||
|
|
def docker_shell(args):
|
||
|
|
try:
|
||
|
|
# First we need to start the docker container in a waiting mode,
|
||
|
|
# so it doesn't do anything, but most importantly keeps working
|
||
|
|
# while the shell session is in progress.
|
||
|
|
docker_run(args, "--wait", "--detach")
|
||
|
|
# Since the docker container is running, we can actually connect to it
|
||
|
|
call("docker exec -it satest bash", shell=True)
|
||
|
|
|
||
|
|
except KeyboardInterrupt:
|
||
|
|
pass
|
||
|
|
|
||
|
|
finally:
|
||
|
|
docker_cleanup()
|
||
|
|
|
||
|
|
|
||
|
|
def docker_run(args, command, docker_args=""):
|
||
|
|
try:
|
||
|
|
return call("docker run --rm --name satest "
|
||
|
|
"-v {llvm}:/llvm-project "
|
||
|
|
"-v {build}:/build "
|
||
|
|
"-v {clang}:/analyzer "
|
||
|
|
"-v {scripts}:/scripts "
|
||
|
|
"-v {projects}:/projects "
|
||
|
|
"{docker_args} "
|
||
|
|
"satest-image:latest {command}"
|
||
|
|
.format(llvm=args.llvm_project_dir,
|
||
|
|
build=args.build_dir,
|
||
|
|
clang=args.clang_dir,
|
||
|
|
scripts=SCRIPTS_DIR,
|
||
|
|
projects=PROJECTS_DIR,
|
||
|
|
docker_args=docker_args,
|
||
|
|
command=command),
|
||
|
|
shell=True)
|
||
|
|
|
||
|
|
except KeyboardInterrupt:
|
||
|
|
docker_cleanup()
|
||
|
|
|
||
|
|
|
||
|
|
def docker_cleanup():
|
||
|
|
print("Please wait for docker to clean up")
|
||
|
|
call("docker stop satest", shell=True)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser()
|
||
|
|
subparsers = parser.add_subparsers()
|
||
|
|
|
||
|
|
# add subcommand
|
||
|
|
add_parser = subparsers.add_parser(
|
||
|
|
"add",
|
||
|
|
help="Add a new project for the analyzer testing.")
|
||
|
|
# TODO: Add an option not to build.
|
||
|
|
# TODO: Set the path to the Repository directory.
|
||
|
|
add_parser.add_argument("name", nargs=1, help="Name of the new project")
|
||
|
|
add_parser.add_argument("--mode", action="store", default=1, type=int,
|
||
|
|
choices=[0, 1, 2],
|
||
|
|
help="Build mode: 0 for single file project, "
|
||
|
|
"1 for scan_build, "
|
||
|
|
"2 for single file c++11 project")
|
||
|
|
add_parser.add_argument("--source", action="store", default="script",
|
||
|
|
choices=["script", "git", "zip"],
|
||
|
|
help="Source type of the new project: "
|
||
|
|
"'git' for getting from git "
|
||
|
|
"(please provide --origin and --commit), "
|
||
|
|
"'zip' for unpacking source from a zip file, "
|
||
|
|
"'script' for downloading source by running "
|
||
|
|
"a custom script")
|
||
|
|
add_parser.add_argument("--origin", action="store", default="",
|
||
|
|
help="Origin link for a git repository")
|
||
|
|
add_parser.add_argument("--commit", action="store", default="",
|
||
|
|
help="Git hash for a commit to checkout")
|
||
|
|
add_parser.set_defaults(func=add)
|
||
|
|
|
||
|
|
# build subcommand
|
||
|
|
build_parser = subparsers.add_parser(
|
||
|
|
"build",
|
||
|
|
help="Build projects from the project map and compare results with "
|
||
|
|
"the reference.")
|
||
|
|
build_parser.add_argument("--strictness", dest="strictness",
|
||
|
|
type=int, default=0,
|
||
|
|
help="0 to fail on runtime errors, 1 to fail "
|
||
|
|
"when the number of found bugs are different "
|
||
|
|
"from the reference, 2 to fail on any "
|
||
|
|
"difference from the reference. Default is 0.")
|
||
|
|
build_parser.add_argument("-r", dest="regenerate", action="store_true",
|
||
|
|
default=False,
|
||
|
|
help="Regenerate reference output.")
|
||
|
|
build_parser.add_argument("--override-compiler", action="store_true",
|
||
|
|
default=False, help="Call scan-build with "
|
||
|
|
"--override-compiler option.")
|
||
|
|
build_parser.add_argument("-j", "--jobs", dest="jobs",
|
||
|
|
type=int, default=0,
|
||
|
|
help="Number of projects to test concurrently")
|
||
|
|
build_parser.add_argument("--extra-analyzer-config",
|
||
|
|
dest="extra_analyzer_config", type=str,
|
||
|
|
default="",
|
||
|
|
help="Arguments passed to to -analyzer-config")
|
||
|
|
build_parser.add_argument("--projects", action="store", default="",
|
||
|
|
help="Comma-separated list of projects to test")
|
||
|
|
build_parser.add_argument("--max-size", action="store", default=None,
|
||
|
|
help="Maximum size for the projects to test")
|
||
|
|
build_parser.add_argument("-v", "--verbose", action="count", default=0)
|
||
|
|
build_parser.set_defaults(func=build)
|
||
|
|
|
||
|
|
# compare subcommand
|
||
|
|
cmp_parser = subparsers.add_parser(
|
||
|
|
"compare",
|
||
|
|
help="Comparing two static analyzer runs in terms of "
|
||
|
|
"reported warnings and execution time statistics.")
|
||
|
|
cmp_parser.add_argument("--root-old", dest="root_old",
|
||
|
|
help="Prefix to ignore on source files for "
|
||
|
|
"OLD directory",
|
||
|
|
action="store", type=str, default="")
|
||
|
|
cmp_parser.add_argument("--root-new", dest="root_new",
|
||
|
|
help="Prefix to ignore on source files for "
|
||
|
|
"NEW directory",
|
||
|
|
action="store", type=str, default="")
|
||
|
|
cmp_parser.add_argument("--verbose-log", dest="verbose_log",
|
||
|
|
help="Write additional information to LOG "
|
||
|
|
"[default=None]",
|
||
|
|
action="store", type=str, default=None,
|
||
|
|
metavar="LOG")
|
||
|
|
cmp_parser.add_argument("--stats-only", action="store_true",
|
||
|
|
dest="stats_only", default=False,
|
||
|
|
help="Only show statistics on reports")
|
||
|
|
cmp_parser.add_argument("--show-stats", action="store_true",
|
||
|
|
dest="show_stats", default=False,
|
||
|
|
help="Show change in statistics")
|
||
|
|
cmp_parser.add_argument("--histogram", action="store", default=None,
|
||
|
|
help="Show histogram of paths differences. "
|
||
|
|
"Requires matplotlib")
|
||
|
|
cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
|
||
|
|
cmp_parser.add_argument("new", nargs=1, help="Directory with new results")
|
||
|
|
cmp_parser.set_defaults(func=compare)
|
||
|
|
|
||
|
|
# update subcommand
|
||
|
|
upd_parser = subparsers.add_parser(
|
||
|
|
"update",
|
||
|
|
help="Update static analyzer reference results based on the previous "
|
||
|
|
"run of SATest build. Assumes that SATest build was just run.")
|
||
|
|
upd_parser.add_argument("--git", action="store_true",
|
||
|
|
help="Stage updated results using git.")
|
||
|
|
upd_parser.set_defaults(func=update)
|
||
|
|
|
||
|
|
# docker subcommand
|
||
|
|
dock_parser = subparsers.add_parser(
|
||
|
|
"docker",
|
||
|
|
help="Run regression system in the docker.")
|
||
|
|
|
||
|
|
dock_parser.add_argument("--build-image", action="store_true",
|
||
|
|
help="Build docker image for running tests.")
|
||
|
|
dock_parser.add_argument("--shell", action="store_true",
|
||
|
|
help="Start a shell on docker.")
|
||
|
|
dock_parser.add_argument("--llvm-project-dir", action="store",
|
||
|
|
default=DEFAULT_LLVM_DIR,
|
||
|
|
help="Path to LLVM source code. Defaults "
|
||
|
|
"to the repo where this script is located. ")
|
||
|
|
dock_parser.add_argument("--build-dir", action="store", default="",
|
||
|
|
help="Path to a directory where docker should "
|
||
|
|
"build LLVM code.")
|
||
|
|
dock_parser.add_argument("--clang-dir", action="store", default="",
|
||
|
|
help="Path to find/install LLVM installation.")
|
||
|
|
dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[],
|
||
|
|
help="Additionall args that will be forwarded "
|
||
|
|
"to the docker's entrypoint.")
|
||
|
|
dock_parser.set_defaults(func=docker)
|
||
|
|
|
||
|
|
# benchmark subcommand
|
||
|
|
bench_parser = subparsers.add_parser(
|
||
|
|
"benchmark",
|
||
|
|
help="Run benchmarks by building a set of projects multiple times.")
|
||
|
|
|
||
|
|
bench_parser.add_argument("-i", "--iterations", action="store",
|
||
|
|
type=int, default=20,
|
||
|
|
help="Number of iterations for building each "
|
||
|
|
"project.")
|
||
|
|
bench_parser.add_argument("-o", "--output", action="store",
|
||
|
|
default="benchmark.csv",
|
||
|
|
help="Output csv file for the benchmark results")
|
||
|
|
bench_parser.add_argument("--projects", action="store", default="",
|
||
|
|
help="Comma-separated list of projects to test")
|
||
|
|
bench_parser.add_argument("--max-size", action="store", default=None,
|
||
|
|
help="Maximum size for the projects to test")
|
||
|
|
bench_parser.set_defaults(func=benchmark)
|
||
|
|
|
||
|
|
bench_subparsers = bench_parser.add_subparsers()
|
||
|
|
bench_compare_parser = bench_subparsers.add_parser(
|
||
|
|
"compare",
|
||
|
|
help="Compare benchmark runs.")
|
||
|
|
bench_compare_parser.add_argument("--old", action="store", required=True,
|
||
|
|
help="Benchmark reference results to "
|
||
|
|
"compare agains.")
|
||
|
|
bench_compare_parser.add_argument("--new", action="store", required=True,
|
||
|
|
help="New benchmark results to check.")
|
||
|
|
bench_compare_parser.add_argument("-o", "--output",
|
||
|
|
action="store", required=True,
|
||
|
|
help="Output file for plots.")
|
||
|
|
bench_compare_parser.set_defaults(func=benchmark_compare)
|
||
|
|
|
||
|
|
args = parser.parse_args()
|
||
|
|
args.func(parser, args)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|