bernsteinbear.com
this repo has no description
1#!/usr/bin/env python3
2import argparse
3import hashlib
4import json
5import logging
6import os
7import subprocess
8import sys
9
10from util import get_repo_root, normalize_revision, run, SCRIPT_PATH
11
12
13RESULTSDIR = "/var/tmp/benchresults"
14
15
16def benchmark(repo_root, revision, binary, resultsdir, args):
17 results = []
18 if args.run_django:
19 cgoutdir = f"{resultsdir}/cg/django"
20 run(["mkdir", "-p", cgoutdir])
21 cmd = [
22 f"{repo_root}/benchmarks/benchmarks/django/benchmark.py",
23 "-i",
24 binary,
25 "--callgrind-out-dir",
26 cgoutdir,
27 "--json",
28 "--callgrind",
29 ]
30 proc = run(cmd, cwd="/", log_level=logging.INFO)
31 results += json.loads(proc.stdout)
32
33 if args.run_benchmarks:
34 cgoutdir = f"{resultsdir}/cg"
35 cmd = [
36 f"{repo_root}/benchmarks/run.py",
37 "-i",
38 binary,
39 "-p",
40 f"{repo_root}/benchmarks/benchmarks",
41 "--json",
42 "--tool=callgrind",
43 "--callgrind-out-dir",
44 cgoutdir,
45 ]
46 proc = run(cmd, cwd="/", log_level=logging.INFO)
47 results += json.loads(proc.stdout)
48
49 if args.run_benchmark:
50 cgoutdir = f"{resultsdir}/cg"
51 cmd = [
52 f"{repo_root}/benchmarks/run.py",
53 "-i",
54 binary,
55 "-p",
56 f"{repo_root}/benchmarks/benchmarks",
57 "--benchmark",
58 args.run_benchmark,
59 "--json",
60 "--tool=callgrind",
61 "--callgrind-out-dir",
62 cgoutdir,
63 ]
64 proc = run(cmd, cwd="/", log_level=logging.INFO)
65 results += json.loads(proc.stdout)
66
67 for result in results:
68 # Remove interpreter path as it unnecessarily produces
69 # differences because of changing temporary directories.
70 if "interpreter" in result:
71 del result["interpreter"]
72 result["interpreter_name"] = "pyro"
73 result["version"] = revision
74
75 return results
76
77
78if __name__ == "__main__":
79 description = "Checkout, build and benchmark a single revision"
80 parser = argparse.ArgumentParser(description=description)
81 parser.add_argument("revision")
82 parser.add_argument(
83 "--ignore-cache", dest="check_cache", default=True, action="store_false"
84 )
85 parser.add_argument(
86 "--no-update-cache", dest="update_cache", default=True, action="store_false"
87 )
88 parser.add_argument(
89 "--only-cache", dest="only_cache", default=False, action="store_true"
90 )
91 parser.add_argument("--run-django", default=None, action="store_true")
92 parser.add_argument("--run-benchmarks", default=None, action="store_true")
93 parser.add_argument("--run-benchmark", default=None, action="store")
94 parser.add_argument("-v", "--verbose", action="store_true")
95 args = parser.parse_args()
96
97 log_level = logging.DEBUG if args.verbose else logging.INFO
98 logging.basicConfig(format="%(message)s", level=log_level)
99
100 repo_root = get_repo_root(SCRIPT_PATH)
101
102 if (
103 args.run_django is None
104 and args.run_benchmarks is None
105 and args.run_benchmark is None
106 ):
107 args.run_django = True
108
109 cache_env = (
110 open(__file__, "r").read()
111 + f"django: {args.run_django}"
112 + f"benchmarks: {args.run_benchmarks}"
113 + f"benchmark: {args.run_benchmark}"
114 )
115 cache_env_hash = hashlib.sha256(cache_env.encode()).hexdigest()[:8]
116
117 revision = normalize_revision(repo_root, args.revision)
118 resultsdir = f"{RESULTSDIR}/{revision}_{cache_env_hash}"
119 cachefile = f"{resultsdir}/result.json"
120
121 results = None
122 if args.check_cache:
123 try:
124 results = json.load(open(cachefile))
125 sys.stderr.write(f"Using cached results from {cachefile}\n")
126 except Exception:
127 pass
128
129 if results is None and not args.only_cache:
130 cmd = [f"{SCRIPT_PATH}/build_rev.py"]
131 if args.verbose:
132 cmd += ["--verbose"]
133 cmd += [revision]
134 p = run(cmd)
135 binary = p.stdout.strip()
136 if not binary.startswith("/"):
137 logging.critical(f"Build script did not produce a path. Output: {binary}")
138
139 results = benchmark(repo_root, revision, binary, resultsdir, args)
140 if args.update_cache and results:
141 try:
142 run(["mkdir", "-p", os.path.dirname(cachefile)])
143 with open(cachefile, "w") as fp:
144 json.dump(results, fp=fp, indent=2, sort_keys=True)
145 except Exception:
146 logging.warning(f"Failed to save results to {cachefile}\n")
147
148 out = sys.stdout
149 json.dump(results, fp=out, indent=2, sort_keys=True)
150 out.write("\n")