bernsteinbear.com
this repo has no description
1#!/usr/bin/env python3
2import argparse
3import json
4import logging
5import subprocess
6import sys
7from collections import defaultdict
8
9from util import get_repo_root, normalize_revision, run, SCRIPT_PATH
10
11
12def run_benchmarks(revs, revnames, args):
13 rev_samples = []
14 for rev in revs:
15 cmd = [f"{SCRIPT_PATH}/benchmark_rev.py", *args.benchmark_rev_args, rev]
16 proc = run(cmd)
17 try:
18 samples = json.loads(proc.stdout)
19 rev_samples.append(samples)
20 except Exception:
21 sys.stderr.write(f"Failed to parse benchmark data:\n{proc.stdout}\n")
22 sys.exit(1)
23
24 return rev_samples
25
26
27def aggregate_results(rev_samples, revnames): # noqa: C901
28 samples_per_benchmark = defaultdict(list)
29 for rev in rev_samples:
30 found_benchmarks = set()
31 for result in rev:
32 if "benchmark" not in result:
33 sys.stderr.write("Warning: result without benchmark name\n")
34 continue
35 benchmark = result["benchmark"]
36 if benchmark in found_benchmarks:
37 sys.stderr.write("Multiple results for same benchmark?\n")
38 continue
39 found_benchmarks.add(benchmark)
40 samples_per_benchmark[benchmark].append(result)
41
42 results = {}
43 for _benchmark, rev_samples in sorted(samples_per_benchmark.items()):
44 keys = set()
45 for samples in rev_samples:
46 keys.update(samples.keys())
47
48 bresults = {}
49 for key in keys:
50 all_same = True
51 common_value = None
52 values = []
53 for samples in rev_samples:
54 value = samples.get(key)
55 values.append(value)
56 if value is None:
57 continue
58 if common_value is None:
59 common_value = value
60 elif value != common_value:
61 all_same = False
62 if all_same:
63 bresults[key] = common_value
64 continue
65 for revname, value in zip(revnames, values):
66 if value is None:
67 continue
68 bresults[f"{key} {revname}"] = value
69
70 # Compute delta to rev0
71 samples0 = rev_samples[0]
72 for key, value0 in samples0.items():
73 if key in bresults:
74 continue
75 if not isinstance(value0, (float, int)) or value0 == 0:
76 continue
77 for revname, samples in zip(revnames[1:], rev_samples[1:]):
78 if key not in samples:
79 continue
80 value1 = samples[key]
81 delta_key = f"{key} ∆" + ("" if len(revnames) == 2 else f" {revname}")
82 rel_delta = (value1 - value0) / float(value0)
83 bresults[delta_key] = f"{rel_delta * 100.:2.1f}%"
84 results[bresults["benchmark"]] = bresults
85 return results
86
87
88if __name__ == "__main__":
89 description = "Checkout, build, benchmark and compare multiple revisions"
90 epilog = f"""
91
92Example:
93 {sys.argv[0]} .~1 . # compare parent with current revision
94"""
95 parser = argparse.ArgumentParser(description=description, epilog=epilog)
96 parser.add_argument("revisions", metavar="REVISION", nargs="+", default="")
97 parser.add_argument(
98 "--run-django",
99 dest="benchmark_rev_args",
100 default=[],
101 action="append_const",
102 const="--run-django",
103 )
104 parser.add_argument(
105 "--run-benchmarks",
106 dest="benchmark_rev_args",
107 action="append_const",
108 const="--run-benchmarks",
109 )
110 parser.add_argument(
111 "--run-benchmark",
112 dest="benchmark_rev_args",
113 action="append",
114 type=lambda arg: f"--run-benchmark={arg}",
115 )
116 parser.add_argument(
117 "--ignore-cache",
118 dest="benchmark_rev_args",
119 action="append_const",
120 const="--ignore-cache",
121 )
122 parser.add_argument("-v", "--verbose", action="store_true")
123 args = parser.parse_args()
124
125 log_level = logging.DEBUG if args.verbose else logging.INFO
126 logging.basicConfig(format="%(message)s", level=log_level, stream=sys.stderr)
127
128 if args.verbose:
129 args.benchmark_rev_args.insert(0, "--verbose")
130
131 if len(args.revisions) < 2:
132 parser.print_help()
133 sys.stdout.write("\n\nError: Need at least two revision numbers\n")
134 sys.exit(1)
135
136 repo_root = get_repo_root(SCRIPT_PATH)
137
138 revs = args.revisions
139 if len(revs) == 2:
140 revnames = ["before", "now"]
141 else:
142 revnames = [str(x) for x in range(len(revs))]
143 revs = [normalize_revision(repo_root, rev) for rev in revs]
144
145 rev_samples = run_benchmarks(revs, revnames, args)
146
147 results = aggregate_results(rev_samples, revnames)
148
149 out = sys.stdout
150 json.dump(results, fp=out, indent=2, sort_keys=True, ensure_ascii=False)
151 out.write("\n")