bernsteinbear.com
this repo has no description
1#!/usr/bin/env python3
2# Copyright (c) Facebook, Inc. and its affiliates. (http://www.facebook.com)
3import os
4import sys
5import time
6
7
8class TimeTool:
9 # Minimum number of seconds we should run benchmark before
10 # results are considered significant.
11 MIN_TIME = 0.5
12
13 # The number of runs of each benchmark. If greater than 1, the
14 # mean and standard deviation of the runs will be reported
15 REPETITIONS = 1
16
17 # Upper bound on the number of iterations
18 MAX_ITERATIONS = 1000000000
19
20 def __init__(self):
21 self.min_time = TimeTool.MIN_TIME
22 self.reps = TimeTool.REPETITIONS
23 self.max_iters = TimeTool.MAX_ITERATIONS
24
25 @staticmethod
26 def mean(numbers):
27 total = 0.0
28 for x in numbers:
29 total += x
30 return total / len(numbers)
31
32 @staticmethod
33 def stdev(numbers):
34 m = TimeTool.mean(numbers)
35 variance = 0.0
36 for x in numbers:
37 r = x - m
38 variance += r ** 2
39 variance /= len(numbers) - 1
40 return variance ** 0.5
41
42 def _should_report_results(self, iters, tm):
43 if iters >= self.max_iters or tm >= self.min_time:
44 return True
45 return False
46
47 def _do_n_iterations(self, iters):
48 total_time = 0.0
49 while iters:
50 begin = time.time()
51 self.module.run()
52 end = time.time()
53 total_time += end - begin
54 iters -= 1
55 return total_time
56
57 def _predict_num_iters(self, iters, tm):
58 # See how much iterations should be increased by.
59 # Note: Avoid division by zero with max(seconds, 1ns).
60 multiplier = self.min_time * 1.4 / max(tm, 1e-9)
61
62 # If our last run was at least 10% of self.min_time then we
63 # use the multiplier directly.
64 # Otherwise we use at most 10 times expansion.
65 # Note: When the last run was at least 10% of the min time the max
66 # expansion should be 14x.
67 is_significant = (tm / self.min_time) > 0.1
68 multiplier = multiplier if is_significant else min(10.0, multiplier)
69 if multiplier <= 1.0:
70 multiplier = 2.0
71
72 # So what seems to be the sufficiently-large iteration count? Round up.
73 max_next_iters = 0.5 + max(multiplier * iters, iters + 1.0)
74
75 # But we do have *some* sanity limits though..
76 next_iters = min(max_next_iters, self.max_iters)
77
78 return int(next_iters)
79
80 def _do_one_repetition(self):
81 iters = 1
82 while True:
83 tm = self._do_n_iterations(iters)
84 if self._should_report_results(iters, tm):
85 break
86 iters = self._predict_num_iters(iters, tm)
87 return (tm / iters, iters)
88
89 def run(self, module):
90 self.module = module
91 tm_results = []
92 total_iters = 0
93 repetitions = self.reps
94 while repetitions > 0:
95 tm, iters = self._do_one_repetition()
96 tm_results.append(tm)
97 total_iters += iters
98 repetitions -= 1
99 if self.reps == 1:
100 return {
101 "time_sec": tm_results[0],
102 "num_iters": total_iters,
103 "num_repetitions": self.reps,
104 }
105 return {
106 "time_sec_mean": TimeTool.mean(tm_results),
107 "time_sec_stdev": TimeTool.stdev(tm_results),
108 "num_iters": total_iters,
109 "num_repetitions": self.reps,
110 }
111
112
113def main(argv):
114 benchmark_path = os.path.realpath(sys.argv[1])
115 directory, _, name_and_ext = benchmark_path.rpartition("/")
116 name, _, _ = name_and_ext.rpartition(".")
117 sys.path.append(directory)
118 module = __import__(name)
119 module_file = getattr(module, "__file__", "<builtin>")
120 if os.path.realpath(module_file) != benchmark_path:
121 print(
122 f"Module {name} was imported from {module_file}, not "
123 f"{benchmark_path} as expected. Does your benchmark name conflict "
124 "with a builtin module?",
125 file=sys.stderr,
126 )
127 sys.exit(1)
128 sys.path.pop()
129 time_tool = TimeTool()
130 result = time_tool.run(module)
131 for k, v in result.items():
132 print(k, ",", v)
133
134
135if __name__ == "__main__":
136 main(sys.argv[1:])