diff --git a/test/perf/run.py b/test/perf/run.py index 0b286e73..bd203ba2 100644 --- a/test/perf/run.py +++ b/test/perf/run.py @@ -24,6 +24,12 @@ from safety import SecretScanner import resources +# Scenarios run once per invocation. pool waits on the product's real 2 min +# idle release and 1 min spare cool-down until the scaled cool-down is fixed; +# its gate is one sample, so it has no noise estimate. +SINGLE_RUN = {"pool": 1} + + def summary(values): middle = statistics.median(values) return { @@ -150,6 +156,8 @@ def derived(runs): "noise": stats["mad"], "samples": stats["samples"], } + if len(values) == 1: + metrics[name]["single_sample"] = True if runs["boot"][0]["checkpoints"]: add( @@ -322,8 +330,9 @@ def main(): try: for name in names: raw[name] = [] - for index in range(args.runs): - print(f"{args.mode} {name} {index + 1}/{args.runs}", flush=True) + runs = SINGLE_RUN.get(name, args.runs) + for index in range(runs): + print(f"{args.mode} {name} {index + 1}/{runs}", flush=True) raw[name].append( run_scenario( name,