A game about forced loneliness, made by TACStudios
tacstudios.tngl.sh
1using System;
2using System.Collections.Generic;
3using System.Reflection;
4using Unity.Collections;
5using Unity.Collections.LowLevel.Unsafe;
6
7namespace Unity.PerformanceTesting.Benchmark
8{
9 internal static class BenchmarkRunner
10 {
11 static string progressTitle;
12
13 static void StartProgress(string title, int typeIndex, int typeCount, string typeName) =>
14 progressTitle = $"Benchmarking {title} {typeIndex + 1}/{typeCount} - {typeName}";
15
16 static void EndProgress()
17 {
18#if UNITY_EDITOR
19 UnityEditor.EditorUtility.ClearProgressBar();
20#endif
21 }
22
23 static void SetProgressText(string text, float unitProgress)
24 {
25#if UNITY_EDITOR
26 if (UnityEditor.EditorUtility.DisplayCancelableProgressBar(progressTitle, text, unitProgress))
27 throw new Exception("User cancelled benchmark operation");
28#endif
29 }
30
31 /// <summary>
32 /// Contains a combination of a BenchmarkComparison attributed enum and the Type with perf. measurements
33 /// to determine names for the benchmark.
34 ///
35 /// Also contains reflected info on the enum defined and external benchmark values used to organize
36 /// benchmark tests and results, though this will not vary between different Types with perf. measurments.
37 /// These constant values are also associated with a classification of enum-defined vs external, and
38 /// baseline vs not.
39 ///
40 /// There may only be one baseline per benchmark comparison type.
41 /// </summary>
42 class BenchmarkComparisonTypeData
43 {
44 public string defaultName;
45 public Type enumType;
46
47 public string[] names;
48 public int[] values;
49 public BenchmarkResultType[] resultTypes;
50
51 public SampleUnit resultUnit;
52 public int resultDecimalPlaces;
53 public BenchmarkRankingStatistic resultStatistic;
54
55 public BenchmarkComparisonTypeData(int variants)
56 {
57 names = new string[variants];
58 values = new int[variants];
59 resultTypes = new BenchmarkResultType[variants];
60 enumType = null;
61 defaultName = null;
62 resultUnit = SampleUnit.Millisecond;
63 resultDecimalPlaces = 3;
64 resultStatistic = BenchmarkRankingStatistic.Median;
65 }
66 }
67
68 /// <summary>
69 /// Given a System.Type that contains performance test methods, reflect the setup to a benchmark comparison.
70 /// Throws on any errors with the setup.
71 /// </summary>
72 unsafe static BenchmarkComparisonTypeData GatherComparisonStructure(Type t)
73 {
74 //--------
75 // Determine and validate the benchmark comparison this type is intended for
76 //--------
77 Type benchmarkEnumType = null;
78 foreach(var attributeData in t.GetCustomAttributesData())
79 {
80 if (attributeData.AttributeType == typeof(BenchmarkAttribute))
81 {
82 benchmarkEnumType = (Type)attributeData.ConstructorArguments[0].Value;
83 break;
84 }
85 }
86 if (benchmarkEnumType == null)
87 throw new ArgumentException($"Exactly one [{nameof(BenchmarkAttribute)}] must exist on the type {t.Name} to generate benchmark data");
88
89 // Find the baseline and the formatting for its title name (could be external to the enum or included)
90 CustomAttributeData attrBenchmarkComparison = null;
91 List<CustomAttributeData> attrBenchmarkComparisonExternal = new List<CustomAttributeData>();
92 CustomAttributeData attrBenchmarkFormat = null;
93 foreach (var attributeData in benchmarkEnumType.GetCustomAttributesData())
94 {
95 if (attributeData.AttributeType == typeof(BenchmarkComparisonAttribute))
96 {
97 attrBenchmarkComparison = attributeData;
98 }
99 // Find any other external comparisons
100 else if (attributeData.AttributeType == typeof(BenchmarkComparisonExternalAttribute))
101 {
102 attrBenchmarkComparisonExternal.Add(attributeData);
103 }
104 // Find optional formatting of table results
105 else if (attributeData.AttributeType == typeof(BenchmarkComparisonDisplayAttribute))
106 {
107 attrBenchmarkFormat = attributeData;
108 }
109 }
110 if (attrBenchmarkComparison == null)
111 throw new ArgumentException($"Exactly one [{nameof(BenchmarkComparisonAttribute)}] must exist on the enum {benchmarkEnumType.Name} to generate benchmark data and define the baseline");
112
113 //--------
114 // Collect values and name formatting for enum and external
115 //--------
116
117 // Enum field values
118 var enumFields = benchmarkEnumType.GetFields(BindingFlags.Static | BindingFlags.Public);
119 var enumCount = enumFields.Length;
120 var enumValues = stackalloc int[enumCount];
121 var enumValuesSet = new HashSet<int>(enumCount);
122 for (int i = 0; i < enumCount; i++)
123 {
124 int value = (int)enumFields[i].GetRawConstantValue();
125 enumValues[i] = value;
126 enumValuesSet.Add(value);
127 }
128
129 var enumFormats = new List<string>(enumCount);
130 foreach(var x in enumFields)
131 {
132 int oldCount = enumFormats.Count;
133 foreach (var attributeData in x.GetCustomAttributesData())
134 {
135 if (attributeData.AttributeType == typeof(BenchmarkNameAttribute))
136 {
137 enumFormats.Add((string)attributeData.ConstructorArguments[0].Value);
138 break;
139 }
140 }
141 if (oldCount == enumFormats.Count)
142 throw new ArgumentException($"{x.Name} as well as all other enum values in {benchmarkEnumType.Name} must have a single [{nameof(BenchmarkNameAttribute)}] defined");
143 }
144
145 // External values
146 var externalValues = new List<int>(attrBenchmarkComparisonExternal.Count);
147 foreach(var x in attrBenchmarkComparisonExternal)
148 {
149 var externalValue = (int)x.ConstructorArguments[0].Value;
150 if (enumValuesSet.Contains(externalValue))
151 throw new ArgumentException($"Externally-defined benchmark values for {benchmarkEnumType.Name} must not be a duplicate of another enum-defined or externally-defined benchmark value for {benchmarkEnumType.Name}");
152 }
153 var externalFormats = new List<string>(attrBenchmarkComparisonExternal.Count);
154 foreach(var x in attrBenchmarkComparisonExternal)
155 {
156 externalFormats.Add((string)x.ConstructorArguments[1].Value);
157 }
158
159 var externalCount = externalValues.Count;
160
161 // Baseline value
162 int baselineValue = (int)attrBenchmarkComparison.ConstructorArguments[0].Value;
163 string externalBaselineFormat = null;
164 if (attrBenchmarkComparison.ConstructorArguments.Count == 1)
165 {
166 if (!enumValuesSet.Contains(baselineValue))
167 throw new ArgumentException($"{baselineValue} not found in enum {benchmarkEnumType.Name}. Either specify an existing value as the baseline, or add a formatting string for the externally defined baseline value.");
168 }
169 else
170 {
171 if (enumValuesSet.Contains(baselineValue))
172 throw new ArgumentException($"To specify an enum-defined benchmark baseline in {benchmarkEnumType.Name}, pass only the argument {baselineValue} without a name, as the name requires definition in the enum");
173 if (externalValues.Contains(baselineValue))
174 throw new ArgumentException($"To specify an external-defined benchmark baseline in {benchmarkEnumType.Name}, define only in [{nameof(BenchmarkComparisonAttribute)}] and omit also defining with [{nameof(BenchmarkComparisonExternalAttribute)}]");
175 externalBaselineFormat = (string)attrBenchmarkComparison.ConstructorArguments[1].Value;
176 }
177
178 // Total
179 int variantCount = enumCount + externalCount + (externalBaselineFormat == null ? 0 : 1);
180
181 //--------
182 // Collect name overrides on the specific type with benchmarking methods
183 //--------
184
185 string defaultNameOverride = null;
186 var nameOverride = new Dictionary<int, string>();
187 foreach (var attr in t.CustomAttributes)
188 {
189 if (attr.AttributeType == typeof(BenchmarkNameOverrideAttribute))
190 {
191 if (attr.ConstructorArguments.Count == 1)
192 {
193 if (defaultNameOverride != null)
194 throw new ArgumentException($"No more than one default name override is allowed for {t.Name} using [{nameof(BenchmarkNameOverrideAttribute)}]");
195 defaultNameOverride = (string)attr.ConstructorArguments[0].Value;
196 }
197 else
198 {
199 int valueOverride = (int)attr.ConstructorArguments[0].Value;
200 if (nameOverride.ContainsKey(valueOverride))
201 throw new ArgumentException($"No more than one name override is allowed for benchmark comparison value {valueOverride} using [{nameof(BenchmarkNameOverrideAttribute)}]");
202 nameOverride[valueOverride] = (string)attr.ConstructorArguments[1].Value;
203 }
204 }
205 }
206
207 //--------
208 // Record all the information
209 //--------
210
211 var ret = new BenchmarkComparisonTypeData(variantCount);
212 ret.defaultName = defaultNameOverride ?? t.Name;
213 ret.enumType = benchmarkEnumType;
214
215 // Result optional custom formatting
216 if (attrBenchmarkFormat != null)
217 {
218 ret.resultUnit = (SampleUnit)attrBenchmarkFormat.ConstructorArguments[0].Value;
219 ret.resultDecimalPlaces = (int)attrBenchmarkFormat.ConstructorArguments[1].Value;
220 ret.resultStatistic = (BenchmarkRankingStatistic)attrBenchmarkFormat.ConstructorArguments[2].Value;
221 }
222
223 // Enum field values
224 for (int i = 0; i < enumCount; i++)
225 {
226 ret.names[i] = enumFormats[i];
227 ret.values[i] = enumValues[i];
228 ret.resultTypes[i] = baselineValue == ret.values[i] ? BenchmarkResultType.NormalBaseline : BenchmarkResultType.Normal;
229 }
230
231 // External values
232 for (int i = 0; i < externalCount; i++)
233 {
234 ret.names[enumCount + i] = externalFormats[i];
235 ret.values[enumCount + i] = externalValues[i];
236 ret.resultTypes[enumCount + i] = BenchmarkResultType.External;
237 }
238
239 // External baseline value if it exists
240 if (externalBaselineFormat != null)
241 {
242 ret.names[variantCount - 1] = externalBaselineFormat;
243 ret.values[variantCount - 1] = baselineValue;
244 ret.resultTypes[variantCount - 1] = BenchmarkResultType.ExternalBaseline;
245 }
246
247 for (int i = 0; i < variantCount; i++)
248 {
249 if (nameOverride.TryGetValue(ret.values[i], out string name))
250 ret.names[i] = string.Format(ret.names[i], name);
251 else
252 ret.names[i] = string.Format(ret.names[i], ret.defaultName);
253 }
254
255 if (new HashSet<int>(ret.values).Count != ret.values.Length)
256 throw new ArgumentException($"Each enum value and external value in {benchmarkEnumType.Name} must be unique");
257
258 return ret;
259 }
260
261 /// <summary>
262 /// Reflects all possible arguments to a performance test method. Finds the parameter which benchmark comparisons
263 /// are based around (must be an enum type decorated with [BenchmarkComparison] attribute).
264 ///
265 /// There is a (usually small) finite set of arguments possible in performance test methods due to
266 /// requiring [Values(a, b, c)] attribute on any parameter that isn't a bool or enum.
267 /// </summary>
268 static void GatherAllArguments(ParameterInfo[] paramInfo, string methodName, BenchmarkComparisonTypeData structure, out int[] argCounts, out CustomAttributeTypedArgument[][] argValues, out string[] argNames, out int paramForComparison)
269 {
270 paramForComparison = -1;
271
272 argCounts = new int[paramInfo.Length];
273 argValues = new CustomAttributeTypedArgument[paramInfo.Length][];
274 argNames = new string[paramInfo.Length];
275 for (int p = 0; p < paramInfo.Length; p++)
276 {
277 // It is correct to throw if a parameter doesn't include Values attribute, NUnit errors as well
278 CustomAttributeData valuesAttribute = null;
279 foreach (var cad in paramInfo[p].GetCustomAttributesData())
280 {
281 if (cad.AttributeType == typeof(NUnit.Framework.ValuesAttribute))
282 {
283 valuesAttribute = cad;
284 break;
285 }
286 }
287 if (valuesAttribute == null)
288 throw new ArgumentException($"No [Values(...)] attribute found for parameter {paramInfo[p].Name} in {methodName}");
289
290 var values = valuesAttribute.ConstructorArguments;
291
292 argNames[p] = paramInfo[p].Name;
293
294 if (paramInfo[p].ParameterType.IsEnum && paramInfo[p].ParameterType.GetCustomAttribute<BenchmarkComparisonAttribute>() != null)
295 {
296 // [Values] <comparisonEnumType> <paramName>
297 //
298 // values.Count must be 0 or inconsistent benchmark measurements might be made.
299 // Alternatively, we could treat as if it had no arguments for benchmarks, and allow performance testing for regressions
300 // to be more specific, but for now it seems like a good idea to perf. test all valid combinations we offer, and in fact
301 // a good idea to enforce that in some manner.
302
303 if (paramInfo[p].ParameterType != structure.enumType)
304 throw new ArgumentException($"The method {methodName} parameterizes benchmark comparison type {paramInfo[p].ParameterType.Name} but only supports {structure.enumType.Name}.");
305
306 if (paramForComparison != -1)
307 throw new ArgumentException($"More than one parameter specifies {structure.enumType.Name}. Only one may exist.");
308
309 paramForComparison = p;
310
311 argCounts[p] = structure.resultTypes.Length;
312 argValues[p] = new CustomAttributeTypedArgument[argCounts[p]];
313
314 // [Values(...)] <comparisonEnumType> <paramName>
315 // This specifies comparison critera, and any excluded values will be shown as not available in the results report
316
317 if (values.Count == 0)
318 {
319 // [Values]
320 // This is the normal usage encompassing all comparison types
321
322 for (int e = 0; e < argCounts[p]; e++)
323 argValues[p][e] = new CustomAttributeTypedArgument(structure.values[e]);
324 }
325 else
326 {
327 // [Values(1-to-3-arguments)] <comparisonEnumType> <paramName>
328 var ctorValues = values;
329
330 if (values.Count == 1 && values[0].ArgumentType == typeof(object[]))
331 {
332 // [Values(more-than-3-arguments)] <comparisonEnumType> <paramName>
333 //
334 // This is for ValuesAttribute(params object[] args)
335
336 var arrayValue = values[0].Value as System.Collections.Generic.IList<CustomAttributeTypedArgument>;
337 ctorValues = arrayValue;
338 }
339
340 for (int e = 0; e < argCounts[p]; e++)
341 {
342 if (structure.resultTypes[e] == BenchmarkResultType.External || structure.resultTypes[e] == BenchmarkResultType.ExternalBaseline)
343 argValues[p][e] = new CustomAttributeTypedArgument(structure.values[e]);
344 else
345 argValues[p][e] = default; // We can later check if ArgumentType is null to determine an unused comparison test
346 }
347
348 // If we don't include NormalBaseline values, it is an error - you can't not include a baseline
349 bool hasNormalBaseline = false;
350 string normalBaselineName = null;
351 for (int i = 0; i < structure.resultTypes.Length; i++)
352 {
353 if (structure.resultTypes[i] == BenchmarkResultType.NormalBaseline)
354 {
355 hasNormalBaseline = true;
356 normalBaselineName = structure.enumType.GetEnumNames()[i];
357 }
358 }
359
360 bool specifiedBaseline = !hasNormalBaseline;
361 for (int ca = 0; ca < ctorValues.Count; ca++)
362 {
363 // Ensure it's not some alternative value cast to the enum type such as an external baseline identifying value
364 // because that would end up as part of the Performance Test Framework tests.
365 if (ctorValues[ca].ArgumentType != structure.enumType)
366 throw new ArgumentException($"Only {structure.enumType} values may be specified. External comparison types are always added automatically.");
367
368 // Find the index this value would have been at, and set the argValue there to the struct.values for it
369 for (int v = 0; v < structure.values.Length; v++)
370 {
371 if (structure.values[v] == (int)ctorValues[ca].Value)
372 {
373 argValues[p][v] = new CustomAttributeTypedArgument(structure.values[v]);
374 if (structure.resultTypes[v] == BenchmarkResultType.NormalBaseline)
375 specifiedBaseline = true;
376 }
377 }
378 }
379
380 if (!specifiedBaseline)
381 throw new ArgumentException($"This comparison type requires the baseline {structure.enumType.Name}.{normalBaselineName} to be measured.");
382 }
383 }
384 else if (values.Count == 0)
385 {
386 // [Values] <type> <paramName>
387 //
388 // This has default behaviour for bools and enums, otherwise error
389
390 if (paramInfo[p].ParameterType == typeof(bool))
391 {
392 argCounts[p] = 2;
393 argValues[p] = new CustomAttributeTypedArgument[]
394 {
395 new CustomAttributeTypedArgument(true),
396 new CustomAttributeTypedArgument(false)
397 };
398 }
399 else if (paramInfo[p].ParameterType.IsEnum)
400 {
401 var enumValues = Enum.GetValues(paramInfo[p].ParameterType);
402 argCounts[p] = enumValues.Length;
403 argValues[p] = new CustomAttributeTypedArgument[argCounts[p]];
404 for (int e = 0; e < argCounts[p]; e++)
405 argValues[p][e] = new CustomAttributeTypedArgument(enumValues.GetValue(e));
406 }
407 else
408 throw new ArgumentException($"[Values] attribute of parameter {paramInfo[p].Name} in {methodName} is empty");
409 }
410 else if (values.Count == 1 && values[0].ArgumentType == typeof(object[]))
411 {
412 // [Values(more-than-3-arguments)] <type> <paramName>
413 //
414 // This is for ValuesAttribute(params object[] args)
415
416 var arrayValue = values[0].Value as System.Collections.Generic.IList<CustomAttributeTypedArgument>;
417 argValues[p] = new CustomAttributeTypedArgument[arrayValue.Count];
418 arrayValue.CopyTo(argValues[p], 0);
419 argCounts[p] = arrayValue.Count;
420 }
421 else
422 {
423 // [Values(1-to-3-arguments)] <type> <paramName>
424 argValues[p] = new CustomAttributeTypedArgument[values.Count];
425 values.CopyTo(argValues[p], 0);
426 argCounts[p] = values.Count;
427 }
428 }
429
430 if (paramForComparison == -1)
431 throw new ArgumentException($"No benchmark comparison is parameterized. One must be specified");
432 }
433
434 /// <summary>
435 /// Given
436 /// a) X number of permutations for all arguments to each parameter in a performance test method
437 /// b) the possible arguments to each parameter
438 /// c) the parameter defining the benchmark comparison
439 ///
440 /// Return
441 /// a) the argument set (called variant) for Permutation[0 to X-1]
442 /// b) the isolated benchmark comparison index, based on the benchmark comparison enum values, for this variant
443 /// </summary>
444 static BenchmarkResultType GetVariantArguments(int variantIndex, BenchmarkComparisonTypeData structure, int paramForComparison, CustomAttributeTypedArgument[][] argValues, int[] argCounts, out object[] args, out int comparisonIndex)
445 {
446 comparisonIndex = 0;
447
448 int numParams = argValues.Length;
449
450 // Calculate ValuesAttribute indices for each parameter
451 // Calculate actual comparison index to ensure only benchmarks comparison are bunched together
452 int[] argValueIndex = new int[numParams];
453 for (int p = 0, argSet = variantIndex, comparisonMult = 1; p < numParams; p++)
454 {
455 argValueIndex[p] = argSet % argCounts[p];
456 argSet = (argSet - argValueIndex[p]) / argCounts[p];
457
458 if (p != paramForComparison)
459 {
460 comparisonIndex += argValueIndex[p] * comparisonMult;
461 comparisonMult *= argCounts[p];
462 }
463 }
464
465 // Find each argument using above ValuesAttribute indices
466 args = new object[numParams];
467 if (argValues[paramForComparison][argValueIndex[paramForComparison]].ArgumentType == null)
468 return BenchmarkResultType.Ignored;
469
470 for (int p = 0; p < numParams; p++)
471 args[p] = argValues[p][argValueIndex[p]].Value;
472
473 return structure.resultTypes[argValueIndex[paramForComparison]];
474 }
475
476 /// <summary>
477 /// Runs benchmarking for all defined benchmark methods in a type.
478 /// </summary>
479 static BenchmarkReportGroup GatherGroupData(Type t, BenchmarkComparisonTypeData structure)
480 {
481 var group = new BenchmarkReportGroup(structure.defaultName, structure.names, structure.resultTypes, structure.resultDecimalPlaces);
482 uint groupFootnoteBit = BenchmarkResults.kFlagFootnotes;
483
484 var allMethods = t.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic);
485 var methods = new List<MethodInfo>(allMethods.Length);
486 foreach (var m in allMethods)
487 {
488 if (m.GetCustomAttribute<NUnit.Framework.TestAttribute>() != null && m.GetCustomAttribute<PerformanceAttribute>() != null)
489 methods.Add(m);
490 }
491
492 var inst = Activator.CreateInstance(t);
493 for (int m = 0; m < methods.Count; m++)
494 {
495 var method = methods[m];
496
497 // Get ValueAttributes information for all parameters
498 GatherAllArguments(method.GetParameters(), $"{t.Name}.{method.Name}", structure,
499 out var argCounts, out var argValues, out var argNames, out var paramForComparison);
500
501 // Record any footnotes for this method
502 uint comparisonFootnoteFlags = 0;
503 foreach (var cad in method.GetCustomAttributesData())
504 {
505 if (cad.AttributeType != typeof(BenchmarkTestFootnoteAttribute))
506 continue;
507
508 var footnoteText = new NativeText($"{method.Name}(", Allocator.Persistent);
509 int paramsShown = 0;
510 for (int p = 0; p < argNames.Length; p++)
511 {
512 if (p == paramForComparison)
513 continue;
514
515 if (paramsShown++ > 0)
516 footnoteText.Append(", ");
517 footnoteText.Append(argNames[p]);
518 }
519 footnoteText.Append(")");
520 if (cad.ConstructorArguments.Count == 1)
521 footnoteText.Append($" -- {(string)cad.ConstructorArguments[0].Value}");
522 group.customFootnotes.Add(groupFootnoteBit, footnoteText);
523 comparisonFootnoteFlags |= groupFootnoteBit;
524 groupFootnoteBit <<= 1;
525 }
526
527 // Calculate number of variations based on all ValuesAttributes + parameters
528 int totalVariants = 1;
529 for (int p = 0; p < argCounts.Length; p++)
530 totalVariants *= argCounts[p];
531 int numComparisons = totalVariants / argCounts[paramForComparison];
532
533 BenchmarkReportComparison[] comparison = new BenchmarkReportComparison[numComparisons];
534
535 for (int i = 0; i < totalVariants; i++)
536 {
537 SetProgressText($"Running benchmark {i + 1}/{totalVariants} for {method.Name}", (float)(m + 1) / methods.Count);
538
539 // comparisonIndex indicates the variation of a complete benchmark comparison. i.e.
540 // you could be benchmarking between 3 different variants (such as NativeArray vs UnsafeArray vs C# Array)
541 // but you may also have 4 versions of that (such as 1000 elements, 10000 elements, 100000, and 1000000)
542 BenchmarkResultType resultType = GetVariantArguments(i, structure, paramForComparison, argValues, argCounts,
543 out var args, out int comparisonIndex);
544 if (resultType == BenchmarkResultType.Ignored)
545 {
546 if (comparison[comparisonIndex].comparisonName.IsEmpty)
547 comparison[comparisonIndex] = new BenchmarkReportComparison(method.Name);
548 comparison[comparisonIndex].results.Add(BenchmarkResults.Ignored);
549 continue;
550 }
551
552 if (comparison[comparisonIndex].comparisonName.IsEmpty)
553 {
554 string paramsString = null;
555 for (int p = 0; p < argCounts.Length; p++)
556 {
557 if (p == paramForComparison)
558 continue;
559 if (paramsString == null)
560 paramsString = $"({args[p]}";
561 else
562 paramsString += $", {args[p]}";
563 }
564
565 if (paramsString != null)
566 comparison[comparisonIndex] = new BenchmarkReportComparison($"{method.Name}{paramsString})");
567 else
568 comparison[comparisonIndex] = new BenchmarkReportComparison(method.Name);
569 }
570
571 // Call the performance method
572 method.Invoke(inst, args);
573
574 var results = BenchmarkMeasure.CalculateLastResults(structure.resultUnit, structure.resultStatistic);
575 comparison[comparisonIndex].results.Add(results);
576 }
577
578 // Add all sets of comparisons to the full group
579 for (int i = 0; i < numComparisons; i++)
580 {
581 comparison[i].footnoteFlags |= comparisonFootnoteFlags;
582 comparison[i].RankResults(structure.resultTypes);
583 group.comparisons.Add(comparison[i]);
584 }
585 }
586
587 return group;
588 }
589
590 /// <summary>
591 /// Runs benchmarking for all given types.
592 /// </summary>
593 /// <param name="title">The title to the full report</param>
594 /// <param name="benchmarkTypes">An array of types each marked with <see cref="BenchmarkAttribute"/></param>
595 /// <returns></returns>
596 public static BenchmarkReports RunBenchmarks(string title, Type[] benchmarkTypes)
597 {
598 BenchmarkMeasure.ForBenchmarks = true;
599 BenchmarkReports reports = default;
600
601 try
602 {
603 reports = new BenchmarkReports(title);
604
605 for (int i = 0; i < benchmarkTypes.Length; i++)
606 {
607 StartProgress(title, i, benchmarkTypes.Length, benchmarkTypes[i].Name);
608 SetProgressText("Gathering benchmark data", 0);
609 var benchmarkStructure = GatherComparisonStructure(benchmarkTypes[i]);
610 var group = GatherGroupData(benchmarkTypes[i], benchmarkStructure);
611 reports.groups.Add(group);
612 }
613 }
614 finally
615 {
616 BenchmarkMeasure.ForBenchmarks = false;
617 EndProgress();
618 }
619
620 return reports;
621 }
622 }
623}