A game about forced loneliness, made by TACStudios
tacstudios.tngl.sh
1using NUnit.Framework;
2using System;
3using System.Collections.Generic;
4using System.Diagnostics;
5using System.Reflection;
6using UnityEngine;
7
8namespace Unity.PerformanceTesting.Benchmark
9{
10 /// <summary>
11 /// Generates and saves a markdown file after running benchmarks.
12 /// </summary>
13 public static class BenchmarkGenerator
14 {
15 // This must have the same number of elements as there are bits in the flags parameter for GetFlagSuperscripts
16 static string[] superscripts = { "¹", "²", "³", "⁴", "⁵", "⁶", "⁷", "⁸", "⁹",
17 "¹⁰", "¹¹", "¹²", "¹³", "¹⁴", "¹⁵", "¹⁶", "¹⁷", "¹⁸", "¹⁹",
18 "²⁰", "²¹", "²²", "²³", "²⁴", "²⁵", "²⁶", "²⁷", "²⁸", "²⁹",
19 "³⁰", "³¹", "³²"
20 };
21 static string[] superscriptDesc =
22 {
23 "Optimizations were disabled to perform this benchmark",
24 "Benchmark run on parallel job workers - results may vary",
25 };
26 static string GetFlagSuperscripts(uint flags)
27 {
28 string ret = "";
29 for (int f = 0; f < sizeof(uint) * 8; f++)
30 {
31 if ((flags & (1 << f)) != 0)
32 {
33 if (ret.Length > 0)
34 ret += "˒";
35 ret += superscripts[f];
36 }
37 }
38 return ret;
39 }
40
41 /// <summary>
42 /// First, runs benchmarks for all benchmark methods in all types attributed with [Benchmark(benchmarkEnumType)].
43 /// Then, generates a report in markdown with these results, and saves to the requested file path.<para />
44 /// A common integration method is to call this directly from a menu item handler.
45 /// </summary>
46 /// <param name="title">The title of the entire benchmark report</param>
47 /// <param name="benchmarkEnumType">An enum with a <see cref="BenchmarkComparisonAttribute"/> which is specified in all <see cref="BenchmarkAttribute"/>s marking
48 /// classes which contain performance methods to be benchmarked. All performance test methods in the class
49 /// must contain a parameter of the enum marked with <see cref="BenchmarkComparisonAttribute"/> which is specified in the class's
50 /// <see cref="BenchmarkAttribute"/>, and may not contain any other parameter with another enum marked with <see cref="BenchmarkComparisonAttribute"/>.</param>
51 /// <param name="filePath">The output file path to save the generated markdown to.</param>
52 /// <param name="description">A global description for the entire benchmark report, or null.</param>
53 /// <param name="notesTitle">The title for a global "notes" section for the entire benchmark report, or null.</param>
54 /// <param name="notes">An array of notes in the previously mentioned global "notes" section for the entire benchmark report, or null.</param>
55 /// <exception cref="ArgumentException">Thrown for any errors in defining the benchmarks.</exception>
56 public static void GenerateMarkdown(string title, Type benchmarkEnumType, string filePath, string description = null, string notesTitle = null, string[] notes = null)
57 {
58 var attrBenchmarkComparison = benchmarkEnumType.GetCustomAttribute<BenchmarkComparisonAttribute>();
59 if (attrBenchmarkComparison == null)
60 throw new ArgumentException($"{benchmarkEnumType.Name} is not a valid benchmark comparison enum type as it is not decorated with [{nameof(BenchmarkComparisonAttribute)}]");
61
62 Stopwatch timer = new Stopwatch();
63 timer.Start();
64 var assemblies = AppDomain.CurrentDomain.GetAssemblies();
65 var benchmarkTypes = new List<Type>();
66
67 foreach (Assembly assembly in assemblies)
68 {
69 var types = assembly.GetTypes();
70 foreach(var t in types)
71 {
72 var cads = t.GetCustomAttributesData();
73 foreach (var cad in cads)
74 {
75 if (cad.AttributeType != typeof(BenchmarkAttribute))
76 continue;
77
78 if ((Type)cad.ConstructorArguments[0].Value == benchmarkEnumType &&
79 (bool)cad.ConstructorArguments[1].Value == false)
80 benchmarkTypes.Add(t);
81 }
82 }
83 }
84 UnityEngine.Debug.Log($"Took {timer.Elapsed}s to find all types with [Benchmark(typeof({benchmarkEnumType.Name}))]");
85
86 timer.Restart();
87 GenerateMarkdown(title, benchmarkTypes.ToArray(), filePath, description, notesTitle, notes);
88 UnityEngine.Debug.Log($"Took {timer.Elapsed}s to benchmark all types with [Benchmark(typeof({benchmarkEnumType.Name}))]");
89 }
90
91 /// <summary>
92 /// First, runs benchmarks for all benchmark methods in all given types.<br />
93 /// Then, generates a report in markdown with these results, and saves to the requested file path.
94 /// </summary>
95 /// <param name="title">The title of the entire benchmark report</param>
96 /// <param name="benchmarkTypes">An array of Types each annotated with a <see cref="BenchmarkAttribute"/> for comparison. Each Type may
97 /// refer to a class with different arguments to the <see cref="BenchmarkAttribute"/> if desired, but all performance test methods in the class
98 /// must each contain a parameter of the enum marked with <see cref="BenchmarkComparisonAttribute"/> which is specified in the class's
99 /// <see cref="BenchmarkAttribute"/>, and may not contain any other parameter with another enum marked with <see cref="BenchmarkComparisonAttribute"/>.</param>
100 /// <param name="filePath">The output file path to save the generated markdown to.</param>
101 /// <param name="description">A global description for the entire benchmark report, or null.</param>
102 /// <param name="notesTitle">The title for a global "notes" section for the entire benchmark report, or null.</param>
103 /// <param name="notes">An array of notes in the previously mentioned global "notes" section for the entire benchmark report, or null.</param>
104 /// <exception cref="ArgumentException">Thrown for any errors in defining the benchmarks.</exception>
105 public static void GenerateMarkdown(string title, Type[] benchmarkTypes, string filePath, string description = null, string notesTitle = null, string[] notes = null)
106 {
107 using (var reports = BenchmarkRunner.RunBenchmarks(title, benchmarkTypes))
108 {
109 MarkdownBuilder md = new MarkdownBuilder();
110 md.Header(1, $"Performance Comparison: {reports.reportName}");
111
112 int versionFilter = Application.unityVersion.IndexOf('-');
113 md.Note($"<span style=\"color:red\">This file is auto-generated</span>",
114 $"All measurments were taken on {SystemInfo.processorType} with {SystemInfo.processorCount} logical cores.",
115 $"Unity Editor version: {Application.unityVersion.Substring(0, versionFilter == -1 ? Application.unityVersion.Length : versionFilter)}",
116 "To regenerate this file locally use: **DOTS -> Unity.Collections -> Generate ***** menu.");
117
118 // Generate ToC
119
120 const string kSectionBenchmarkResults = "Benchmark Results";
121
122 md.Header(2, "Table of Contents");
123 md.ListItem(0).LinkHeader(kSectionBenchmarkResults).Br();
124 foreach (var group in reports.groups)
125 md.ListItem(1).LinkHeader(group.groupName.ToString()).Br();
126
127 // Generate benchmark tables
128
129 md.Header(2, kSectionBenchmarkResults);
130
131 // Report description and notes first
132 if (description != null && description.Length > 0)
133 {
134 md.AppendLine(description);
135 md.BrParagraph();
136 }
137
138 if (notes != null && notes.Length > 0)
139 {
140 if (notesTitle != null && notesTitle.Length > 0)
141 md.Note(notesTitle, notes);
142 else
143 md.Note(notes);
144 }
145
146 // Report each group results as ordered in the table of contents
147 foreach (var group in reports.groups)
148 {
149 md.BrParagraph().Header(3, $"*{group.groupName}*");
150 string[] titles = new string[group.variantNames.Length];
151 for (int i = 0; i < titles.Length; i++)
152 {
153 titles[i] = group.variantNames[i].ToString();
154 switch (group.resultTypes[i])
155 {
156 case BenchmarkResultType.ExternalBaseline:
157 case BenchmarkResultType.External:
158 titles[i] = $"*{titles[i]}*";
159 break;
160 }
161 }
162 md.TableHeader(false, "Functionality", true, titles);
163 uint tableFlags = 0;
164
165 // Find max amount of alignment spacing needed
166 int[] ratioSpace = new int[group.variantNames.Length];
167 foreach (var comparison in group.comparisons)
168 {
169 for (int i = 0; i < ratioSpace.Length; i++)
170 {
171 if (comparison.results[i].ranking == BenchmarkRankingType.Ignored)
172 continue;
173 int ratio10 = Mathf.RoundToInt((float)(comparison.results[i].baselineRatio * 10));
174 int pow10 = 0;
175 while (ratio10 >= 100)
176 {
177 pow10++;
178 ratio10 /= 10;
179 }
180 ratioSpace[i] = Mathf.Max(ratioSpace[i], pow10);
181 }
182 }
183
184 foreach (var comparison in group.comparisons)
185 {
186 uint rowFlags = comparison.footnoteFlags;
187 int items = comparison.results.Length;
188 var tableData = new string[items];
189 for (int i = 0; i < items; i++)
190 {
191 if (comparison.results[i].ranking == BenchmarkRankingType.Ignored)
192 {
193 tableData[i] = "---";
194 continue;
195 }
196
197 string format = $"{{0:F{group.resultDecimalPlaces}}}";
198 string result = $"{string.Format(format, comparison.results[i].Comparator)}{comparison.results[i].UnitSuffix}";
199 string speedup = $"({comparison.results[i].baselineRatio:F1}x)";
200 rowFlags |= comparison.results[i].resultFlags;
201
202 int ratio10 = Mathf.RoundToInt((float)(comparison.results[i].baselineRatio * 10));
203
204 if (ratio10 > 10)
205 speedup = $"<span style=\"color:green\">{speedup}</span>";
206 else if (ratio10 < 10)
207 speedup = $"<span style=\"color:red\">{speedup}</span>";
208 else
209 speedup = $"<span style=\"color:grey\">{speedup}</span>";
210
211 int alignSpaces = ratioSpace[i];
212 while (ratio10 >= 100)
213 {
214 alignSpaces--;
215 ratio10 /= 10;
216 }
217
218 speedup = $"{new string(' ', alignSpaces)}{speedup}";
219
220 tableData[i] = $"{result} {speedup}";
221
222 switch (group.resultTypes[i])
223 {
224 case BenchmarkResultType.ExternalBaseline:
225 case BenchmarkResultType.External:
226 tableData[i] = $"*{tableData[i]}*";
227 break;
228 }
229 switch (comparison.results[i].ranking)
230 {
231 case BenchmarkRankingType.Normal:
232 tableData[i] = $"{tableData[i]} "; // those 2 spaces are unicode en-space because >1 ASCII code spaces collapse
233 break;
234 case BenchmarkRankingType.Best:
235 tableData[i] = $"{tableData[i]} 🟢";
236 break;
237 case BenchmarkRankingType.Worst:
238 tableData[i] = $"{tableData[i]} 🟠";
239 break;
240 }
241 }
242
243 tableFlags |= rowFlags;
244 if (rowFlags != 0)
245 md.TableRow($"`{comparison.comparisonName}`*{GetFlagSuperscripts(rowFlags)}*", tableData);
246 else
247 md.TableRow($"`{comparison.comparisonName}`", tableData);
248 }
249
250 md.Br();
251 for (int f = 0; f < 32; f++)
252 {
253 if ((tableFlags & (1 << f)) != 0)
254 {
255 if (f < superscriptDesc.Length)
256 md.AppendLine($"*{superscripts[f]}* {superscriptDesc[f]}");
257 else
258 md.AppendLine($"*{superscripts[f]}* {group.customFootnotes[1u << f]}");
259 }
260 }
261 md.HorizontalLine();
262 }
263
264 md.Save(filePath);
265
266 }
267 }
268 }
269}