aria.rip
this repo has no description
1<!-- livebook:{"app_settings":{"slug":"asdf"}} -->
2
3# Dissertation Visualisations
4
5```elixir
6Mix.install([
7 {:tucan, "~> 0.3.0"},
8 {:kino_vega_lite, "~> 0.1.8"},
9 {:json, "~> 1.4"},
10 {:explorer, "~> 0.8.0"},
11 {:kino_explorer, "~> 0.1.11"},
12 {:math, "~> 0.7.0"}
13])
14```
15
16## Setup
17
18```elixir
19# Some common variables
20require Explorer.DataFrame
21require Explorer.Series
22require VegaLite
23alias Explorer.DataFrame, as: DF
24alias Explorer.Series, as: SE
25job_id = "current"
26job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
27sections_dir = Path.join(job_dir, "sections")
28cm_dir = Path.join([job_dir, "candelabra", "benchmark_results"])
29criterion_dir = Path.join(job_dir, "criterion")
30```
31
32<!-- livebook:{"branch_parent_index":0} -->
33
34## Cost models
35
36We read in the cost models from the JSON output.
37
38```elixir
39{:ok, cost_model_files} = File.ls(cm_dir)
40
41cost_model_files =
42 cost_model_files
43 |> Enum.map(fn fname -> Path.join(cm_dir, fname) |> Path.absname() end)
44
45# Should be one for each library implementation
46cost_model_files
47```
48
49```elixir
50# Find the coefficients, ie the actual cost models
51cost_models =
52 cost_model_files
53 |> Enum.map(fn fname ->
54 impl = Path.basename(fname) |> String.replace("_", ":")
55 contents = File.read!(fname)
56 contents = JSON.decode!(contents)
57
58 contents["model"]["by_op"]
59 |> Enum.map(fn {op, %{"coeffs" => coeffs}} ->
60 %{
61 op: op,
62 impl: impl,
63 coeffs: coeffs
64 }
65 end)
66 |> DF.new()
67 end)
68 |> DF.concat_rows()
69```
70
71```elixir
72# Get the raw data points
73cost_model_points =
74 cost_model_files
75 |> Enum.map(fn fname ->
76 impl = Path.basename(fname) |> String.replace("_", ":")
77 contents = File.read!(fname)
78 contents = JSON.decode!(contents)
79
80 contents["results"]["by_op"]
81 |> Enum.flat_map(fn {op, results} ->
82 Enum.map(results, fn [n, cost] ->
83 %{
84 op: op,
85 impl: String.split(impl, "::") |> List.last(),
86 n: n,
87 t: cost
88 }
89 end)
90 end)
91 |> DF.new()
92 end)
93 |> DF.concat_rows()
94```
95
96```elixir
97# Discard points outside one standard deviation, as we do when fitting
98cost_model_points =
99 cost_model_points
100 |> DF.group_by(["impl", "op", "n"])
101 |> DF.mutate(avg: mean(t), dev: standard_deviation(t))
102 |> DF.filter(abs(t - avg) < dev)
103 |> DF.discard(["avg", "dev"])
104 |> DF.mutate(t: cast(t, {:duration, :nanosecond}))
105```
106
107We can now plot our graphs. The below module provides most of the code, with cells below it specifying our actual graphs.
108
109```elixir
110defmodule CostModel do
111 @defaults %{y_domain: nil, ns: 1..60_000//100, draw_points: true}
112 @all_impls Enum.sort([
113 "SortedVec",
114 "SortedVecSet",
115 "SortedVecMap",
116 "Vec",
117 "VecSet",
118 "VecMap",
119 "BTreeSet",
120 "BTreeMap",
121 "HashSet",
122 "HashMap",
123 "LinkedList"
124 ])
125
126 # Make the names in the legends shorter and more readable
127 def friendly_impl_name(impl) do
128 String.split(impl, "::") |> List.last()
129 end
130
131 # Get a dataframe of points lying on the cost model, one point for each of `ns`.
132 def points_for(cost_models, ns, impl, op) do
133 # Get coefficients
134 %{"coeffs" => [coeffs]} =
135 DF.filter(cost_models, impl == ^impl and op == ^op)
136 |> DF.to_columns()
137
138 Enum.map(ns, fn n ->
139 t =
140 (coeffs
141 |> Enum.take(3)
142 |> Enum.with_index()
143 |> Enum.map(fn {coeff, idx} -> coeff * n ** idx end)
144 |> Enum.sum()) + Enum.at(coeffs, 3) * Math.log2(n)
145
146 %{
147 impl: friendly_impl_name(impl),
148 op: op,
149 n: n,
150 t: max(t, 0)
151 }
152 end)
153 |> DF.new()
154 end
155
156 # Plot the specified cost model, optionally specifying the x/y domains and omitting points
157 def plot(cost_models, cost_model_points, impls, op, opts \\ []) do
158 %{y_domain: y_domain, ns: ns, draw_points: draw_points} = Enum.into(opts, @defaults)
159
160 plot =
161 Tucan.layers(
162 # The actual cost model function
163 [
164 cost_models
165 |> DF.filter(op == ^op)
166 |> DF.distinct(["impl"])
167 |> DF.to_rows()
168 |> Enum.map(fn %{"impl" => impl} -> points_for(cost_models, ns, impl, op) end)
169 |> DF.concat_rows()
170 |> DF.filter(impl in ^impls)
171 |> DF.sort_by(impl)
172 |> Tucan.lineplot("n", "t", color_by: "impl", clip: true)
173 ] ++
174 if(draw_points,
175 # The raw points, if necessary
176 do: [
177 cost_model_points
178 |> DF.filter(op == ^op and impl in ^impls)
179 |> DF.group_by(["impl", "n"])
180 |> DF.sort_by(impl)
181 |> DF.mutate(t: cast(t, :f32))
182 |> Tucan.scatter(
183 "n",
184 "t",
185 color_by: "impl",
186 clip: true
187 )
188 ],
189 else: []
190 )
191 )
192
193 # Adjust x/y domain and set title, etc
194 plot =
195 plot
196 |> Tucan.Axes.set_y_title("Estimated cost")
197 |> Tucan.Axes.set_x_title("Size of container (n)")
198 |> Tucan.set_size(500, 250)
199 |> Tucan.Legend.set_title(:color, "Implementation")
200 |> Tucan.Scale.set_x_domain(ns.first, ns.last)
201
202 case y_domain do
203 [lo, hi] -> Tucan.Scale.set_y_domain(plot, lo, hi)
204 _ -> plot
205 end
206 end
207
208 # Plot the cost models for `op` across all implementations, grouped by the 2D array `impl_splits`
209 def split_plot(cost_models, cost_model_points, impl_splits, op) do
210 Enum.map(impl_splits, &plot(cost_models, cost_model_points, &1, op))
211 |> Tucan.vconcat()
212 # Ensures we don't share a legend for them all
213 |> VegaLite.resolve(:scale, color: :independent)
214 end
215end
216```
217
218Below are our actual graphs, which are displayed and exported to JSON files in the thesis directory.
219
220<!-- livebook:{"reevaluate_automatically":true} -->
221
222```elixir
223graph =
224 CostModel.split_plot(
225 cost_models,
226 cost_model_points,
227 [
228 ["Vec", "LinkedList"],
229 ["SortedVec", "SortedVecSet", "SortedVecMap", "VecSet", "VecMap"],
230 ["BTreeSet", "BTreeMap", "HashSet", "HashMap"]
231 ],
232 "insert"
233 )
234
235VegaLite.Export.save!(graph, "../thesis/assets/insert.json")
236
237graph
238```
239
240<!-- livebook:{"reevaluate_automatically":true} -->
241
242```elixir
243graph =
244 CostModel.plot(
245 cost_models,
246 cost_model_points,
247 ["VecSet", "SortedVecSet", "HashSet", "BTreeSet"],
248 "insert",
249 ns: 1..3000//10,
250 y_domain: [0, 200],
251 draw_points: false
252 )
253
254VegaLite.Export.save!(graph, "../thesis/assets/insert_small_n.json")
255
256graph
257```
258
259```elixir
260graph =
261 CostModel.split_plot(
262 cost_models,
263 cost_model_points,
264 [
265 ["SortedVec", "SortedVecSet", "SortedVecMap"],
266 [
267 "Vec",
268 "LinkedList",
269 "VecMap",
270 "VecSet"
271 ],
272 ["BTreeSet", "BTreeMap", "HashSet", "HashMap"]
273 ],
274 "contains"
275 )
276
277VegaLite.Export.save!(graph, "../thesis/assets/contains.json")
278
279graph
280```
281
282The below block can be used to inspect the cost models of certain operations and implementations
283
284```elixir
285impls = ["SortedVec", "SortedVecSet", "SortedVecMap"]
286op = "insert"
287
288CostModel.plot(
289 cost_models,
290 cost_model_points,
291 impls,
292 op
293)
294```
295
296<!-- livebook:{"branch_parent_index":0} -->
297
298## Benchmarks
299
300We read in benchmark data from criterion's JSON output.
301
302```elixir
303# Read in the results of every individual criterion benchmark
304raw_benchmarks =
305 File.ls!(criterion_dir)
306 |> Enum.map(fn name ->
307 File.ls!(Path.join(criterion_dir, name))
308 |> Enum.map(fn p -> %{bench: name, subbench: p} end)
309 end)
310 |> List.flatten()
311 |> Enum.map(fn %{bench: bench, subbench: subbench} ->
312 File.ls!(Path.join([criterion_dir, bench, subbench]))
313 |> Enum.filter(fn x -> String.contains?(x, "Mapping2D") end)
314 |> Enum.map(fn x -> Path.join([criterion_dir, bench, subbench, x]) end)
315 |> Enum.map(fn dir ->
316 raw_results =
317 Path.join(dir, "estimates.json")
318 |> File.read!()
319 |> JSON.decode!()
320
321 %{
322 bench_id: bench <> "/" <> subbench,
323 proj: String.split(bench, "-") |> hd,
324 using:
325 Regex.scan(~r/\"(\w*)\", ([^)]*)/, Path.basename(dir))
326 |> Enum.map(fn [_, ctn, impl] -> %{ctn: ctn, impl: impl} end),
327 mean: raw_results["mean"]["point_estimate"],
328 stderr: raw_results["mean"]["standard_error"]
329 }
330 end)
331 end)
332 |> List.flatten()
333 |> DF.new()
334 |> DF.mutate(
335 mean: cast(mean, {:duration, :nanosecond}),
336 stderr: cast(stderr, {:duration, :nanosecond})
337 )
338```
339
340```elixir
341# Helper function for making the `using` field look nicer
342display_using = fn using ->
343 using
344 |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} -> ctn <> "=" <> impl end)
345 |> Enum.join(", ")
346end
347```
348
349```elixir
350# Aggregate benchmark results by project, since we can only do assignments by project
351# Unfortunately we can't group by lists, so we need to do some weird shit.
352# This is basically equivalent to:
353# benchmarks = raw_benchmarks
354# |> DF.group_by(["proj", "using"])
355# |> DF.summarise(time: sum(mean))
356
357# Build list of using values to index into
358usings =
359 raw_benchmarks["using"]
360 |> SE.to_list()
361 |> Enum.uniq()
362
363benchmarks =
364 raw_benchmarks
365 # Make a column corresponding to using that isn't a list
366 |> DF.put(
367 "using_idx",
368 raw_benchmarks["using"]
369 |> SE.to_list()
370 |> Enum.map(fn using -> Enum.find_index(usings, &(&1 == using)) end)
371 )
372 # Get the total benchmark time for each project and assignment
373 |> DF.group_by(["proj", "using_idx"])
374 |> DF.summarise(time: sum(cast(mean, :f32)))
375 # Convert using_idx back to original using values
376 |> DF.to_rows()
377 |> Enum.map(fn row = %{"using_idx" => using_idx} ->
378 Map.put(row, "using", Enum.at(usings, using_idx))
379 end)
380 |> DF.new()
381 |> DF.select(["proj", "time", "using"])
382```
383
384We read our cost estimates from the log output.
385
386```elixir
387# Cost estimates by project, ctn, and implementation
388projs = SE.distinct(benchmarks["proj"])
389
390cost_estimates =
391 SE.transform(projs, fn proj_name ->
392 [_, table | _] =
393 Path.join(sections_dir, "compare-" <> proj_name)
394 |> File.read!()
395 |> String.split("& file \\\\\n\\hline\n")
396
397 table
398 |> String.split("\n\\end{tabular}")
399 |> hd
400 |> String.split("\n")
401 |> Enum.map(fn x -> String.split(x, " & ") end)
402 |> Enum.map(fn [ctn, impl, cost | _] ->
403 %{
404 proj: proj_name,
405 ctn: ctn,
406 impl:
407 impl
408 |> String.replace("\\_", "_"),
409 cost:
410 if String.contains?(cost, ".") do
411 String.to_float(cost)
412 else
413 String.to_integer(cost)
414 end
415 }
416 end)
417 end)
418 |> SE.to_list()
419 |> List.flatten()
420 |> DF.new()
421```
422
423```elixir
424# Double-check that we have all of the cost estimates for everything mentioned in the assignments
425estimate_impls = SE.distinct(cost_estimates["impl"])
426
427true =
428 (raw_benchmarks
429 |> DF.explode("using")
430 |> DF.unnest("using"))["impl"]
431 |> SE.distinct()
432 |> SE.to_list()
433 |> Enum.all?(&SE.any?(SE.equal(estimate_impls, &1)))
434```
435
436We then find the estimated cost of every assignment that we benchmarked
437
438```elixir
439# Gets the cost of assignment from cost estimates
440cost_of_assignment = fn proj, assignment ->
441 assignment
442 |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} ->
443 DF.filter(cost_estimates, proj == ^proj and ctn == ^ctn and impl == ^impl)["cost"][0]
444 end)
445 |> Enum.sum()
446end
447
448cost_of_assignment.("example_stack", [%{"ctn" => "StackCon", "impl" => "std::vec::Vec"}])
449```
450
451```elixir
452# For each benchmarked assignment, estimate the cost.
453estimated_costs =
454 benchmarks
455 |> DF.to_rows_stream()
456 |> Enum.map(fn %{"proj" => proj, "using" => using} ->
457 %{
458 proj: proj,
459 using: using,
460 estimated_cost: cost_of_assignment.(proj, using)
461 }
462 end)
463 |> DF.new()
464```
465
466Now we can compare our benchmark results to our estimated costs.
467
468We first filter out adaptive containers, to later consider them separately.
469
470```elixir
471# Don't worry about adaptive containers for now
472singular_estimated_costs =
473 estimated_costs
474 |> DF.to_rows_stream()
475 |> Enum.filter(fn %{"using" => using} ->
476 Enum.all?(using, fn %{"impl" => impl} -> !String.contains?(impl, "until") end)
477 end)
478 |> DF.new()
479
480singular_benchmarks =
481 benchmarks
482 |> DF.to_rows_stream()
483 |> Enum.filter(fn %{"using" => using} ->
484 Enum.all?(using, fn %{"impl" => impl} -> !String.contains?(impl, "until") end)
485 end)
486 |> DF.new()
487
488DF.n_rows(singular_benchmarks)
489```
490
491```elixir
492# Tools for printing out latex
493defmodule Latex do
494 def escape_latex(val) do
495 if is_number(val) do
496 "$" <> to_string(val) <> "$"
497 else
498 String.replace(to_string(val), ~r/(\\|{|}|_|\^|#|&|\$|%|~)/, "\\\\\\1")
499 end
500 end
501
502 def table(df) do
503 cols = DF.names(df)
504
505 "\\begin{tabular}{|" <>
506 String.duplicate("c|", length(cols)) <>
507 "}\n" <>
508 Enum.join(Enum.map(cols, &escape_latex/1), " & ") <>
509 " \\\\\n\\hline\n" <>
510 (DF.to_rows(df)
511 |> Enum.map(fn row ->
512 cols
513 |> Enum.map(&escape_latex(row[&1]))
514 |> Enum.join(" & ")
515 end)
516 |> Enum.join(" \\\\\n")) <>
517 " \\\\\n\\end{tabular}"
518 end
519end
520```
521
522Compare the fastest and slowest assignments for each project
523
524```elixir
525singular_benchmarks
526|> DF.group_by("proj")
527|> DF.summarise(max: max(time), min: min(time))
528|> DF.mutate(spread: round((max - min) * ^(10 ** -6), 2), slowdown: round(max / min - 1, 1))
529|> DF.discard(["max", "min"])
530|> DF.sort_by(proj)
531|> DF.rename(%{
532 "proj" => "Project",
533 "spread" => "Maximum slowdown (ms)",
534 "slowdown" => "Maximum relative slowdown"
535})
536
537# |> Latex.table()
538# |> IO.puts()
539```
540
541Compare the predicted and actual best implementation for each container type
542
543```elixir
544selection_comparison =
545 singular_benchmarks
546 |> DF.explode("using")
547 |> DF.unnest("using")
548 |> DF.group_by(["proj"])
549 |> DF.filter(time == min(time))
550 |> DF.join(
551 cost_estimates
552 |> DF.filter(not contains(impl, "until"))
553 |> DF.group_by(["proj", "ctn"])
554 |> DF.filter(cost == min(cost))
555 |> DF.rename(%{"impl" => "predicted_impl"})
556 )
557 |> DF.select(["proj", "ctn", "impl", "predicted_impl"])
558 |> DF.rename(%{"impl" => "best_impl"})
559```
560
561```elixir
562selection_comparison
563|> DF.put(
564 "best_impl",
565 SE.transform(selection_comparison["best_impl"], &CostModel.friendly_impl_name/1)
566)
567|> DF.put(
568 "predicted_impl",
569 SE.transform(selection_comparison["predicted_impl"], &CostModel.friendly_impl_name/1)
570)
571|> DF.put(
572 "mark",
573 SE.not_equal(selection_comparison["best_impl"], selection_comparison["predicted_impl"])
574 |> SE.transform(&if &1, do: "*", else: "")
575)
576|> DF.ungroup()
577|> DF.sort_by(proj)
578|> DF.rename(%{
579 "mark" => " ",
580 "proj" => "Project",
581 "ctn" => "Container Type",
582 "best_impl" => "Best implementation",
583 "predicted_impl" => "Predicted best"
584})
585
586# |> Latex.table()
587# |> IO.puts()
588```
589
590We now look at adaptive containers, starting by seeing when they get suggested
591
592```elixir
593# Container types where an adaptive container was suggested
594adaptive_suggestions =
595 estimated_costs
596 |> DF.explode("using")
597 |> DF.unnest("using")
598 |> DF.filter(contains(impl, "until"))
599 |> DF.distinct(["proj", "ctn", "impl"])
600
601adaptive_suggestions
602# Hacky way to make things look nicer
603|> DF.mutate(impl: replace(impl, "std::collections::", ""))
604|> DF.mutate(impl: replace(impl, "std::vec::", ""))
605|> DF.mutate(impl: replace(impl, "primrose_library::", ""))
606|> DF.sort_by(asc: proj, asc: ctn)
607|> DF.rename(%{
608 "proj" => "Project",
609 "ctn" => "Container Type",
610 "impl" => "Suggestion"
611})
612
613# |> Latex.table()
614# |> IO.puts()
615```
616
617Get benchmarks for projects we suggested an adaptive container for, and find the benchmark 'size' as a new column
618
619```elixir
620adaptive_projs = DF.distinct(adaptive_suggestions, ["proj"])["proj"]
621adaptive_estimated_costs = estimated_costs |> DF.filter(proj in ^adaptive_projs)
622
623adaptive_raw_benchmarks =
624 raw_benchmarks
625 |> DF.filter(proj in ^adaptive_projs)
626
627adaptive_raw_benchmarks =
628 adaptive_raw_benchmarks
629 |> DF.put(
630 "n",
631 adaptive_raw_benchmarks["bench_id"]
632 |> SE.split("/")
633 |> SE.transform(&Enum.at(&1, 1))
634 )
635 |> DF.put(
636 "using",
637 adaptive_raw_benchmarks["using"]
638 |> SE.transform(display_using)
639 )
640```
641
642We then summarise the results for each benchmark size, for assignments that either involve an adaptive container or are the best possible assignment
643
644```elixir
645format_dur = fn dur ->
646 String.split(to_string(dur), " ") |> hd
647end
648
649best_usings =
650 adaptive_raw_benchmarks
651 # get best set of assignments for each project
652 |> DF.group_by(["proj", "using"])
653 |> DF.filter(not contains(using, "until"))
654 |> DF.summarise(total: sum(cast(mean, :f32)))
655 |> DF.group_by(["proj"])
656 |> DF.filter(total == min(total))
657 |> DF.discard("total")
658 |> DF.rename(%{"using" => "best_using"})
659 # select adaptive container and the best assignment for each project
660 |> DF.join(adaptive_raw_benchmarks)
661 |> DF.filter(using == best_using or contains(using, "until"))
662
663# summary data point
664best_usings =
665 best_usings
666 |> DF.put("mean", SE.transform(best_usings["mean"], format_dur))
667 |> DF.put("stderr", SE.transform(best_usings["stderr"], format_dur))
668 |> DF.mutate(value: mean <> " +/- " <> stderr)
669 |> DF.select(["proj", "using", "n", "value"])
670```
671
672Finally, we print them out per-project for clarity
673
674```elixir
675for proj <- SE.distinct(best_usings["proj"]) |> SE.to_enum() do
676 best_usings
677 |> DF.filter(proj == ^proj)
678 |> DF.select(["proj", "using", "n", "value"])
679 |> DF.pivot_wider("n", "value")
680 |> Latex.table()
681 |> IO.puts()
682end
683```