wire / bench / runner.nix

# SPDX-License-Identifier: AGPL-3.0-or-later
# Copyright 2024-2025 wire Contributors

{
  lib,
  inputs,
  ...
}:
let
  inherit (lib)
    mapAttrsToList
    flatten
    ;
in
{
  config.perSystem =
    {
      pkgs,
      self',
      system,
      ...
    }:
    let
      benchDirFileset = lib.fileset.toSource {
        root = ../..;
        fileset = lib.fileset.union ./. (
          lib.fileset.fileFilter (
            file: (file.hasExt "nix") || (file.hasExt "txt") || (file.hasExt "lock")
          ) ../.
        );
      };

      nodes =
        builtins.listToAttrs (
          builtins.map (index: {
            value = {
              imports = [
                ./vm.nix
              ];

              _module.args = {
                index = builtins.toString index;
              };
            };
            name = "node_${builtins.toString index}";
          }) (lib.range 0 (import ./num-nodes.nix))
        )
        // {
          deployer = {
            imports = [
              ./vm.nix
            ];

            environment.systemPackages = [
              pkgs.git

              (pkgs.writeShellScriptBin "setup-benchmark" ''
                mkdir -p $HOME/wire
                cp -r ${benchDirFileset}/*-source/* $HOME/wire

                cp -r $HOME/wire/bench/wire-flake $HOME/wire-flake
                cp -r $HOME/wire/bench/colmena-flake $HOME/colmena-flake

                chmod -R +w $HOME/*

                cd $HOME/wire
                git init .
                git add -A
              '')

              (pkgs.writeShellScriptBin "run-benchmark" ''
                bench_dir=$HOME/wire/bench

                wire_args="apply test --path $bench_dir/wire -vv --ssh-accept-host -p 10"
                wire_args_flake="apply test --path $HOME/wire-flake -vv --ssh-accept-host -p 10"

                colmena_args="apply test --config $bench_dir/colmena/hive.nix -v -p 10"
                colmena_args_flake="apply test --config $HOME/colmena-flake/flake.nix -v -p 10"

                ${lib.getExe pkgs.hyperfine} --warmup 1 --show-output --runs 5 \
                  --export-markdown stats.md \
                  --export-json run.json \
                  "${lib.getExe self'.packages.wire-small} $wire_args_flake" -n "wire@HEAD - flake" \
                  "${lib.getExe' inputs.colmena_benchmarking.packages.x86_64-linux.colmena "colmena"} $colmena_args_flake" \
                      -n "colmena@pinned - flake" \
                  "${lib.getExe self'.packages.wire-small} $wire_args" -n "wire@HEAD - hive.nix"
              '')
            ];

            _module.args = {
              index = "deployer";
            };
          };
        };

      evalConfig = import (pkgs.path + "/nixos/lib/eval-config.nix");

      evalVM =
        module:
        evalConfig {
          inherit system;
          modules = [ module ];
        };
    in
    {
      checks.bench = pkgs.testers.runNixOSTest {
        inherit nodes;

        name = "benchmark";

        defaults =
          {
            ...
          }:
          let
            # hive = builtins.scopedImport {
            #   __nixPath = _b: null;
            #   __findFile = _path: name: if name == "nixpkgs" then pkgs.path else throw "oops!!";
            # } "${injectedFlakeDir}/${path}/hive.nix";

            # fetch **all** dependencies of a flake
            # it's called fetchLayer because my naming skills are awful
            fetchLayer =
              input:
              let
                subLayers = if input ? inputs then map fetchLayer (builtins.attrValues input.inputs) else [ ];
              in
              [
                input.outPath
              ]
              ++ subLayers;
          in
          {
            virtualisation.additionalPaths = flatten [
              (mapAttrsToList (_: val: (evalVM val).config.system.build.toplevel.drvPath) nodes)
              (mapAttrsToList (_: fetchLayer) inputs)
            ];

            nix.settings.experimental-features = [
              "nix-command"
              "flakes"
            ];
          };
        node.specialArgs = {
          snakeOil = import "${pkgs.path}/nixos/tests/ssh-keys.nix" pkgs;
          inherit (self'.packages) wire-small-dev;
        };
        skipTypeCheck = true;
        testScript = ''
          start_all()

          for i in range(0,${builtins.toString (import ./num-nodes.nix)}):
              machine = globals().get(f"node_{i}")
              machine.wait_for_unit("sshd.service") # type: ignore

          node_deployer.succeed("setup-benchmark");
          node_deployer.succeed("run-benchmark");

          node_deployer.copy_from_vm("run.json")
          node_deployer.copy_from_vm("stats.json")
        '';
      };
    };
}

# "${
#   lib.getExe (builtins.getFlake "github:mrshmllow/wire/stable").packages.${system}.wire-small
# } $wire_args" -n "wire@stable - hive.nix" \
# "${
#   lib.getExe (builtins.getFlake "github:mrshmllow/wire/stable").packages.${system}.wire-small
# } $wire_args_flake" -n "wire@stable - flake" \