nixos/victoriatraces: init service module

Add service module for VictoriaTraces, following the same patterns
as the existing victorialogs and victoriametrics modules.

nixos/modules/module-list.nix

···
   ./services/databases/tigerbeetle.nix
   ./services/databases/victorialogs.nix
   ./services/databases/victoriametrics.nix
+  ./services/databases/victoriatraces.nix
   ./services/desktops/accountsservice.nix
   ./services/desktops/ayatana-indicators.nix
   ./services/desktops/bamf.nix

nixos/modules/services/databases/victoriatraces.nix

···
+{
+  config,
+  pkgs,
+  lib,
+  utils,
+  ...
+}:
+let
+  inherit (lib)
+    escapeShellArgs
+    getBin
+    hasPrefix
+    literalExpression
+    mkBefore
+    mkEnableOption
+    mkIf
+    mkOption
+    mkPackageOption
+    optionalString
+    types
+    ;
+  cfg = config.services.victoriatraces;
+  startCLIList = [
+    "${cfg.package}/bin/victoria-traces"
+    "-storageDataPath=/var/lib/${cfg.stateDir}"
+    "-httpListenAddr=${cfg.listenAddress}"
+  ]
+  ++ lib.optionals (cfg.basicAuthUsername != null) [
+    "-httpAuth.username=${cfg.basicAuthUsername}"
+  ]
+  ++ lib.optionals (cfg.basicAuthPasswordFile != null) [
+    "-httpAuth.password=file://%d/basic_auth_password"
+  ];
+in
+{
+  options.services.victoriatraces = {
+    enable = mkEnableOption "VictoriaTraces is an open source distributed traces storage and query engine from VictoriaMetrics";
+    package = mkPackageOption pkgs "victoriatraces" { };
+    listenAddress = mkOption {
+      default = ":10428";
+      type = types.str;
+      description = ''
+        TCP address to listen for incoming http requests.
+      '';
+    };
+    stateDir = mkOption {
+      type = types.str;
+      default = "victoriatraces";
+      description = ''
+        Directory below `/var/lib` to store VictoriaTraces data.
+        This directory will be created automatically using systemd's StateDirectory mechanism.
+      '';
+    };
+    retentionPeriod = mkOption {
+      type = types.str;
+      default = "7d";
+      example = "30d";
+      description = ''
+        Retention period for trace data. Data older than retentionPeriod is automatically deleted.
+      '';
+    };
+    basicAuthUsername = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.str;
+      description = ''
+        Basic Auth username used to protect VictoriaTraces instance by authorization
+      '';
+    };
+
+    basicAuthPasswordFile = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.str;
+      description = ''
+        File that contains the Basic Auth password used to protect VictoriaTraces instance by authorization
+      '';
+    };
+    extraOptions = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = literalExpression ''
+        [
+          "-loggerLevel=WARN"
+          "-retention.maxDiskSpaceUsageBytes=1073741824"
+        ]
+      '';
+      description = ''
+        Extra options to pass to VictoriaTraces. See {command}`victoria-traces -help` for
+        possible options.
+      '';
+    };
+  };
+  config = mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion =
+          (cfg.basicAuthUsername == null && cfg.basicAuthPasswordFile == null)
+          || (cfg.basicAuthUsername != null && cfg.basicAuthPasswordFile != null);
+        message = "Both basicAuthUsername and basicAuthPasswordFile must be set together to enable basicAuth functionality, or neither should be set.";
+      }
+    ];
+
+    systemd.services.victoriatraces = {
+      description = "VictoriaTraces distributed traces database";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      startLimitBurst = 5;
+
+      serviceConfig = {
+        ExecStart = lib.concatStringsSep " " [
+          (escapeShellArgs (startCLIList ++ [ "-retentionPeriod=${cfg.retentionPeriod}" ]))
+          (utils.escapeSystemdExecArgs cfg.extraOptions)
+        ];
+        DynamicUser = true;
+        LoadCredential = lib.optional (
+          cfg.basicAuthPasswordFile != null
+        ) "basic_auth_password:${cfg.basicAuthPasswordFile}";
+        RestartSec = 1;
+        Restart = "on-failure";
+        RuntimeDirectory = "victoriatraces";
+        RuntimeDirectoryMode = "0700";
+        StateDirectory = cfg.stateDir;
+        StateDirectoryMode = "0700";
+
+        # Increase the limit to avoid errors like 'too many open files' when handling many trace spans
+        LimitNOFILE = 1048576;
+
+        # Hardening
+        DeviceAllow = [ "/dev/null rw" ];
+        DevicePolicy = "strict";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "full";
+        RemoveIPC = true;
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+          "AF_UNIX"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+      };
+
+      postStart =
+        let
+          bindAddr = (optionalString (hasPrefix ":" cfg.listenAddress) "127.0.0.1") + cfg.listenAddress;
+        in
+        mkBefore ''
+          until ${getBin pkgs.curl}/bin/curl -s -o /dev/null http://${bindAddr}/ping; do
+            sleep 1;
+          done
+        '';
+    };
+  };
+}

nixos/tests/all-tests.nix

···
   vengi-tools = runTest ./vengi-tools.nix;
   victorialogs = import ./victorialogs { inherit runTest; };
   victoriametrics = import ./victoriametrics { inherit runTest; };
+  victoriatraces = import ./victoriatraces { inherit runTest; };
   vikunja = runTest ./vikunja.nix;
   virtualbox = handleTestOn [ "x86_64-linux" ] ./virtualbox.nix { };
   vm-variant = handleTest ./vm-variant.nix { };

nixos/tests/victoriatraces/default.nix

···
+{ runTest }:
+{
+  service-endpoints = runTest ./service-endpoints.nix;
+  otlp-ingestion = runTest ./otlp-ingestion.nix;
+}

nixos/tests/victoriatraces/otlp-ingestion.nix

···
+{ lib, pkgs, ... }:
+let
+  # Simple protobuf trace
+  traceGenerator = pkgs.writeScriptBin "trace-generator" ''
+    #!${
+      pkgs.python3.withPackages (
+        ps: with ps; [
+          requests
+          protobuf
+          opentelemetry-proto
+          opentelemetry-api
+          opentelemetry-sdk
+          opentelemetry-exporter-otlp-proto-http
+        ]
+      )
+    }/bin/python3
+
+    import time
+    from opentelemetry import trace
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+    from opentelemetry.sdk.resources import Resource
+
+    resource = Resource.create({
+        "service.name": "test-service",
+        "service.version": "1.0.0"
+    })
+
+    provider = TracerProvider(resource=resource)
+
+    otlp_exporter = OTLPSpanExporter(
+        endpoint="http://localhost:10428/insert/opentelemetry/v1/traces",
+        headers={},
+    )
+
+    provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
+    trace.set_tracer_provider(provider)
+
+    tracer = trace.get_tracer("test-tracer", "1.0.0")
+
+    # Test span
+    with tracer.start_as_current_span("test-span") as span:
+        span.set_attribute("http.method", "GET")
+        span.set_attribute("http.url", "/test")
+        time.sleep(0.1)
+
+    provider.force_flush()
+
+    print("Test trace sent")
+  '';
+in
+{
+  name = "victoriatraces-otlp-ingestion";
+  meta.maintainers = with lib.maintainers; [ cmacrae ];
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      services.victoriatraces = {
+        enable = true;
+        retentionPeriod = "1d";
+      };
+
+      environment.systemPackages = with pkgs; [
+        curl
+        jq
+        traceGenerator
+      ];
+    };
+
+  testScript = ''
+    machine.wait_for_unit("victoriatraces.service")
+    machine.wait_for_open_port(10428)
+    machine.succeed("curl --fail http://localhost:10428/")
+    machine.succeed("trace-generator")
+
+    # Wait for trace to be indexed
+    machine.wait_until_succeeds("""
+      curl -s 'http://localhost:10428/select/jaeger/api/services' | \
+      jq -e '.data[] | select(. == "test-service")'
+    """, timeout=10)
+
+    # Query for traces from our test service
+    machine.succeed("""
+      curl -s 'http://localhost:10428/select/jaeger/api/traces?service=test-service' | \
+      jq -e '.data[0].spans[0].operationName' | grep -q 'test-span'
+    """)
+
+    # Verify the trace has the expected attributes
+    machine.succeed("""
+      curl -s 'http://localhost:10428/select/jaeger/api/traces?service=test-service' | \
+      jq -e '.data[0].spans[0].tags[] | select(.key == "http.method") | .value == "GET"'
+    """)
+  '';
+}

nixos/tests/victoriatraces/service-endpoints.nix

···
+{ lib, ... }:
+{
+  name = "victoriatraces-service-endpoints";
+  meta.maintainers = with lib.maintainers; [ cmacrae ];
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      services.victoriatraces = {
+        enable = true;
+        extraOptions = [
+          "-loggerLevel=WARN"
+        ];
+      };
+
+      environment.systemPackages = with pkgs; [
+        curl
+        jq
+      ];
+    };
+
+  testScript = ''
+    machine.wait_for_unit("victoriatraces.service")
+    machine.wait_for_open_port(10428)
+
+    with subtest("Service endpoints are accessible"):
+        machine.succeed("curl --fail http://localhost:10428/")
+        machine.succeed("curl --fail http://localhost:10428/select/vmui")
+        machine.succeed("curl --fail http://localhost:10428/metrics | grep -E '^(vm_|process_)'")
+        machine.succeed("curl --fail http://localhost:10428/select/jaeger/api/services")
+
+    with subtest("OTLP trace ingestion endpoint accepts requests"):
+        machine.succeed("curl --fail -X POST http://localhost:10428/insert/opentelemetry/v1/traces -H 'Content-Type: application/x-protobuf'")
+        machine.succeed("test -d /var/lib/victoriatraces")
+  '';
+}