nixos/invidious: add option to run more invidious instances

nixos/modules/services/web-apps/invidious.nix

···
   generatedHmacKeyFile = "/var/lib/invidious/hmac_key";
   generateHmac = cfg.hmacKeyFile == null;
 
-  serviceConfig = {
-    systemd.services.invidious = {
-      description = "Invidious (An alternative YouTube front-end)";
-      wants = [ "network-online.target" ];
-      after = [ "network-online.target" ];
-      wantedBy = [ "multi-user.target" ];
+  commonInvidousServiceConfig = {
+    description = "Invidious (An alternative YouTube front-end)";
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ] ++ lib.optional cfg.database.createLocally "postgresql.service";
+    requires = lib.optional cfg.database.createLocally "postgresql.service";
+    wantedBy = [ "multi-user.target" ];
 
-      preStart = lib.optionalString generateHmac ''
-        if [[ ! -e "${generatedHmacKeyFile}" ]]; then
-          ${pkgs.pwgen}/bin/pwgen 20 1 > "${generatedHmacKeyFile}"
-          chmod 0600 "${generatedHmacKeyFile}"
-        fi
-      '';
+    serviceConfig = {
+      RestartSec = "2s";
+      DynamicUser = true;
+      User = lib.mkIf (cfg.database.createLocally || cfg.serviceScale > 1) "invidious";
+      StateDirectory = "invidious";
+      StateDirectoryMode = "0750";
 
-      script = ''
-        configParts=()
-      ''
-      # autogenerated hmac_key
-      + lib.optionalString generateHmac ''
-        configParts+=("$(${pkgs.jq}/bin/jq -R '{"hmac_key":.}' <"${generatedHmacKeyFile}")")
-      ''
-      # generated settings file
-      + ''
-        configParts+=("$(< ${lib.escapeShellArg settingsFile})")
-      ''
-      # optional database password file
-      + lib.optionalString (cfg.database.host != null) ''
-        configParts+=("$(${pkgs.jq}/bin/jq -R '{"db":{"password":.}}' ${lib.escapeShellArg cfg.database.passwordFile})")
-      ''
-      # optional extra settings file
-      + lib.optionalString (cfg.extraSettingsFile != null) ''
-        configParts+=("$(< ${lib.escapeShellArg cfg.extraSettingsFile})")
-      ''
-      # explicitly specified hmac key file
-      + lib.optionalString (cfg.hmacKeyFile != null) ''
-        configParts+=("$(< ${lib.escapeShellArg cfg.hmacKeyFile})")
-      ''
-      # merge all parts into a single configuration with later elements overriding previous elements
-      + ''
-        export INVIDIOUS_CONFIG="$(${pkgs.jq}/bin/jq -s 'reduce .[] as $item ({}; . * $item)' <<<"''${configParts[*]}")"
-        exec ${cfg.package}/bin/invidious
-      '';
+      CapabilityBoundingSet = "";
+      PrivateDevices = true;
+      PrivateUsers = true;
+      ProtectHome = true;
+      ProtectKernelLogs = true;
+      ProtectProc = "invisible";
+      RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
+      RestrictNamespaces = true;
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
 
-      serviceConfig = {
-        RestartSec = "2s";
-        DynamicUser = true;
-        StateDirectory = "invidious";
-        StateDirectoryMode = "0750";
-
-        CapabilityBoundingSet = "";
-        PrivateDevices = true;
-        PrivateUsers = true;
-        ProtectHome = true;
-        ProtectKernelLogs = true;
-        ProtectProc = "invisible";
-        RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
-        RestrictNamespaces = true;
-        SystemCallArchitectures = "native";
-        SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
+      # Because of various issues Invidious must be restarted often, at least once a day, ideally
+      # every hour.
+      # This option enables the automatic restarting of the Invidious instance.
+      # To ensure multiple instances of Invidious are not restarted at the exact same time, a
+      # randomized extra offset of up to 5 minutes is added.
+      Restart = lib.mkDefault "always";
+      RuntimeMaxSec = lib.mkDefault "1h";
+      RuntimeRandomizedExtraSec = lib.mkDefault "5min";
+    };
+  };
+  mkInvidiousService = scaleIndex:
+    lib.foldl' lib.recursiveUpdate commonInvidousServiceConfig [
+      # only generate the hmac file in the first service
+      (lib.optionalAttrs (scaleIndex == 0) {
+        preStart = lib.optionalString generateHmac ''
+          if [[ ! -e "${generatedHmacKeyFile}" ]]; then
+            ${pkgs.pwgen}/bin/pwgen 20 1 > "${generatedHmacKeyFile}"
+            chmod 0600 "${generatedHmacKeyFile}"
+          fi
+        '';
+      })
+      # configure the secondary services to run after the first service
+      (lib.optionalAttrs (scaleIndex > 0) {
+        after = commonInvidousServiceConfig.after ++ [ "invidious.service" ];
+        wants = commonInvidousServiceConfig.wants ++ [ "invidious.service" ];
+      })
+      {
+        script = ''
+          configParts=()
+        ''
+        # autogenerated hmac_key
+        + lib.optionalString generateHmac ''
+          configParts+=("$(${pkgs.jq}/bin/jq -R '{"hmac_key":.}' <"${generatedHmacKeyFile}")")
+        ''
+        # generated settings file
+        + ''
+          configParts+=("$(< ${lib.escapeShellArg settingsFile})")
+        ''
+        # optional database password file
+        + lib.optionalString (cfg.database.host != null) ''
+          configParts+=("$(${pkgs.jq}/bin/jq -R '{"db":{"password":.}}' ${lib.escapeShellArg cfg.database.passwordFile})")
+        ''
+        # optional extra settings file
+        + lib.optionalString (cfg.extraSettingsFile != null) ''
+          configParts+=("$(< ${lib.escapeShellArg cfg.extraSettingsFile})")
+        ''
+        # explicitly specified hmac key file
+        + lib.optionalString (cfg.hmacKeyFile != null) ''
+          configParts+=("$(< ${lib.escapeShellArg cfg.hmacKeyFile})")
+        ''
+        # configure threads for secondary instances
+        + lib.optionalString (scaleIndex > 0) ''
+          configParts+=('{"channel_threads":0, "feed_threads":0}')
+        ''
+        # configure different ports for the instances
+        + ''
+          configParts+=('{"port":${toString (cfg.port + scaleIndex)}}')
+        ''
+        # merge all parts into a single configuration with later elements overriding previous elements
+        + ''
+          export INVIDIOUS_CONFIG="$(${pkgs.jq}/bin/jq -s 'reduce .[] as $item ({}; . * $item)' <<<"''${configParts[*]}")"
+          exec ${cfg.package}/bin/invidious
+        '';
+      }
+    ];
 
-        # Because of various issues Invidious must be restarted often, at least once a day, ideally
-        # every hour.
-        # This option enables the automatic restarting of the Invidious instance.
-        Restart = lib.mkDefault "always";
-        RuntimeMaxSec = lib.mkDefault "1h";
-      };
-    };
+  serviceConfig = {
+    systemd.services = builtins.listToAttrs (builtins.genList
+      (scaleIndex: {
+        name = "invidious" + lib.optionalString (scaleIndex > 0) "-${builtins.toString scaleIndex}";
+        value = mkInvidiousService scaleIndex;
+      })
+      cfg.serviceScale);
 
     services.invidious.settings = {
-      inherit (cfg) port;
-
       # Automatically initialises and migrates the database if necessary
       check_tables = true;
 
···
       inherit (cfg) domain;
     });
 
-    assertions = [{
-      assertion = cfg.database.host != null -> cfg.database.passwordFile != null;
-      message = "If database host isn't null, database password needs to be set";
-    }];
+    assertions = [
+      {
+        assertion = cfg.database.host != null -> cfg.database.passwordFile != null;
+        message = "If database host isn't null, database password needs to be set";
+      }
+      {
+        assertion = cfg.serviceScale >= 1;
+        message = "Service can't be scaled below one instance";
+      }
+    ];
   };
 
   # Settings necessary for running with an automatically managed local database
···
         local ${cfg.settings.db.dbname} ${cfg.settings.db.user} peer map=invidious
       '';
     };
-
-    systemd.services.invidious = {
-      requires = [ "postgresql.service" ];
-      after = [ "postgresql.service" ];
-
-      serviceConfig = {
-        User = "invidious";
-      };
-    };
   };
 
   nginxConfig = lib.mkIf cfg.nginx.enable {
···
     services.nginx = {
       enable = true;
       virtualHosts.${cfg.domain} = {
-        locations."/".proxyPass = "http://127.0.0.1:${toString cfg.port}";
+        locations."/".proxyPass =
+          if cfg.serviceScale == 1 then
+            "http://127.0.0.1:${toString cfg.port}"
+          else "http://upstream-invidious";
 
         enableACME = lib.mkDefault true;
         forceSSL = lib.mkDefault true;
+      };
+      upstreams = lib.mkIf (cfg.serviceScale > 1) {
+        "upstream-invidious".servers = builtins.listToAttrs (builtins.genList
+          (scaleIndex: {
+            name = "127.0.0.1:${toString (cfg.port + scaleIndex)}";
+            value = { };
+          })
+          cfg.serviceScale);
       };
     };
 
···
 
         It gets merged with the settings specified in {option}`services.invidious.settings`
         and can be used to store secrets like `hmac_key` outside of the nix store.
+      '';
+    };
+
+    serviceScale = lib.mkOption {
+      type = types.int;
+      default = 1;
+      description = lib.mdDoc ''
+        How many invidious instances to run.
+
+        See https://docs.invidious.io/improve-public-instance/#2-multiple-invidious-processes for more details
+        on how this is intended to work. All instances beyond the first one have the options `channel_threads`
+        and `feed_threads` set to 0 to avoid conflicts with multiple instances refreshing subscriptions. Instances
+        will be configured to bind to consecutive ports starting with {option}`services.invidious.port` for the
+        first instance.
       '';
     };