nixos/mattermost: modernize, support MySQL and mmctl

Based on #198040. Prioritizes backwards compatibility, including
database and plugin compatibility, while adding more sensible
defaults like database peer authentication.

Expand the scope of tests to include plugins (including building
from source) and testing that a piece of media uploads and downloads
to make sure the storage directory doesn't vanish.

nixos/modules/services/web-apps/mattermost.nix

···
   ...
 }:
 
-with lib;
+let
+  inherit (lib.strings)
+    hasInfix
+    hasSuffix
+    escapeURL
+    concatStringsSep
+    escapeShellArg
+    escapeShellArgs
+    versionAtLeast
+    optionalString
+    ;
+
+  inherit (lib.meta) getExe;
+
+  inherit (lib.lists) singleton;
+
+  inherit (lib.attrsets) mapAttrsToList recursiveUpdate optionalAttrs;
+
+  inherit (lib.options) mkOption mkPackageOption mkEnableOption;
 
-let
+  inherit (lib.modules)
+    mkRenamedOptionModule
+    mkMerge
+    mkIf
+    mkDefault
+    ;
+
+  inherit (lib.trivial) warnIf throwIf;
+
+  inherit (lib) types;
 
   cfg = config.services.mattermost;
 
-  database = "postgres://${cfg.localDatabaseUser}:${cfg.localDatabasePassword}@localhost:5432/${cfg.localDatabaseName}?sslmode=disable&connect_timeout=10";
+  # The directory to store mutable data within dataDir.
+  mutableDataDir = "${cfg.dataDir}/data";
 
-  postgresPackage = config.services.postgresql.package;
+  # The plugin directory. Note that this is the *post-unpack* plugin directory,
+  # since Mattermost unpacks plugins to put them there. (Hence, mutable data.)
+  pluginDir = "${mutableDataDir}/plugins";
+
+  # Mattermost uses this as a staging directory to unpack plugins, among possibly other things.
+  # Ensure that it's inside mutableDataDir since it can get rather large.
+  tempDir = "${mutableDataDir}/tmp";
 
-  createDb =
+  # Creates a database URI.
+  mkDatabaseUri =
     {
-      statePath ? cfg.statePath,
-      localDatabaseUser ? cfg.localDatabaseUser,
-      localDatabasePassword ? cfg.localDatabasePassword,
-      localDatabaseName ? cfg.localDatabaseName,
-      useSudo ? true,
+      scheme,
+      user ? null,
+      password ? null,
+      escapeUserAndPassword ? true,
+      host ? null,
+      escapeHost ? true,
+      port ? null,
+      path ? null,
+      query ? { },
     }:
-    ''
-      if ! test -e ${escapeShellArg "${statePath}/.db-created"}; then
-        ${lib.optionalString useSudo "${pkgs.sudo}/bin/sudo -u ${escapeShellArg config.services.postgresql.superUser} \\"}
-          ${postgresPackage}/bin/psql postgres -c \
-            "CREATE ROLE ${localDatabaseUser} WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD '${localDatabasePassword}'"
-        ${lib.optionalString useSudo "${pkgs.sudo}/bin/sudo -u ${escapeShellArg config.services.postgresql.superUser} \\"}
-          ${postgresPackage}/bin/createdb \
-            --owner ${escapeShellArg localDatabaseUser} ${escapeShellArg localDatabaseName}
-        touch ${escapeShellArg "${statePath}/.db-created"}
-      fi
-    '';
+    let
+      nullToEmpty = val: if val == null then "" else toString val;
+
+      # Converts a list of URI attrs to a query string.
+      toQuery = mapAttrsToList (
+        name: value: if value == null then null else (escapeURL name) + "=" + (escapeURL (toString value))
+      );
+
+      schemePart = if scheme == null then "" else "${escapeURL scheme}://";
+      userPart =
+        let
+          realUser = if escapeUserAndPassword then escapeURL user else user;
+          realPassword = if escapeUserAndPassword then escapeURL password else password;
+        in
+        if user == null && password == null then
+          ""
+        else if user != null && password != null then
+          "${realUser}:${realPassword}"
+        else if user != null then
+          realUser
+        else
+          throw "Either user or username and password must be provided";
+      hostPart =
+        let
+          realHost = if escapeHost then escapeURL (nullToEmpty host) else nullToEmpty host;
+        in
+        if userPart == "" then realHost else "@" + realHost;
+      portPart = if port == null then "" else ":" + (toString port);
+      pathPart = if path == null then "" else "/" + path;
+      queryPart = if query == { } then "" else "?" + concatStringsSep "&" (toQuery query);
+    in
+    schemePart + userPart + hostPart + portPart + pathPart + queryPart;
+
+  database =
+    let
+      hostIsPath = hasInfix "/" cfg.database.host;
+    in
+    if cfg.database.driver == "postgres" then
+      if cfg.database.peerAuth then
+        mkDatabaseUri {
+          scheme = cfg.database.driver;
+          inherit (cfg.database) user;
+          path = escapeURL cfg.database.name;
+          query = {
+            host = cfg.database.socketPath;
+          } // cfg.database.extraConnectionOptions;
+        }
+      else
+        mkDatabaseUri {
+          scheme = cfg.database.driver;
+          inherit (cfg.database) user password;
+          host = if hostIsPath then null else cfg.database.host;
+          port = if hostIsPath then null else cfg.database.port;
+          path = escapeURL cfg.database.name;
+          query =
+            optionalAttrs hostIsPath { host = cfg.database.host; } // cfg.database.extraConnectionOptions;
+        }
+    else if cfg.database.driver == "mysql" then
+      if cfg.database.peerAuth then
+        mkDatabaseUri {
+          scheme = null;
+          inherit (cfg.database) user;
+          escapeUserAndPassword = false;
+          host = "unix(${cfg.database.socketPath})";
+          escapeHost = false;
+          path = escapeURL cfg.database.name;
+          query = cfg.database.extraConnectionOptions;
+        }
+      else
+        mkDatabaseUri {
+          scheme = null;
+          inherit (cfg.database) user password;
+          escapeUserAndPassword = false;
+          host =
+            if hostIsPath then
+              "unix(${cfg.database.host})"
+            else
+              "tcp(${cfg.database.host}:${toString cfg.database.port})";
+          escapeHost = false;
+          path = escapeURL cfg.database.name;
+          query = cfg.database.extraConnectionOptions;
+        }
+    else
+      throw "Invalid database driver: ${cfg.database.driver}";
 
   mattermostPluginDerivations =
     with pkgs;
···
     else
       stdenv.mkDerivation {
         name = "${cfg.package.name}-plugins";
-        nativeBuildInputs = [
-          autoPatchelfHook
-        ] ++ mattermostPluginDerivations;
-        buildInputs = [
-          cfg.package
-        ];
+        nativeBuildInputs = [ autoPatchelfHook ] ++ mattermostPluginDerivations;
+        buildInputs = [ cfg.package ];
         installPhase = ''
-          mkdir -p $out/data/plugins
+          mkdir -p $out
           plugins=(${
             escapeShellArgs (map (plugin: "${plugin}/share/plugin.tar.gz") mattermostPluginDerivations)
           })
           for plugin in "''${plugins[@]}"; do
-            hash="$(sha256sum "$plugin" | cut -d' ' -f1)"
+            hash="$(sha256sum "$plugin" | awk '{print $1}')"
             mkdir -p "$hash"
             tar -C "$hash" -xzf "$plugin"
             autoPatchelf "$hash"
-            GZIP_OPT=-9 tar -C "$hash" -cvzf "$out/data/plugins/$hash.tar.gz" .
+            GZIP_OPT=-9 tar -C "$hash" -cvzf "$out/$hash.tar.gz" .
             rm -rf "$hash"
           done
         '';
···
       };
 
   mattermostConfWithoutPlugins = recursiveUpdate {
-    ServiceSettings.SiteURL = cfg.siteUrl;
-    ServiceSettings.ListenAddress = cfg.listenAddress;
+    ServiceSettings = {
+      SiteURL = cfg.siteUrl;
+      ListenAddress = "${cfg.host}:${toString cfg.port}";
+      LocalModeSocketLocation = cfg.socket.path;
+      EnableLocalMode = cfg.socket.enable;
+    };
     TeamSettings.SiteName = cfg.siteName;
-    SqlSettings.DriverName = "postgres";
-    SqlSettings.DataSource = database;
-    PluginSettings.Directory = "${cfg.statePath}/plugins/server";
-    PluginSettings.ClientDirectory = "${cfg.statePath}/plugins/client";
-  } cfg.extraConfig;
+    SqlSettings.DriverName = cfg.database.driver;
+    SqlSettings.DataSource =
+      if cfg.database.fromEnvironment then
+        null
+      else
+        warnIf (!cfg.database.peerAuth && cfg.database.password != null) ''
+          Database password is set in Mattermost config! This password will end up in the Nix store.
+
+          You may be able to simply set the following, if the database is on the same host
+          and peer authentication is enabled:
+
+          services.mattermost.database.peerAuth = true;
+
+          Note that this is the default if you set system.stateVersion to 25.05 or later
+          and the database host is localhost.
+
+          Alternatively, you can write the following to ${
+            if cfg.environmentFile == null then "your environment file" else cfg.environmentFile
+          }:
+
+          MM_SQLSETTINGS_DATASOURCE=${database}
+
+          Then set the following options:
+          services.mattermost.environmentFile = "<your environment file>";
+          services.mattermost.database.fromEnvironment = true;
+        '' database;
+    FileSettings.Directory = cfg.dataDir;
+    PluginSettings.Directory = "${pluginDir}/server";
+    PluginSettings.ClientDirectory = "${pluginDir}/client";
+    LogSettings.FileLocation = cfg.logDir;
+  } cfg.settings;
 
   mattermostConf = recursiveUpdate mattermostConfWithoutPlugins (
-    lib.optionalAttrs (mattermostPlugins != null) {
-      PluginSettings = {
-        Enable = true;
-      };
-    }
+    if mattermostPlugins == null then
+      { }
+    else
+      {
+        PluginSettings = {
+          Enable = true;
+        };
+      }
   );
 
   mattermostConfJSON = pkgs.writeText "mattermost-config.json" (builtins.toJSON mattermostConf);
 
 in
-
 {
+  imports = [
+    (mkRenamedOptionModule
+      [
+        "services"
+        "mattermost"
+        "listenAddress"
+      ]
+      [
+        "services"
+        "mattermost"
+        "host"
+      ]
+    )
+    (mkRenamedOptionModule
+      [
+        "services"
+        "mattermost"
+        "localDatabaseCreate"
+      ]
+      [
+        "services"
+        "mattermost"
+        "database"
+        "create"
+      ]
+    )
+    (mkRenamedOptionModule
+      [
+        "services"
+        "mattermost"
+        "localDatabasePassword"
+      ]
+      [
+        "services"
+        "mattermost"
+        "database"
+        "password"
+      ]
+    )
+    (mkRenamedOptionModule
+      [
+        "services"
+        "mattermost"
+        "localDatabaseUser"
+      ]
+      [
+        "services"
+        "mattermost"
+        "database"
+        "user"
+      ]
+    )
+    (mkRenamedOptionModule
+      [
+        "services"
+        "mattermost"
+        "localDatabaseName"
+      ]
+      [
+        "services"
+        "mattermost"
+        "database"
+        "name"
+      ]
+    )
+    (mkRenamedOptionModule
+      [
+        "services"
+        "mattermost"
+        "extraConfig"
+      ]
+      [
+        "services"
+        "mattermost"
+        "settings"
+      ]
+    )
+    (mkRenamedOptionModule
+      [
+        "services"
+        "mattermost"
+        "statePath"
+      ]
+      [
+        "services"
+        "mattermost"
+        "dataDir"
+      ]
+    )
+  ];
+
   options = {
     services.mattermost = {
       enable = mkEnableOption "Mattermost chat server";
 
       package = mkPackageOption pkgs "mattermost" { };
-
-      statePath = mkOption {
-        type = types.str;
-        default = "/var/lib/mattermost";
-        description = "Mattermost working directory";
-      };
 
       siteUrl = mkOption {
         type = types.str;
···
         description = "Name of this Mattermost site.";
       };
 
-      listenAddress = mkOption {
+      host = mkOption {
         type = types.str;
-        default = ":8065";
-        example = "[::1]:8065";
+        default = "127.0.0.1";
+        example = "0.0.0.0";
+        description = ''
+          Host or address that this Mattermost instance listens on.
+        '';
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 8065;
+        description = ''
+          Port for Mattermost server to listen on.
+        '';
+      };
+
+      dataDir = mkOption {
+        type = types.path;
+        default = "/var/lib/mattermost";
+        description = ''
+          Mattermost working directory.
+        '';
+      };
+
+      socket = {
+        enable = mkEnableOption "Mattermost control socket";
+
+        path = mkOption {
+          type = types.path;
+          default = "${cfg.dataDir}/mattermost.sock";
+          defaultText = ''''${config.mattermost.dataDir}/mattermost.sock'';
+          description = ''
+            Default location for the Mattermost control socket used by `mmctl`.
+          '';
+        };
+
+        export = mkEnableOption "Export socket control to system environment variables";
+      };
+
+      logDir = mkOption {
+        type = types.path;
+        default =
+          if versionAtLeast config.system.stateVersion "25.05" then
+            "/var/log/mattermost"
+          else
+            "${cfg.dataDir}/logs";
+        defaultText = ''
+          if versionAtLeast config.system.stateVersion "25.05" then "/var/log/mattermost"
+          else "''${config.services.mattermost.dataDir}/logs";
+        '';
+        description = ''
+          Mattermost log directory.
+        '';
+      };
+
+      configDir = mkOption {
+        type = types.path;
+        default =
+          if versionAtLeast config.system.stateVersion "25.05" then
+            "/etc/mattermost"
+          else
+            "${cfg.dataDir}/config";
+        defaultText = ''
+          if versionAtLeast config.system.stateVersion "25.05" then
+            "/etc/mattermost"
+          else
+            "''${config.services.mattermost.dataDir}/config";
+        '';
         description = ''
-          Address and port this Mattermost instance listens to.
+          Mattermost config directory.
         '';
       };
 
···
         '';
       };
 
-      extraConfig = mkOption {
-        type = types.attrs;
-        default = { };
-        description = ''
-          Additional configuration options as Nix attribute set in config.json schema.
-        '';
-      };
-
       plugins = mkOption {
-        type = types.listOf (
-          types.oneOf [
-            types.path
-            types.package
-          ]
-        );
+        type = with types; listOf (either path package);
         default = [ ];
         example = "[ ./com.github.moussetc.mattermost.plugin.giphy-2.0.0.tar.gz ]";
         description = ''
···
           .tar.gz files.
         '';
       };
+
+      environment = mkOption {
+        type = with types; attrsOf (either int str);
+        default = { };
+        description = ''
+          Extra environment variables to export to the Mattermost process, in the systemd unit.
+        '';
+        example = {
+          MM_SERVICESETTINGS_SITEURL = "http://example.com";
+        };
+      };
+
       environmentFile = mkOption {
-        type = types.nullOr types.path;
+        type = with types; nullOr path;
         default = null;
         description = ''
           Environment file (see {manpage}`systemd.exec(5)`
           "EnvironmentFile=" section for the syntax) which sets config options
-          for mattermost (see [the mattermost documentation](https://docs.mattermost.com/configure/configuration-settings.html#environment-variables)).
+          for mattermost (see [the Mattermost documentation](https://docs.mattermost.com/configure/configuration-settings.html#environment-variables)).
 
           Settings defined in the environment file will overwrite settings
           set via nix or via the {option}`services.mattermost.extraConfig`
···
         '';
       };
 
-      localDatabaseCreate = mkOption {
-        type = types.bool;
-        default = true;
-        description = ''
-          Create a local PostgreSQL database for Mattermost automatically.
-        '';
-      };
+      database = {
+        driver = mkOption {
+          type = types.enum [
+            "postgres"
+            "mysql"
+          ];
+          default = "postgres";
+          description = ''
+            The database driver to use (Postgres or MySQL).
+          '';
+        };
 
-      localDatabaseName = mkOption {
-        type = types.str;
-        default = "mattermost";
-        description = ''
-          Local Mattermost database name.
-        '';
-      };
+        create = mkOption {
+          type = types.bool;
+          default = true;
+          description = ''
+            Create a local PostgreSQL or MySQL database for Mattermost automatically.
+          '';
+        };
 
-      localDatabaseUser = mkOption {
-        type = types.str;
-        default = "mattermost";
-        description = ''
-          Local Mattermost database username.
-        '';
-      };
+        peerAuth = mkOption {
+          type = types.bool;
+          default = versionAtLeast config.system.stateVersion "25.05" && cfg.database.host == "localhost";
+          defaultText = ''
+            versionAtLeast config.system.stateVersion "25.05" && config.services.mattermost.database.host == "localhost"
+          '';
+          description = ''
+            If set, will use peer auth instead of connecting to a Postgres server.
+            Use services.mattermost.database.socketPath to configure the socket path.
+          '';
+        };
+
+        socketPath = mkOption {
+          type = types.path;
+          default =
+            if cfg.database.driver == "postgres" then "/run/postgresql" else "/run/mysqld/mysqld.sock";
+          defaultText = ''
+            if config.services.mattermost.database.driver == "postgres" then "/run/postgresql" else "/run/mysqld/mysqld.sock";
+          '';
+          description = ''
+            The database (Postgres or MySQL) socket path.
+          '';
+        };
+
+        fromEnvironment = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Use services.mattermost.environmentFile to configure the database instead of writing the database URI
+            to the Nix store. Useful if you use password authentication with peerAuth set to false.
+          '';
+        };
+
+        name = mkOption {
+          type = types.str;
+          default = "mattermost";
+          description = ''
+            Local Mattermost database name.
+          '';
+        };
+
+        host = mkOption {
+          type = types.str;
+          default = "localhost";
+          example = "127.0.0.1";
+          description = ''
+            Host to use for the database. Can also be set to a path if you'd like to connect
+            to a socket using a username and password.
+          '';
+        };
+
+        port = mkOption {
+          type = types.port;
+          default = if cfg.database.driver == "postgres" then 5432 else 3306;
+          defaultText = ''
+            if config.services.mattermost.database.type == "postgres" then 5432 else 3306
+          '';
+          example = 3306;
+          description = ''
+            Port to use for the database.
+          '';
+        };
+
+        user = mkOption {
+          type = types.str;
+          default = "mattermost";
+          description = ''
+            Local Mattermost database username.
+          '';
+        };
+
+        password = mkOption {
+          type = types.str;
+          default = "mmpgsecret";
+          description = ''
+            Password for local Mattermost database user. If set and peerAuth is not true,
+            will cause a warning nagging you to use environmentFile instead since it will
+            end up in the Nix store.
+          '';
+        };
 
-      localDatabasePassword = mkOption {
-        type = types.str;
-        default = "mmpgsecret";
-        description = ''
-          Password for local Mattermost database user.
-        '';
+        extraConnectionOptions = mkOption {
+          type = with types; attrsOf (either int str);
+          default =
+            if cfg.database.driver == "postgres" then
+              {
+                sslmode = "disable";
+                connect_timeout = 30;
+              }
+            else if cfg.database.driver == "mysql" then
+              {
+                charset = "utf8mb4,utf8";
+                writeTimeout = "30s";
+                readTimeout = "30s";
+              }
+            else
+              throw "Invalid database driver ${cfg.database.driver}";
+          defaultText = ''
+            if config.mattermost.database.driver == "postgres" then
+              {
+                sslmode = "disable";
+                connect_timeout = 30;
+              }
+            else if config.mattermost.database.driver == "mysql" then
+              {
+                charset = "utf8mb4,utf8";
+                writeTimeout = "30s";
+                readTimeout = "30s";
+              }
+            else
+              throw "Invalid database driver";
+          '';
+          description = ''
+            Extra options that are placed in the connection URI's query parameters.
+          '';
+        };
       };
 
       user = mkOption {
···
         '';
       };
 
+      settings = mkOption {
+        type = types.attrs;
+        default = { };
+        description = ''
+          Additional configuration options as Nix attribute set in config.json schema.
+        '';
+      };
+
       matterircd = {
         enable = mkEnableOption "Mattermost IRC bridge";
         package = mkPackageOption pkgs "matterircd" { };
···
 
   config = mkMerge [
     (mkIf cfg.enable {
-      users.users = optionalAttrs (cfg.user == "mattermost") {
-        mattermost = {
+      users.users = {
+        ${cfg.user} = {
           group = cfg.group;
-          uid = config.ids.uids.mattermost;
-          home = cfg.statePath;
+          uid = mkIf (cfg.user == "mattermost") config.ids.uids.mattermost;
+          home = cfg.dataDir;
+          isSystemUser = true;
+          packages = [ cfg.package ];
         };
       };
 
-      users.groups = optionalAttrs (cfg.group == "mattermost") {
-        mattermost.gid = config.ids.gids.mattermost;
+      users.groups = {
+        ${cfg.group} = {
+          gid = mkIf (cfg.group == "mattermost") config.ids.gids.mattermost;
+        };
       };
 
-      services.postgresql.enable = cfg.localDatabaseCreate;
+      services.postgresql = mkIf (cfg.database.driver == "postgres" && cfg.database.create) {
+        enable = true;
+        ensureDatabases = singleton cfg.database.name;
+        ensureUsers = singleton {
+          name =
+            throwIf
+              (cfg.database.peerAuth && (cfg.database.user != cfg.user || cfg.database.name != cfg.database.user))
+              ''
+                Mattermost database peer auth is enabled and the user, database user, or database name mismatch.
+                Peer authentication will not work.
+              ''
+              cfg.database.user;
+          ensureDBOwnership = true;
+        };
+      };
 
-      # The systemd service will fail to execute the preStart hook
-      # if the WorkingDirectory does not exist
-      systemd.tmpfiles.settings."10-mattermost".${cfg.statePath}.d = { };
+      services.mysql = mkIf (cfg.database.driver == "mysql" && cfg.database.create) {
+        enable = true;
+        package = mkDefault pkgs.mariadb;
+        ensureDatabases = singleton cfg.database.name;
+        ensureUsers = singleton {
+          name = cfg.database.user;
+          ensurePermissions = {
+            "${cfg.database.name}.*" = "ALL PRIVILEGES";
+          };
+        };
+        settings = rec {
+          mysqld = {
+            collation-server = mkDefault "utf8mb4_general_ci";
+            init-connect = mkDefault "SET NAMES utf8mb4";
+            character-set-server = mkDefault "utf8mb4";
+          };
+          mysqld_safe = mysqld;
+        };
+      };
 
-      systemd.services.mattermost = {
+      environment = {
+        variables = mkIf cfg.socket.export {
+          MMCTL_LOCAL = "true";
+          MMCTL_LOCAL_SOCKET_PATH = cfg.socket.path;
+        };
+      };
+
+      systemd.tmpfiles.rules =
+        [
+          "d ${cfg.dataDir} 0750 ${cfg.user} ${cfg.group} - -"
+          "d ${cfg.logDir} 0750 ${cfg.user} ${cfg.group} - -"
+          "d ${cfg.configDir} 0750 ${cfg.user} ${cfg.group} - -"
+          "d ${mutableDataDir} 0750 ${cfg.user} ${cfg.group} - -"
+
+          # Make sure tempDir exists and is not a symlink.
+          "R- ${tempDir} - - - - -"
+          "d= ${tempDir} 0750 ${cfg.user} ${cfg.group} - -"
+
+          # Ensure that pluginDir is a directory, as it could be a symlink on prior versions.
+          "r- ${pluginDir} - - - - -"
+          "d= ${pluginDir} 0750 ${cfg.user} ${cfg.group} - -"
+
+          # Ensure that the plugin directories exist.
+          "d= ${mattermostConf.PluginSettings.Directory} 0750 ${cfg.user} ${cfg.group} - -"
+          "d= ${mattermostConf.PluginSettings.ClientDirectory} 0750 ${cfg.user} ${cfg.group} - -"
+
+          # Link in some of the immutable data directories.
+          "L+ ${cfg.dataDir}/bin - - - - ${cfg.package}/bin"
+          "L+ ${cfg.dataDir}/fonts - - - - ${cfg.package}/fonts"
+          "L+ ${cfg.dataDir}/i18n - - - - ${cfg.package}/i18n"
+          "L+ ${cfg.dataDir}/templates - - - - ${cfg.package}/templates"
+          "L+ ${cfg.dataDir}/client - - - - ${cfg.package}/client"
+        ]
+        ++ (
+          if mattermostPlugins == null then
+            # Create the plugin tarball directory if it's a symlink.
+            [
+              "r- ${cfg.dataDir}/plugins - - - - -"
+              "d= ${cfg.dataDir}/plugins 0750 ${cfg.user} ${cfg.group} - -"
+            ]
+          else
+            # Symlink the plugin tarball directory, removing anything existing.
+            [ "L+ ${cfg.dataDir}/plugins - - - - ${mattermostPlugins}" ]
+        );
+
+      systemd.services.mattermost = rec {
         description = "Mattermost chat service";
         wantedBy = [ "multi-user.target" ];
-        after = [
-          "network.target"
-          "postgresql.service"
+        after = mkMerge [
+          [ "network.target" ]
+          (mkIf (cfg.database.driver == "postgres" && cfg.database.create) [ "postgresql.service" ])
+          (mkIf (cfg.database.driver == "mysql" && cfg.database.create) [ "mysql.service" ])
+        ];
+        requires = after;
+
+        environment = mkMerge [
+          {
+            # Use tempDir as this can get rather large, especially if Mattermost unpacks a large number of plugins.
+            TMPDIR = tempDir;
+          }
+          cfg.environment
         ];
 
         preStart =
           ''
-            mkdir -p "${cfg.statePath}"/{data,config,logs,plugins}
-            mkdir -p "${cfg.statePath}/plugins"/{client,server}
-            ln -sf ${cfg.package}/{bin,fonts,i18n,templates,client} "${cfg.statePath}"
+            dataDir=${escapeShellArg cfg.dataDir}
+            configDir=${escapeShellArg cfg.configDir}
+            logDir=${escapeShellArg cfg.logDir}
+            package=${escapeShellArg cfg.package}
+            nixConfig=${escapeShellArg mattermostConfJSON}
           ''
-          + lib.optionalString (mattermostPlugins != null) ''
-            rm -rf "${cfg.statePath}/data/plugins"
-            ln -sf ${mattermostPlugins}/data/plugins "${cfg.statePath}/data"
+          + optionalString (versionAtLeast config.system.stateVersion "25.05") ''
+            # Migrate configs in the pre-25.05 directory structure.
+            oldConfig="$dataDir/config/config.json"
+            newConfig="$configDir/config.json"
+            if [ "$oldConfig" != "$newConfig" ] && [ -f "$oldConfig" ] && [ ! -f "$newConfig" ]; then
+              # Migrate the legacy config location to the new config location
+              echo "Moving legacy config at $oldConfig to $newConfig" >&2
+              mkdir -p "$configDir"
+              mv "$oldConfig" "$newConfig"
+              touch "$configDir/.initial-created"
+            fi
+
+            # Logs too.
+            oldLogs="$dataDir/logs"
+            newLogs="$logDir"
+            if [ "$oldLogs" != "$newLogs" ] && [ -d "$oldLogs" ]; then
+              # Migrate the legacy log location to the new log location.
+              # Allow this to fail if there aren't any logs to move.
+              echo "Moving legacy logs at $oldLogs to $newLogs" >&2
+              mkdir -p "$newLogs"
+              mv "$oldLogs"/* "$newLogs" || true
+            fi
           ''
-          + lib.optionalString (!cfg.mutableConfig) ''
-            rm -f "${cfg.statePath}/config/config.json"
-            ${pkgs.jq}/bin/jq -s '.[0] * .[1]' ${cfg.package}/config/config.json ${mattermostConfJSON} > "${cfg.statePath}/config/config.json"
+          + optionalString (!cfg.mutableConfig) ''
+            ${getExe pkgs.jq} -s '.[0] * .[1]' "$package/config/config.json" "$nixConfig" > "$configDir/config.json"
           ''
-          + lib.optionalString cfg.mutableConfig ''
-            if ! test -e "${cfg.statePath}/config/.initial-created"; then
-              rm -f ${cfg.statePath}/config/config.json
-              ${pkgs.jq}/bin/jq -s '.[0] * .[1]' ${cfg.package}/config/config.json ${mattermostConfJSON} > "${cfg.statePath}/config/config.json"
-              touch "${cfg.statePath}/config/.initial-created"
+          + optionalString cfg.mutableConfig ''
+            if [ ! -e "$configDir/.initial-created" ]; then
+              ${getExe pkgs.jq} -s '.[0] * .[1]' "$package/config/config.json" "$nixConfig" > "$configDir/config.json"
+              touch "$configDir/.initial-created"
             fi
           ''
-          + lib.optionalString (cfg.mutableConfig && cfg.preferNixConfig) ''
-            new_config="$(${pkgs.jq}/bin/jq -s '.[0] * .[1]' "${cfg.statePath}/config/config.json" ${mattermostConfJSON})"
+          + optionalString (cfg.mutableConfig && cfg.preferNixConfig) ''
+            echo "$(${getExe pkgs.jq} -s '.[0] * .[1]' "$configDir/config.json" "$nixConfig")" > "$configDir/config.json"
+          '';
+
+        serviceConfig = mkMerge [
+          {
+            User = cfg.user;
+            Group = cfg.group;
+            ExecStart = "${getExe cfg.package} --config ${cfg.configDir}/config.json";
+            ReadWritePaths = [
+              cfg.dataDir
+              cfg.logDir
+              cfg.configDir
+            ];
+            UMask = "0027";
+            Restart = "always";
+            RestartSec = 10;
+            LimitNOFILE = 49152;
+            LockPersonality = true;
+            NoNewPrivileges = true;
+            PrivateDevices = true;
+            PrivateTmp = true;
+            PrivateUsers = true;
+            ProtectClock = true;
+            ProtectControlGroups = true;
+            ProtectHome = true;
+            ProtectHostname = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            ProtectProc = "invisible";
+            ProtectSystem = "strict";
+            RestrictNamespaces = true;
+            RestrictSUIDSGID = true;
+            EnvironmentFile = cfg.environmentFile;
+            WorkingDirectory = cfg.dataDir;
+          }
+          (mkIf (cfg.dataDir == "/var/lib/mattermost") {
+            StateDirectory = baseNameOf cfg.dataDir;
+            StateDirectoryMode = "0750";
+          })
+          (mkIf (cfg.logDir == "/var/log/mattermost") {
+            LogsDirectory = baseNameOf cfg.logDir;
+            LogsDirectoryMode = "0750";
+          })
+          (mkIf (cfg.configDir == "/etc/mattermost") {
+            ConfigurationDirectory = baseNameOf cfg.configDir;
+            ConfigurationDirectoryMode = "0750";
+          })
+        ];
 
-            rm -f "${cfg.statePath}/config/config.json"
-            echo "$new_config" > "${cfg.statePath}/config/config.json"
-          ''
-          + lib.optionalString cfg.localDatabaseCreate (createDb { })
-          + ''
-            # Don't change permissions recursively on the data, current, and symlinked directories (see ln -sf command above).
-            # This dramatically decreases startup times for installations with a lot of files.
-            find . -maxdepth 1 -not -name data -not -name client -not -name templates -not -name i18n -not -name fonts -not -name bin -not -name . \
-              -exec chown "${cfg.user}:${cfg.group}" -R {} \; -exec chmod u+rw,g+r,o-rwx -R {} \;
+        unitConfig.JoinsNamespaceOf = mkMerge [
+          (mkIf (cfg.database.driver == "postgres" && cfg.database.create) [ "postgresql.service" ])
+          (mkIf (cfg.database.driver == "mysql" && cfg.database.create) [ "mysql.service" ])
+        ];
+      };
 
-            chown "${cfg.user}:${cfg.group}" "${cfg.statePath}/data" .
-            chmod u+rw,g+r,o-rwx "${cfg.statePath}/data" .
+      assertions = [
+        {
+          # Make sure the URL doesn't have a trailing slash
+          assertion = !(hasSuffix "/" cfg.siteUrl);
+          message = ''
+            services.mattermost.siteUrl should not have a trailing "/".
+          '';
+        }
+        {
+          # Make sure this isn't a host/port pair
+          assertion = !(hasInfix ":" cfg.host && !(hasInfix "[" cfg.host) && !(hasInfix "]" cfg.host));
+          message = ''
+            services.mattermost.host should not include a port. Use services.mattermost.host for the address
+            or hostname, and services.mattermost.port to specify the port separately.
           '';
-
-        serviceConfig = {
-          PermissionsStartOnly = true;
-          User = cfg.user;
-          Group = cfg.group;
-          ExecStart = "${cfg.package}/bin/mattermost";
-          WorkingDirectory = "${cfg.statePath}";
-          Restart = "always";
-          RestartSec = "10";
-          LimitNOFILE = "49152";
-          EnvironmentFile = cfg.environmentFile;
-        };
-        unitConfig.JoinsNamespaceOf = mkIf cfg.localDatabaseCreate "postgresql.service";
-      };
+        }
+      ];
     })
     (mkIf cfg.matterircd.enable {
       systemd.services.matterircd = {
···
         serviceConfig = {
           User = "nobody";
           Group = "nogroup";
-          ExecStart = "${cfg.matterircd.package}/bin/matterircd ${escapeShellArgs cfg.matterircd.parameters}";
+          ExecStart = "${getExe cfg.matterircd.package} ${escapeShellArgs cfg.matterircd.parameters}";
           WorkingDirectory = "/tmp";
           PrivateTmp = true;
           Restart = "always";
···
       };
     })
   ];
+
+  meta.maintainers = with lib.maintainers; [ numinit ];
 }

nixos/tests/all-tests.nix

···
   matrix-synapse-workers = handleTest ./matrix/synapse-workers.nix {};
   mautrix-meta-postgres = handleTest ./matrix/mautrix-meta-postgres.nix {};
   mautrix-meta-sqlite = handleTest ./matrix/mautrix-meta-sqlite.nix {};
-  mattermost = handleTest ./mattermost.nix {};
+  mattermost = handleTest ./mattermost {};
   mealie = handleTest ./mealie.nix {};
   mediamtx = handleTest ./mediamtx.nix {};
   mediatomb = handleTest ./mediatomb.nix {};

nixos/tests/mattermost.nix

···
-import ./make-test-python.nix (
-  { pkgs, lib, ... }:
-  let
-    host = "smoke.test";
-    port = "8065";
-    url = "http://${host}:${port}";
-    siteName = "NixOS Smoke Tests, Inc.";
-
-    makeMattermost =
-      mattermostConfig:
-      { config, ... }:
-      {
-        environment.systemPackages = [
-          pkgs.mattermost
-          pkgs.curl
-          pkgs.jq
-        ];
-        networking.hosts = {
-          "127.0.0.1" = [ host ];
-        };
-        services.mattermost = lib.recursiveUpdate {
-          enable = true;
-          inherit siteName;
-          listenAddress = "0.0.0.0:${port}";
-          siteUrl = url;
-          extraConfig = {
-            SupportSettings.AboutLink = "https://nixos.org";
-          };
-        } mattermostConfig;
-      };
-  in
-  {
-    name = "mattermost";
-
-    nodes = {
-      mutable = makeMattermost {
-        mutableConfig = true;
-        extraConfig.SupportSettings.HelpLink = "https://search.nixos.org";
-      };
-      mostlyMutable = makeMattermost {
-        mutableConfig = true;
-        preferNixConfig = true;
-        plugins =
-          let
-            mattermostDemoPlugin = pkgs.fetchurl {
-              url = "https://github.com/mattermost/mattermost-plugin-demo/releases/download/v0.9.0/com.mattermost.demo-plugin-0.9.0.tar.gz";
-              sha256 = "1h4qi34gcxcx63z8wiqcf2aaywmvv8lys5g8gvsk13kkqhlmag25";
-            };
-          in
-          [
-            mattermostDemoPlugin
-          ];
-      };
-      immutable = makeMattermost {
-        package = pkgs.mattermost.overrideAttrs (prev: {
-          webapp = prev.webapp.overrideAttrs (prevWebapp: {
-            # Ensure that users can add patches.
-            postPatch =
-              prevWebapp.postPatch or ""
-              + ''
-                substituteInPlace channels/src/root.html --replace-fail "Mattermost" "Patched Mattermost"
-              '';
-          });
-        });
-        mutableConfig = false;
-        extraConfig.SupportSettings.HelpLink = "https://search.nixos.org";
-      };
-      environmentFile = makeMattermost {
-        mutableConfig = false;
-        extraConfig.SupportSettings.AboutLink = "https://example.org";
-        environmentFile = pkgs.writeText "mattermost-env" ''
-          MM_SUPPORTSETTINGS_ABOUTLINK=https://nixos.org
-        '';
-      };
-    };
-
-    testScript =
-      let
-        expectConfig =
-          jqExpression:
-          pkgs.writeShellScript "expect-config" ''
-            set -euo pipefail
-            echo "Expecting config to match: "${lib.escapeShellArg jqExpression} >&2
-            curl ${lib.escapeShellArg url} >/dev/null
-            config="$(curl ${lib.escapeShellArg "${url}/api/v4/config/client?format=old"})"
-            echo "Config: $(echo "$config" | ${pkgs.jq}/bin/jq)" >&2
-            [[ "$(echo "$config" | ${pkgs.jq}/bin/jq -r ${lib.escapeShellArg ".SiteName == $siteName and .Version == ($mattermostName / $sep)[-1] and (${jqExpression})"} --arg siteName ${lib.escapeShellArg siteName} --arg mattermostName ${lib.escapeShellArg pkgs.mattermost.name} --arg sep '-')" = "true" ]]
-          '';
-
-        setConfig =
-          jqExpression:
-          pkgs.writeShellScript "set-config" ''
-            set -euo pipefail
-            mattermostConfig=/var/lib/mattermost/config/config.json
-            newConfig="$(${pkgs.jq}/bin/jq -r ${lib.escapeShellArg jqExpression} $mattermostConfig)"
-            rm -f $mattermostConfig
-            echo "$newConfig" > "$mattermostConfig"
-          '';
-
-      in
-      ''
-        start_all()
-
-        ## Mutable node tests ##
-        mutable.wait_for_unit("mattermost.service")
-        mutable.wait_for_open_port(8065)
-        mutable.succeed("curl -L http://localhost:8065/index.html | grep '${siteName}'")
-        mutable.succeed("curl -L http://localhost:8065/index.html | grep 'Mattermost'")
-
-        # Get the initial config
-        mutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"''}")
-
-        # Edit the config
-        mutable.succeed("${setConfig ''.SupportSettings.AboutLink = "https://mattermost.com"''}")
-        mutable.succeed("${setConfig ''.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"''}")
-        mutable.systemctl("restart mattermost.service")
-        mutable.wait_for_open_port(8065)
-
-        # AboutLink and HelpLink should be changed
-        mutable.succeed("${expectConfig ''.AboutLink == "https://mattermost.com" and .HelpLink == "https://nixos.org/nixos/manual"''}")
-
-        ## Mostly mutable node tests ##
-        mostlyMutable.wait_for_unit("mattermost.service")
-        mostlyMutable.wait_for_open_port(8065)
-        mostlyMutable.succeed("curl -L http://localhost:8065/index.html | grep '${siteName}'")
-        mostlyMutable.succeed("curl -L http://localhost:8065/index.html | grep 'Mattermost'")
-
-        # Get the initial config
-        mostlyMutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org"''}")
-
-        # Edit the config
-        mostlyMutable.succeed("${setConfig ''.SupportSettings.AboutLink = "https://mattermost.com"''}")
-        mostlyMutable.succeed("${setConfig ''.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"''}")
-        mostlyMutable.systemctl("restart mattermost.service")
-        mostlyMutable.wait_for_open_port(8065)
-
-        # AboutLink should be overridden by NixOS configuration; HelpLink should be what we set above
-        mostlyMutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual"''}")
-
-        ## Immutable node tests ##
-        immutable.wait_for_unit("mattermost.service")
-        immutable.wait_for_open_port(8065)
-        # Since we patched it, it doesn't replace the site name at runtime anymore
-        immutable.succeed("curl -L http://localhost:8065/index.html | grep 'Patched Mattermost'")
-
-        # Get the initial config
-        immutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"''}")
-
-        # Edit the config
-        immutable.succeed("${setConfig ''.SupportSettings.AboutLink = "https://mattermost.com"''}")
-        immutable.succeed("${setConfig ''.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"''}")
-        immutable.systemctl("restart mattermost.service")
-        immutable.wait_for_open_port(8065)
-
-        # Our edits should be ignored on restart
-        immutable.succeed("${expectConfig ''.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"''}")
-
-
-        ## Environment File node tests ##
-        environmentFile.wait_for_unit("mattermost.service")
-        environmentFile.wait_for_open_port(8065)
-        environmentFile.succeed("curl -L http://localhost:8065/index.html | grep '${siteName}'")
-        environmentFile.succeed("curl -L http://localhost:8065/index.html | grep 'Mattermost'")
-
-        # Settings in the environment file should override settings set otherwise
-        environmentFile.succeed("${expectConfig ''.AboutLink == "https://nixos.org"''}")
-      '';
-  }
-)

nixos/tests/mattermost/default.nix

···
+import ../make-test-python.nix (
+  { pkgs, lib, ... }:
+  let
+    host = "smoke.test";
+    port = 8065;
+    url = "http://${host}:${toString port}";
+    siteName = "NixOS Smoke Tests, Inc.";
+
+    makeMattermost =
+      mattermostConfig: extraConfig:
+      lib.mkMerge [
+        (
+          { config, ... }:
+          {
+            environment = {
+              systemPackages = [
+                pkgs.mattermost
+                pkgs.curl
+                pkgs.jq
+              ];
+            };
+            networking.hosts = {
+              "127.0.0.1" = [ host ];
+            };
+
+            # Assume that Postgres won't update across stateVersion.
+            services.postgresql = {
+              package = lib.mkForce pkgs.postgresql;
+              initialScript = lib.mkIf (!config.services.mattermost.database.peerAuth) (
+                pkgs.writeText "init.sql" ''
+                  create role ${config.services.mattermost.database.user} with login nocreatedb nocreaterole encrypted password '${config.services.mattermost.database.password}';
+                ''
+              );
+            };
+
+            system.stateVersion = lib.mkDefault "25.05";
+
+            services.mattermost = lib.recursiveUpdate {
+              enable = true;
+              inherit siteName;
+              host = "0.0.0.0";
+              inherit port;
+              siteUrl = url;
+              socket = {
+                enable = true;
+                export = true;
+              };
+              database = {
+                peerAuth = lib.mkDefault true;
+              };
+              settings = {
+                SupportSettings.AboutLink = "https://nixos.org";
+                PluginSettings.AutomaticPrepackagedPlugins = false;
+                AnnouncementSettings = {
+                  # Disable this since it doesn't work in the sandbox and causes a timeout.
+                  AdminNoticesEnabled = false;
+                  UserNoticesEnabled = false;
+                };
+              };
+            } mattermostConfig;
+
+            # Upgrade to the latest Mattermost.
+            specialisation.latest.configuration = {
+              services.mattermost.package = lib.mkForce pkgs.mattermostLatest;
+              system.stateVersion = lib.mkVMOverride "25.05";
+            };
+          }
+        )
+        extraConfig
+      ];
+
+    makeMysql =
+      mattermostConfig: extraConfig:
+      lib.mkMerge [
+        mattermostConfig
+        (
+          { pkgs, config, ... }:
+          {
+            services.mattermost.database = {
+              driver = lib.mkForce "mysql";
+              peerAuth = lib.mkForce true;
+            };
+          }
+        )
+        extraConfig
+      ];
+  in
+  {
+    name = "mattermost";
+
+    nodes = rec {
+      postgresMutable =
+        makeMattermost
+          {
+            mutableConfig = true;
+            settings.SupportSettings.HelpLink = "https://search.nixos.org";
+          }
+          {
+            # Last version to support the "old" config layout.
+            system.stateVersion = lib.mkForce "24.11";
+
+            # First version to support the "new" config layout.
+            specialisation.upgrade.configuration.system.stateVersion = lib.mkVMOverride "25.05";
+          };
+      postgresMostlyMutable = makeMattermost {
+        mutableConfig = true;
+        preferNixConfig = true;
+        plugins = with pkgs; [
+          # Build the demo plugin.
+          (mattermost.buildPlugin {
+            pname = "mattermost-plugin-starter-template";
+            version = "0.1.0";
+            src = fetchFromGitHub {
+              owner = "mattermost";
+              repo = "mattermost-plugin-starter-template";
+              # Newer versions have issues with their dependency lockfile.
+              rev = "7c98e89ac1a268ce8614bc665571b7bbc9a70df2";
+              hash = "sha256-uyfxB0GZ45qL9ssWUord0eKQC6S0TlCTtjTOXWtK4H0=";
+            };
+            vendorHash = "sha256-Jl4F9YkHNqiFP9/yeyi4vTntqxMk/J1zhEP6QLSvJQA=";
+            npmDepsHash = "sha256-z08nc4XwT+uQjQlZiUydJyh8mqeJoYdPFWuZpw9k99s=";
+          })
+
+          # Build the todos plugin.
+          (mattermost.buildPlugin {
+            pname = "mattermost-plugin-todo";
+            version = "0.8-pre";
+            src = fetchFromGitHub {
+              owner = "mattermost-community";
+              repo = "mattermost-plugin-todo";
+              # 0.7.1 didn't work, seems to use an older set of node dependencies.
+              rev = "f25dc91ea401c9f0dcd4abcebaff10eb8b9836e5";
+              hash = "sha256-OM+m4rTqVtolvL5tUE8RKfclqzoe0Y38jLU60Pz7+HI=";
+            };
+            vendorHash = "sha256-5KpechSp3z/Nq713PXYruyNxveo6CwrCSKf2JaErbgg=";
+            npmDepsHash = "sha256-o2UOEkwb8Vx2lDWayNYgng0GXvmS6lp/ExfOq3peyMY=";
+            extraGoModuleAttrs = {
+              npmFlags = [ "--legacy-peer-deps" ];
+            };
+          })
+        ];
+      } { };
+      postgresImmutable = makeMattermost {
+        package = pkgs.mattermost.overrideAttrs (prev: {
+          webapp = prev.webapp.overrideAttrs (prevWebapp: {
+            # Ensure that users can add patches.
+            postPatch =
+              prevWebapp.postPatch or ""
+              + ''
+                substituteInPlace channels/src/root.html --replace-fail "Mattermost" "Patched Mattermost"
+              '';
+          });
+        });
+        mutableConfig = false;
+
+        # Make sure something other than the default works.
+        user = "mmuser";
+        group = "mmgroup";
+
+        database = {
+          # Ensure that this gets tested on Postgres.
+          peerAuth = false;
+        };
+        settings.SupportSettings.HelpLink = "https://search.nixos.org";
+      } { };
+      postgresEnvironmentFile = makeMattermost {
+        mutableConfig = false;
+        database.fromEnvironment = true;
+        settings.SupportSettings.AboutLink = "https://example.org";
+        environmentFile = pkgs.writeText "mattermost-env" ''
+          MM_SQLSETTINGS_DATASOURCE=postgres:///mattermost?host=/run/postgresql
+          MM_SUPPORTSETTINGS_ABOUTLINK=https://nixos.org
+        '';
+      } { };
+
+      mysqlMutable = makeMysql postgresMutable { };
+      mysqlMostlyMutable = makeMysql postgresMostlyMutable { };
+      mysqlImmutable = makeMysql postgresImmutable {
+        # Let's try to use this on MySQL.
+        services.mattermost.database = {
+          peerAuth = lib.mkForce true;
+          user = lib.mkForce "mmuser";
+          name = lib.mkForce "mmuser";
+        };
+      };
+      mysqlEnvironmentFile = makeMysql postgresEnvironmentFile {
+        services.mattermost.environmentFile = lib.mkForce (
+          pkgs.writeText "mattermost-env" ''
+            MM_SQLSETTINGS_DATASOURCE=mattermost@unix(/run/mysqld/mysqld.sock)/mattermost?charset=utf8mb4,utf8&writeTimeout=30s
+            MM_SUPPORTSETTINGS_ABOUTLINK=https://nixos.org
+          ''
+        );
+      };
+    };
+
+    testScript =
+      { nodes, ... }:
+      let
+        expectConfig = pkgs.writeShellScript "expect-config" ''
+          set -euo pipefail
+          config="$(curl ${lib.escapeShellArg "${url}/api/v4/config/client?format=old"})"
+          echo "Config: $(echo "$config" | ${pkgs.jq}/bin/jq)" >&2
+          [[ "$(echo "$config" | ${pkgs.jq}/bin/jq -r ${lib.escapeShellArg ".SiteName == $siteName and .Version == $mattermostVersion and "}"($1)" --arg siteName ${lib.escapeShellArg siteName} --arg mattermostVersion "$2" --arg sep '-')" = "true" ]]
+        '';
+
+        setConfig = pkgs.writeShellScript "set-config" ''
+          set -eo pipefail
+          mattermostConfig=/etc/mattermost/config.json
+          nixosVersion="$2"
+          if [ -z "$nixosVersion" ]; then
+            nixosVersion="$(nixos-version)"
+          fi
+          nixosVersion="$(echo "$nixosVersion" | sed -nr 's/^([0-9]{2})\.([0-9]{2}).*/\1\2/p')"
+          echo "NixOS version: $nixosVersion" >&2
+          if [ "$nixosVersion" -lt 2505 ]; then
+            mattermostConfig=/var/lib/mattermost/config/config.json
+          fi
+          newConfig="$(${pkgs.jq}/bin/jq -r "$1" "$mattermostConfig")"
+          echo "New config @ $mattermostConfig: $(echo "$newConfig" | ${pkgs.jq}/bin/jq)" >&2
+          truncate -s 0 "$mattermostConfig"
+          echo "$newConfig" >> "$mattermostConfig"
+        '';
+
+        expectPlugins = pkgs.writeShellScript "expect-plugins" ''
+          set -euo pipefail
+          case "$1" in
+            ""|*[!0-9]*)
+              plugins="$(curl ${lib.escapeShellArg "${url}/api/v4/plugins/webapp"})"
+              echo "Plugins: $(echo "$plugins" | ${pkgs.jq}/bin/jq)" >&2
+              [[ "$(echo "$plugins" | ${pkgs.jq}/bin/jq -r "$1")" == "true" ]]
+              ;;
+            *)
+              code="$(curl -s -o /dev/null -w "%{http_code}" ${lib.escapeShellArg "${url}/api/v4/plugins/webapp"})"
+              [[ "$code" == "$1" ]]
+              ;;
+          esac
+        '';
+
+        ensurePost = pkgs.writeShellScript "ensure-post" ''
+          set -euo pipefail
+
+          url="$1"
+          failIfNotFound="$2"
+
+          # Make sure the user exists
+          thingExists='(type == "array" and length > 0)'
+          userExists="($thingExists and ("'.[0].username == "nixos"))'
+          if mmctl user list --json | jq | tee /dev/stderr | jq -e "$userExists | not"; then
+            if [ "$failIfNotFound" -ne 0 ]; then
+              echo "User didn't exist!" >&2
+              exit 1
+            else
+              mmctl user create \
+                --email [e-mail] \
+                --username nixos --password nixosrules --system-admin --email-verified >&2
+
+              # Make sure the user exists.
+              while mmctl user list --json | jq | tee /dev/stderr | jq -e "$userExists | not"; do
+                sleep 1
+              done
+            fi
+          fi
+
+          # Auth.
+          mmctl auth login "$url" --name nixos --username nixos --password nixosrules
+
+          # Make sure the team exists
+          teamExists="($thingExists and ("'.[0].display_name == "NixOS Smoke Tests, Inc."))'
+          if mmctl team list --json | jq | tee /dev/stderr | jq -e "$teamExists | not"; then
+            if [ "$failIfNotFound" -ne 0 ]; then
+              echo "Team didn't exist!" >&2
+              exit 1
+            else
+              mmctl team create \
+                --name nixos \
+                --display-name "NixOS Smoke Tests, Inc."
+
+              # Teams take a second to create.
+              while mmctl team list --json | jq | tee /dev/stderr | jq -e "$teamExists | not"; do
+                sleep 1
+              done
+
+              # Add the user.
+              mmctl team users add nixos [e-mail]
+            fi
+          fi
+
+          authToken="$(cat ~/.config/mmctl/config | jq -r '.nixos.authToken')"
+          authHeader="Authorization: Bearer $authToken"
+          acceptHeader="Accept: application/json; charset=UTF-8"
+
+          # Make sure the test post exists.
+          postContents="pls enjoy this NixOS meme I made"
+          postAttachment=${./test.jpg}
+          postAttachmentSize="$(stat -c%s $postAttachment)"
+          postAttachmentHash="$(sha256sum $postAttachment | awk '{print $1}')"
+          postAttachmentId=""
+          postPredicate='select(.message == $message and (.file_ids | length) > 0 and (.metadata.files[0].size | tonumber) == ($size | tonumber))'
+          postExists="($thingExists and ("'(.[] | '"$postPredicate"' | length) > 0))'
+          if mmctl post list nixos:off-topic --json | jq | tee /dev/stderr | jq --arg message "$postContents" --arg size "$postAttachmentSize" -e "$postExists | not"; then
+            if [ "$failIfNotFound" -ne 0 ]; then
+              echo "Post didn't exist!" >&2
+              exit 1
+            else
+              # Can't use mmcli for this seemingly.
+              channelId="$(mmctl channel list nixos --json | jq | tee /dev/stderr | jq -r '.[] | select(.name == "off-topic") | .id')"
+              echo "Channel ID: $channelId" >&2
+
+              # Upload the file.
+              echo "Uploading file at $postAttachment (size: $postAttachmentSize)..." >&2
+              postAttachmentId="$(curl "$url/api/v4/files" -X POST -H "$acceptHeader" -H "$authHeader" \
+                -F "files=@$postAttachment" -F "channel_id=$channelId" -F "client_ids=test" | jq | tee /dev/stderr | jq -r '.file_infos[0].id')"
+
+              # Create the post with it attached.
+              postJson="$(echo '{}' | jq -c --arg channelId "$channelId" --arg message "$postContents" --arg fileId "$postAttachmentId" \
+                '{channel_id: $channelId, message: $message, file_ids: [$fileId]}')"
+              echo "Creating post with contents $postJson..." >&2
+              curl "$url/api/v4/posts" -X POST -H "$acceptHeader" -H "$authHeader" --json "$postJson" | jq >&2
+            fi
+          fi
+
+          if mmctl post list nixos:off-topic --json | jq | tee /dev/stderr | jq --arg message "$postContents" --arg size "$postAttachmentSize" -e "$postExists"; then
+            # Get the attachment ID.
+            getPostAttachmentId=".[] | $postPredicate | .file_ids[0]"
+            postAttachmentId="$(mmctl post list nixos:off-topic --json | jq | tee /dev/stderr | \
+              jq --arg message "$postContents" --arg size "$postAttachmentSize" -r "$getPostAttachmentId")"
+
+            echo "Expected post attachment hash: $postAttachmentHash" >&2
+            actualPostAttachmentHash="$(curl "$url/api/v4/files/$postAttachmentId?download=1" -H "$authHeader" | sha256sum | awk '{print $1}')"
+            echo "Actual post attachment hash: $postAttachmentHash" >&2
+            if [ "$actualPostAttachmentHash" != "$postAttachmentHash" ]; then
+              echo "Post attachment hash mismatched!" >&2
+              exit 1
+            else
+              echo "Post attachment hash was OK!" >&2
+              exit 0
+            fi
+          else
+            echo "Post didn't exist when it should have!" >&2
+            exit 1
+          fi
+        '';
+      in
+      ''
+        import shlex
+
+        def wait_mattermost_up(node, site_name="${siteName}"):
+          node.systemctl("start mattermost.service")
+          node.wait_for_unit("mattermost.service")
+          node.wait_for_open_port(8065)
+          node.succeed(f"curl {shlex.quote('${url}')} >/dev/null")
+          node.succeed(f"curl {shlex.quote('${url}')}/index.html | grep {shlex.quote(site_name)}")
+
+        def restart_mattermost(node, site_name="${siteName}"):
+          node.systemctl("restart mattermost.service")
+          wait_mattermost_up(node, site_name)
+
+        def expect_config(node, mattermost_version, *configs):
+          for config in configs:
+            node.succeed(f"${expectConfig} {shlex.quote(config)} {shlex.quote(mattermost_version)}")
+
+        def expect_plugins(node, jq_or_code):
+          node.succeed(f"${expectPlugins} {shlex.quote(str(jq_or_code))}")
+
+        def ensure_post(node, fail_if_not_found=False):
+          node.succeed(f"${ensurePost} {shlex.quote('${url}')} {1 if fail_if_not_found else 0}")
+
+        def set_config(node, *configs, nixos_version='25.05'):
+          for config in configs:
+            args = [shlex.quote("${setConfig}")]
+            args.append(shlex.quote(config))
+            if nixos_version:
+              args.append(shlex.quote(str(nixos_version)))
+            node.succeed(' '.join(args))
+
+        def run_mattermost_tests(mutableToplevel: str, mutable,
+                                 mostlyMutableToplevel: str, mostlyMutable,
+                                 immutableToplevel: str, immutable,
+                                 environmentFileToplevel: str, environmentFile):
+          esr, latest = '${pkgs.mattermost.version}', '${pkgs.mattermostLatest.version}'
+
+          ## Mutable node tests ##
+          mutable.start()
+          wait_mattermost_up(mutable)
+
+          # Get the initial config
+          expect_config(mutable, esr, '.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"')
+
+          # Edit the config and make a post
+          set_config(
+            mutable,
+            '.SupportSettings.AboutLink = "https://mattermost.com"',
+            '.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"',
+            nixos_version='24.11' # Default 'mutable' config is an old version
+          )
+          ensure_post(mutable)
+          restart_mattermost(mutable)
+
+          # AboutLink and HelpLink should be changed, and the post should exist
+          expect_config(mutable, esr, '.AboutLink == "https://mattermost.com" and .HelpLink == "https://nixos.org/nixos/manual"')
+          ensure_post(mutable, fail_if_not_found=True)
+
+          # Switch to the newer config
+          mutable.succeed(f"{mutableToplevel}/specialisation/upgrade/bin/switch-to-configuration switch")
+          wait_mattermost_up(mutable)
+
+          # AboutLink and HelpLink should be changed, still, and the post should still exist
+          expect_config(mutable, esr, '.AboutLink == "https://mattermost.com" and .HelpLink == "https://nixos.org/nixos/manual"')
+          ensure_post(mutable, fail_if_not_found=True)
+
+          # Switch to the latest Mattermost version
+          mutable.succeed(f"{mutableToplevel}/specialisation/latest/bin/switch-to-configuration switch")
+          wait_mattermost_up(mutable)
+
+          # AboutLink and HelpLink should be changed, still, and the post should still exist
+          expect_config(mutable, latest, '.AboutLink == "https://mattermost.com" and .HelpLink == "https://nixos.org/nixos/manual"')
+          ensure_post(mutable, fail_if_not_found=True)
+
+          mutable.shutdown()
+
+          ## Mostly mutable node tests ##
+          mostlyMutable.start()
+          wait_mattermost_up(mostlyMutable)
+
+          # Get the initial config
+          expect_config(mostlyMutable, esr, '.AboutLink == "https://nixos.org"')
+
+          # No plugins.
+          expect_plugins(mostlyMutable, 'length == 0')
+
+          # Edit the config and make a post
+          set_config(
+            mostlyMutable,
+            '.SupportSettings.AboutLink = "https://mattermost.com"',
+            '.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"',
+            '.PluginSettings.PluginStates."com.mattermost.plugin-todo".Enable = true'
+          )
+          ensure_post(mostlyMutable)
+          restart_mattermost(mostlyMutable)
+
+          # AboutLink should be overridden by NixOS configuration; HelpLink should be what we set above
+          expect_config(mostlyMutable, esr, '.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual"')
+
+          # Single plugin that's now enabled.
+          expect_plugins(mostlyMutable, 'length == 1')
+
+          # Post should exist.
+          ensure_post(mostlyMutable, fail_if_not_found=True)
+
+          # Switch to the latest Mattermost version
+          mostlyMutable.succeed(f"{mostlyMutableToplevel}/specialisation/latest/bin/switch-to-configuration switch")
+          wait_mattermost_up(mostlyMutable)
+
+          # AboutLink should be overridden and the post should still exist
+          expect_config(mostlyMutable, latest, '.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual"')
+          ensure_post(mostlyMutable, fail_if_not_found=True)
+
+          mostlyMutable.shutdown()
+
+          ## Immutable node tests ##
+          immutable.start()
+          wait_mattermost_up(immutable, "Patched Mattermost")
+
+          # Get the initial config
+          expect_config(immutable, esr, '.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"')
+
+          # Edit the config and make a post
+          set_config(
+            immutable,
+            '.SupportSettings.AboutLink = "https://mattermost.com"',
+            '.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"'
+          )
+          ensure_post(immutable)
+          restart_mattermost(immutable, "Patched Mattermost")
+
+          # Our edits should be ignored on restart
+          expect_config(immutable, esr, '.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"')
+
+          # No plugins.
+          expect_plugins(immutable, 'length == 0')
+
+          # Post should exist.
+          ensure_post(immutable, fail_if_not_found=True)
+
+          # Switch to the latest Mattermost version
+          immutable.succeed(f"{immutableToplevel}/specialisation/latest/bin/switch-to-configuration switch")
+          wait_mattermost_up(immutable)
+
+          # AboutLink and HelpLink should be changed, still, and the post should still exist
+          expect_config(immutable, latest, '.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"')
+          ensure_post(immutable, fail_if_not_found=True)
+
+          immutable.shutdown()
+
+          ## Environment File node tests ##
+          environmentFile.start()
+          wait_mattermost_up(environmentFile)
+          ensure_post(environmentFile)
+
+          # Settings in the environment file should override settings set otherwise, and the post should exist
+          expect_config(environmentFile, esr, '.AboutLink == "https://nixos.org"')
+          ensure_post(environmentFile, fail_if_not_found=True)
+
+          # Switch to the latest Mattermost version
+          environmentFile.succeed(f"{environmentFileToplevel}/specialisation/latest/bin/switch-to-configuration switch")
+          wait_mattermost_up(environmentFile)
+
+          # AboutLink should be changed still, and the post should still exist
+          expect_config(environmentFile, latest, '.AboutLink == "https://nixos.org"')
+          ensure_post(environmentFile, fail_if_not_found=True)
+
+          environmentFile.shutdown()
+
+        run_mattermost_tests(
+          "${nodes.mysqlMutable.system.build.toplevel}",
+          mysqlMutable,
+          "${nodes.mysqlMostlyMutable.system.build.toplevel}",
+          mysqlMostlyMutable,
+          "${nodes.mysqlImmutable.system.build.toplevel}",
+          mysqlImmutable,
+          "${nodes.mysqlEnvironmentFile.system.build.toplevel}",
+          mysqlEnvironmentFile
+        )
+
+        run_mattermost_tests(
+          "${nodes.postgresMutable.system.build.toplevel}",
+          postgresMutable,
+          "${nodes.postgresMostlyMutable.system.build.toplevel}",
+          postgresMostlyMutable,
+          "${nodes.postgresImmutable.system.build.toplevel}",
+          postgresImmutable,
+          "${nodes.postgresEnvironmentFile.system.build.toplevel}",
+          postgresEnvironmentFile
+        )
+      '';
+  }
+)