nixos/meilisearch: generic settings; handle secrets better. + fix racy test (#424481)

nixos/doc/manual/redirects.json

···
     "index.html#module-services-meilisearch-quickstart-search"
   ],
   "module-services-meilisearch-defaults": [
-    "index.html#module-services-meilisearch-defaults"
-  ],
-  "module-services-meilisearch-missing": [
+    "index.html#module-services-meilisearch-defaults",
     "index.html#module-services-meilisearch-missing"
   ],
   "module-services-networking-yggdrasil": [

nixos/modules/services/search/meilisearch.md

···
 
 - The default nixos package doesn't come with the [dashboard](https://docs.meilisearch.com/learn/getting_started/quick_start.html#search), since the dashboard features makes some assets downloads at compile time.
 
-- Anonymized Analytics sent to meilisearch are disabled by default.
+- `no_analytics` is set to true by default.
 
-- Default deployment is development mode. It doesn't require a secret master key. All routes are not protected and accessible.
+- `http_addr` is derived from {option}`services.meilisearch.listenAddress` and {option}`services.meilisearch.listenPort`. The two sub-fields are separate because this makes it easier to consume in certain other modules.
 
-## Missing {#module-services-meilisearch-missing}
+- `db_path` is set to `/var/lib/meilisearch` by default. Upstream, the default value is equivalent to `/var/lib/meilisearch/data.ms`.
+
+- `dump_dir` and `snapshot_dir` are set to `/var/lib/meilisearch/dumps` and `/var/lib/meilisearch/snapshots`, respectively. This is equivalent to the upstream defaults.
 
-- the snapshot feature is not yet configurable from the module, it's just a matter of adding the relevant environment variables.
+- All other options inherit their upstream defaults. In particular, the default configuration uses `env = "development"`, which doesn't require a master key, in which case all routes are unprotected.

nixos/modules/services/search/meilisearch.nix

···
 let
   cfg = config.services.meilisearch;
 
+  settingsFormat = pkgs.formats.toml { };
+
+  # These secrets are used in the config file and can be set to paths.
+  secrets-with-path =
+    builtins.map
+      (
+        { environment, name }:
+        {
+          inherit name environment;
+          setting = cfg.settings.${name};
+        }
+      )
+      [
+        {
+          environment = "MEILI_SSL_CERT_PATH";
+          name = "ssl_cert_path";
+        }
+        {
+          environment = "MEILI_SSL_KEY_PATH";
+          name = "ssl_key_path";
+        }
+        {
+          environment = "MEILI_SSL_AUTH_PATH";
+          name = "ssl_auth_path";
+        }
+        {
+          environment = "MEILI_SSL_OCSP_PATH";
+          name = "ssl_ocsp_path";
+        }
+      ];
+
+  # We also handle `master_key` separately.
+  # It cannot be set to a path, so we template it.
+  master-key-placeholder = "@MASTER_KEY@";
+
+  configFile = settingsFormat.generate "config.toml" (
+    builtins.removeAttrs (
+      if cfg.masterKeyFile != null then
+        cfg.settings // { master_key = master-key-placeholder; }
+      else
+        builtins.removeAttrs cfg.settings [ "master_key" ]
+    ) (map (secret: secret.name) secrets-with-path)
+  );
+
 in
 {
-
   meta.maintainers = with lib.maintainers; [
     Br1ght0ne
     happysalada
   ];
   meta.doc = ./meilisearch.md;
 
-  ###### interface
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "meilisearch" "environment" ]
+      [ "services" "meilisearch" "settings" "env" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "meilisearch" "logLevel" ]
+      [ "services" "meilisearch" "settings" "log_level" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "meilisearch" "noAnalytics" ]
+      [ "services" "meilisearch" "settings" "no_analytics" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "meilisearch" "maxIndexSize" ]
+      [ "services" "meilisearch" "settings" "max_index_size" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "meilisearch" "payloadSizeLimit" ]
+      [ "services" "meilisearch" "settings" "http_payload_size_limit" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "meilisearch" "dumplessUpgrade" ]
+      [ "services" "meilisearch" "settings" "experimental_dumpless_upgrade" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "meilisearch" "masterKeyEnvironmentFile" ] ''
+      Use `services.meilisearch.masterKeyFile` instead. It does not require you to prefix the file with "MEILI_MASTER_KEY=".
+      If you were abusing this option to set other options, you can now configure them with `services.meilisearch.settings`.
+    '')
+  ];
 
   options.services.meilisearch = {
-    enable = lib.mkEnableOption "MeiliSearch - a RESTful search API";
+    enable = lib.mkEnableOption "Meilisearch - a RESTful search API";
 
     package = lib.mkPackageOption pkgs "meilisearch" {
       extraDescription = ''
···
     };
 
     listenAddress = lib.mkOption {
-      description = "MeiliSearch listen address.";
-      default = "127.0.0.1";
+      default = "localhost";
       type = lib.types.str;
+      description = ''
+        The IP address that Meilisearch will listen on.
+
+        It can also be a hostname like "localhost". If it resolves to an IPv4 and IPv6 address, Meilisearch will listen on both.
+      '';
     };
 
     listenPort = lib.mkOption {
-      description = "MeiliSearch port to listen on.";
       default = 7700;
       type = lib.types.port;
-    };
-
-    environment = lib.mkOption {
-      description = "Defines the running environment of MeiliSearch.";
-      default = "development";
-      type = lib.types.enum [
-        "development"
-        "production"
-      ];
+      description = ''
+        The port that Meilisearch will listen on.
+      '';
     };
 
-    # TODO change this to LoadCredentials once possible
-    masterKeyEnvironmentFile = lib.mkOption {
+    masterKeyFile = lib.mkOption {
       description = ''
         Path to file which contains the master key.
         By doing so, all routes will be protected and will require a key to be accessed.
         If no master key is provided, all routes can be accessed without requiring any key.
-        The format is the following:
-        MEILI_MASTER_KEY=my_secret_key
       '';
       default = null;
-      type = with lib.types; nullOr path;
-    };
-
-    noAnalytics = lib.mkOption {
-      description = ''
-        Deactivates analytics.
-        Analytics allow MeiliSearch to know how many users are using MeiliSearch,
-        which versions and which platforms are used.
-        This process is entirely anonymous.
-      '';
-      default = true;
-      type = lib.types.bool;
-    };
-
-    logLevel = lib.mkOption {
-      description = ''
-        Defines how much detail should be present in MeiliSearch's logs.
-        MeiliSearch currently supports four log levels, listed in order of increasing verbosity:
-        - 'ERROR': only log unexpected events indicating MeiliSearch is not functioning as expected
-        - 'WARN:' log all unexpected events, regardless of their severity
-        - 'INFO:' log all events. This is the default value
-        - 'DEBUG': log all events and including detailed information on MeiliSearch's internal processes.
-          Useful when diagnosing issues and debugging
-      '';
-      default = "INFO";
-      type = lib.types.str;
+      type = lib.types.nullOr lib.types.path;
     };
 
-    maxIndexSize = lib.mkOption {
+    settings = lib.mkOption {
       description = ''
-        Sets the maximum size of the index.
-        Value must be given in bytes or explicitly stating a base unit.
-        For example, the default value can be written as 107374182400, '107.7Gb', or '107374 Mb'.
-        Default is 100 GiB
+        Configuration settings for Meilisearch.
+        Look at the documentation for available options:
+        https://github.com/meilisearch/meilisearch/blob/main/config.toml
+        https://www.meilisearch.com/docs/learn/self_hosted/configure_meilisearch_at_launch#all-instance-options
       '';
-      default = "107374182400";
-      type = lib.types.str;
-    };
 
-    payloadSizeLimit = lib.mkOption {
-      description = ''
-        Sets the maximum size of accepted JSON payloads.
-        Value must be given in bytes or explicitly stating a base unit.
-        For example, the default value can be written as 107374182400, '107.7Gb', or '107374 Mb'.
-        Default is ~ 100 MB
-      '';
-      default = "104857600";
-      type = lib.types.str;
-    };
+      default = { };
 
-    # TODO: turn on by default when it stops being experimental
-    dumplessUpgrade = lib.mkOption {
-      default = false;
-      example = true;
-      description = ''
-        Whether to enable (experimental) dumpless upgrade.
+      type = lib.types.submodule {
+        freeformType = settingsFormat.type;
 
-        Allows upgrading from Meilisearch >=v1.12 to Meilisearch >=v1.13 without manually
-        dumping and importing the database.
+        imports = builtins.map (secret: {
+          # give them proper types, just so they're easier to consume from this file
+          options.${secret.name} = lib.mkOption {
+            # but they should not show up in documentation as special in any way.
+            visible = false;
 
-        More information at https://www.meilisearch.com/docs/learn/update_and_migration/updating#dumpless-upgrade
-      '';
-      type = lib.types.bool;
+            type = lib.types.nullOr lib.types.path;
+            default = null;
+          };
+        }) secrets-with-path;
+      };
     };
+  };
 
-  };
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !cfg.settings ? master_key;
+        message = ''
+          Do not set `services.meilisearch.settings.master_key` in your configuration.
+          Use `services.meilisearch.masterKeyFile` instead.
+        '';
+      }
+    ];
 
-  ###### implementation
+    services.meilisearch.settings = {
+      # we use `listenAddress` and `listenPort` to derive the `http_addr` setting.
+      # this is the only setting we treat like this.
+      # we do this because some dependent services like Misskey/Sharkey need separate host,port for no good reason.
+      http_addr = "${cfg.listenAddress}:${toString cfg.listenPort}";
+
+      # upstream's default for `db_path` is `/var/lib/meilisearch/data.ms/`, but ours is different for no reason.
+      db_path = lib.mkDefault "/var/lib/meilisearch";
+      # these are equivalent to the upstream defaults, because we set a working directory.
+      # they are only set here for consistency with `db_path`.
+      dump_dir = lib.mkDefault "/var/lib/meilisearch/dumps";
+      snapshot_dir = lib.mkDefault "/var/lib/meilisearch/snapshots";
 
-  config = lib.mkIf cfg.enable {
+      # this is intentionally different from upstream's default.
+      no_analytics = lib.mkDefault true;
+    };
 
     warnings = lib.optional (lib.versionOlder cfg.package.version "1.12") ''
       Meilisearch 1.11 will be removed in NixOS 25.11. As it was the last
···
     environment.systemPackages = [ cfg.package ];
 
     systemd.services.meilisearch = {
-      description = "MeiliSearch daemon";
+      description = "Meilisearch daemon";
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
-      environment = {
-        MEILI_DB_PATH = "/var/lib/meilisearch";
-        MEILI_HTTP_ADDR = "${cfg.listenAddress}:${toString cfg.listenPort}";
-        MEILI_NO_ANALYTICS = lib.boolToString cfg.noAnalytics;
-        MEILI_ENV = cfg.environment;
-        MEILI_DUMP_DIR = "/var/lib/meilisearch/dumps";
-        MEILI_LOG_LEVEL = cfg.logLevel;
-        MEILI_MAX_INDEX_SIZE = cfg.maxIndexSize;
-        MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE = lib.boolToString cfg.dumplessUpgrade;
-      };
+
+      preStart = lib.mkMerge [
+        ''
+          install -m 700 '${configFile}' "$RUNTIME_DIRECTORY/config.toml"
+        ''
+        (lib.mkIf (cfg.masterKeyFile != null) ''
+          ${lib.getExe pkgs.replace-secret} '${master-key-placeholder}' "$CREDENTIALS_DIRECTORY/master_key" "$RUNTIME_DIRECTORY/config.toml"
+        '')
+      ];
+
+      environment = builtins.listToAttrs (
+        builtins.map (secret: {
+          name = secret.environment;
+          value = lib.mkIf (secret.setting != null) "%d/${secret.name}";
+        }) secrets-with-path
+      );
+
       serviceConfig = {
-        ExecStart = "${cfg.package}/bin/meilisearch";
+        LoadCredential = lib.mkMerge (
+          [
+            (lib.mkIf (cfg.masterKeyFile != null) [ "master_key:${cfg.masterKeyFile}" ])
+          ]
+          ++ builtins.map (
+            secret: lib.mkIf (secret.setting != null) [ "${secret.name}:${secret.setting}" ]
+          ) secrets-with-path
+        );
+        ExecStart = "${lib.getExe cfg.package} --config-file-path \${RUNTIME_DIRECTORY}/config.toml";
         DynamicUser = true;
         StateDirectory = "meilisearch";
-        EnvironmentFile = lib.mkIf (cfg.masterKeyEnvironmentFile != null) cfg.masterKeyEnvironmentFile;
+        WorkingDirectory = "%S/meilisearch";
+        RuntimeDirectory = "meilisearch";
+        RuntimeDirectoryMode = "0700";
       };
     };
   };

nixos/modules/services/web-apps/sharkey.nix

···
         description = ''
           Whether to automatically set up a local Meilisearch instance and configure Sharkey to use it.
 
-          You need to ensure `services.meilisearch.masterKeyEnvironmentFile` is correctly configured for a working
+          You need to ensure `services.meilisearch.masterKeyFile` is correctly configured for a working
           Meilisearch setup. You also need to configure Sharkey to use an API key obtained from Meilisearch with the
           `MK_CONFIG_MEILISEARCH_APIKEY` environment variable, and set `services.sharkey.settings.meilisearch.index` to
           the created index. See https://docs.joinsharkey.org/docs/customisation/search/meilisearch/ for how to create
···
       (mkIf cfg.setupMeilisearch {
         services.meilisearch = {
           enable = mkDefault true;
-          environment = mkDefault "production";
+          settings.env = mkDefault "production";
         };
 
         services.sharkey.settings = {

nixos/tests/meilisearch.nix

···
     machine.wait_for_unit("meilisearch")
     machine.wait_for_open_port(7700)
 
+    def wait_task(cmd):
+        response = json.loads(machine.succeed(cmd))
+        task_uid = response["taskUid"]
+        machine.wait_until_succeeds(
+            f"curl ${apiUrl}/tasks/{task_uid} | jq -e '.status | IN(\"succeeded\", \"failed\", \"canceled\")'"
+        )
+        machine.succeed(f"curl ${apiUrl}/tasks/{task_uid} | jq -e '.status == \"succeeded\"'")
+        return response
+
     with subtest("check version"):
         version = json.loads(machine.succeed("curl ${apiUrl}/version"))
         assert version["pkgVersion"] == "${pkgs.meilisearch.version}"
 
     with subtest("create index"):
-        machine.succeed(
-            "curl -X POST -H 'Content-Type: application/json' ${apiUrl}/indexes --data @${indexJSON}"
-        )
+        wait_task("curl -X POST -H 'Content-Type: application/json' ${apiUrl}/indexes --data @${indexJSON}")
         indexes = json.loads(machine.succeed("curl ${apiUrl}/indexes"))
         assert indexes["total"] == 1, "index wasn't created"
 
     with subtest("add documents"):
-        response = json.loads(
-            machine.succeed(
-                "curl -X POST -H 'Content-Type: application/json' ${apiUrl}/indexes/${uid}/documents --data-binary @${moviesJSON}"
-            )
-        )
-        task_uid = response["taskUid"]
-        machine.wait_until_succeeds(
-            f"curl ${apiUrl}/tasks/{task_uid} | jq -e '.status == \"succeeded\"'"
-        )
+        wait_task("curl -X POST -H 'Content-Type: application/json' ${apiUrl}/indexes/${uid}/documents --data-binary @${moviesJSON}")
 
     with subtest("search"):
         response = json.loads(

nixos/tests/web-apps/sharkey.nix

···
         };
       };
 
-      services.meilisearch.masterKeyEnvironmentFile = pkgs.writeText "meilisearch-key" ''
-        MEILI_MASTER_KEY=${meilisearchKey}
-      '';
+      services.meilisearch.masterKeyFile = pkgs.writeText "meilisearch-key" meilisearchKey;
     };
 
   testScript =