Merge pull request #221318 from mweinelt/synapse-extras

nixos/matrix-synapse: Allow passing extras, discover extras from config

nixos/doc/manual/release-notes/rl-2311.section.md

···
 
 - The `services.ananicy.extraRules` option now has the type of `listOf attrs` instead of `string`.
 
+- The `matrix-synapse` package & module have undergone some significant internal changes, for most setups no intervention is needed, though:
+  - The option [`services.matrix-synapse.package`](#opt-services.matrix-synapse.package) is now read-only. For modifying the package, use an overlay which modifies `matrix-synapse-unwrapped` instead. More on that below.
+  - The `enableSystemd` & `enableRedis` arguments have been removed and `matrix-synapse` has been renamed to `matrix-synapse-unwrapped`. Also, several optional dependencies (such as `psycopg2` or `authlib`) have been removed.
+  - These optional dependencies are automatically added via a wrapper (`pkgs.matrix-synapse.override { extras = ["redis"]; }` for `hiredis` & `txredisapi` for instance) if the relevant config section is declared in `services.matrix-synapse.settings`. For instance, if `services.matrix-synapse.settings.redis.enabled` is set to `true`, `"redis"` will be automatically added to the `extras` list of `pkgs.matrix-synapse`.
+  - A list of all extras (and the extras enabled by default) can be found at the [option's reference for `services.matrix-synapse.extras`](#opt-services.matrix-synapse.extras).
+  - In some cases (e.g. for running synapse workers) it was necessary to re-use the `PYTHONPATH` of `matrix-synapse.service`'s environment to have all plugins available. This isn't necessary anymore, instead `config.services.matrix-synapse.package` can be used as it points to the wrapper with properly configured `extras` and also all plugins defined via [`services.matrix-synapse.plugins`](#opt-services.matrix-synapse.plugins) available. This is also the reason for why the option is read-only now, it's supposed to be set by the module only.
+
 - `etcd` has been updated to 3.5, you will want to read the [3.3 to 3.4](https://etcd.io/docs/v3.5/upgrades/upgrade_3_4/) and [3.4 to 3.5](https://etcd.io/docs/v3.5/upgrades/upgrade_3_5/) upgrade guides
 
 - `consul` has been updated to `1.16.0`. See the [release note](https://github.com/hashicorp/consul/releases/tag/v1.16.0) for more details. Once a new Consul version has started and upgraded its data directory, it generally cannot be downgraded to the previous version.

nixos/modules/services/matrix/synapse.nix

···
   # remove null values from the final configuration
   finalSettings = lib.filterAttrsRecursive (_: v: v != null) cfg.settings;
   configFile = format.generate "homeserver.yaml" finalSettings;
-  logConfigFile = format.generate "log_config.yaml" cfg.logConfig;
-
-  pluginsEnv = cfg.package.python.buildEnv.override {
-    extraLibs = cfg.plugins;
-  };
 
   usePostgresql = cfg.settings.database.name == "psycopg2";
   hasLocalPostgresDB = let args = cfg.settings.database.args; in
···
             "${bindAddress}"
         }:${builtins.toString listener.port}/"
     '';
+
+  defaultExtras = [
+    "systemd"
+    "postgres"
+    "url-preview"
+    "user-search"
+  ];
+
+  wantedExtras = cfg.extras
+    ++ lib.optional (cfg.settings ? oidc_providers) "oidc"
+    ++ lib.optional (cfg.settings ? jwt_config) "jwt"
+    ++ lib.optional (cfg.settings ? saml2_config) "saml2"
+    ++ lib.optional (cfg.settings ? opentracing) "opentracing"
+    ++ lib.optional (cfg.settings ? redis) "redis"
+    ++ lib.optional (cfg.settings ? sentry) "sentry"
+    ++ lib.optional (cfg.settings ? user_directory) "user-search"
+    ++ lib.optional (cfg.settings.url_preview_enabled) "url-preview"
+    ++ lib.optional (cfg.settings.database.name == "psycopg2") "postgres";
+
+  wrapped = pkgs.matrix-synapse.override {
+    extras = wantedExtras;
+    inherit (cfg) plugins;
+  };
 in {
 
   imports = [
···
 
       package = mkOption {
         type = types.package;
-        default = pkgs.matrix-synapse;
-        defaultText = literalExpression "pkgs.matrix-synapse";
+        readOnly = true;
+        description = lib.mdDoc ''
+          Reference to the `matrix-synapse` wrapper with all extras
+          (e.g. for `oidc` or `saml2`) added to the `PYTHONPATH` of all executables.
+
+          This option is useful to reference the "final" `matrix-synapse` package that's
+          actually used by `matrix-synapse.service`. For instance, when using
+          workers, it's possible to run
+          `''${config.services.matrix-synapse.package}/bin/synapse_worker` and
+          no additional PYTHONPATH needs to be specified for extras or plugins configured
+          via `services.matrix-synapse`.
+
+          However, this means that this option is supposed to be only declared
+          by the `services.matrix-synapse` module itself and is thus read-only.
+          In order to modify `matrix-synapse` itself, use an overlay to override
+          `pkgs.matrix-synapse-unwrapped`.
+        '';
+      };
+
+      extras = mkOption {
+        type = types.listOf (types.enum (lib.attrNames pkgs.matrix-synapse-unwrapped.optional-dependencies));
+        default = defaultExtras;
+        example = literalExpression ''
+          [
+            "cache-memory" # Provide statistics about caching memory consumption
+            "jwt"          # JSON Web Token authentication
+            "opentracing"  # End-to-end tracing support using Jaeger
+            "oidc"         # OpenID Connect authentication
+            "postgres"     # PostgreSQL database backend
+            "redis"        # Redis support for the replication stream between worker processes
+            "saml2"        # SAML2 authentication
+            "sentry"       # Error tracking and performance metrics
+            "systemd"      # Provide the JournalHandler used in the default log_config
+            "url-preview"  # Support for oEmbed URL previews
+            "user-search"  # Support internationalized domain names in user-search
+          ]
+        '';
         description = lib.mdDoc ''
-          Overridable attribute of the matrix synapse server package to use.
+          Explicitly install extras provided by matrix-synapse. Most
+          will require some additional configuration.
+
+          Extras will automatically be enabled, when the relevant
+          configuration sections are present.
+
+          Please note that this option is additive: i.e. when adding a new item
+          to this list, the defaults are still kept. To override the defaults as well,
+          use `lib.mkForce`.
         '';
       };
 
···
         default = {};
         description = mdDoc ''
           The primary synapse configuration. See the
-          [sample configuration](https://github.com/matrix-org/synapse/blob/v${cfg.package.version}/docs/sample_config.yaml)
+          [sample configuration](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_config.yaml)
           for possible values.
 
           Secrets should be passed in by using the `extraConfigFiles` option.
···
     ];
 
     services.matrix-synapse.configFile = configFile;
+    services.matrix-synapse.package = wrapped;
+
+    # default them, so they are additive
+    services.matrix-synapse.extras = defaultExtras;
 
     users.users.matrix-synapse = {
       group = "matrix-synapse";
···
           --keys-directory ${cfg.dataDir} \
           --generate-keys
       '';
-      environment = {
-        PYTHONPATH = makeSearchPathOutput "lib" cfg.package.python.sitePackages [ pluginsEnv ];
-      } // optionalAttrs (cfg.withJemalloc) {
+      environment = optionalAttrs (cfg.withJemalloc) {
         LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
       };
       serviceConfig = {

nixos/tests/matrix/synapse.nix

···
 
   nodes = {
     # Since 0.33.0, matrix-synapse doesn't allow underscores in server names
-    serverpostgres = { pkgs, nodes, ... }: let
+    serverpostgres = { pkgs, nodes, config, ... }: let
       mailserverIP = nodes.mailserver.config.networking.primaryIPAddress;
     in
     {
···
           database = {
             name = "psycopg2";
             args.password = "synapse";
+          };
+          redis = {
+            enabled = true;
+            host = "localhost";
+            port = config.services.redis.servers.matrix-synapse.port;
           };
           tls_certificate_path = "${cert}";
           tls_private_key_path = "${key}";
···
             LC_COLLATE = "C"
             LC_CTYPE = "C";
         '';
+      };
+
+      services.redis.servers.matrix-synapse = {
+        enable = true;
+        port = 6380;
       };
 
       networking.extraHosts = ''
···
     serverpostgres.wait_for_unit("matrix-synapse.service")
     serverpostgres.wait_until_succeeds(
         "curl --fail -L --cacert ${ca_pem} https://localhost:8448/"
+    )
+    serverpostgres.wait_until_succeeds(
+        "journalctl -u matrix-synapse.service | grep -q 'Connected to redis'"
     )
     serverpostgres.require_unit_state("postgresql.service")
     serverpostgres.succeed("register_new_matrix_user -u ${testUser} -p ${testPassword} -a -k ${registrationSharedSecret} https://localhost:8448/")

pkgs/servers/matrix-synapse/default.nix

···
-{ lib, stdenv, fetchFromGitHub, python3, openssl, cargo, rustPlatform, rustc
-, enableSystemd ? lib.meta.availableOn stdenv.hostPlatform python3.pkgs.systemd
+{ lib
+, stdenv
+, fetchFromGitHub
+, python3
+, openssl
+, cargo
+, rustPlatform
+, rustc
 , nixosTests
-, enableRedis ? true
 , callPackage
 }:
 
···
   plugins = python3.pkgs.callPackage ./plugins { };
   tools = callPackage ./tools { };
 in
-with python3.pkgs;
-buildPythonApplication rec {
+python3.pkgs.buildPythonApplication rec {
   pname = "matrix-synapse";
   version = "1.89.0";
   format = "pyproject";
···
     sed -i '/^setuptools_rust =/d' pyproject.toml
   '';
 
-  nativeBuildInputs = [
+  nativeBuildInputs = with python3.pkgs; [
     poetry-core
     rustPlatform.cargoSetupHook
     setuptools-rust
···
     rustc
   ];
 
-  buildInputs = [ openssl ];
+  buildInputs = [
+    openssl
+  ];
 
-  propagatedBuildInputs = [
-    authlib
+  propagatedBuildInputs = with python3.pkgs; [
+    attrs
     bcrypt
     bleach
     canonicaljson
-    daemonize
+    cryptography
     ijson
     immutabledict
     jinja2
     jsonschema
-    lxml
     matrix-common
     msgpack
     netaddr
+    packaging
     phonenumbers
     pillow
     prometheus-client
-    psutil
-    psycopg2
     pyasn1
+    pyasn1-modules
     pydantic
-    pyicu
     pymacaroons
-    pynacl
     pyopenssl
-    pysaml2
     pyyaml
-    requests
-    setuptools
+    service-identity
     signedjson
     sortedcontainers
     treq
     twisted
     typing-extensions
     unpaddedbase64
-  ] ++ lib.optional enableSystemd systemd
-    ++ lib.optionals enableRedis [ hiredis txredisapi ];
+  ]
+  ++ twisted.optional-dependencies.tls;
 
-  nativeCheckInputs = [ mock parameterized openssl ];
+  passthru.optional-dependencies = with python3.pkgs; {
+    postgres = if isPyPy then [
+      psycopg2cffi
+    ] else [
+      psycopg2
+    ];
+    saml2 = [
+      pysaml2
+    ];
+    oidc = [
+      authlib
+    ];
+    systemd = [
+      systemd
+    ];
+    url-preview = [
+      lxml
+    ];
+    sentry = [
+      sentry-sdk
+    ];
+    opentracing = [
+      jaeger-client
+      opentracing
+    ];
+    jwt = [
+      authlib
+    ];
+    redis = [
+      hiredis
+      txredisapi
+    ];
+    cache-memory = [
+      pympler
+    ];
+    user-search = [
+      pyicu
+    ];
+  };
+
+  nativeCheckInputs = [
+    openssl
+  ] ++ (with python3.pkgs; [
+    mock
+    parameterized
+  ])
+  ++ lib.flatten (lib.attrValues passthru.optional-dependencies);
 
   doCheck = !stdenv.isDarwin;
 
···
 
   meta = with lib; {
     homepage = "https://matrix.org";
+    changelog = "https://github.com/matrix-org/synapse/releases/tag/v${version}";
     description = "Matrix reference homeserver";
     license = licenses.asl20;
     maintainers = teams.matrix.members;

pkgs/servers/matrix-synapse/plugins/ldap3.nix

···
 , fetchPypi
 , ldap3
 , ldaptor
-, matrix-synapse
+, matrix-synapse-unwrapped
 , pytestCheckHook
 , service-identity
 , setuptools
···
 
   propagatedBuildInputs = [ service-identity ldap3 twisted ];
 
-  nativeCheckInputs = [ ldaptor matrix-synapse pytestCheckHook ];
+  nativeCheckInputs = [ ldaptor matrix-synapse-unwrapped pytestCheckHook ];
 
   pythonImportsCheck = [ "ldap_auth_provider" ];
 

pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix

···
-{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse }:
+{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped }:
 
 buildPythonPackage rec {
   pname = "matrix-synapse-mjolnir-antispam";
···
 
   sourceRoot = "./${src.name}/synapse_antispam";
 
-  buildInputs = [ matrix-synapse ];
+  buildInputs = [ matrix-synapse-unwrapped ];
 
   doCheck = false; # no tests
   pythonImportsCheck = [ "mjolnir" ];

pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix

···
-{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse, twisted }:
+{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped, twisted }:
 
 buildPythonPackage rec {
   pname = "matrix-synapse-shared-secret-auth";
···
   doCheck = false;
   pythonImportsCheck = [ "shared_secret_authenticator" ];
 
-  buildInputs = [ matrix-synapse ];
+  buildInputs = [ matrix-synapse-unwrapped ];
   propagatedBuildInputs = [ twisted ];
 
   meta = with lib; {

pkgs/servers/matrix-synapse/wrapper.nix

···
+{ lib
+, stdenv
+, makeWrapper
+, matrix-synapse-unwrapped
+, extras ? [
+    "postgres"
+    "url-preview"
+    "user-search"
+  ] ++ lib.optional (lib.meta.availableOn stdenv.hostPlatform matrix-synapse-unwrapped.python.pkgs.systemd) "systemd"
+, plugins ? [ ]
+, ...
+}:
+
+let
+  extraPackages = lib.concatMap (extra: matrix-synapse-unwrapped.optional-dependencies.${extra}) (lib.unique extras);
+
+  pluginsEnv = matrix-synapse-unwrapped.python.buildEnv.override {
+    extraLibs = plugins;
+  };
+
+  searchPath = lib.makeSearchPathOutput "lib" matrix-synapse-unwrapped.python.sitePackages (extraPackages ++ [ pluginsEnv ]);
+in
+stdenv.mkDerivation {
+  name = (lib.appendToName "wrapped" matrix-synapse-unwrapped).name;
+
+  nativeBuildInputs = [
+    makeWrapper
+  ];
+
+  buildCommand = ''
+    for bin in ${matrix-synapse-unwrapped}/bin/*; do
+      echo $bin
+      makeWrapper "$bin" "$out/bin/$(basename $bin)" \
+        --set PYTHONPATH ${searchPath}
+    done;
+  '';
+
+  passthru = {
+    unwrapped = matrix-synapse-unwrapped;
+
+    # for backward compatibility
+    inherit (matrix-synapse-unwrapped) plugins tools;
+  };
+}

pkgs/top-level/all-packages.nix

···
 
   matrix-sliding-sync = callPackage ../servers/matrix-synapse/sliding-sync { };
 
-  matrix-synapse = callPackage ../servers/matrix-synapse { };
-
-  matrix-synapse-plugins = recurseIntoAttrs matrix-synapse.plugins;
-
-  matrix-synapse-tools = recurseIntoAttrs matrix-synapse.tools;
+  matrix-synapse = callPackage ../servers/matrix-synapse/wrapper.nix { };
+  matrix-synapse-unwrapped = callPackage ../servers/matrix-synapse/default.nix { };
+  matrix-synapse-plugins = recurseIntoAttrs matrix-synapse-unwrapped.plugins;
+  matrix-synapse-tools = recurseIntoAttrs matrix-synapse-unwrapped.tools;
 
   matrix-appservice-irc = callPackage ../servers/matrix-synapse/matrix-appservice-irc { };