Merge pull request #186276 from typetetris/feature/add-timescaledb-toolkit

timescaledb_toolkit: init at 1.14.0

nixos/tests/all-tests.nix

···
   terminal-emulators = handleTest ./terminal-emulators.nix {};
   tiddlywiki = handleTest ./tiddlywiki.nix {};
   tigervnc = handleTest ./tigervnc.nix {};
+  timescaledb = handleTest ./timescaledb.nix {};
   timezone = handleTest ./timezone.nix {};
   tinc = handleTest ./tinc {};
   tinydns = handleTest ./tinydns.nix {};

nixos/tests/timescaledb.nix

···
+# mostly copied from ./postgresql.nix as it seemed unapproriate to
+# test additional extensions for postgresql there.
+
+{ system ? builtins.currentSystem
+, config ? { }
+, pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
+
+let
+  postgresql-versions = import ../../pkgs/servers/sql/postgresql pkgs;
+  test-sql = pkgs.writeText "postgresql-test" ''
+    CREATE EXTENSION timescaledb;
+    CREATE EXTENSION timescaledb_toolkit;
+
+    CREATE TABLE sth (
+      time TIMESTAMPTZ NOT NULL,
+      value DOUBLE PRECISION
+    );
+
+    SELECT create_hypertable('sth', 'time');
+
+    INSERT INTO sth (time, value) VALUES
+    ('2003-04-12 04:05:06 America/New_York', 1.0),
+    ('2003-04-12 04:05:07 America/New_York', 2.0),
+    ('2003-04-12 04:05:08 America/New_York', 3.0),
+    ('2003-04-12 04:05:09 America/New_York', 4.0),
+    ('2003-04-12 04:05:10 America/New_York', 5.0)
+    ;
+
+    WITH t AS (
+      SELECT
+        time_bucket('1 day'::interval, time) AS dt,
+        stats_agg(value) AS stats
+      FROM sth
+      GROUP BY time_bucket('1 day'::interval, time)
+    )
+    SELECT
+      average(stats)
+    FROM t;
+  '';
+  make-postgresql-test = postgresql-name: postgresql-package: makeTest {
+    name = postgresql-name;
+    meta = with pkgs.lib.maintainers; {
+      maintainers = [ typetetris ];
+    };
+
+    nodes.machine = { ... }:
+      {
+        services.postgresql = {
+          enable = true;
+          package = postgresql-package;
+          extraPlugins = with postgresql-package.pkgs; [
+            timescaledb
+            timescaledb_toolkit
+          ];
+          settings = { shared_preload_libraries = "timescaledb, timescaledb_toolkit"; };
+        };
+      };
+
+    testScript = ''
+      def check_count(statement, lines):
+          return 'test $(sudo -u postgres psql postgres -tAc "{}"|wc -l) -eq {}'.format(
+              statement, lines
+          )
+
+
+      machine.start()
+      machine.wait_for_unit("postgresql")
+
+      with subtest("Postgresql with extensions timescaledb and timescaledb_toolkit is available just after unit start"):
+          machine.succeed(
+              "sudo -u postgres psql -f ${test-sql}"
+          )
+
+      machine.fail(check_count("SELECT * FROM sth;", 3))
+      machine.succeed(check_count("SELECT * FROM sth;", 5))
+      machine.fail(check_count("SELECT * FROM sth;", 4))
+
+      machine.shutdown()
+    '';
+
+  };
+  applicablePostgresqlVersions = filterAttrs (_: value: versionAtLeast value.version "12") postgresql-versions;
+in
+mapAttrs'
+  (name: package: {
+    inherit name;
+    value = make-postgresql-test name package;
+  })
+  applicablePostgresqlVersions

pkgs/development/tools/rust/cargo-pgx/buildPgxExtension.nix

···
+# preBuildAndTest and some small other bits
+# taken from https://github.com/tcdi/pgx/blob/v0.4.5/nix/extension.nix
+# (but now heavily modified)
+# which uses MIT License with the following license file
+#
+# MIT License
+#
+# Portions Copyright 2019-2021 ZomboDB, LLC.
+# Portions Copyright 2021-2022 Technology Concepts & Design, Inc. <[e-mail]>.
+# All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+{ lib
+, cargo-pgx
+, pkg-config
+, rustPlatform
+, stdenv
+, Security
+, writeShellScriptBin
+}:
+
+# The idea behind: Use it mostly like rustPlatform.buildRustPackage and so
+# we hand most of the arguments down.
+#
+# Additional arguments are:
+#   - `postgresql` postgresql package of the version of postgresql this extension should be build for.
+#                  Needs to be the build platform variant.
+#   - `useFakeRustfmt` Whether to use a noop fake command as rustfmt. cargo-pgx tries to call rustfmt.
+#                      If the generated rust bindings aren't needed to use the extension, its a
+#                      unnecessary and heavy dependency. If you set this to true, you also
+#                      have to add `rustfmt` to `nativeBuildInputs`.
+
+{ buildAndTestSubdir ? null
+, buildType ? "release"
+, buildFeatures ? [ ]
+, cargoBuildFlags ? [ ]
+, postgresql
+# cargo-pgx calls rustfmt on generated bindings, this is not strictly necessary, so we avoid the
+# dependency here. Set to false and provide rustfmt in nativeBuildInputs, if you need it, e.g.
+# if you include the generated code in the output via postInstall.
+, useFakeRustfmt ? true
+, ...
+} @ args:
+let
+  rustfmtInNativeBuildInputs = lib.lists.any (dep: lib.getName dep == "rustfmt") (args.nativeBuildInputs or []);
+in
+
+assert lib.asserts.assertMsg ((args.installPhase or "") == "")
+  "buildPgxExtensions overwrites the installPhase, so providing one does nothing";
+assert lib.asserts.assertMsg ((args.buildPhase or "") == "")
+  "buildPgxExtensions overwrites the buildPhase, so providing one does nothing";
+assert lib.asserts.assertMsg (useFakeRustfmt -> !rustfmtInNativeBuildInputs)
+  "The parameter useFakeRustfmt is set to true, but rustfmt is included in nativeBuildInputs. Either set useFakeRustfmt to false or remove rustfmt from nativeBuildInputs.";
+assert lib.asserts.assertMsg (!useFakeRustfmt -> rustfmtInNativeBuildInputs)
+  "The parameter useFakeRustfmt is set to false, but rustfmt is not included in nativeBuildInputs. Either set useFakeRustfmt to true or add rustfmt from nativeBuildInputs.";
+
+let
+  fakeRustfmt = writeShellScriptBin "rustfmt" ''
+    exit 0
+    '';
+  maybeDebugFlag = lib.optionalString (buildType != "release") "--debug";
+  maybeEnterBuildAndTestSubdir = lib.optionalString (buildAndTestSubdir != null) ''
+    export CARGO_TARGET_DIR="$(pwd)/target"
+    pushd "${buildAndTestSubdir}"
+  '';
+  maybeLeaveBuildAndTestSubdir = lib.optionalString (buildAndTestSubdir != null) "popd";
+
+  pgxPostgresMajor = lib.versions.major postgresql.version;
+  preBuildAndTest = ''
+    export PGX_HOME=$(mktemp -d)
+    export PGDATA="$PGX_HOME/data-${pgxPostgresMajor}/"
+    cargo-pgx pgx init "--pg${pgxPostgresMajor}" ${postgresql}/bin/pg_config
+    echo "unix_socket_directories = '$(mktemp -d)'" > "$PGDATA/postgresql.conf"
+
+    # This is primarily for Mac or other Nix systems that don't use the nixbld user.
+    export USER="$(whoami)"
+    pg_ctl start
+    createuser -h localhost --superuser --createdb "$USER" || true
+    pg_ctl stop
+  '';
+
+  argsForBuildRustPackage = builtins.removeAttrs args [ "postgresql" "useFakeRustfmt" ];
+
+  # so we don't accidentally `(rustPlatform.buildRustPackage argsForBuildRustPackage) // { ... }` because
+  # we forgot parentheses
+  finalArgs = argsForBuildRustPackage // {
+    buildInputs = (args.buildInputs or [ ]) ++ lib.optionals stdenv.isDarwin [ Security ];
+
+    nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [
+      cargo-pgx
+      postgresql
+      pkg-config
+      rustPlatform.bindgenHook
+    ] ++ lib.optionals useFakeRustfmt [ fakeRustfmt ];
+
+    buildPhase = ''
+      runHook preBuild
+
+      echo "Executing cargo-pgx buildPhase"
+      ${preBuildAndTest}
+      ${maybeEnterBuildAndTestSubdir}
+
+      NIX_PGLIBDIR="${postgresql}/lib" \
+      PGX_BUILD_FLAGS="--frozen -j $NIX_BUILD_CORES ${builtins.concatStringsSep " " cargoBuildFlags}" \
+      cargo-pgx pgx package \
+        --pg-config ${postgresql}/bin/pg_config \
+        ${maybeDebugFlag} \
+        --features "${builtins.concatStringsSep " " buildFeatures}" \
+        --out-dir "$out"
+
+      ${maybeLeaveBuildAndTestSubdir}
+
+      runHook postBuild
+    '';
+
+    preCheck = preBuildAndTest + args.preCheck or "";
+
+    installPhase = ''
+      runHook preInstall
+
+      echo "Executing buildPgxExtension install"
+
+      ${maybeEnterBuildAndTestSubdir}
+
+      cargo-pgx pgx stop all
+
+      mv $out/${postgresql}/* $out
+      rm -rf $out/nix
+
+      ${maybeLeaveBuildAndTestSubdir}
+
+      runHook postInstall
+    '';
+
+    PGX_PG_SYS_SKIP_BINDING_REWRITE = "1";
+    CARGO_BUILD_INCREMENTAL = "false";
+    RUST_BACKTRACE = "full";
+
+    checkNoDefaultFeatures = true;
+    checkFeatures = (args.checkFeatures or [ ]) ++ [ "pg_test pg${pgxPostgresMajor}" ];
+  };
+in
+rustPlatform.buildRustPackage finalArgs

pkgs/servers/sql/postgresql/ext/timescaledb_toolkit.nix

···
+{ lib
+, fetchFromGitHub
+, buildPgxExtension
+, postgresql
+, stdenv
+, nixosTests
+}:
+
+buildPgxExtension rec {
+  inherit postgresql;
+
+  pname = "timescaledb_toolkit";
+  version = "1.14.0";
+
+  src = fetchFromGitHub {
+    owner = "timescale";
+    repo = "timescaledb-toolkit";
+    rev = version;
+    sha256 = "sha256-ADmYALsCzZGqTX0XSkCif7ndvXwa8nEqddQpty4hbZ0=";
+  };
+
+  cargoSha256 = "sha256-ukjJ11LmfG+k8D20rj68i43gOWUN80nf3hIAjUWXihI=";
+  buildAndTestSubdir = "extension";
+
+  passthru.tests = {
+    timescaledb_toolkit = nixosTests.timescaledb;
+  };
+
+  # tests take really long
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Provide additional tools to ease all things analytic when using TimescaleDB";
+    homepage = "https://github.com/timescale/timescaledb-toolkit";
+    maintainers = with maintainers; [ typetetris ];
+    platforms = postgresql.meta.platforms;
+    license = licenses.asl20;
+
+    # as it needs to be used with timescaledb, simply use the condition from there
+    broken = versionOlder postgresql.version "12";
+  };
+}

pkgs/servers/sql/postgresql/packages.nix

···
 
     timescaledb = super.callPackage ./ext/timescaledb.nix { };
 
+    timescaledb_toolkit = super.callPackage ./ext/timescaledb_toolkit.nix { };
+
     tsearch_extras = super.callPackage ./ext/tsearch_extras.nix { };
 
     tds_fdw = super.callPackage ./ext/tds_fdw.nix { };

pkgs/top-level/all-packages.nix

···
   cargo-pgx = callPackage ../development/tools/rust/cargo-pgx {
     inherit (darwin.apple_sdk.frameworks) Security;
   };
+  buildPgxExtension = callPackage ../development/tools/rust/cargo-pgx/buildPgxExtension.nix {
+    inherit (darwin.apple_sdk.frameworks) Security;
+  };
   cargo-release = callPackage ../development/tools/rust/cargo-release { };
   cargo-rr = callPackage ../development/tools/rust/cargo-rr { };
   cargo-tarpaulin = callPackage ../development/tools/analysis/cargo-tarpaulin {