Merge pull request #292099 from hercules-ci/update-nixops_unstable

nixops_unstable to nixops_unstable_minimal.withPlugins migration + update

nixos/tests/nixops/default.nix

···
     #  - Alternatively, blocked on a NixOps 2 release
     #    https://github.com/NixOS/nixops/issues/1242
     # stable = testsLegacyNetwork { nixopsPkg = pkgs.nixops; };
-    unstable = testsForPackage { nixopsPkg = pkgs.nixops_unstable; };
+    unstable = testsForPackage { nixopsPkg = pkgs.nixops_unstable_minimal; };
 
     # inherit testsForPackage;
   };
···
           pkgs.hello
           pkgs.figlet
         ];
+        virtualisation.memorySize = 2048;
 
         # TODO: make this efficient, https://github.com/NixOS/nixpkgs/issues/180529
         system.includeBuildDependencies = true;

pkgs/applications/networking/cluster/nixops/default.nix

···
-{ python3 }:
+{ lib, python3, emptyFile }:
 
 let
-  python = python3.override {
-    packageOverrides = self: super: {
-      nixops = self.callPackage ./unwrapped.nix { };
-    } // (plugins self);
-  };
+  inherit (lib) extends;
+
+  # doc: https://github.com/NixOS/nixpkgs/pull/158781/files#diff-854251fa1fe071654921224671c8ba63c95feb2f96b2b3a9969c81676780053a
+  encapsulate = layerZero:
+    let
+      fixed = layerZero ({ extend = f: encapsulate (extends f layerZero); } // fixed);
+    in fixed.public;
+
+  nixopsContextBase = this: {
+
+    python = python3.override {
+      packageOverrides = self: super: {
+        nixops = self.callPackage ./unwrapped.nix { };
+      } // (this.plugins self super);
+    };
+
+    plugins = ps: _super: with ps; rec {
+      nixops-aws = callPackage ./plugins/nixops-aws.nix { };
+      nixops-digitalocean = callPackage ./plugins/nixops-digitalocean.nix { };
+      nixops-encrypted-links = callPackage ./plugins/nixops-encrypted-links.nix { };
+      nixops-gce = callPackage ./plugins/nixops-gce.nix { };
+      nixops-hercules-ci = callPackage ./plugins/nixops-hercules-ci.nix { };
+      nixops-hetzner = callPackage ./plugins/nixops-hetzner.nix { };
+      nixops-hetznercloud = callPackage ./plugins/nixops-hetznercloud.nix { };
+      nixops-libvirtd = callPackage ./plugins/nixops-libvirtd.nix { };
+      nixops-vbox = callPackage ./plugins/nixops-vbox.nix { };
+      nixos-modules-contrib = callPackage ./plugins/nixos-modules-contrib.nix { };
+
+      # aliases for backwards compatibility
+      nixops-gcp = nixops-gce;
+      nixops-virtd = nixops-libvirtd;
+      nixopsvbox = nixops-vbox;
+    };
+
+    # We should not reapply the overlay, but it tends to work out. (It's been this way since poetry2nix was dropped.)
+    availablePlugins = this.plugins this.python.pkgs this.python.pkgs;
+
+    selectedPlugins = [];
+
+    # selector is a function mapping pythonPackages to a list of plugins
+    # e.g. nixops_unstable.withPlugins (ps: with ps; [ nixops-aws ])
+    withPlugins = selector:
+      this.extend (this: _old: {
+        selectedPlugins = selector this.availablePlugins;
+      });
+
+    rawPackage = this.python.pkgs.toPythonApplication (this.python.pkgs.nixops.overridePythonAttrs (old: {
+      propagatedBuildInputs = old.propagatedBuildInputs ++ this.selectedPlugins;
+
+      # Propagating dependencies leaks them through $PYTHONPATH which causes issues
+      # when used in nix-shell.
+      postFixup = ''
+        rm $out/nix-support/propagated-build-inputs
+      '';
+    }));
+
+    # Extra package attributes that aren't derivation attributes, just like `mkDerivation`'s `passthru`.
+    extraPackageAttrs = {
+      inherit (this) selectedPlugins availablePlugins withPlugins python;
+      tests = this.rawPackage.tests // {
+        nixos = this.rawPackage.tests.nixos.passthru.override {
+          nixopsPkg = this.rawPackage;
+        };
+        commutative_addAvailablePlugins_withPlugins =
+          assert
+            (this.public.addAvailablePlugins (self: super: { inherit emptyFile; })).withPlugins (ps: [ emptyFile ])
+            ==
+            # Note that this value proves that the package is not instantiated until the end, where it's valid again.
+            (this.public.withPlugins (ps: [ emptyFile ])).addAvailablePlugins (self: super: { inherit emptyFile; });
+          emptyFile;
+      }
+        # Make sure we also test with a configuration that's been extended with a plugin.
+        // lib.optionalAttrs (this.selectedPlugins == [ ]) {
+        withAPlugin =
+          lib.recurseIntoAttrs
+            (this.withPlugins (ps: with ps; [ nixops-encrypted-links ])).tests;
+      };
+      overrideAttrs = f: this.extend (this: oldThis: {
+        rawPackage = oldThis.rawPackage.overrideAttrs f;
+      });
+      /**
+       * nixops.addAvailablePlugins: Overlay -> Package
+       *
+       * Add available plugins to the package. You probably also want to enable
+       * them with the `withPlugins` method.
+       */
+      addAvailablePlugins = newPlugins: this.extend (finalThis: oldThis: {
+        plugins = lib.composeExtensions oldThis.plugins newPlugins;
+      });
 
-  plugins = ps: with ps; rec {
-    nixops-aws = callPackage ./plugins/nixops-aws.nix { };
-    nixops-digitalocean = callPackage ./plugins/nixops-digitalocean.nix { };
-    nixops-encrypted-links = callPackage ./plugins/nixops-encrypted-links.nix { };
-    nixops-gce = callPackage ./plugins/nixops-gce.nix { };
-    nixops-hercules-ci = callPackage ./plugins/nixops-hercules-ci.nix { };
-    nixops-hetzner = callPackage ./plugins/nixops-hetzner.nix { };
-    nixops-hetznercloud = callPackage ./plugins/nixops-hetznercloud.nix { };
-    nixops-libvirtd = callPackage ./plugins/nixops-libvirtd.nix { };
-    nixops-vbox = callPackage ./plugins/nixops-vbox.nix { };
-    nixos-modules-contrib = callPackage ./plugins/nixos-modules-contrib.nix { };
+      # For those who need or dare.
+      internals = this;
+    };
 
-    # aliases for backwards compatibility
-    nixops-gcp = nixops-gce;
-    nixops-virtd = nixops-libvirtd;
-    nixopsvbox = nixops-vbox;
+    package = lib.lazyDerivation { outputs = [ "out" "dist" ]; derivation = this.rawPackage; } // this.extraPackageAttrs;
+
+    public = this.package;
   };
 
-  # selector is a function mapping pythonPackages to a list of plugins
-  # e.g. nixops_unstable.withPlugins (ps: with ps; [ nixops-aws ])
-  withPlugins = selector: let
-    selected = selector (plugins python.pkgs);
-  in python.pkgs.toPythonApplication (python.pkgs.nixops.overridePythonAttrs (old: {
-    propagatedBuildInputs = old.propagatedBuildInputs ++ selected;
+  minimal = encapsulate nixopsContextBase;
 
-    # Propagating dependencies leaks them through $PYTHONPATH which causes issues
-    # when used in nix-shell.
-    postFixup = ''
-      rm $out/nix-support/propagated-build-inputs
-    '';
+in
+{
+  nixops_unstable_minimal = minimal;
 
-    passthru = old.passthru // {
-      plugins = plugins python.pkgs;
-      inherit withPlugins python;
-    };
-  }));
-in withPlugins (ps: [
-  ps.nixops-aws
-  ps.nixops-digitalocean
-  ps.nixops-encrypted-links
-  ps.nixops-gce
-  ps.nixops-hercules-ci
-  ps.nixops-hetzner
-  ps.nixops-hetznercloud
-  ps.nixops-libvirtd
-  ps.nixops-vbox
-])
+  # Not recommended; too fragile.
+  nixops_unstable_full = minimal.withPlugins (ps: [
+    ps.nixops-aws
+    ps.nixops-digitalocean
+    ps.nixops-encrypted-links
+    ps.nixops-gce
+    ps.nixops-hercules-ci
+    ps.nixops-hetzner
+    ps.nixops-hetznercloud
+    ps.nixops-libvirtd
+    ps.nixops-vbox
+  ]);
+}

pkgs/applications/networking/cluster/nixops/plugins/nixops-aws.nix

···
 
 buildPythonPackage {
   pname = "nixops-aws";
-  version = "unstable-2023-08-09";
+  version = "unstable-2024-02-29";
   pyproject = true;
 
   src = fetchFromGitHub {
     owner = "NixOS";
     repo = "nixops-aws";
-    rev = "8802d1cda9004ec1362815292c2a8ab95e6d64e8";
-    hash = "sha256-i0KjFrwpDHRch9jorccdVwnjAQiORClDUqm2R2xvwuU=";
+    rev = "d173b2f14ec767d782ceab45fb22b32fe3b5a1f7";
+    hash = "sha256-ocTtc7POt1bugb9Bki2ew2Eh5uc933GftNw1twoOJsc=";
   };
 
   postPatch = ''

pkgs/applications/networking/cluster/nixops/unwrapped.nix

···
 
 buildPythonApplication rec {
   pname = "nixops";
-  version = "unstable-2023-12-17";
+  version = "unstable-2024-02-28";
   pyproject = true;
 
   src = fetchFromGitHub {
     owner = "NixOS";
     repo = "nixops";
-    rev = "053668e849bb369973cf265b7e8f38e66ef70138";
-    hash = "sha256-Kus1Ls1tT8fVGLX0NakRXmjuz5/J/tfqU4TLOkiZqvo=";
+    rev = "08feccb14074c5434f3e483d19a7f7d9bfcdb669";
+    hash = "sha256-yWeF5apQJdChjYVSOyH6LYjJYGa1RL68LRHrSgZ9l8U=";
   };
 
   postPatch = ''
···
   pythonImportsCheck = [ "nixops" ];
 
   passthru = {
-    tests.nixops = nixosTests.nixops.unstable;
+    tests.nixos = nixosTests.nixops.unstable;
     updateScript = unstableGitUpdater {};
   };
 

pkgs/top-level/aliases.nix

···
   nix_2_4 = nixVersions.nix_2_4;
   nix_2_5 = nixVersions.nix_2_5;
   nix_2_6 = nixVersions.nix_2_6;
-  nixops = throw "'nixops' has been removed. Please use 'nixops_unstable' for the time being."; # Added 2023-10-26
+  nixops = throw "'nixops' has been removed. Please use 'nixops_unstable_minimal' for the time being. E.g. nixops_unstable_minimal.withPlugins (ps: [ ps.nixops-gce ])"; # Added 2023-10-26
   nixopsUnstable = nixops_unstable; # Added 2022-03-03
+
+  # When the nixops_unstable alias is removed, nixops_unstable_minimal can be renamed to nixops_unstable.
+  nixops_unstable = throw "nixops_unstable has been replaced. Please use for example 'nixops_unstable_minimal.withPlugins (ps: [ ps.nixops-gce ps.nixops-encrypted-links ])' instead"; # Added 2024-02-28
+
   nixosTest = testers.nixosTest; # Added 2022-05-05
   nmap-unfree = nmap; # Added 2021-04-06
   nodejs_14 = throw "nodejs_14 has been removed as it is EOL."; # Added 2023-10-30

pkgs/top-level/all-packages.nix

···
 
   nixStatic = pkgsStatic.nix;
 
-  nixops_unstable = callPackage ../applications/networking/cluster/nixops { };
+  inherit (callPackages ../applications/networking/cluster/nixops { })
+    nixops_unstable_minimal
+
+    # Not recommended; too fragile
+    nixops_unstable_full;
+
+  # Useful with ofborg, e.g. commit prefix `nixops_unstablePlugins.nixops-aws: ...` to trigger automatically.
+  nixops_unstablePlugins = recurseIntoAttrs nixops_unstable_minimal.availablePlugins;
 
   /*
     Evaluate a NixOS configuration using this evaluation of Nixpkgs.