Merge pull request #256226 from ElvishJerricco/systemd-stage-1-testing-backdoor

nixos/doc/manual/release-notes/rl-2311.section.md

···
 - `teleport` has been upgraded from major version 12 to major version 14. Please see upstream [upgrade instructions](https://goteleport.com/docs/management/operations/upgrading/) and release notes for versions [13](https://goteleport.com/docs/changelog/#1300-050823) and [14](https://goteleport.com/docs/changelog/#1400-092023). Note that Teleport does not officially support upgrades across more than one major version at a time. If you're running Teleport server components, it is recommended to first upgrade to an intermediate 13.x version by setting `services.teleport.package = pkgs.teleport_13`. Afterwards, this option can be removed to upgrade to the default version (14).
 
 - The Linux kernel module `msr` (see [`msr(4)`](https://man7.org/linux/man-pages/man4/msr.4.html)), which provides an interface to read and write the model-specific registers (MSRs) of an x86 CPU, can now be configured via `hardware.cpu.x86.msr`.
+
+- There is a new NixOS option when writing NixOS tests `testing.initrdBackdoor`, that enables `backdoor.service` in initrd. Requires `boot.initrd.systemd.enable` to be enabled. Boot will pause in stage 1 at `initrd.target`, and will listen for commands from the `Machine` python interface, just like stage 2 normally does. This enables commands to be sent to test and debug stage 1. Use `machine.switch_root()` to leave stage 1 and proceed to stage 2.

nixos/lib/test-driver/test_driver/machine.py

···
     def run_callbacks(self) -> None:
         for callback in self.callbacks:
             callback()
+
+    def switch_root(self) -> None:
+        """
+        Transition from stage 1 to stage 2. This requires the
+        machine to be configured with `testing.initrdBackdoor = true`
+        and `boot.initrd.systemd.enable = true`.
+        """
+        self.wait_for_unit("initrd.target")
+        self.execute(
+            "systemctl isolate --no-block initrd-switch-root.target 2>/dev/null >/dev/null",
+            check_return=False,
+            check_output=False,
+        )
+        self.wait_for_console_text(r"systemd\[1\]:.*Switching root\.")
+        self.connected = False
+        self.connect()

nixos/modules/testing/test-instrumentation.nix

···
 with lib;
 
 let
+  cfg = config.testing;
+
   qemu-common = import ../../lib/qemu-common.nix { inherit lib pkgs; };
+
+  backdoorService = {
+    wantedBy = [ "sysinit.target" ];
+    unitConfig.DefaultDependencies = false;
+    conflicts = [ "shutdown.target" "initrd-switch-root.target" ];
+    before = [ "shutdown.target" "initrd-switch-root.target" ];
+    requires = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ];
+    after = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ];
+    script =
+      ''
+        export USER=root
+        export HOME=/root
+        export DISPLAY=:0.0
+
+        if [[ -e /etc/profile ]]; then
+            source /etc/profile
+        fi
+
+        # Don't use a pager when executing backdoor
+        # actions. Because we use a tty, commands like systemctl
+        # or nix-store get confused into thinking they're running
+        # interactively.
+        export PAGER=
+
+        cd /tmp
+        exec < /dev/hvc0 > /dev/hvc0
+        while ! exec 2> /dev/${qemu-common.qemuSerialDevice}; do sleep 0.1; done
+        echo "connecting to host..." >&2
+        stty -F /dev/hvc0 raw -echo # prevent nl -> cr/nl conversion
+        # The following line is essential since it signals to
+        # the test driver that the shell is ready.
+        # See: the connect method in the Machine class.
+        echo "Spawning backdoor root shell..."
+        # Passing the terminal device makes bash run non-interactively.
+        # Otherwise we get errors on the terminal because bash tries to
+        # setup things like job control.
+        # Note: calling bash explicitly here instead of sh makes sure that
+        # we can also run non-NixOS guests during tests.
+        PS1= exec /usr/bin/env bash --norc /dev/hvc0
+      '';
+      serviceConfig.KillSignal = "SIGHUP";
+  };
+
 in
 
 {
 
+  options.testing = {
+
+    initrdBackdoor = lib.mkEnableOption (lib.mdDoc ''
+      enable backdoor.service in initrd. Requires
+      boot.initrd.systemd.enable to be enabled. Boot will pause in
+      stage 1 at initrd.target, and will listen for commands from the
+      Machine python interface, just like stage 2 normally does. This
+      enables commands to be sent to test and debug stage 1. Use
+      machine.switch_root() to leave stage 1 and proceed to stage 2.
+    '');
+
+  };
+
   config = {
 
-    systemd.services.backdoor =
-      { wantedBy = [ "multi-user.target" ];
-        requires = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ];
-        after = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ];
-        script =
-          ''
-            export USER=root
-            export HOME=/root
-            export DISPLAY=:0.0
+    assertions = [
+      {
+        assertion = cfg.initrdBackdoor -> config.boot.initrd.systemd.enable;
+        message = ''
+          testing.initrdBackdoor requires boot.initrd.systemd.enable to be enabled.
+        '';
+      }
+    ];
 
-            source /etc/profile
+    systemd.services.backdoor = backdoorService;
+
+    boot.initrd.systemd = lib.mkMerge [
+      {
+        contents."/etc/systemd/journald.conf".text = ''
+          [Journal]
+          ForwardToConsole=yes
+          MaxLevelConsole=debug
+        '';
+
+        extraConfig = config.systemd.extraConfig;
+      }
+
+      (lib.mkIf cfg.initrdBackdoor {
+        # Implemented in machine.switch_root(). Suppress the unit by
+        # making it a noop without removing it, which would break
+        # initrd-parse-etc.service
+        services.initrd-cleanup.serviceConfig.ExecStart = [
+          # Reset
+          ""
+          # noop
+          "/bin/true"
+        ];
 
-            # Don't use a pager when executing backdoor
-            # actions. Because we use a tty, commands like systemctl
-            # or nix-store get confused into thinking they're running
-            # interactively.
-            export PAGER=
+        services.backdoor = backdoorService;
 
-            cd /tmp
-            exec < /dev/hvc0 > /dev/hvc0
-            while ! exec 2> /dev/${qemu-common.qemuSerialDevice}; do sleep 0.1; done
-            echo "connecting to host..." >&2
-            stty -F /dev/hvc0 raw -echo # prevent nl -> cr/nl conversion
-            # The following line is essential since it signals to
-            # the test driver that the shell is ready.
-            # See: the connect method in the Machine class.
-            echo "Spawning backdoor root shell..."
-            # Passing the terminal device makes bash run non-interactively.
-            # Otherwise we get errors on the terminal because bash tries to
-            # setup things like job control.
-            # Note: calling bash explicitly here instead of sh makes sure that
-            # we can also run non-NixOS guests during tests.
-            PS1= exec /usr/bin/env bash --norc /dev/hvc0
-          '';
-        serviceConfig.KillSignal = "SIGHUP";
-      };
+        contents."/usr/bin/env".source = "${pkgs.coreutils}/bin/env";
+      })
+    ];
 
     # Prevent agetty from being instantiated on the serial device, since it
     # interferes with the backdoor (writes to it will randomly fail
···
         MaxLevelConsole=debug
       '';
 
-    boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
-      [Journal]
-      ForwardToConsole=yes
-      MaxLevelConsole=debug
-    '';
-
     systemd.extraConfig = ''
       # Don't clobber the console with duplicate systemd messages.
       ShowStatus=no
···
       DefaultTimeoutStartSec=300
       DefaultDeviceTimeoutSec=300
     '';
-
-    boot.initrd.systemd.extraConfig = config.systemd.extraConfig;
 
     boot.consoleLogLevel = 7;
 

nixos/tests/systemd-initrd-modprobe.nix

···
   name = "systemd-initrd-modprobe";
 
   nodes.machine = { pkgs, ... }: {
+    testing.initrdBackdoor = true;
     boot.initrd.systemd.enable = true;
     boot.initrd.kernelModules = [ "loop" ]; # Load module in initrd.
     boot.extraModprobeConfig = ''
···
   };
 
   testScript = ''
+    machine.wait_for_unit("initrd.target")
+    max_loop = machine.succeed("cat /sys/module/loop/parameters/max_loop")
+    assert int(max_loop) == 42, "Parameter should be respected for initrd kernel modules"
+
+    # Make sure it sticks in stage 2
+    machine.switch_root()
     machine.wait_for_unit("multi-user.target")
     max_loop = machine.succeed("cat /sys/module/loop/parameters/max_loop")
     assert int(max_loop) == 42, "Parameter should be respected for initrd kernel modules"

nixos/tests/systemd-initrd-networkd-ssh.nix

···
 
   nodes = {
     server = { config, pkgs, ... }: {
-      environment.systemPackages = [ pkgs.cryptsetup ];
-      boot.loader.systemd-boot.enable = true;
-      boot.loader.timeout = 0;
-      virtualisation = {
-        emptyDiskImages = [ 4096 ];
-        useBootLoader = true;
-        # Booting off the encrypted disk requires an available init script from
-        # the Nix store
-        mountHostNixStore = true;
-        useEFIBoot = true;
-      };
-
-      specialisation.encrypted-root.configuration = {
-        virtualisation.rootDevice = "/dev/mapper/root";
-        virtualisation.fileSystems."/".autoFormat = true;
-        boot.initrd.luks.devices = lib.mkVMOverride {
-          root.device = "/dev/vdb";
-        };
-        boot.initrd.systemd.enable = true;
-        boot.initrd.network = {
+      testing.initrdBackdoor = true;
+      boot.initrd.systemd.enable = true;
+      boot.initrd.systemd.contents."/etc/msg".text = "foo";
+      boot.initrd.network = {
+        enable = true;
+        ssh = {
           enable = true;
-          ssh = {
-            enable = true;
-            authorizedKeys = [ (lib.readFile ./initrd-network-ssh/id_ed25519.pub) ];
-            port = 22;
-            # Terrible hack so it works with useBootLoader
-            hostKeys = [ { outPath = "${./initrd-network-ssh/ssh_host_ed25519_key}"; } ];
-          };
+          authorizedKeys = [ (lib.readFile ./initrd-network-ssh/id_ed25519.pub) ];
+          port = 22;
+          hostKeys = [ ./initrd-network-ssh/ssh_host_ed25519_key ];
         };
       };
     };
···
         status, _ = client.execute("nc -z server 22")
         return status == 0
 
-    server.wait_for_unit("multi-user.target")
-    server.succeed(
-        "echo somepass | cryptsetup luksFormat --type=luks2 /dev/vdb",
-        "bootctl set-default nixos-generation-1-specialisation-encrypted-root.conf",
-        "sync",
-    )
-    server.shutdown()
-    server.start()
-
     client.wait_for_unit("network.target")
     with client.nested("waiting for SSH server to come up"):
         retry(ssh_is_up)
 
-    client.succeed(
-        "echo somepass | ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'systemd-tty-ask-password-agent' & exit"
+    msg = client.succeed(
+        "ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'cat /etc/msg'"
     )
+    assert "foo" in msg
 
+    server.switch_root()
     server.wait_for_unit("multi-user.target")
-    server.succeed("mount | grep '/dev/mapper/root on /'")
   '';
 })

nixos/tests/systemd-initrd-networkd.nix

···
-import ./make-test-python.nix ({ pkgs, lib, ... }: {
-  name = "systemd-initrd-network";
-  meta.maintainers = [ lib.maintainers.elvishjerricco ];
+{ system ? builtins.currentSystem
+, config ? {}
+, pkgs ? import ../.. { inherit system config; }
+, lib ? pkgs.lib
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+
+let
+  inherit (lib.maintainers) elvishjerricco;
+
+  common = {
+    boot.initrd.systemd = {
+      enable = true;
+      network.wait-online.timeout = 10;
+      network.wait-online.anyInterface = true;
+      targets.network-online.requiredBy = [ "initrd.target" ];
+      services.systemd-networkd-wait-online.requiredBy =
+        [ "network-online.target" ];
+      initrdBin = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ];
+    };
+    testing.initrdBackdoor = true;
+    boot.initrd.network.enable = true;
+  };
+
+  mkFlushTest = flush: script: makeTest {
+    name = "systemd-initrd-network-${lib.optionalString (!flush) "no-"}flush";
+    meta.maintainers = [ elvishjerricco ];
+
+    nodes.machine = {
+      imports = [ common ];
 
-  nodes = let
-    mkFlushTest = flush: script: { ... }: {
-      boot.initrd.systemd.enable = true;
-      boot.initrd.network = {
-        enable = true;
-        flushBeforeStage2 = flush;
-      };
+      boot.initrd.network.flushBeforeStage2 = flush;
       systemd.services.check-flush = {
         requiredBy = ["multi-user.target"];
         before = ["network-pre.target" "multi-user.target"];
···
         inherit script;
       };
     };
-  in {
-    basic = { ... }: {
-      boot.initrd.network.enable = true;
 
-      boot.initrd.systemd = {
-        enable = true;
-        # Enable network-online to fail the test in case of timeout
-        network.wait-online.timeout = 10;
-        network.wait-online.anyInterface = true;
-        targets.network-online.requiredBy = [ "initrd.target" ];
-        services.systemd-networkd-wait-online.requiredBy =
-          [ "network-online.target" ];
-
-          initrdBin = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ];
-          services.check = {
-            requiredBy = [ "initrd.target" ];
-            before = [ "initrd.target" ];
-            after = [ "network-online.target" ];
-            serviceConfig.Type = "oneshot";
-            path = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ];
-            script = ''
-              ip addr | grep 10.0.2.15 || exit 1
-              ping -c1 10.0.2.2 || exit 1
-            '';
-          };
-      };
-    };
+    testScript = ''
+      machine.wait_for_unit("network-online.target")
+      machine.succeed(
+          "ip addr | grep 10.0.2.15",
+          "ping -c1 10.0.2.2",
+      )
+      machine.switch_root()
 
-    doFlush = mkFlushTest true ''
-      if ip addr | grep 10.0.2.15; then
-        echo "Network configuration survived switch-root; flushBeforeStage2 failed"
-        exit 1
-      fi
+      machine.wait_for_unit("multi-user.target")
     '';
+  };
 
-    dontFlush = mkFlushTest false ''
-      if ! (ip addr | grep 10.0.2.15); then
-        echo "Network configuration didn't survive switch-root"
-        exit 1
-      fi
+in {
+  basic = makeTest {
+    name = "systemd-initrd-network";
+    meta.maintainers = [ elvishjerricco ];
+
+    nodes.machine = common;
+
+    testScript = ''
+      machine.wait_for_unit("network-online.target")
+      machine.succeed(
+          "ip addr | grep 10.0.2.15",
+          "ping -c1 10.0.2.2",
+      )
+      machine.switch_root()
+
+      # Make sure the systemd-network user was set correctly in initrd
+      machine.wait_for_unit("multi-user.target")
+      machine.succeed("[ $(stat -c '%U,%G' /run/systemd/netif/links) = systemd-network,systemd-network ]")
+      machine.succeed("ip addr show >&2")
+      machine.succeed("ip route show >&2")
     '';
   };
 
-  testScript = ''
-    start_all()
-    basic.wait_for_unit("multi-user.target")
-    doFlush.wait_for_unit("multi-user.target")
-    dontFlush.wait_for_unit("multi-user.target")
-    # Make sure the systemd-network user was set correctly in initrd
-    basic.succeed("[ $(stat -c '%U,%G' /run/systemd/netif/links) = systemd-network,systemd-network ]")
-    basic.succeed("ip addr show >&2")
-    basic.succeed("ip route show >&2")
+  doFlush = mkFlushTest true ''
+    if ip addr | grep 10.0.2.15; then
+      echo "Network configuration survived switch-root; flushBeforeStage2 failed"
+      exit 1
+    fi
   '';
-})
+
+  dontFlush = mkFlushTest false ''
+    if ! (ip addr | grep 10.0.2.15); then
+      echo "Network configuration didn't survive switch-root"
+      exit 1
+    fi
+  '';
+}

nixos/tests/systemd-initrd-simple.nix

···
   name = "systemd-initrd-simple";
 
   nodes.machine = { pkgs, ... }: {
-    boot.initrd.systemd = {
-      enable = true;
-      emergencyAccess = true;
-    };
+    testing.initrdBackdoor = true;
+    boot.initrd.systemd.enable = true;
     virtualisation.fileSystems."/".autoResize = true;
   };
 
   testScript = ''
     import subprocess
+
+    with subtest("testing initrd backdoor"):
+        machine.wait_for_unit("initrd.target")
+        machine.succeed("systemctl status initrd-fs.target")
+        machine.switch_root()
 
     with subtest("handover to stage-2 systemd works"):
         machine.wait_for_unit("multi-user.target")
···
         subprocess.check_call(["qemu-img", "resize", "vm-state-machine/machine.qcow2", "+1G"])
 
         machine.start()
+        machine.switch_root()
         newAvail = machine.succeed("df --output=avail / | sed 1d")
 
         assert int(oldAvail) < int(newAvail), "File system did not grow"