Merge pull request #236389 from Enzime/darwin-builder

darwin-builder: use port 31022 by default

doc/builders/special/darwin-builder.section.md

···
-# darwin.builder {#sec-darwin-builder}
+# darwin.linux-builder {#sec-darwin-builder}
 
-`darwin.builder` provides a way to bootstrap a Linux builder on a macOS machine.
+`darwin.linux-builder` provides a way to bootstrap a Linux builder on a macOS machine.
 
 This requires macOS version 12.4 or later.
 
-This also requires that port 22 on your machine is free (since Nix does not
-permit specifying a non-default SSH port for builders).
+The builder runs on host port 31022 by default.
+You can change it by overriding `virtualisation.darwin-builder.hostPort`.
+See the [example](#sec-darwin-builder-example-flake).
 
 You will also need to be a trusted user for your Nix installation.  In other
 words, your `/etc/nix/nix.conf` should have something like:
···
 To launch the builder, run the following flake:
 
 ```ShellSession
-$ nix run nixpkgs#darwin.builder
+$ nix run nixpkgs#darwin.linux-builder
 ```
 
 That will prompt you to enter your `sudo` password:
···
 ```
 # - Replace ${ARCH} with either aarch64 or x86_64 to match your host machine
 # - Replace ${MAX_JOBS} with the maximum number of builds (pick 4 if you're not sure)
-builders = ssh-ng://builder@localhost ${ARCH}-linux /etc/nix/builder_ed25519 ${MAX_JOBS} - - - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUpCV2N4Yi9CbGFxdDFhdU90RStGOFFVV3JVb3RpQzVxQkorVXVFV2RWQ2Igcm9vdEBuaXhvcwo=
+builders = ssh-ng://builder@linux-builder ${ARCH}-linux /etc/nix/builder_ed25519 ${MAX_JOBS} - - - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUpCV2N4Yi9CbGFxdDFhdU90RStGOFFVV3JVb3RpQzVxQkorVXVFV2RWQ2Igcm9vdEBuaXhvcwo=
 
 # Not strictly necessary, but this will reduce your disk utilization
 builders-use-substitutes = true
+```
+
+To allow Nix to connect to a builder not running on port 22, you will also need to create a new file at `/etc/ssh/ssh_config.d/100-linux-builder.conf`:
+
+```
+Host linux-builder
+  Hostname localhost
+  HostKeyAlias linux-builder
+  Port 31022
 ```
 
 … and then restart your Nix daemon to apply the change:

nixos/modules/profiles/macos-builder.nix

···
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 
 let
   keysDirectory = "/var/keys";
···
        '';
     };
     hostPort = mkOption {
-      default = 22;
+      default = 31022;
       type = types.int;
-      example = 31022;
+      example = 22;
       description = ''
         The localhost host port to forward TCP to the guest port.
       '';
···
 
         hostPkgs = config.virtualisation.host.pkgs;
 
-  script = hostPkgs.writeShellScriptBin "create-builder" (
+        script = hostPkgs.writeShellScriptBin "create-builder" (
           # When running as non-interactively as part of a DarwinConfiguration the working directory
           # must be set to a writeable directory.
         (if cfg.workingDirectory != "." then ''
           ${hostPkgs.coreutils}/bin/mkdir --parent "${cfg.workingDirectory}"
           cd "${cfg.workingDirectory}"
-  '' else "") + ''
+        '' else "") + ''
           KEYS="''${KEYS:-./keys}"
           ${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
           PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
···
           if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
             (set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
           fi
-          KEYS="$(${hostPkgs.nix}/bin/nix-store --add "$KEYS")" ${config.system.build.vm}/bin/run-nixos-vm
+          KEYS="$(${hostPkgs.nix}/bin/nix-store --add "$KEYS")" ${lib.getExe config.system.build.vm}
         '');
 
       in
···
         Please inspect the trace of the following command to figure out which module
         has a dependency on stateVersion.
 
-          nix-instantiate --attr darwin.builder --show-trace
+          nix-instantiate --attr darwin.linux-builder --show-trace
       '');
     };
 
···
       # This ensures that anything built on the guest isn't lost when the guest is
       # restarted.
       writableStoreUseTmpfs = false;
+
+      # Pass certificates from host to the guest otherwise when custom CA certificates
+      # are required we can't use the cached builder.
+      useHostCerts = true;
     };
   };
 }

nixos/modules/security/ca.nix

···
 {
 
   options = {
+    security.pki.installCACerts = mkEnableOption "Add CA certificates to system" // {
+      default = true;
+      internal = true;
+    };
 
     security.pki.certificateFiles = mkOption {
       type = types.listOf types.path;
···
 
   };
 
-  config = {
+  config = mkIf cfg.installCACerts {
 
     # NixOS canonical location + Debian/Ubuntu/Arch/Gentoo compatibility.
     environment.etc."ssl/certs/ca-certificates.crt".source = caBundle;

nixos/modules/virtualisation/qemu-vm.nix

···
       # Create a directory for exchanging data with the VM.
       mkdir -p "$TMPDIR/xchg"
 
+      ${lib.optionalString cfg.useHostCerts
+      ''
+        mkdir -p "$TMPDIR/certs"
+        if [ -e "$NIX_SSL_CERT_FILE" ]; then
+          cp -L "$NIX_SSL_CERT_FILE" "$TMPDIR"/certs/ca-certificates.crt
+        else
+          echo \$NIX_SSL_CERT_FILE should point to a valid file if virtualisation.useHostCerts is enabled.
+        fi
+      ''}
+
       ${lib.optionalString cfg.useEFIBoot
       ''
         # Expose EFI variables, it's useful even when we are not using a bootloader (!).
···
           '';
       };
 
-
     virtualisation.bios =
       mkOption {
         type = types.nullOr types.package;
···
             An alternate BIOS (such as `qboot`) with which to start the VM.
             Should contain a file named `bios.bin`.
             If `null`, QEMU's builtin SeaBIOS will be used.
+          '';
+      };
+
+    virtualisation.useHostCerts =
+      mkOption {
+        type = types.bool;
+        default = false;
+        description =
+          lib.mdDoc ''
+            If enabled, when `NIX_SSL_CERT_FILE` is set on the host,
+            pass the CA certificates from the host to the VM.
           '';
       };
 
···
         source = ''"''${SHARED_DIR:-$TMPDIR/xchg}"'';
         target = "/tmp/shared";
       };
+      certs = mkIf cfg.useHostCerts {
+        source = ''"$TMPDIR"/certs'';
+        target = "/etc/ssl/certs";
+      };
     };
+
+    security.pki.installCACerts = mkIf cfg.useHostCerts false;
 
     virtualisation.qemu.networkingOptions =
       let

pkgs/top-level/darwin-packages.nix

···
 , generateSplicesForMkScope, makeScopeWithSplicing
 , stdenv
 , preLibcCrossHeaders
+, config
 }:
 
 let
···
   discrete-scroll = callPackage ../os-specific/darwin/discrete-scroll { };
 
   # See doc/builders/special/darwin-builder.section.md
-  builder =
+  linux-builder = lib.makeOverridable ({ modules }:
     let
       toGuest = builtins.replaceStrings [ "darwin" ] [ "linux" ];
 
···
         configuration = {
           imports = [
             ../../nixos/modules/profiles/macos-builder.nix
-          ];
+          ] ++ modules;
 
           virtualisation.host = { inherit pkgs; };
         };
···
       };
 
     in
-      nixos.config.system.build.macos-builder-installer;
+      nixos.config.system.build.macos-builder-installer) { modules = [ ]; };
+
+} // lib.optionalAttrs config.allowAliases {
+  builder = throw "'darwin.builder' has been changed and renamed to 'darwin.linux-builder'. The default ssh port is now 31022. Please update your configuration or override the port back to 22. See https://nixos.org/manual/nixpkgs/unstable/#sec-darwin-builder"; # added 2023-07-06
 })