amazon-image: fetch metadata only in stage-2

This also removes automatic enablement/mounting of instance store swap
devices and ext3 filesystems. This behaviour is strongly opinionated
and shouldn't be enabled by default.

The unionfs behaviour never took effect anyway, because the AMI
manifest path only exists for instance store-backed AMIs, which have
not been supported by nixpkgs since
84742e22934d697e0476fab5a6c8886723ff92ef (2019).

nixos/doc/manual/from_md/release-notes/rl-2305.section.xml

···
   </section>
   <section xml:id="sec-release-23.05-incompatibilities">
     <title>Backward Incompatibilities</title>
-    <itemizedlist spacing="compact">
+    <itemizedlist>
       <listitem>
         <para>
           <literal>carnix</literal> and <literal>cratesIO</literal> has
···
           and
           <link xlink:href="https://github.com/kolloch/crate2nix">crate2nix</link>
           instead.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The EC2 image module no longer fetches instance metadata in
+          stage-1. This results in a significantly smaller initramfs,
+          since network drivers no longer need to be included, and
+          faster boots, since metadata fetching can happen in parallel
+          with startup of other services. This breaks services which
+          rely on metadata being present by the time stage-2 is entered.
+          Anything which reads EC2 metadata from
+          <literal>/etc/ec2-metadata</literal> should now have an
+          <literal>after</literal> dependency on
+          <literal>fetch-ec2-metadata.service</literal>
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The EC2 image module previously detected and automatically
+          mounted ext3-formatted instance store devices and partitions
+          in stage-1 (initramfs), storing <literal>/tmp</literal> on the
+          first discovered device. This behaviour, which only catered to
+          very specific use cases and could not be disabled, has been
+          removed. Users relying on this should provide their own
+          implementation, and probably use ext4 and perform the mount in
+          stage-2.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
+          The EC2 image module previously detected and activated
+          swap-formatted instance store devices and partitions in
+          stage-1 (initramfs). This behaviour has been removed. Users
+          relying on this should provide their own implementation.
         </para>
       </listitem>
     </itemizedlist>

nixos/doc/manual/release-notes/rl-2305.section.md

···
 
 - `carnix` and `cratesIO` has been removed due to being unmaintained, use alternatives such as [naersk](https://github.com/nix-community/naersk) and [crate2nix](https://github.com/kolloch/crate2nix) instead.
 
+- The EC2 image module no longer fetches instance metadata in stage-1. This results in a significantly smaller initramfs, since network drivers no longer need to be included, and faster boots, since metadata fetching can happen in parallel with startup of other services.
+  This breaks services which rely on metadata being present by the time stage-2 is entered. Anything which reads EC2 metadata from `/etc/ec2-metadata` should now have an `after` dependency on `fetch-ec2-metadata.service`
+
+- The EC2 image module previously detected and automatically mounted ext3-formatted instance store devices and partitions in stage-1 (initramfs), storing `/tmp` on the first discovered device. This behaviour, which only catered to very specific use cases and could not be disabled, has been removed. Users relying on this should provide their own implementation, and probably use ext4 and perform the mount in stage-2.
+
+- The EC2 image module previously detected and activated swap-formatted instance store devices and partitions in stage-1 (initramfs). This behaviour has been removed. Users relying on this should provide their own implementation.
+
 ## Other Notable Changes {#sec-release-23.05-notable-changes}
 
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

nixos/modules/virtualisation/amazon-image.nix

···
 
 let
   cfg = config.ec2;
-  metadataFetcher = import ./ec2-metadata-fetcher.nix {
-    inherit (pkgs) curl;
-    targetRoot = "$targetRoot/";
-    wgetExtraOptions = "-q";
-  };
 in
 
 {
···
     boot.extraModulePackages = [
       config.boot.kernelPackages.ena
     ];
-    boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ];
-    boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ];
+    boot.initrd.kernelModules = [ "xen-blkfront" ];
+    boot.initrd.availableKernelModules = [ "nvme" ];
     boot.kernelParams = [ "console=ttyS0,115200n8" "random.trust_cpu=on" ];
 
     # Prevent the nouveau kernel module from being loaded, as it
···
       terminal_input console serial
     '';
 
-    boot.initrd.network.enable = true;
-
-    # Mount all formatted ephemeral disks and activate all swap devices.
-    # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
-    # because the set of devices is dependent on the instance type
-    # (e.g. "m1.small" has one ephemeral filesystem and one swap device,
-    # while "m1.large" has two ephemeral filesystems and no swap
-    # devices).  Also, put /tmp and /var on /disk0, since it has a lot
-    # more space than the root device.  Similarly, "move" /nix to /disk0
-    # by layering a unionfs-fuse mount on top of it so we have a lot more space for
-    # Nix operations.
-    boot.initrd.postMountCommands =
-      ''
-        ${metadataFetcher}
-
-        diskNr=0
-        diskForUnionfs=
-        for device in /dev/xvd[abcde]*; do
-            if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
-            fsType=$(blkid -o value -s TYPE "$device" || true)
-            if [ "$fsType" = swap ]; then
-                echo "activating swap device $device..."
-                swapon "$device" || true
-            elif [ "$fsType" = ext3 ]; then
-                mp="/disk$diskNr"
-                diskNr=$((diskNr + 1))
-                if mountFS "$device" "$mp" "" ext3; then
-                    if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
-                fi
-            else
-                echo "skipping unknown device type $device"
-            fi
-        done
-
-        if [ -n "$diskForUnionfs" ]; then
-            mkdir -m 755 -p $targetRoot/$diskForUnionfs/root
-
-            mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
-            mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
-
-            if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then
-                mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
-                mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var
-
-                mkdir -p /unionfs-chroot/ro-nix
-                mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix
-
-                mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
-                mkdir -p /unionfs-chroot/rw-nix
-                mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix
-
-                unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
-            fi
-        fi
-      '';
-
-    boot.initrd.extraUtilsCommands =
-      ''
-        # We need swapon in the initrd.
-        copy_bin_and_libs ${pkgs.util-linux}/sbin/swapon
-      '';
+    systemd.services.fetch-ec2-metadata = {
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.wget ];
+      script = pkgs.callPackage ./ec2-metadata-fetcher.nix {
+        targetRoot = "/";
+        wgetExtraOptions = "";
+      };
+      serviceConfig.Type = "oneshot";
+    };
 
     # Allow root logins only using the SSH key that the user specified
     # at instance creation time.
···
 
     # Always include cryptsetup so that Charon can use it.
     environment.systemPackages = [ pkgs.cryptsetup ];
-
-    boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];
 
     # EC2 has its own NTP server provided by the hypervisor
     networking.timeServers = [ "169.254.169.123" ];

nixos/modules/virtualisation/ec2-data.nix

···
 
         wantedBy = [ "multi-user.target" "sshd.service" ];
         before = [ "sshd.service" ];
+        after = ["fetch-ec2-metadata.service"];
 
         path = [ pkgs.iproute2 ];