nix/vm: store service data in a shared folder on the host

This also switches away from `nixos-shell` in the process as by this
point it wasn't really adding much to our setup except inflexibility.

Signed-off-by: Winter <[e-mail]>

.gitignore

···
 *.rdb
 .envrc
 # Created if following hacking.md
-genjwks.out
+genjwks.out
+/nix/vm-data

docs/hacking.md

···
 You will also need to set the `$TANGLED_VM_SPINDLE_OWNER`
 variable to some value. If you don't want to [set up a
 spindle](#running-a-spindle), you can use any placeholder
-value. 
+value.
 
-You can now start a lightweight NixOS VM using
-`nixos-shell` like so:
+You can now start a lightweight NixOS VM like so:
 
 ```bash
-nix run .#vm
-# or nixos-shell --flake .#vm
+nix run --impure .#vm
 
-# hit Ctrl-a + c + q to exit the VM
+# type `poweroff` at the shell to exit the VM
 ```
 
 This starts a knot on port 6000, a spindle on port 6555

flake.nix

···
         program = ''${tailwind-watcher}/bin/run'';
       };
       vm = let
-        system =
+        guestSystem =
           if pkgs.stdenv.hostPlatform.isAarch64
-          then "aarch64"
-          else "x86_64";
-
-        nixos-shell = pkgs.nixos-shell.overrideAttrs (old: {
-          patches =
-            (old.patches or [])
-            ++ [
-              # https://github.com/Mic92/nixos-shell/pull/94
-              (pkgs.fetchpatch {
-                name = "fix-foreign-vm.patch";
-                url = "https://github.com/Mic92/nixos-shell/commit/113e4cc55ae236b5b0b1fbd8b321e9b67c77580e.patch";
-                hash = "sha256-eauetBK0wXAOcd9PYbExokNCiwz2QyFnZ4FnwGi9VCo=";
-              })
-            ];
-        });
+          then "aarch64-linux"
+          else "x86_64-linux";
       in {
         type = "app";
-        program = toString (pkgs.writeShellScript "vm" ''
-          ${nixos-shell}/bin/nixos-shell --flake .#vm-${system} --guest-system ${system}-linux
-        '');
+        program =
+          (pkgs.writeShellApplication {
+            name = "launch-vm";
+            text = ''
+              rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1)
+              cd "$rootDir"
+
+              mkdir -p nix/vm-data/{knot,repos,spindle,spindle-logs}
+
+              export TANGLED_VM_DATA_DIR="$rootDir/nix/vm-data"
+              exec ${pkgs.lib.getExe
+                (import ./nix/vm.nix {
+                  inherit nixpkgs self;
+                  system = guestSystem;
+                  hostSystem = system;
+                }).config.system.build.vm}
+            '';
+          })
+          + /bin/launch-vm;
       };
       gomod2nix = {
         type = "app";
···
       imports = [./nix/modules/spindle.nix];
 
       services.tangled-spindle.package = lib.mkDefault self.packages.${pkgs.system}.spindle;
-    };
-    nixosConfigurations.vm-x86_64 = import ./nix/vm.nix {
-      inherit self nixpkgs;
-      system = "x86_64-linux";
-    };
-    nixosConfigurations.vm-aarch64 = import ./nix/vm.nix {
-      inherit self nixpkgs;
-      system = "aarch64-linux";
     };
   };
 }

nix/vm.nix

···
 {
   nixpkgs,
   system,
+  hostSystem,
   self,
 }: let
   envVar = name: let
···
       self.nixosModules.knot
       self.nixosModules.spindle
       ({
+        lib,
         config,
         pkgs,
         ...
       }: {
-        nixos-shell = {
-          inheritPath = false;
-          mounts = {
-            mountHome = false;
-            mountNixProfile = false;
-          };
-        };
-        virtualisation = {
+        virtualisation.vmVariant.virtualisation = {
+          host.pkgs = import nixpkgs {system = hostSystem;};
+
+          graphics = false;
           memorySize = 2048;
           diskSize = 10 * 1024;
           cores = 2;
···
               guest.port = 6555;
             }
           ];
+          sharedDirectories = {
+            # We can't use the 9p mounts directly for most of these
+            # as SQLite is incompatible with them. So instead we
+            # mount the shared directories to a different location
+            # and copy the contents around on service start/stop.
+            knotData = {
+              source = "$TANGLED_VM_DATA_DIR/knot";
+              target = "/mnt/knot-data";
+            };
+            spindleData = {
+              source = "$TANGLED_VM_DATA_DIR/spindle";
+              target = "/mnt/spindle-data";
+            };
+            spindleLogs = {
+              source = "$TANGLED_VM_DATA_DIR/spindle-logs";
+              target = "/var/log/spindle";
+            };
+          };
         };
+        # This is fine because any and all ports that are forwarded to host are explicitly marked above, we don't need a separate guest firewall
+        networking.firewall.enable = false;
         services.getty.autologinUser = "root";
         environment.systemPackages = with pkgs; [curl vim git sqlite litecli];
-        systemd.tmpfiles.rules = let
-          u = config.services.tangled-knot.gitUser;
-          g = config.services.tangled-knot.gitUser;
-        in [
-          "d /var/lib/knot 0770 ${u} ${g} - -" # Create the directory first
-          "f+ /var/lib/knot/secret 0660 ${u} ${g} - KNOT_SERVER_SECRET=${envVar "TANGLED_VM_KNOT_SECRET"}"
-        ];
         services.tangled-knot = {
           enable = true;
           motd = "Welcome to the development knot!\n";
           server = {
-            secretFile = "/var/lib/knot/secret";
+            secretFile = builtins.toFile "knot-secret" ("KNOT_SERVER_SECRET=" + (envVar "TANGLED_VM_KNOT_SECRET"));
             hostname = "localhost:6000";
             listenAddr = "0.0.0.0:6000";
           };
···
               provider = "sqlite";
             };
           };
+        };
+        users = {
+          # So we don't have to deal with permission clashing between
+          # blank disk VMs and existing state
+          users.${config.services.tangled-knot.gitUser}.uid = 666;
+          groups.${config.services.tangled-knot.gitUser}.gid = 666;
+
+          # TODO: separate spindle user
+        };
+        systemd.services = let
+          mkDataSyncScripts = source: target: {
+            enableStrictShellChecks = true;
+
+            preStart = lib.mkBefore ''
+              mkdir -p ${target}
+              ${lib.getExe pkgs.rsync} -a ${source}/ ${target}
+            '';
+
+            postStop = lib.mkAfter ''
+              ${lib.getExe pkgs.rsync} -a ${target}/ ${source}
+            '';
+
+            serviceConfig.PermissionsStartOnly = true;
+          };
+        in {
+          knot = mkDataSyncScripts "/mnt/knot-data" config.services.tangled-knot.stateDir;
+          spindle = mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled-spindle.server.dbPath);
         };
       })
     ];