nix/vm: run AT Protocol stack inside VM for offline dev #6

+279 -31

2 changed files

Diff round #1

expand all

flake.nix

nix

vm.nix

flake.nix

··· 195 195 cp -fr --no-preserve=ownership ${packages'.appview-static-files}/* appview/pages/static 196 196 export TANGLED_OAUTH_CLIENT_KID="$(date +%s)" 197 197 export TANGLED_OAUTH_CLIENT_SECRET="$(${packages'.goat}/bin/goat key generate -t P-256 | grep -A1 "Secret Key" | tail -n1 | awk '{print $1}')" 198 + 199 + # When TANGLED_VM_LOCAL_DEV=1, point the appview at local AT Protocol services 200 + if [[ "''${TANGLED_VM_LOCAL_DEV:-0}" != "0" ]]; then 201 + export TANGLED_PLC_URL="http://localhost:2582" 202 + export TANGLED_PDS_HOST="http://localhost:2583" 203 + export TANGLED_PDS_ADMIN_SECRET="tangled-local-dev" 204 + export TANGLED_JETSTREAM_ENDPOINT="ws://localhost:6008/subscribe" 205 + echo "local-dev: appview configured for local AT Protocol stack" 206 + fi 198 207 ''; 199 208 env.CGO_ENABLED = 1; 200 209 };

+270 -31

nix/vm.nix

··· 17 17 then var 18 18 else default; 19 19 20 - plcUrl = envVarOr "TANGLED_VM_PLC_URL" "https://plc.directory"; 21 - jetstream = envVarOr "TANGLED_VM_JETSTREAM_ENDPOINT" "wss://jetstream1.us-west.bsky.network/subscribe"; 20 + # When true, run a local PLC directory, PDS, and Jetstream inside the VM 21 + # so that the entire stack works offline (after the initial nix build). 22 + localDev = (envVarOr "TANGLED_VM_LOCAL_DEV" "0") != "0"; 23 + 24 + plcUrl = 25 + if localDev 26 + then "http://localhost:2582" 27 + else envVarOr "TANGLED_VM_PLC_URL" "https://plc.directory"; 28 + jetstream = 29 + if localDev 30 + then "ws://localhost:6008/subscribe" 31 + else envVarOr "TANGLED_VM_JETSTREAM_ENDPOINT" "wss://jetstream1.us-west.bsky.network/subscribe"; 32 + 33 + pdsPort = 2583; 34 + plcPort = 2582; 35 + jetstreamPort = 6008; 36 + 37 + # Static secrets for local development only — NOT for production use 38 + pdsAdminPassword = "tangled-local-dev"; 39 + pdsJwtSecret = "tangled-local-dev-jwt-secret-DO-NOT-USE-IN-PRODUCTION"; 40 + # A pre-generated secp256k1 private key for local dev PLC rotation key 41 + pdsPlcRotationKey = "b2e9683fd0e78a52f82c30e9c5f5e44fd55c6fb22e0f4e1ec92c54e1fe4a0509"; 22 42 in 23 43 nixpkgs.lib.nixosSystem { 24 44 inherit system; ··· 30 50 config, 31 51 pkgs, 32 52 ... 33 - }: { 53 + }: let 54 + did-plc-server = pkgs.callPackage ./pkgs/did-plc-server.nix {}; 55 + pds-local-dev = pkgs.callPackage ./pkgs/pds-local-dev.nix {}; 56 + jetstream-pkg = pkgs.callPackage ./pkgs/jetstream.nix {}; 57 + 58 + # Bootstrap script: creates a dev account on the PDS and writes 59 + # the owner DID to an environment file that knot/spindle load. 60 + bootstrapScript = pkgs.writeShellScript "tangled-bootstrap" '' 61 + set -euo pipefail 62 + ENV_FILE="/var/lib/tangled-dev/env" 63 + DID_FILE="/var/lib/tangled-dev/owner-did" 64 + 65 + # If the DID file already exists from a previous boot, skip bootstrap 66 + if [[ -f "$DID_FILE" ]]; then 67 + echo "tangled-bootstrap: owner DID already exists: $(cat "$DID_FILE")" 68 + did="$(cat "$DID_FILE")" 69 + else 70 + echo "tangled-bootstrap: waiting for PDS to be ready..." 71 + for i in $(seq 1 30); do 72 + if ${pkgs.curl}/bin/curl -sf http://localhost:${toString pdsPort}/xrpc/_health > /dev/null 2>&1; then 73 + break 74 + fi 75 + sleep 1 76 + done 77 + 78 + echo "tangled-bootstrap: creating dev account..." 79 + resp=$(${pkgs.curl}/bin/curl -sf -X POST \ 80 + -H "Content-Type: application/json" \ 81 + -d '{"email":"dev@localhost","handle":"dev.test","password":"password"}' \ 82 + "http://localhost:${toString pdsPort}/xrpc/com.atproto.server.createAccount") 83 + 84 + did=$(echo "$resp" | ${pkgs.jq}/bin/jq -r '.did') 85 + if [[ -z "$did" || "$did" == "null" ]]; then 86 + echo "tangled-bootstrap: ERROR: failed to create account" 87 + echo "$resp" 88 + exit 1 89 + fi 90 + 91 + echo "$did" > "$DID_FILE" 92 + fi 93 + 94 + # Write environment file for knot and spindle 95 + printf 'KNOT_SERVER_OWNER=%s\nSPINDLE_SERVER_OWNER=%s\n' "$did" "$did" > "$ENV_FILE" 96 + 97 + echo "" 98 + echo "========================================" 99 + echo " Tangled Local Dev Bootstrap Complete" 100 + echo " Owner DID: $did" 101 + echo " Handle: dev.test" 102 + echo " Password: password" 103 + echo " PDS: http://localhost:${toString pdsPort}" 104 + echo "========================================" 105 + echo "" 106 + ''; 107 + in { 34 108 virtualisation.vmVariant.virtualisation = { 35 109 host.pkgs = import nixpkgs {system = hostSystem;}; 36 110 37 111 graphics = false; 38 - memorySize = 2048; 112 + memorySize = 113 + if localDev 114 + then 4096 115 + else 2048; 39 116 diskSize = 10 * 1024; 40 117 cores = 2; 41 - forwardPorts = [ 42 - # ssh 43 - { 44 - from = "host"; 45 - host.port = 2222; 46 - guest.port = 22; 47 - } 48 - # knot 49 - { 50 - from = "host"; 51 - host.port = 6444; 52 - guest.port = 6444; 53 - } 54 - # spindle 55 - { 56 - from = "host"; 57 - host.port = 6555; 58 - guest.port = 6555; 59 - } 60 - ]; 118 + forwardPorts = 119 + [ 120 + # ssh 121 + { 122 + from = "host"; 123 + host.port = 2222; 124 + guest.port = 22; 125 + } 126 + # knot 127 + { 128 + from = "host"; 129 + host.port = 6444; 130 + guest.port = 6444; 131 + } 132 + # spindle 133 + { 134 + from = "host"; 135 + host.port = 6555; 136 + guest.port = 6555; 137 + } 138 + ] 139 + ++ lib.optionals localDev [ 140 + # PLC directory 141 + { 142 + from = "host"; 143 + host.port = plcPort; 144 + guest.port = plcPort; 145 + } 146 + # PDS 147 + { 148 + from = "host"; 149 + host.port = pdsPort; 150 + guest.port = pdsPort; 151 + } 152 + # Jetstream 153 + { 154 + from = "host"; 155 + host.port = jetstreamPort; 156 + guest.port = jetstreamPort; 157 + } 158 + ]; 61 159 sharedDirectories = { 62 160 # We can't use the 9p mounts directly for most of these 63 161 # as SQLite is incompatible with them. So instead we ··· 81 179 networking.firewall.enable = false; 82 180 time.timeZone = "Europe/London"; 83 181 services.getty.autologinUser = "root"; 84 - environment.systemPackages = with pkgs; [curl vim git sqlite litecli]; 182 + environment.systemPackages = with pkgs; [curl vim git sqlite litecli jq]; 85 183 services.tangled.knot = { 86 184 enable = true; 87 185 motd = "Welcome to the development knot!\n"; 88 186 server = { 89 - owner = envVar "TANGLED_VM_KNOT_OWNER"; 187 + owner = 188 + if localDev 189 + then "bootstrap-pending" 190 + else envVar "TANGLED_VM_KNOT_OWNER"; 90 191 hostname = envVarOr "TANGLED_VM_KNOT_HOST" "localhost:6444"; 91 192 plcUrl = plcUrl; 92 193 jetstreamEndpoint = jetstream; 93 194 listenAddr = "0.0.0.0:6444"; 195 + dev = localDev; 94 196 }; 95 197 }; 96 198 services.tangled.spindle = { 97 199 enable = true; 98 200 server = { 99 - owner = envVar "TANGLED_VM_SPINDLE_OWNER"; 201 + owner = 202 + if localDev 203 + then "bootstrap-pending" 204 + else envVar "TANGLED_VM_SPINDLE_OWNER"; 100 205 hostname = envVarOr "TANGLED_VM_SPINDLE_HOST" "localhost:6555"; 101 206 plcUrl = plcUrl; 102 207 jetstreamEndpoint = jetstream; ··· 109 214 }; 110 215 }; 111 216 }; 217 + 218 + # ── Local PLC + PDS + Jetstream (offline dev mode) ────────────── 219 + # 220 + # When TANGLED_VM_LOCAL_DEV=1, the VM runs a fully self-contained 221 + # AT Protocol stack: 222 + # PostgreSQL → PLC directory → PDS → Jetstream → Knot/Spindle 223 + # 224 + # A bootstrap service creates a dev account and writes the owner 225 + # DID to an environment file that knot/spindle load at startup. 226 + 227 + # PostgreSQL for PLC directory (persistent DID storage) 228 + services.postgresql = lib.mkIf localDev { 229 + enable = true; 230 + package = pkgs.postgresql_16; 231 + ensureDatabases = ["plc"]; 232 + ensureUsers = [ 233 + { 234 + name = "plc"; 235 + ensureDBOwnership = true; 236 + } 237 + ]; 238 + # Trust local connections (dev-only, no passwords needed) 239 + authentication = lib.mkForce '' 240 + # TYPE DATABASE USER ADDRESS METHOD 241 + local all all trust 242 + host all all 127.0.0.1/32 trust 243 + host all all ::1/128 trust 244 + ''; 245 + }; 246 + 112 247 users = { 113 248 # So we don't have to deal with permission clashing between 114 249 # blank disk VMs and existing state ··· 117 252 118 253 # TODO: separate spindle user 119 254 }; 255 + 120 256 systemd.services = let 121 257 mkDataSyncScripts = source: target: { 122 258 enableStrictShellChecks = true; ··· 132 268 133 269 serviceConfig.PermissionsStartOnly = true; 134 270 }; 135 - in { 136 - knot = mkDataSyncScripts "/mnt/knot-data" config.services.tangled.knot.stateDir; 137 - spindle = mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled.spindle.server.dbPath); 138 - }; 271 + # In localDev mode, knot/spindle depend on the bootstrap service 272 + # which creates a dev account and writes the owner DID to an env 273 + # file. Requires= ensures knot/spindle won't start if bootstrap 274 + # failed (e.g. PDS never came up). After= ensures ordering. 275 + localDevDeps = lib.optionalAttrs localDev { 276 + after = ["tangled-bootstrap.service"]; 277 + requires = ["tangled-bootstrap.service"]; 278 + serviceConfig.EnvironmentFile = ["/var/lib/tangled-dev/env"]; 279 + }; 280 + in 281 + { 282 + knot = 283 + (mkDataSyncScripts "/mnt/knot-data" config.services.tangled.knot.stateDir) 284 + // localDevDeps; 285 + spindle = 286 + (mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled.spindle.server.dbPath)) 287 + // localDevDeps; 288 + } 289 + // lib.optionalAttrs localDev { 290 + # PLC directory server 291 + plc-server = { 292 + description = "DID PLC Directory Server (local development)"; 293 + after = ["network.target" "postgresql.service"]; 294 + wants = ["postgresql.service"]; 295 + wantedBy = ["multi-user.target"]; 296 + 297 + serviceConfig = { 298 + Environment = [ 299 + "DATABASE_URL=postgres://plc@127.0.0.1/plc" 300 + "PORT=${toString plcPort}" 301 + "LOG_ENABLED=true" 302 + "LOG_LEVEL=info" 303 + "DEBUG_MODE=1" 304 + ]; 305 + ExecStart = "${did-plc-server}/bin/plc-server"; 306 + Restart = "on-failure"; 307 + RestartSec = "5s"; 308 + }; 309 + }; 310 + 311 + # Bluesky PDS (patched for HTTP local dev) 312 + bluesky-pds = { 313 + description = "Bluesky PDS (local development)"; 314 + after = ["network.target" "plc-server.service"]; 315 + wants = ["plc-server.service"]; 316 + wantedBy = ["multi-user.target"]; 317 + 318 + serviceConfig = { 319 + StateDirectory = "bluesky-pds"; 320 + WorkingDirectory = "/var/lib/bluesky-pds"; 321 + Environment = [ 322 + "PDS_HOSTNAME=localhost:${toString pdsPort}" 323 + "PDS_PORT=${toString pdsPort}" 324 + "PDS_DATA_DIRECTORY=/var/lib/bluesky-pds" 325 + "PDS_DID_PLC_URL=http://localhost:${toString plcPort}" 326 + "PDS_BLOBSTORE_DISK_LOCATION=/var/lib/bluesky-pds/blobs" 327 + "PDS_JWT_SECRET=${pdsJwtSecret}" 328 + "PDS_ADMIN_PASSWORD=${pdsAdminPassword}" 329 + "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=${pdsPlcRotationKey}" 330 + "PDS_CRAWLERS=" 331 + "PDS_INVITE_REQUIRED=false" 332 + "PDS_DEV_MODE=true" 333 + "PDS_LOG_ENABLED=true" 334 + "NODE_ENV=development" 335 + ]; 336 + ExecStart = "${pds-local-dev}/bin/pds"; 337 + Restart = "on-failure"; 338 + RestartSec = "5s"; 339 + }; 340 + }; 341 + 342 + # Jetstream (native Nix package — subscribes to local PDS firehose) 343 + jetstream = { 344 + description = "AT Protocol Jetstream (local development)"; 345 + after = ["network.target" "bluesky-pds.service"]; 346 + wants = ["bluesky-pds.service"]; 347 + wantedBy = ["multi-user.target"]; 348 + 349 + serviceConfig = { 350 + StateDirectory = "jetstream"; 351 + Environment = [ 352 + "JETSTREAM_WS_URL=ws://127.0.0.1:${toString pdsPort}/xrpc/com.atproto.sync.subscribeRepos" 353 + "JETSTREAM_LISTEN_ADDR=:${toString jetstreamPort}" 354 + "JETSTREAM_METRICS_LISTEN_ADDR=:6009" 355 + "JETSTREAM_DATA_DIR=/var/lib/jetstream" 356 + ]; 357 + ExecStart = "${jetstream-pkg}/bin/jetstream"; 358 + Restart = "on-failure"; 359 + RestartSec = "5s"; 360 + }; 361 + }; 362 + 363 + # Bootstrap: create dev account and write owner DID env file 364 + tangled-bootstrap = { 365 + description = "Tangled local dev bootstrap (account creation)"; 366 + after = ["bluesky-pds.service"]; 367 + wants = ["bluesky-pds.service"]; 368 + wantedBy = ["multi-user.target"]; 369 + 370 + serviceConfig = { 371 + Type = "oneshot"; 372 + RemainAfterExit = true; 373 + StateDirectory = "tangled-dev"; 374 + ExecStart = bootstrapScript; 375 + }; 376 + }; 377 + }; 139 378 }) 140 379 ]; 141 380 }

History

2 rounds 0 comments

nolith.dev submitted #1 10d

interdiff

1 commit

expand

9a34104b

nix/vm: run AT Protocol stack inside VM for offline dev

no conflicts, ready to merge

expand 0 comments

nolith.dev submitted #0 10d

diff

1 commit

expand

8e651b93

nix/vm: run AT Protocol stack inside VM for offline dev

expand 0 comments