nolith.dev / tangled-core
forked from tangled.org/core
Monorepo for Tangled
fork atom

nix/vm: persist AT Protocol state via 9p and create default label definitions #8

open opened by nolith.dev targeting master from local-dev

Add 9p shared directory for AT Protocol state (PLC/PDS/Jetstream data and owner DID) so that identities survive VM reboots even when the qcow2 disk is deleted.

Key changes:

  • Bootstrap script handles existing accounts (login fallback when createAccount fails with persisted PDS data)
  • Creates default label definitions (wontfix, good-first-issue, duplicate, documentation, assignee) via com.atproto.repo.putRecord
  • PostgreSQL (PLC) backed up via pg_dump on shutdown, restored via psql on boot only when the database is empty (qcow2 was deleted)
  • PDS_HOSTNAME set to "localhost" (not "localhost:port") to avoid https:// URL generation that breaks OAuth loopback validation
  • PDS/Jetstream data synced via mkDataSyncScripts
  • Owner DID written to both VM disk and 9p mount for host-side access
  • Use dev@example.com and tangled-dev.test handle for bootstrap account

Signed-off-by: Alessio Caiazza code.git@caiazza.info

STACK 10
#10 flake: add local-dev app, DID-based label defaults in devshell
0
#1
#9 local-dev: add tmux orchestrator for offline development
0
#1
#8 nix/vm: persist AT Protocol state via 9p and create default label definitions
0
#1
#7 docs: add fully offline development guide
0
#1
#6 nix/vm: run AT Protocol stack inside VM for offline dev
0
#1
#5 nix: add PLC, PDS, and Jetstream packages
0
#1
#4 appview/oauth: add DevStartAuthFlow for HTTP dev mode
0
#1
#3 appview,knot,spindle: use DevDirectory in dev mode
0
#1
#2 idresolver: add DevDirectory for .test TLD handle resolution
0
#1
#1 idresolver: add PDSClient() helper to reduce duplication
0
#1
Labels

None yet.

assignee

None yet.

Participants 1
AT URI
at://did:plc:nzep3slobztdph3kxswzbing/sh.tangled.repo.pull/3mgmn4naukj22
+227 -82
Diff #1
+227 -82
nix/vm.nix
··· 61 61 set -euo pipefail 62 62 ENV_FILE="/var/lib/tangled-dev/env" 63 63 DID_FILE="/var/lib/tangled-dev/owner-did" 64 + ATPROTO_DID_FILE="/mnt/atproto/owner-did" 65 + curl="${pkgs.curl}/bin/curl" 66 + jq="${pkgs.jq}/bin/jq" 67 + PDS="http://localhost:${toString pdsPort}" 64 68 65 - # If the DID file already exists from a previous boot, skip bootstrap 69 + # Check for DID from a previous session — either on VM disk or 70 + # in the 9p-persisted atproto directory (survives qcow2 deletion). 71 + existing_did="" 66 72 if [[ -f "$DID_FILE" ]]; then 67 - echo "tangled-bootstrap: owner DID already exists: $(cat "$DID_FILE")" 68 - did="$(cat "$DID_FILE")" 69 - else 70 - echo "tangled-bootstrap: waiting for PDS to be ready..." 71 - for i in $(seq 1 30); do 72 - if ${pkgs.curl}/bin/curl -sf http://localhost:${toString pdsPort}/xrpc/_health > /dev/null 2>&1; then 73 - break 74 - fi 75 - sleep 1 76 - done 73 + existing_did="$(cat "$DID_FILE")" 74 + elif [[ -f "$ATPROTO_DID_FILE" ]]; then 75 + existing_did="$(cat "$ATPROTO_DID_FILE")" 76 + fi 77 + 78 + echo "tangled-bootstrap: waiting for PDS to be ready..." 79 + for i in $(seq 1 30); do 80 + if $curl -sf "$PDS/xrpc/_health" > /dev/null 2>&1; then 81 + break 82 + fi 83 + sleep 1 84 + done 85 + 86 + if [[ -n "$existing_did" ]]; then 87 + echo "tangled-bootstrap: owner DID already exists: $existing_did" 88 + did="$existing_did" 77 89 90 + # Re-login to get a fresh accessJwt for label creation 91 + echo "tangled-bootstrap: logging in to get access token..." 92 + login_resp=$($curl -s -X POST \ 93 + -H "Content-Type: application/json" \ 94 + -d '{"identifier":"tangled-dev.test","password":"password"}' \ 95 + "$PDS/xrpc/com.atproto.server.createSession") 96 + access_jwt=$(echo "$login_resp" | $jq -r '.accessJwt') 97 + else 78 98 echo "tangled-bootstrap: creating dev account..." 79 - resp=$(${pkgs.curl}/bin/curl -sf -X POST \ 99 + resp=$($curl -s -X POST \ 80 100 -H "Content-Type: application/json" \ 81 - -d '{"email":"dev@localhost","handle":"dev.test","password":"password"}' \ 82 - "http://localhost:${toString pdsPort}/xrpc/com.atproto.server.createAccount") 101 + -d '{"email":"dev@example.com","handle":"tangled-dev.test","password":"password"}' \ 102 + "$PDS/xrpc/com.atproto.server.createAccount") 83 103 84 - did=$(echo "$resp" | ${pkgs.jq}/bin/jq -r '.did') 104 + did=$(echo "$resp" | $jq -r '.did') 105 + access_jwt=$(echo "$resp" | $jq -r '.accessJwt') 85 106 if [[ -z "$did" || "$did" == "null" ]]; then 86 - echo "tangled-bootstrap: ERROR: failed to create account" 87 - echo "$resp" 88 - exit 1 107 + # Account might already exist (9p-persisted PDS with no DID file). 108 + # Try logging in instead. 109 + echo "tangled-bootstrap: createAccount failed, trying login..." 110 + login_resp=$($curl -s -X POST \ 111 + -H "Content-Type: application/json" \ 112 + -d '{"identifier":"tangled-dev.test","password":"password"}' \ 113 + "$PDS/xrpc/com.atproto.server.createSession") 114 + did=$(echo "$login_resp" | $jq -r '.did') 115 + access_jwt=$(echo "$login_resp" | $jq -r '.accessJwt') 116 + 117 + if [[ -z "$did" || "$did" == "null" ]]; then 118 + echo "tangled-bootstrap: ERROR: failed to create account or login" 119 + echo "createAccount response: $resp" 120 + echo "createSession response: $login_resp" 121 + exit 1 122 + fi 89 123 fi 90 124 91 125 echo "$did" > "$DID_FILE" 92 126 fi 93 127 128 + # Write DID to 9p mount so the host can read it 129 + echo "$did" > "$ATPROTO_DID_FILE" 130 + 94 131 # Write environment file for knot and spindle 95 132 printf 'KNOT_SERVER_OWNER=%s\nSPINDLE_SERVER_OWNER=%s\n' "$did" "$did" > "$ENV_FILE" 96 133 134 + # ── Create default label definitions ────────────────────────── 135 + if [[ -n "$access_jwt" && "$access_jwt" != "null" ]]; then 136 + echo "tangled-bootstrap: creating default label definitions..." 137 + now=$(date -u +"%Y-%m-%dT%H:%M:%S.000Z") 138 + 139 + create_label() { 140 + local rkey="$1" name="$2" color="$3" vtype="$4" vformat="$5" multiple="$6" 141 + local scope='["sh.tangled.repo.issue","sh.tangled.repo.pull"]' 142 + local multiple_field="" 143 + if [[ "$multiple" == "true" ]]; then 144 + multiple_field=',"multiple":true' 145 + fi 146 + 147 + local record="{\"\$type\":\"sh.tangled.label.definition\",\"name\":\"$name\",\"color\":\"$color\",\"createdAt\":\"$now\",\"scope\":$scope,\"valueType\":{\"type\":\"$vtype\",\"format\":\"$vformat\"}$multiple_field}" 148 + 149 + local resp 150 + resp=$($curl -sf -X POST \ 151 + -H "Content-Type: application/json" \ 152 + -H "Authorization: Bearer $access_jwt" \ 153 + -d "{\"repo\":\"$did\",\"collection\":\"sh.tangled.label.definition\",\"rkey\":\"$rkey\",\"record\":$record}" \ 154 + "$PDS/xrpc/com.atproto.repo.putRecord" 2>&1) || true 155 + 156 + if echo "$resp" | $jq -e '.uri' > /dev/null 2>&1; then 157 + echo " created: $rkey" 158 + else 159 + echo " $rkey: already exists or skipped" 160 + fi 161 + } 162 + 163 + create_label "wontfix" "wontfix" "#737373" "null" "any" "false" 164 + create_label "good-first-issue" "good first issue" "#22c55e" "null" "any" "false" 165 + create_label "duplicate" "duplicate" "#a855f7" "null" "any" "false" 166 + create_label "documentation" "documentation" "#3b82f6" "null" "any" "false" 167 + create_label "assignee" "assignee" "#f59e0b" "string" "did" "true" 168 + else 169 + echo "tangled-bootstrap: WARNING: no access token, skipping label creation" 170 + fi 171 + 97 172 echo "" 98 173 echo "========================================" 99 174 echo " Tangled Local Dev Bootstrap Complete" 100 175 echo " Owner DID: $did" 101 - echo " Handle: dev.test" 176 + echo " Handle: tangled-dev.test" 102 177 echo " Password: password" 103 - echo " PDS: http://localhost:${toString pdsPort}" 178 + echo " PDS: $PDS" 104 179 echo "========================================" 105 180 echo "" 106 181 ''; ··· 156 231 guest.port = jetstreamPort; 157 232 } 158 233 ]; 159 - sharedDirectories = { 160 - # We can't use the 9p mounts directly for most of these 161 - # as SQLite is incompatible with them. So instead we 162 - # mount the shared directories to a different location 163 - # and copy the contents around on service start/stop. 164 - knotData = { 165 - source = "$TANGLED_VM_DATA_DIR/knot"; 166 - target = "/mnt/knot-data"; 167 - }; 168 - spindleData = { 169 - source = "$TANGLED_VM_DATA_DIR/spindle"; 170 - target = "/mnt/spindle-data"; 171 - }; 172 - spindleLogs = { 173 - source = "$TANGLED_VM_DATA_DIR/spindle-logs"; 174 - target = "/var/log/spindle"; 234 + sharedDirectories = 235 + { 236 + # We can't use the 9p mounts directly for most of these 237 + # as SQLite is incompatible with them. So instead we 238 + # mount the shared directories to a different location 239 + # and copy the contents around on service start/stop. 240 + knotData = { 241 + source = "$TANGLED_VM_DATA_DIR/knot"; 242 + target = "/mnt/knot-data"; 243 + }; 244 + spindleData = { 245 + source = "$TANGLED_VM_DATA_DIR/spindle"; 246 + target = "/mnt/spindle-data"; 247 + }; 248 + spindleLogs = { 249 + source = "$TANGLED_VM_DATA_DIR/spindle-logs"; 250 + target = "/var/log/spindle"; 251 + }; 252 + } 253 + // lib.optionalAttrs localDev { 254 + # AT Protocol state persistence: PLC (postgres), PDS (sqlite+blobs), 255 + # Jetstream (pebbledb), and the owner DID file. 256 + atprotoData = { 257 + source = "$TANGLED_VM_DATA_DIR/atproto"; 258 + target = "/mnt/atproto"; 259 + }; 175 260 }; 176 - }; 177 261 }; 178 262 # This is fine because any and all ports that are forwarded to host are explicitly marked above, we don't need a separate guest firewall 179 263 networking.firewall.enable = false; ··· 287 371 // localDevDeps; 288 372 } 289 373 // lib.optionalAttrs localDev { 374 + # ── AT Protocol state persistence ───────────────────────── 375 + # PDS, Jetstream, and PostgreSQL (PLC) data is synced to/from 376 + # the 9p mount at /mnt/atproto so it survives VM reboots. 377 + # DID stability across reboots is critical — knot/spindle 378 + # data references the owner DID. 379 + # 380 + # PostgreSQL (PLC) persistence via pg_dump/psql. 381 + # The qcow2 disk is the primary storage; the 9p dump is a 382 + # safety net so DID data survives qcow2 deletion. 383 + # 384 + # Boot: pg starts → setup creates DB → restore loads dump (if DB empty) 385 + # Shutdown: backup dumps → pg stops 386 + atproto-pg-restore = { 387 + description = "Restore PLC database from 9p dump"; 388 + after = ["postgresql.service" "postgresql-setup.service"]; 389 + requires = ["postgresql.service"]; 390 + wantedBy = ["multi-user.target"]; 391 + serviceConfig = { 392 + Type = "oneshot"; 393 + RemainAfterExit = true; 394 + ExecStart = pkgs.writeShellScript "atproto-pg-restore" '' 395 + dump="/mnt/atproto/plc.sql" 396 + if [ ! -f "$dump" ]; then 397 + echo "atproto-pg-restore: no dump file, skipping" 398 + exit 0 399 + fi 400 + table_count=$(${pkgs.postgresql_16}/bin/psql -U plc -d plc -tAc \ 401 + "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'") 402 + if [ "$table_count" != "0" ]; then 403 + echo "atproto-pg-restore: database already has $table_count tables, skipping" 404 + exit 0 405 + fi 406 + echo "atproto-pg-restore: loading dump into empty database..." 407 + ${pkgs.postgresql_16}/bin/psql -U plc -d plc < "$dump" 408 + echo "atproto-pg-restore: done" 409 + ''; 410 + }; 411 + }; 412 + atproto-pg-backup = { 413 + description = "Backup PLC database to 9p dump"; 414 + after = ["postgresql.service"]; 415 + wants = ["postgresql.service"]; 416 + wantedBy = ["multi-user.target"]; 417 + serviceConfig = { 418 + Type = "oneshot"; 419 + RemainAfterExit = true; 420 + # ExecStart is a no-op; the real work is in ExecStop 421 + # which fires on shutdown while postgresql is still running. 422 + ExecStart = "${pkgs.coreutils}/bin/true"; 423 + ExecStop = pkgs.writeShellScript "atproto-pg-backup" '' 424 + echo "atproto-pg-backup: dumping PLC database..." 425 + ${pkgs.postgresql_16}/bin/pg_dump -U plc plc > /mnt/atproto/plc.sql 426 + echo "atproto-pg-backup: done ($(wc -c < /mnt/atproto/plc.sql) bytes)" 427 + ''; 428 + }; 429 + }; 430 + 290 431 # PLC directory server 291 432 plc-server = { 292 433 description = "DID PLC Directory Server (local development)"; 293 - after = ["network.target" "postgresql.service"]; 434 + after = ["network.target" "atproto-pg-restore.service"]; 294 435 wants = ["postgresql.service"]; 295 436 wantedBy = ["multi-user.target"]; 296 437 ··· 309 450 }; 310 451 311 452 # Bluesky PDS (patched for HTTP local dev) 312 - bluesky-pds = { 313 - description = "Bluesky PDS (local development)"; 314 - after = ["network.target" "plc-server.service"]; 315 - wants = ["plc-server.service"]; 316 - wantedBy = ["multi-user.target"]; 453 + bluesky-pds = 454 + (mkDataSyncScripts "/mnt/atproto/pds" "/var/lib/bluesky-pds") 455 + // { 456 + description = "Bluesky PDS (local development)"; 457 + after = ["network.target" "plc-server.service"]; 458 + wants = ["plc-server.service"]; 459 + wantedBy = ["multi-user.target"]; 317 460 318 - serviceConfig = { 319 - StateDirectory = "bluesky-pds"; 320 - WorkingDirectory = "/var/lib/bluesky-pds"; 321 - Environment = [ 322 - "PDS_HOSTNAME=localhost:${toString pdsPort}" 323 - "PDS_PORT=${toString pdsPort}" 324 - "PDS_DATA_DIRECTORY=/var/lib/bluesky-pds" 325 - "PDS_DID_PLC_URL=http://localhost:${toString plcPort}" 326 - "PDS_BLOBSTORE_DISK_LOCATION=/var/lib/bluesky-pds/blobs" 327 - "PDS_JWT_SECRET=${pdsJwtSecret}" 328 - "PDS_ADMIN_PASSWORD=${pdsAdminPassword}" 329 - "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=${pdsPlcRotationKey}" 330 - "PDS_CRAWLERS=" 331 - "PDS_INVITE_REQUIRED=false" 332 - "PDS_DEV_MODE=true" 333 - "PDS_LOG_ENABLED=true" 334 - "NODE_ENV=development" 335 - ]; 336 - ExecStart = "${pds-local-dev}/bin/pds"; 337 - Restart = "on-failure"; 338 - RestartSec = "5s"; 461 + serviceConfig = { 462 + StateDirectory = "bluesky-pds"; 463 + WorkingDirectory = "/var/lib/bluesky-pds"; 464 + Environment = [ 465 + "PDS_HOSTNAME=localhost" 466 + "PDS_PORT=${toString pdsPort}" 467 + "PDS_DATA_DIRECTORY=/var/lib/bluesky-pds" 468 + "PDS_DID_PLC_URL=http://localhost:${toString plcPort}" 469 + "PDS_BLOBSTORE_DISK_LOCATION=/var/lib/bluesky-pds/blobs" 470 + "PDS_JWT_SECRET=${pdsJwtSecret}" 471 + "PDS_ADMIN_PASSWORD=${pdsAdminPassword}" 472 + "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=${pdsPlcRotationKey}" 473 + "PDS_CRAWLERS=" 474 + "PDS_INVITE_REQUIRED=false" 475 + "PDS_DEV_MODE=true" 476 + "PDS_LOG_ENABLED=true" 477 + "NODE_ENV=development" 478 + ]; 479 + ExecStart = "${pds-local-dev}/bin/pds"; 480 + Restart = "on-failure"; 481 + RestartSec = "5s"; 482 + }; 339 483 }; 340 - }; 341 484 342 485 # Jetstream (native Nix package — subscribes to local PDS firehose) 343 - jetstream = { 344 - description = "AT Protocol Jetstream (local development)"; 345 - after = ["network.target" "bluesky-pds.service"]; 346 - wants = ["bluesky-pds.service"]; 347 - wantedBy = ["multi-user.target"]; 486 + jetstream = 487 + (mkDataSyncScripts "/mnt/atproto/jetstream" "/var/lib/jetstream") 488 + // { 489 + description = "AT Protocol Jetstream (local development)"; 490 + after = ["network.target" "bluesky-pds.service"]; 491 + wants = ["bluesky-pds.service"]; 492 + wantedBy = ["multi-user.target"]; 348 493 349 - serviceConfig = { 350 - StateDirectory = "jetstream"; 351 - Environment = [ 352 - "JETSTREAM_WS_URL=ws://127.0.0.1:${toString pdsPort}/xrpc/com.atproto.sync.subscribeRepos" 353 - "JETSTREAM_LISTEN_ADDR=:${toString jetstreamPort}" 354 - "JETSTREAM_METRICS_LISTEN_ADDR=:6009" 355 - "JETSTREAM_DATA_DIR=/var/lib/jetstream" 356 - ]; 357 - ExecStart = "${jetstream-pkg}/bin/jetstream"; 358 - Restart = "on-failure"; 359 - RestartSec = "5s"; 494 + serviceConfig = { 495 + StateDirectory = "jetstream"; 496 + Environment = [ 497 + "JETSTREAM_WS_URL=ws://127.0.0.1:${toString pdsPort}/xrpc/com.atproto.sync.subscribeRepos" 498 + "JETSTREAM_LISTEN_ADDR=:${toString jetstreamPort}" 499 + "JETSTREAM_METRICS_LISTEN_ADDR=:6009" 500 + "JETSTREAM_DATA_DIR=/var/lib/jetstream" 501 + ]; 502 + ExecStart = "${jetstream-pkg}/bin/jetstream"; 503 + Restart = "on-failure"; 504 + RestartSec = "5s"; 505 + }; 360 506 }; 361 - }; 362 507 363 508 # Bootstrap: create dev account and write owner DID env file 364 509 tangled-bootstrap = {

History

2 rounds 0 comments
sign up or login to add to the discussion
nolith.dev submitted #1
interdiff
1 commit
expand
a293e80d
nix/vm: persist AT Protocol state via 9p and create default label definitions
no conflicts, ready to merge
expand 0 comments
nolith.dev submitted #0
diff
1 commit
expand
1a8c0373
nix/vm: persist AT Protocol state via 9p and create default label definitions
expand 0 comments