nolith.dev
forked from
tangled.org/core
Monorepo for Tangled
1{
2 nixpkgs,
3 system,
4 hostSystem,
5 self,
6}: let
7 envVar = name: let
8 var = builtins.getEnv name;
9 in
10 if var == ""
11 then throw "\$${name} must be defined, see https://docs.tangled.org/hacking-on-tangled.html#hacking-on-tangled for more details"
12 else var;
13 envVarOr = name: default: let
14 var = builtins.getEnv name;
15 in
16 if var != ""
17 then var
18 else default;
19
20 # When true, run a local PLC directory, PDS, and Jetstream inside the VM
21 # so that the entire stack works offline (after the initial nix build).
22 localDev = (envVarOr "TANGLED_VM_LOCAL_DEV" "0") != "0";
23
24 plcUrl =
25 if localDev
26 then "http://localhost:2582"
27 else envVarOr "TANGLED_VM_PLC_URL" "https://plc.directory";
28 jetstream =
29 if localDev
30 then "ws://localhost:6008/subscribe"
31 else envVarOr "TANGLED_VM_JETSTREAM_ENDPOINT" "wss://jetstream1.us-west.bsky.network/subscribe";
32
33 pdsPort = 2583;
34 plcPort = 2582;
35 jetstreamPort = 6008;
36
37 # Static secrets for local development only — NOT for production use
38 pdsAdminPassword = "tangled-local-dev";
39 pdsJwtSecret = "tangled-local-dev-jwt-secret-DO-NOT-USE-IN-PRODUCTION";
40 # A pre-generated secp256k1 private key for local dev PLC rotation key
41 pdsPlcRotationKey = "b2e9683fd0e78a52f82c30e9c5f5e44fd55c6fb22e0f4e1ec92c54e1fe4a0509";
42in
43 nixpkgs.lib.nixosSystem {
44 inherit system;
45 modules = [
46 self.nixosModules.knot
47 self.nixosModules.spindle
48 ({
49 lib,
50 config,
51 pkgs,
52 ...
53 }: let
54 did-plc-server = pkgs.callPackage ./pkgs/did-plc-server.nix {};
55 pds-local-dev = pkgs.callPackage ./pkgs/pds-local-dev.nix {};
56 jetstream-pkg = pkgs.callPackage ./pkgs/jetstream.nix {};
57
58 # Bootstrap script: creates a dev account on the PDS and writes
59 # the owner DID to an environment file that knot/spindle load.
60 bootstrapScript = pkgs.writeShellScript "tangled-bootstrap" ''
61 set -euo pipefail
62 ENV_FILE="/var/lib/tangled-dev/env"
63 DID_FILE="/var/lib/tangled-dev/owner-did"
64 ATPROTO_DID_FILE="/mnt/atproto/owner-did"
65 curl="${pkgs.curl}/bin/curl"
66 jq="${pkgs.jq}/bin/jq"
67 PDS="http://localhost:${toString pdsPort}"
68
69 # Check for DID from a previous session — either on VM disk or
70 # in the 9p-persisted atproto directory (survives qcow2 deletion).
71 existing_did=""
72 if [[ -f "$DID_FILE" ]]; then
73 existing_did="$(cat "$DID_FILE")"
74 elif [[ -f "$ATPROTO_DID_FILE" ]]; then
75 existing_did="$(cat "$ATPROTO_DID_FILE")"
76 fi
77
78 echo "tangled-bootstrap: waiting for PDS to be ready..."
79 for i in $(seq 1 30); do
80 if $curl -sf "$PDS/xrpc/_health" > /dev/null 2>&1; then
81 break
82 fi
83 sleep 1
84 done
85
86 if [[ -n "$existing_did" ]]; then
87 echo "tangled-bootstrap: owner DID already exists: $existing_did"
88 did="$existing_did"
89
90 # Re-login to get a fresh accessJwt for label creation
91 echo "tangled-bootstrap: logging in to get access token..."
92 login_resp=$($curl -s -X POST \
93 -H "Content-Type: application/json" \
94 -d '{"identifier":"tangled-dev.test","password":"password"}' \
95 "$PDS/xrpc/com.atproto.server.createSession")
96 access_jwt=$(echo "$login_resp" | $jq -r '.accessJwt')
97 else
98 echo "tangled-bootstrap: creating dev account..."
99 resp=$($curl -s -X POST \
100 -H "Content-Type: application/json" \
101 -d '{"email":"dev@example.com","handle":"tangled-dev.test","password":"password"}' \
102 "$PDS/xrpc/com.atproto.server.createAccount")
103
104 did=$(echo "$resp" | $jq -r '.did')
105 access_jwt=$(echo "$resp" | $jq -r '.accessJwt')
106 if [[ -z "$did" || "$did" == "null" ]]; then
107 # Account might already exist (9p-persisted PDS with no DID file).
108 # Try logging in instead.
109 echo "tangled-bootstrap: createAccount failed, trying login..."
110 login_resp=$($curl -s -X POST \
111 -H "Content-Type: application/json" \
112 -d '{"identifier":"tangled-dev.test","password":"password"}' \
113 "$PDS/xrpc/com.atproto.server.createSession")
114 did=$(echo "$login_resp" | $jq -r '.did')
115 access_jwt=$(echo "$login_resp" | $jq -r '.accessJwt')
116
117 if [[ -z "$did" || "$did" == "null" ]]; then
118 echo "tangled-bootstrap: ERROR: failed to create account or login"
119 echo "createAccount response: $resp"
120 echo "createSession response: $login_resp"
121 exit 1
122 fi
123 fi
124
125 echo "$did" > "$DID_FILE"
126 fi
127
128 # Write DID to 9p mount so the host can read it
129 echo "$did" > "$ATPROTO_DID_FILE"
130
131 # Write environment file for knot and spindle
132 printf 'KNOT_SERVER_OWNER=%s\nSPINDLE_SERVER_OWNER=%s\n' "$did" "$did" > "$ENV_FILE"
133
134 # ── Create default label definitions ──────────────────────────
135 if [[ -n "$access_jwt" && "$access_jwt" != "null" ]]; then
136 echo "tangled-bootstrap: creating default label definitions..."
137 now=$(date -u +"%Y-%m-%dT%H:%M:%S.000Z")
138
139 create_label() {
140 local rkey="$1" name="$2" color="$3" vtype="$4" vformat="$5" multiple="$6"
141 local scope='["sh.tangled.repo.issue","sh.tangled.repo.pull"]'
142 local multiple_field=""
143 if [[ "$multiple" == "true" ]]; then
144 multiple_field=',"multiple":true'
145 fi
146
147 local record="{\"\$type\":\"sh.tangled.label.definition\",\"name\":\"$name\",\"color\":\"$color\",\"createdAt\":\"$now\",\"scope\":$scope,\"valueType\":{\"type\":\"$vtype\",\"format\":\"$vformat\"}$multiple_field}"
148
149 local resp
150 resp=$($curl -sf -X POST \
151 -H "Content-Type: application/json" \
152 -H "Authorization: Bearer $access_jwt" \
153 -d "{\"repo\":\"$did\",\"collection\":\"sh.tangled.label.definition\",\"rkey\":\"$rkey\",\"record\":$record}" \
154 "$PDS/xrpc/com.atproto.repo.putRecord" 2>&1) || true
155
156 if echo "$resp" | $jq -e '.uri' > /dev/null 2>&1; then
157 echo " created: $rkey"
158 else
159 echo " $rkey: already exists or skipped"
160 fi
161 }
162
163 create_label "wontfix" "wontfix" "#737373" "null" "any" "false"
164 create_label "good-first-issue" "good first issue" "#22c55e" "null" "any" "false"
165 create_label "duplicate" "duplicate" "#a855f7" "null" "any" "false"
166 create_label "documentation" "documentation" "#3b82f6" "null" "any" "false"
167 create_label "assignee" "assignee" "#f59e0b" "string" "did" "true"
168 else
169 echo "tangled-bootstrap: WARNING: no access token, skipping label creation"
170 fi
171
172 echo ""
173 echo "========================================"
174 echo " Tangled Local Dev Bootstrap Complete"
175 echo " Owner DID: $did"
176 echo " Handle: tangled-dev.test"
177 echo " Password: password"
178 echo " PDS: $PDS"
179 echo "========================================"
180 echo ""
181 '';
182 in {
183 virtualisation.vmVariant.virtualisation = {
184 host.pkgs = import nixpkgs {system = hostSystem;};
185
186 graphics = false;
187 memorySize =
188 if localDev
189 then 4096
190 else 2048;
191 diskSize = 10 * 1024;
192 cores = 2;
193 forwardPorts =
194 [
195 # ssh
196 {
197 from = "host";
198 host.port = 2222;
199 guest.port = 22;
200 }
201 # knot
202 {
203 from = "host";
204 host.port = 6444;
205 guest.port = 6444;
206 }
207 # spindle
208 {
209 from = "host";
210 host.port = 6555;
211 guest.port = 6555;
212 }
213 ]
214 ++ lib.optionals localDev [
215 # PLC directory
216 {
217 from = "host";
218 host.port = plcPort;
219 guest.port = plcPort;
220 }
221 # PDS
222 {
223 from = "host";
224 host.port = pdsPort;
225 guest.port = pdsPort;
226 }
227 # Jetstream
228 {
229 from = "host";
230 host.port = jetstreamPort;
231 guest.port = jetstreamPort;
232 }
233 ];
234 sharedDirectories =
235 {
236 # We can't use the 9p mounts directly for most of these
237 # as SQLite is incompatible with them. So instead we
238 # mount the shared directories to a different location
239 # and copy the contents around on service start/stop.
240 knotData = {
241 source = "$TANGLED_VM_DATA_DIR/knot";
242 target = "/mnt/knot-data";
243 };
244 spindleData = {
245 source = "$TANGLED_VM_DATA_DIR/spindle";
246 target = "/mnt/spindle-data";
247 };
248 spindleLogs = {
249 source = "$TANGLED_VM_DATA_DIR/spindle-logs";
250 target = "/var/log/spindle";
251 };
252 }
253 // lib.optionalAttrs localDev {
254 # AT Protocol state persistence: PLC (postgres), PDS (sqlite+blobs),
255 # Jetstream (pebbledb), and the owner DID file.
256 atprotoData = {
257 source = "$TANGLED_VM_DATA_DIR/atproto";
258 target = "/mnt/atproto";
259 };
260 };
261 };
262 # This is fine because any and all ports that are forwarded to host are explicitly marked above, we don't need a separate guest firewall
263 networking.firewall.enable = false;
264 time.timeZone = "Europe/London";
265 services.getty.autologinUser = "root";
266 environment.systemPackages = with pkgs; [curl vim git sqlite litecli jq];
267 services.tangled.knot = {
268 enable = true;
269 motd = "Welcome to the development knot!\n";
270 server = {
271 owner =
272 if localDev
273 then "bootstrap-pending"
274 else envVar "TANGLED_VM_KNOT_OWNER";
275 hostname = envVarOr "TANGLED_VM_KNOT_HOST" "localhost:6444";
276 plcUrl = plcUrl;
277 jetstreamEndpoint = jetstream;
278 listenAddr = "0.0.0.0:6444";
279 dev = localDev;
280 };
281 };
282 services.tangled.spindle = {
283 enable = true;
284 server = {
285 owner =
286 if localDev
287 then "bootstrap-pending"
288 else envVar "TANGLED_VM_SPINDLE_OWNER";
289 hostname = envVarOr "TANGLED_VM_SPINDLE_HOST" "localhost:6555";
290 plcUrl = plcUrl;
291 jetstreamEndpoint = jetstream;
292 listenAddr = "0.0.0.0:6555";
293 dev = true;
294 queueSize = 100;
295 maxJobCount = 2;
296 secrets = {
297 provider = "sqlite";
298 };
299 };
300 };
301
302 # ── Local PLC + PDS + Jetstream (offline dev mode) ──────────────
303 #
304 # When TANGLED_VM_LOCAL_DEV=1, the VM runs a fully self-contained
305 # AT Protocol stack:
306 # PostgreSQL → PLC directory → PDS → Jetstream → Knot/Spindle
307 #
308 # A bootstrap service creates a dev account and writes the owner
309 # DID to an environment file that knot/spindle load at startup.
310
311 # PostgreSQL for PLC directory (persistent DID storage)
312 services.postgresql = lib.mkIf localDev {
313 enable = true;
314 package = pkgs.postgresql_16;
315 ensureDatabases = ["plc"];
316 ensureUsers = [
317 {
318 name = "plc";
319 ensureDBOwnership = true;
320 }
321 ];
322 # Trust local connections (dev-only, no passwords needed)
323 authentication = lib.mkForce ''
324 # TYPE DATABASE USER ADDRESS METHOD
325 local all all trust
326 host all all 127.0.0.1/32 trust
327 host all all ::1/128 trust
328 '';
329 };
330
331 users = {
332 # So we don't have to deal with permission clashing between
333 # blank disk VMs and existing state
334 users.${config.services.tangled.knot.gitUser}.uid = 666;
335 groups.${config.services.tangled.knot.gitUser}.gid = 666;
336
337 # TODO: separate spindle user
338 };
339
340 systemd.services = let
341 mkDataSyncScripts = source: target: {
342 enableStrictShellChecks = true;
343
344 preStart = lib.mkBefore ''
345 mkdir -p ${target}
346 ${lib.getExe pkgs.rsync} -a ${source}/ ${target}
347 '';
348
349 postStop = lib.mkAfter ''
350 ${lib.getExe pkgs.rsync} -a ${target}/ ${source}
351 '';
352
353 serviceConfig.PermissionsStartOnly = true;
354 };
355 # In localDev mode, knot/spindle depend on the bootstrap service
356 # which creates a dev account and writes the owner DID to an env
357 # file. Requires= ensures knot/spindle won't start if bootstrap
358 # failed (e.g. PDS never came up). After= ensures ordering.
359 localDevDeps = lib.optionalAttrs localDev {
360 after = ["tangled-bootstrap.service"];
361 requires = ["tangled-bootstrap.service"];
362 serviceConfig.EnvironmentFile = ["/var/lib/tangled-dev/env"];
363 };
364 in
365 {
366 knot =
367 (mkDataSyncScripts "/mnt/knot-data" config.services.tangled.knot.stateDir)
368 // localDevDeps;
369 spindle =
370 (mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled.spindle.server.dbPath))
371 // localDevDeps;
372 }
373 // lib.optionalAttrs localDev {
374 # ── AT Protocol state persistence ─────────────────────────
375 # PDS, Jetstream, and PostgreSQL (PLC) data is synced to/from
376 # the 9p mount at /mnt/atproto so it survives VM reboots.
377 # DID stability across reboots is critical — knot/spindle
378 # data references the owner DID.
379 #
380 # PostgreSQL (PLC) persistence via pg_dump/psql.
381 # The qcow2 disk is the primary storage; the 9p dump is a
382 # safety net so DID data survives qcow2 deletion.
383 #
384 # Boot: pg starts → setup creates DB → restore loads dump (if DB empty)
385 # Shutdown: backup dumps → pg stops
386 atproto-pg-restore = {
387 description = "Restore PLC database from 9p dump";
388 after = ["postgresql.service" "postgresql-setup.service"];
389 requires = ["postgresql.service"];
390 wantedBy = ["multi-user.target"];
391 serviceConfig = {
392 Type = "oneshot";
393 RemainAfterExit = true;
394 ExecStart = pkgs.writeShellScript "atproto-pg-restore" ''
395 dump="/mnt/atproto/plc.sql"
396 if [ ! -f "$dump" ]; then
397 echo "atproto-pg-restore: no dump file, skipping"
398 exit 0
399 fi
400 table_count=$(${pkgs.postgresql_16}/bin/psql -U plc -d plc -tAc \
401 "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'")
402 if [ "$table_count" != "0" ]; then
403 echo "atproto-pg-restore: database already has $table_count tables, skipping"
404 exit 0
405 fi
406 echo "atproto-pg-restore: loading dump into empty database..."
407 ${pkgs.postgresql_16}/bin/psql -U plc -d plc < "$dump"
408 echo "atproto-pg-restore: done"
409 '';
410 };
411 };
412 atproto-pg-backup = {
413 description = "Backup PLC database to 9p dump";
414 after = ["postgresql.service"];
415 wants = ["postgresql.service"];
416 wantedBy = ["multi-user.target"];
417 serviceConfig = {
418 Type = "oneshot";
419 RemainAfterExit = true;
420 # ExecStart is a no-op; the real work is in ExecStop
421 # which fires on shutdown while postgresql is still running.
422 ExecStart = "${pkgs.coreutils}/bin/true";
423 ExecStop = pkgs.writeShellScript "atproto-pg-backup" ''
424 echo "atproto-pg-backup: dumping PLC database..."
425 ${pkgs.postgresql_16}/bin/pg_dump -U plc plc > /mnt/atproto/plc.sql
426 echo "atproto-pg-backup: done ($(wc -c < /mnt/atproto/plc.sql) bytes)"
427 '';
428 };
429 };
430
431 # PLC directory server
432 plc-server = {
433 description = "DID PLC Directory Server (local development)";
434 after = ["network.target" "atproto-pg-restore.service"];
435 wants = ["postgresql.service"];
436 wantedBy = ["multi-user.target"];
437
438 serviceConfig = {
439 Environment = [
440 "DATABASE_URL=postgres://plc@127.0.0.1/plc"
441 "PORT=${toString plcPort}"
442 "LOG_ENABLED=true"
443 "LOG_LEVEL=info"
444 "DEBUG_MODE=1"
445 ];
446 ExecStart = "${did-plc-server}/bin/plc-server";
447 Restart = "on-failure";
448 RestartSec = "5s";
449 };
450 };
451
452 # Bluesky PDS (patched for HTTP local dev)
453 bluesky-pds =
454 (mkDataSyncScripts "/mnt/atproto/pds" "/var/lib/bluesky-pds")
455 // {
456 description = "Bluesky PDS (local development)";
457 after = ["network.target" "plc-server.service"];
458 wants = ["plc-server.service"];
459 wantedBy = ["multi-user.target"];
460
461 serviceConfig = {
462 StateDirectory = "bluesky-pds";
463 WorkingDirectory = "/var/lib/bluesky-pds";
464 Environment = [
465 "PDS_HOSTNAME=localhost"
466 "PDS_PORT=${toString pdsPort}"
467 "PDS_DATA_DIRECTORY=/var/lib/bluesky-pds"
468 "PDS_DID_PLC_URL=http://localhost:${toString plcPort}"
469 "PDS_BLOBSTORE_DISK_LOCATION=/var/lib/bluesky-pds/blobs"
470 "PDS_JWT_SECRET=${pdsJwtSecret}"
471 "PDS_ADMIN_PASSWORD=${pdsAdminPassword}"
472 "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=${pdsPlcRotationKey}"
473 "PDS_CRAWLERS="
474 "PDS_INVITE_REQUIRED=false"
475 "PDS_DEV_MODE=true"
476 "PDS_LOG_ENABLED=true"
477 "NODE_ENV=development"
478 ];
479 ExecStart = "${pds-local-dev}/bin/pds";
480 Restart = "on-failure";
481 RestartSec = "5s";
482 };
483 };
484
485 # Jetstream (native Nix package — subscribes to local PDS firehose)
486 jetstream =
487 (mkDataSyncScripts "/mnt/atproto/jetstream" "/var/lib/jetstream")
488 // {
489 description = "AT Protocol Jetstream (local development)";
490 after = ["network.target" "bluesky-pds.service"];
491 wants = ["bluesky-pds.service"];
492 wantedBy = ["multi-user.target"];
493
494 serviceConfig = {
495 StateDirectory = "jetstream";
496 Environment = [
497 "JETSTREAM_WS_URL=ws://127.0.0.1:${toString pdsPort}/xrpc/com.atproto.sync.subscribeRepos"
498 "JETSTREAM_LISTEN_ADDR=:${toString jetstreamPort}"
499 "JETSTREAM_METRICS_LISTEN_ADDR=:6009"
500 "JETSTREAM_DATA_DIR=/var/lib/jetstream"
501 ];
502 ExecStart = "${jetstream-pkg}/bin/jetstream";
503 Restart = "on-failure";
504 RestartSec = "5s";
505 };
506 };
507
508 # Bootstrap: create dev account and write owner DID env file
509 tangled-bootstrap = {
510 description = "Tangled local dev bootstrap (account creation)";
511 after = ["bluesky-pds.service"];
512 wants = ["bluesky-pds.service"];
513 wantedBy = ["multi-user.target"];
514
515 serviceConfig = {
516 Type = "oneshot";
517 RemainAfterExit = true;
518 StateDirectory = "tangled-dev";
519 ExecStart = bootstrapScript;
520 };
521 };
522 };
523 })
524 ];
525 }