+30
-35
modules/pds/nixos.nix
+30
-35
modules/pds/nixos.nix
···
2
2
config,
3
3
lib,
4
4
pkgs,
5
+
options,
5
6
...
6
7
}:
7
8
let
···
23
24
24
25
litestreamConfig = pkgs.writeText "litestream-pds-config.yml" ''
25
26
dbs:
26
-
- dir: ${cfg.pdsDataDir}
27
+
- dir: ${cfg.dataDir}
27
28
pattern: "*.sqlite"
28
29
recursive: true
29
30
watch: true
30
31
replica:
31
32
type: s3
32
-
path: ${cfg.s3Prefix}
33
+
path: ${cfg.backupS3Prefix}
33
34
bucket: ''${S3_BUCKET}
34
35
'';
35
36
···
74
75
fi
75
76
76
77
s3Bucket="''${S3_BUCKET}"
77
-
s3Prefix="${cfg.s3Prefix}"
78
+
s3Prefix="${cfg.backupS3Prefix}"
78
79
79
80
run_aws() {
80
81
local envArgs=()
···
127
128
echo "$databases"
128
129
echo ""
129
130
130
-
mkdir -p "${cfg.pdsDataDir}"
131
+
mkdir -p "${cfg.dataDir}"
131
132
132
133
local restoredCount=0
133
134
for db in $databases; do
134
-
local localPath="${cfg.pdsDataDir}/$db"
135
+
local localPath="${cfg.dataDir}/$db"
135
136
local s3DbPath="$s3Prefix/$db"
136
137
local s3DbUrl="s3://$s3Bucket/$s3DbPath"
137
138
···
176
177
exit 1
177
178
fi
178
179
179
-
if [ -f "${cfg.pdsDataDir}/primary.sqlite" ]; then
180
+
if [ -f "${cfg.dataDir}/primary.sqlite" ]; then
180
181
if ! systemctl is-active --quiet litestream-pds; then
181
182
echo "[PDS HealthCheck] Litestream service is not running"
182
183
exit 1
···
191
192
options.services.pds-with-backups = {
192
193
enable = mkEnableOption "Zero-Touch Recovery PDS with Litestream and S3 blob storage";
193
194
194
-
domain = mkOption {
195
-
type = types.str;
196
-
description = "PDS domain name (e.g., bsky.example.com).";
197
-
example = "bsky.example.com";
198
-
};
199
-
200
-
pdsDataDir = mkOption {
201
-
type = types.str;
195
+
dataDir = mkOption {
196
+
type = types.path;
202
197
default = "/var/lib/pds";
203
198
description = "PDS data directory for SQLite databases.";
204
199
};
···
215
210
example = [ "/run/secrets/pds.env" ];
216
211
};
217
212
218
-
s3Prefix = mkOption {
213
+
backupS3Prefix = mkOption {
219
214
type = types.strMatching "[^/].*[^/]";
220
-
default = "pds";
215
+
default = "backups";
221
216
description = "S3 directory prefix for Litestream replicas.";
222
217
example = "pds-backups";
223
218
};
224
219
225
-
pdsSettings = mkOption {
226
-
type = types.attrs;
220
+
backupLogDir = mkOption {
221
+
type = types.path;
222
+
default = "/var/log/pds-backup";
223
+
description = "Directory for backup and restore logs.";
224
+
internal = true;
225
+
};
226
+
227
+
settings = mkOption {
228
+
type = options.services.bluesky-pds.settings.type;
227
229
default = { };
228
-
description = "Additional settings to pass to bluesky-pds.";
230
+
description = "Additional settings to pass to bluesky-pds:\n\n" ++ options.services.bluesky-pds.settings.description;
229
231
example = {
230
232
PDS_PORT = 3000;
231
-
PDS_DISABLE_PHONE_VERIFICATION = "true";
233
+
PDS_HOSTNAME = "hi.example.com";
232
234
};
233
-
};
234
-
235
-
backupLogDir = mkOption {
236
-
type = types.path;
237
-
default = "/var/log/pds-backup";
238
-
description = "Directory for backup and restore logs.";
239
235
};
240
236
};
241
237
···
244
240
enable = mkDefault true;
245
241
settings = mkMerge [
246
242
{
247
-
PDS_HOSTNAME = cfg.domain;
248
243
PDS_SQLITE_DISABLE_WAL_AUTO_CHECKPOINT = "true";
249
-
PDS_DATA_DIRECTORY = cfg.pdsDataDir;
244
+
PDS_DATA_DIRECTORY = cfg.dataDir;
250
245
}
251
-
cfg.pdsSettings
246
+
cfg.settings
252
247
];
253
248
environmentFiles = secretsFiles;
254
249
};
···
261
256
users.groups.${pdsGroup} = { };
262
257
263
258
systemd.tmpfiles.rules = [
264
-
"d ${cfg.pdsDataDir} 0755 ${pdsUser} ${pdsGroup} -"
259
+
"d ${cfg.dataDir} 0755 ${pdsUser} ${pdsGroup} -"
265
260
"d ${cfg.backupLogDir} 0755 ${pdsUser} ${pdsGroup} -"
266
261
];
267
262
···
284
279
Type = "oneshot";
285
280
ExecStart = "${restoreScript}/bin/pds-litestream-restore";
286
281
EnvironmentFile = secretsFiles;
287
-
User = "root";
288
-
Group = "root";
282
+
User = pdsUser;
283
+
Group = pdsGroup;
289
284
RemainAfterExit = true;
290
285
291
286
NoNewPrivileges = true;
···
318
313
ProtectSystem = "strict";
319
314
ProtectHome = true;
320
315
ReadWritePaths = [
321
-
cfg.pdsDataDir
316
+
cfg.dataDir
322
317
cfg.backupLogDir
323
318
];
324
319
RestrictRealtime = true;
···
337
332
serviceConfig = {
338
333
Type = "oneshot";
339
334
ExecStart = healthCheckScript;
340
-
User = "root";
341
-
Group = "root";
335
+
User = pdsUser;
336
+
Group = pdsGroup;
342
337
343
338
NoNewPrivileges = true;
344
339
ProtectSystem = "strict";
+6
-18
modules/pds/pds-recovery-full.nix
+6
-18
modules/pds/pds-recovery-full.nix
···
13
13
nodes.machine =
14
14
{ pkgs, ... }:
15
15
{
16
-
imports = [ ./default.nix ];
16
+
imports = [ ./nixos.nix ];
17
17
18
18
services.minio = {
19
19
enable = true;
···
22
22
23
23
systemd.tmpfiles.rules = [
24
24
"f /tmp/minio-credentials 0600 root root - MINIO_ROOT_USER=minioadmin\\nMINIO_ROOT_PASSWORD=minioadmin123"
25
-
"f /run/secrets/s3.env 0600 root root - AWS_ACCESS_KEY_ID=minioadmin\\nAWS_SECRET_ACCESS_KEY=minioadmin123\\nAWS_ENDPOINT_URL=http://127.0.0.1:9000\\nS3_BUCKET=pds-test-bucket"
26
-
"f /run/secrets/pds.env 0600 root root - PDS_JWT_SECRET=test-jwt-secret-for-full-testing\\nPDS_ADMIN_PASSWORD=test-admin-password\\nPDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=1111111111111111111111111111111111111111111111111111111111111111"
25
+
"f /run/secrets/s3.env 0600 pds pds - AWS_ACCESS_KEY_ID=minioadmin\\nAWS_SECRET_ACCESS_KEY=minioadmin123\\nAWS_ENDPOINT_URL=http://127.0.0.1:9000\\nS3_BUCKET=pds-test-bucket"
26
+
"f /run/secrets/pds.env 0600 pds pds - PDS_JWT_SECRET=test-jwt-secret-for-full-testing\\nPDS_ADMIN_PASSWORD=test-admin-password\\nPDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=1111111111111111111111111111111111111111111111111111111111111111"
27
27
];
28
28
29
29
services.pds-with-backups = {
30
30
enable = true;
31
-
domain = "test.example.com";
32
-
pdsDataDir = "/var/lib/pds";
33
31
secretsFiles = [
34
32
"/run/secrets/pds.env"
35
33
"/run/secrets/s3.env"
36
34
];
37
-
s3Bucket = "pds-test-bucket";
38
-
s3Prefix = "pds-replica";
39
-
enableStatelessBlobs = false;
40
-
pdsSettings = {
41
-
PDS_PORT = 3000;
42
-
PDS_DISABLE_PHONE_VERIFICATION = "true";
35
+
backupS3Prefix = "pds-replica";
36
+
settings = {
37
+
PDS_HOSTNAME = "advanced.example.com";
43
38
};
44
39
};
45
40
···
62
57
print("\n--- Setting up MinIO ---")
63
58
machine.wait_for_unit("minio.service")
64
59
machine.wait_for_open_port(9000)
65
-
machine.succeed("sleep 5")
66
60
67
61
machine.succeed("test -f /tmp/minio-credentials")
68
62
machine.succeed("test -f /run/secrets/s3.env")
···
80
74
machine.wait_for_unit("bluesky-pds.service")
81
75
machine.wait_for_open_port(3000)
82
76
83
-
machine.succeed("sleep 30")
84
-
85
77
health_response = machine.succeed("curl -s http://127.0.0.1:3000/xrpc/_health")
86
78
try:
87
79
health_data = json.loads(health_response)
···
97
89
print(" [PASS] Litestream service is running")
98
90
99
91
print("\n--- Test 3: Database creation ---")
100
-
machine.succeed("sleep 30")
101
92
pds_files = machine.succeed("ls -la /var/lib/pds/")
102
93
print(f" Files in /var/lib/pds: {pds_files}")
103
94
···
127
118
machine.succeed("systemctl start bluesky-pds")
128
119
machine.wait_for_unit("bluesky-pds.service")
129
120
machine.wait_for_open_port(3000)
130
-
machine.succeed("sleep 30")
131
121
132
122
health_response2 = machine.succeed("curl -s http://127.0.0.1:3000/xrpc/_health")
133
123
try:
···
140
130
print("\n--- Test 7: Litestream resumption ---")
141
131
machine.succeed("systemctl start litestream-pds")
142
132
machine.wait_for_unit("litestream-pds.service")
143
-
machine.succeed("sleep 10")
144
133
145
134
final_minio = machine.succeed("mc ls local/pds-test-bucket/pds-replica/ --recursive 2>/dev/null")
146
135
assert ".sqlite" in final_minio, "Expected ongoing replication"
···
148
137
149
138
print("\n--- Test 8: Health check service ---")
150
139
machine.succeed("systemctl start pds-healthcheck")
151
-
machine.succeed("sleep 2")
152
140
health_log = machine.succeed("journalctl -u pds-healthcheck.service -o cat 2>/dev/null || true")
153
141
assert "All services healthy" in health_log, f"Expected health check message, got: {health_log}"
154
142
print(" [PASS] Health check service working")
+6
-11
modules/pds/pds-recovery-simple.nix
+6
-11
modules/pds/pds-recovery-simple.nix
···
13
13
nodes.machine =
14
14
{ pkgs, lib, ... }:
15
15
{
16
-
imports = [ ./default.nix ];
16
+
imports = [ ./nixos.nix ];
17
17
18
18
services.pds-with-backups = {
19
19
enable = true;
20
-
domain = "test.example.com";
21
-
pdsDataDir = "/var/lib/pds";
22
20
secretsFiles = [
23
21
"/run/secrets/pds.env"
24
22
"/run/secrets/s3.env"
25
23
];
26
-
s3Bucket = "test-bucket";
27
-
s3Prefix = "test-pds";
28
-
enableStatelessBlobs = false;
29
-
pdsSettings = {
30
-
PDS_PORT = 3000;
31
-
PDS_DISABLE_PHONE_VERIFICATION = "true";
24
+
backupS3Prefix = "test-pds";
25
+
settings = {
26
+
PDS_HOSTNAME = "example.com";
32
27
};
33
28
};
34
29
35
30
systemd.tmpfiles.rules = [
36
-
"f /run/secrets/pds.env 0644 root root - PDS_JWT_SECRET=test-jwt-secret\nPDS_ADMIN_PASSWORD=test-password\nPDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=0000000000000000000000000000000000000000000000000000000000000000"
37
-
"f /run/secrets/s3.env 0644 root root - AWS_ACCESS_KEY_ID=test-key\nAWS_SECRET_ACCESS_KEY=test-secret\nAWS_ENDPOINT_URL=https://s3.test.example.com\nS3_BUCKET=test-bucket"
31
+
"f /run/secrets/pds.env 0644 pds pds - PDS_JWT_SECRET=test-jwt-secret\nPDS_ADMIN_PASSWORD=test-password\nPDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=0000000000000000000000000000000000000000000000000000000000000000"
32
+
"f /run/secrets/s3.env 0644 pds pds - AWS_ACCESS_KEY_ID=test-key\nAWS_SECRET_ACCESS_KEY=test-secret\nAWS_ENDPOINT_URL=https://s3.test.example.com\nS3_BUCKET=test-bucket"
38
33
];
39
34
40
35
services.bluesky-pds.enable = true;
+4
-5
systems/reg/pds.nix
+4
-5
systems/reg/pds.nix
···
10
10
format = "dotenv";
11
11
sopsFile = ../../secrets/pds.env;
12
12
restartUnits = [ "bluesky-pds.service" ];
13
+
owner = "pds";
14
+
group = "pds";
13
15
};
14
16
secrets.cloudflare-api = {
15
17
format = "dotenv";
···
19
21
20
22
services.pds-with-backups = {
21
23
enable = true;
22
-
domain = "0xf.fr";
23
24
secretsFiles = [ config.sops.secrets.pds.path ];
24
-
s3Prefix = "backups";
25
-
26
-
pdsSettings = {
27
-
PDS_PORT = 3000;
25
+
settings = {
26
+
PDS_HOSTNAME = "0xf.fr";
28
27
PDS_BLOBSTORE_DISK_LOCATION = null;
29
28
};
30
29
};