commit 54423f5954a4002a1550c197c7ada1e8a55625e3 · quilling.dev/parakeet

+236 -195

flake.nix

··· 80 80 81 81 # Build *just* the cargo dependencies, so we can reuse 82 82 # all of that work (e.g. via cachix) when running in CI 83 - cargoArtifacts = craneLib.buildDepsOnly commonArgs; 83 + # cargoArtifacts = craneLib.buildDepsOnly commonArgs; 84 84 85 - individualCrateArgs = commonArgs // { 86 - inherit cargoArtifacts; 87 - inherit (craneLib.crateNameFromCargoToml { inherit src; }) version; 88 - # NB: we disable tests since we'll run them all via cargo-nextest 89 - doCheck = false; 90 - }; 91 - fileSetForCrate = 92 - crate: 93 - lib.fileset.toSource { 94 - root = ./.; 95 - fileset = lib.fileset.unions [ 96 - ./Cargo.toml 97 - ./Cargo.lock 98 - ./migrations 99 - ./consumer/src/sources/jetstream/zstd_dictionary 100 - (craneLib.fileset.commonCargoSources ./consumer) 101 - ./consumer/src/db/sql 102 - (craneLib.fileset.commonCargoSources ./dataloader-rs) 103 - (craneLib.fileset.commonCargoSources ./did-resolver) 104 - (craneLib.fileset.commonCargoSources ./lexica) 105 - (craneLib.fileset.commonCargoSources ./parakeet) 106 - ./parakeet/src/sql 107 - (craneLib.fileset.commonCargoSources ./parakeet-db) 108 - ./parakeet-db/src/dicts/post_content_v1.dict 109 - (craneLib.fileset.commonCargoSources ./parakeet-lexgen) 110 - (craneLib.fileset.commonCargoSources crate) 111 - ]; 112 - }; 85 + # individualCrateArgs = commonArgs // { 86 + # inherit cargoArtifacts; 87 + # inherit (craneLib.crateNameFromCargoToml { inherit src; }) version; 88 + # # NB: we disable tests since we'll run them all via cargo-nextest 89 + # doCheck = false; 90 + # }; 91 + # fileSetForCrate = 92 + # crate: 93 + # lib.fileset.toSource { 94 + # root = ./.; 95 + # fileset = lib.fileset.unions [ 96 + # ./Cargo.toml 97 + # ./Cargo.lock 98 + # ./migrations 99 + # ./consumer/src/sources/jetstream/zstd_dictionary 100 + # (craneLib.fileset.commonCargoSources ./consumer) 101 + # ./consumer/src/db/sql 102 + # (craneLib.fileset.commonCargoSources ./dataloader-rs) 103 + # (craneLib.fileset.commonCargoSources ./did-resolver) 104 + # (craneLib.fileset.commonCargoSources ./lexica) 105 + # (craneLib.fileset.commonCargoSources ./parakeet) 106 + # ./parakeet/src/sql 107 + # (craneLib.fileset.commonCargoSources ./parakeet-db) 108 + # ./parakeet-db/src/dicts/post_content_v1.dict 109 + # (craneLib.fileset.commonCargoSources ./parakeet-lexgen) 110 + # (craneLib.fileset.commonCargoSources crate) 111 + # ]; 112 + # }; 113 113 114 114 # Build the actual crate itself, reusing the dependency 115 115 # artifacts from above. 116 - consumer = craneLib.buildPackage ( 117 - individualCrateArgs 118 - // { 119 - pname = "consumer"; 120 - cargoExtraArgs = "-p consumer"; 121 - src = fileSetForCrate ./consumer; 122 - postInstall = '' 123 - mkdir -p $out/{bin,lib/consumer} 124 - ''; 125 - } 126 - ); 127 - dataloader = craneLib.buildPackage ( 128 - individualCrateArgs 129 - // { 130 - pname = "dataloader"; 131 - cargoExtraArgs = "-p dataloader"; 132 - src = fileSetForCrate ./dataloader-rs; 133 - } 134 - ); 135 - did-resolver = craneLib.buildPackage ( 136 - individualCrateArgs 137 - // { 138 - pname = "did-resolver"; 139 - cargoExtraArgs = "-p did-resolver"; 140 - src = fileSetForCrate ./did-resolver; 141 - } 142 - ); 143 - lexica = craneLib.buildPackage ( 144 - individualCrateArgs 145 - // { 146 - pname = "lexica"; 147 - cargoExtraArgs = "-p lexica"; 148 - src = fileSetForCrate ./lexica; 149 - } 150 - ); 151 - parakeet = craneLib.buildPackage ( 152 - individualCrateArgs 153 - // { 154 - pname = "parakeet"; 155 - cargoExtraArgs = "-p parakeet"; 156 - src = fileSetForCrate ./parakeet; 157 - } 158 - ); 159 - parakeet-db = craneLib.buildPackage ( 160 - individualCrateArgs 161 - // { 162 - pname = "parakeet-db"; 163 - cargoExtraArgs = "-p parakeet-db"; 164 - src = fileSetForCrate ./parakeet-db; 165 - } 166 - ); 167 - parakeet-lexgen = craneLib.buildPackage ( 168 - individualCrateArgs 169 - // { 170 - pname = "parakeet-lexgen"; 171 - cargoExtraArgs = "-p parakeet-lexgen"; 172 - src = fileSetForCrate ./parakeet-lexgen; 173 - } 174 - ); 116 + # COMMENTED OUT FOR DEV 117 + # consumer = craneLib.buildPackage ( 118 + # individualCrateArgs 119 + # // { 120 + # pname = "consumer"; 121 + # cargoExtraArgs = "-p consumer"; 122 + # src = fileSetForCrate ./consumer; 123 + # postInstall = '' 124 + # mkdir -p $out/{bin,lib/consumer} 125 + # ''; 126 + # } 127 + # ); 128 + # dataloader = craneLib.buildPackage ( 129 + # individualCrateArgs 130 + # // { 131 + # pname = "dataloader"; 132 + # cargoExtraArgs = "-p dataloader"; 133 + # src = fileSetForCrate ./dataloader-rs; 134 + # } 135 + # ); 136 + # did-resolver = craneLib.buildPackage ( 137 + # individualCrateArgs 138 + # // { 139 + # pname = "did-resolver"; 140 + # cargoExtraArgs = "-p did-resolver"; 141 + # src = fileSetForCrate ./did-resolver; 142 + # } 143 + # ); 144 + # lexica = craneLib.buildPackage ( 145 + # individualCrateArgs 146 + # // { 147 + # pname = "lexica"; 148 + # cargoExtraArgs = "-p lexica"; 149 + # src = fileSetForCrate ./lexica; 150 + # } 151 + # ); 152 + # parakeet = craneLib.buildPackage ( 153 + # individualCrateArgs 154 + # // { 155 + # pname = "parakeet"; 156 + # cargoExtraArgs = "-p parakeet"; 157 + # src = fileSetForCrate ./parakeet; 158 + # } 159 + # ); 160 + # parakeet-db = craneLib.buildPackage ( 161 + # individualCrateArgs 162 + # // { 163 + # pname = "parakeet-db"; 164 + # cargoExtraArgs = "-p parakeet-db"; 165 + # src = fileSetForCrate ./parakeet-db; 166 + # } 167 + # ); 168 + # parakeet-lexgen = craneLib.buildPackage ( 169 + # individualCrateArgs 170 + # // { 171 + # pname = "parakeet-lexgen"; 172 + # cargoExtraArgs = "-p parakeet-lexgen"; 173 + # src = fileSetForCrate ./parakeet-lexgen; 174 + # } 175 + # ); 176 + 177 + # Dummy derivations to satisfy references 178 + consumer = pkgs.writeTextDir "bin/consumer" "echo 'Consumer not built in dev mode'"; 179 + dataloader = pkgs.writeTextDir "bin/dataloader" "echo 'Dataloader not built in dev mode'"; 180 + did-resolver = pkgs.writeTextDir "bin/did-resolver" "echo 'DID resolver not built in dev mode'"; 181 + lexica = pkgs.writeTextDir "bin/lexica" "echo 'Lexica not built in dev mode'"; 182 + parakeet = pkgs.writeTextDir "bin/parakeet" "echo 'Parakeet not built in dev mode'"; 183 + parakeet-db = pkgs.writeTextDir "bin/parakeet-db" "echo 'Parakeet-db not built in dev mode'"; 184 + parakeet-lexgen = pkgs.writeTextDir "bin/parakeet-lexgen" "echo 'Parakeet-lexgen not built in dev mode'"; 175 185 in 176 186 { 177 187 checks = { ··· 204 214 # Inherit inputs from checks. 205 215 checks = self.checks.${system}; 206 216 217 + # Manually provision devshell 218 + nativeBuildInputs = with pkgs; [ 219 + pkg-config 220 + ]; 221 + buildInputs = [ 222 + pkgs.openssl 223 + pkgs.postgresql 224 + pkgs.libpq 225 + pkgs.clang 226 + pkgs.libclang 227 + pkgs.lld 228 + pkgs.protobuf 229 + ]; 230 + CLANG_PATH = "${pkgs.llvmPackages_18.clang}/bin/clang"; 231 + PROTOC_INCLUDE = "${pkgs.protobuf}/include"; 232 + PROTOC = "${pkgs.protobuf}/bin/protoc"; 233 + 207 234 # Additional dev-shell environment variables can be set directly 208 235 RUST_BACKTRACE = 0; 209 236 NIXOS_OZONE_WL = 1; ··· 227 254 tokei 228 255 ast-grep 229 256 scc 257 + openssl 258 + postgresql 259 + libpq 260 + clang 261 + libclang 262 + lld 263 + protobuf 264 + pkg-config 230 265 ]; 231 266 }; 232 267 } ··· 277 312 }; 278 313 }; 279 314 config = mkIf cfg.enable { 280 - environment.systemPackages = [ 281 - self.packages.${pkgs.system}.consumer 282 - ]; 283 - systemd.services.consumer = { 284 - description = "consumer"; 285 - after = [ "network-online.target" "parakeet.service" ]; 286 - wants = [ "network-online.target" ]; 287 - requires = [ "parakeet.service" ]; 288 - wantedBy = [ "multi-user.target" ]; 289 - serviceConfig = { 290 - ExecStart = "${self.packages.${pkgs.system}.consumer}/bin/consumer --backfill --indexer"; 291 - Type = "exec"; 292 - WorkingDirectory = cfg.workingDirectory; 315 + # environment.systemPackages = [ 316 + # self.packages.${pkgs.system}.consumer 317 + # ]; 318 + # systemd.services.consumer = { 319 + # description = "consumer"; 320 + # after = [ 321 + # "network-online.target" 322 + # "parakeet.service" 323 + # ]; 324 + # wants = [ "network-online.target" ]; 325 + # requires = [ "parakeet.service" ]; 326 + # wantedBy = [ "multi-user.target" ]; 327 + # serviceConfig = { 328 + # ExecStart = "${self.packages.${pkgs.system}.consumer}/bin/consumer --backfill --indexer"; 329 + # Type = "exec"; 330 + # WorkingDirectory = cfg.workingDirectory; 293 331 294 - EnvironmentFile = cfg.environmentFiles; 295 - User = "parakeet"; 296 - Group = "parakeet"; 297 - StateDirectory = "parakeet"; 298 - StateDirectoryMode = "0755"; 299 - Restart = "always"; 332 + # EnvironmentFile = cfg.environmentFiles; 333 + # User = "parakeet"; 334 + # Group = "parakeet"; 335 + # StateDirectory = "parakeet"; 336 + # StateDirectoryMode = "0755"; 337 + # Restart = "always"; 300 338 301 - # Hardening 302 - RemoveIPC = true; 303 - CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; 304 - NoNewPrivileges = true; 305 - PrivateDevices = true; 306 - ProtectClock = true; 307 - ProtectKernelLogs = true; 308 - ProtectControlGroups = true; 309 - ProtectKernelModules = true; 310 - PrivateMounts = true; 311 - SystemCallArchitectures = [ "native" ]; 312 - MemoryDenyWriteExecute = false; # required by V8 JIT 313 - RestrictNamespaces = true; 314 - RestrictSUIDSGID = true; 315 - ProtectHostname = true; 316 - LockPersonality = true; 317 - ProtectKernelTunables = true; 318 - RestrictAddressFamilies = [ 319 - "AF_UNIX" 320 - "AF_INET" 321 - "AF_INET6" 322 - ]; 323 - RestrictRealtime = true; 324 - DeviceAllow = [ "" ]; 325 - ProtectSystem = "full"; 326 - ProtectProc = "invisible"; 327 - ProcSubset = "pid"; 328 - ProtectHome = true; 329 - PrivateUsers = true; 330 - PrivateTmp = true; 331 - UMask = "0077"; 332 - }; 333 - }; 334 - systemd.services.parakeet = { 335 - description = "parakeet"; 336 - after = [ "network-online.target" "postgresql.service" ]; 337 - wants = [ "network-online.target" ]; 338 - requires = [ "postgresql.service" ]; 339 - wantedBy = [ "multi-user.target" ]; 340 - serviceConfig = { 341 - ExecStart = "${cfg.package}/bin/parakeet"; 342 - Type = "exec"; 343 - WorkingDirectory = cfg.workingDirectory; 339 + # # Hardening 340 + # RemoveIPC = true; 341 + # CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; 342 + # NoNewPrivileges = true; 343 + # PrivateDevices = true; 344 + # ProtectClock = true; 345 + # ProtectKernelLogs = true; 346 + # ProtectControlGroups = true; 347 + # ProtectKernelModules = true; 348 + # PrivateMounts = true; 349 + # SystemCallArchitectures = [ "native" ]; 350 + # MemoryDenyWriteExecute = false; # required by V8 JIT 351 + # RestrictNamespaces = true; 352 + # RestrictSUIDSGID = true; 353 + # ProtectHostname = true; 354 + # LockPersonality = true; 355 + # ProtectKernelTunables = true; 356 + # RestrictAddressFamilies = [ 357 + # "AF_UNIX" 358 + # "AF_INET" 359 + # "AF_INET6" 360 + # ]; 361 + # RestrictRealtime = true; 362 + # DeviceAllow = [ "" ]; 363 + # ProtectSystem = "full"; 364 + # ProtectProc = "invisible"; 365 + # ProcSubset = "pid"; 366 + # ProtectHome = true; 367 + # PrivateUsers = true; 368 + # PrivateTmp = true; 369 + # UMask = "0077"; 370 + # }; 371 + # }; 372 + # systemd.services.parakeet = { 373 + # description = "parakeet"; 374 + # after = [ 375 + # "network-online.target" 376 + # "postgresql.service" 377 + # ]; 378 + # wants = [ "network-online.target" ]; 379 + # requires = [ "postgresql.service" ]; 380 + # wantedBy = [ "multi-user.target" ]; 381 + # serviceConfig = { 382 + # ExecStart = "${cfg.package}/bin/parakeet"; 383 + # Type = "exec"; 384 + # WorkingDirectory = cfg.workingDirectory; 344 385 345 - EnvironmentFile = cfg.environmentFiles; 346 - User = "parakeet"; 347 - Group = "parakeet"; 348 - StateDirectory = "parakeet"; 349 - StateDirectoryMode = "0755"; 350 - Restart = "always"; 386 + # EnvironmentFile = cfg.environmentFiles; 387 + # User = "parakeet"; 388 + # Group = "parakeet"; 389 + # StateDirectory = "parakeet"; 390 + # StateDirectoryMode = "0755"; 391 + # Restart = "always"; 351 392 352 - # Hardening 353 - RemoveIPC = true; 354 - CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; 355 - NoNewPrivileges = true; 356 - PrivateDevices = true; 357 - ProtectClock = true; 358 - ProtectKernelLogs = true; 359 - ProtectControlGroups = true; 360 - ProtectKernelModules = true; 361 - PrivateMounts = true; 362 - SystemCallArchitectures = [ "native" ]; 363 - MemoryDenyWriteExecute = false; # required by V8 JIT 364 - RestrictNamespaces = true; 365 - RestrictSUIDSGID = true; 366 - ProtectHostname = true; 367 - LockPersonality = true; 368 - ProtectKernelTunables = true; 369 - RestrictAddressFamilies = [ 370 - "AF_UNIX" 371 - "AF_INET" 372 - "AF_INET6" 373 - ]; 374 - RestrictRealtime = true; 375 - DeviceAllow = [ "" ]; 376 - ProtectSystem = "full"; 377 - ProtectProc = "invisible"; 378 - ProcSubset = "pid"; 379 - ProtectHome = true; 380 - PrivateUsers = true; 381 - PrivateTmp = true; 382 - UMask = "0077"; 383 - }; 384 - }; 393 + # # Hardening 394 + # RemoveIPC = true; 395 + # CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; 396 + # NoNewPrivileges = true; 397 + # PrivateDevices = true; 398 + # ProtectClock = true; 399 + # ProtectKernelLogs = true; 400 + # ProtectControlGroups = true; 401 + # ProtectKernelModules = true; 402 + # PrivateMounts = true; 403 + # SystemCallArchitectures = [ "native" ]; 404 + # MemoryDenyWriteExecute = false; # required by V8 JIT 405 + # RestrictNamespaces = true; 406 + # RestrictSUIDSGID = true; 407 + # ProtectHostname = true; 408 + # LockPersonality = true; 409 + # ProtectKernelTunables = true; 410 + # RestrictAddressFamilies = [ 411 + # "AF_UNIX" 412 + # "AF_INET" 413 + # "AF_INET6" 414 + # ]; 415 + # RestrictRealtime = true; 416 + # DeviceAllow = [ "" ]; 417 + # ProtectSystem = "full"; 418 + # ProtectProc = "invisible"; 419 + # ProcSubset = "pid"; 420 + # ProtectHome = true; 421 + # PrivateUsers = true; 422 + # PrivateTmp = true; 423 + # UMask = "0077"; 424 + # }; 425 + # }; 385 426 users = { 386 427 users.parakeet = { 387 428 group = "parakeet"; ··· 408 449 extraPlugins = with pkgs.postgresql16Packages; [ 409 450 # Note: pg_stat_statements is built into PostgreSQL (contrib module) 410 451 # and doesn't need to be listed here - just enable via CREATE EXTENSION 411 - pgvector # Vector similarity search (future-proofing) 412 - timescaledb # Time-series optimization 413 - pgrouting # Graph analysis for social graph 414 - postgis # Required dependency for pgrouting 452 + pgvector # Vector similarity search (future-proofing) 453 + timescaledb # Time-series optimization 454 + pgrouting # Graph analysis for social graph 455 + postgis # Required dependency for pgrouting 415 456 ]; 416 457 settings = { 417 458 # Preload extensions that require early initialization ··· 419 460 shared_preload_libraries = "pg_stat_statements,timescaledb"; 420 461 421 462 # pg_stat_statements configuration 422 - "pg_stat_statements.max" = "10000"; # Track up to 10k unique queries 423 - "pg_stat_statements.track" = "all"; # Track all queries (top-level + nested) 463 + "pg_stat_statements.max" = "10000"; # Track up to 10k unique queries 464 + "pg_stat_statements.track" = "all"; # Track all queries (top-level + nested) 424 465 }; 425 466 }; 426 467 };