From ed52f2e113d6011311335d1dabdb01ffae49e9bf Mon Sep 17 00:00:00 2001 From: Scott Hadfield Date: Sun, 27 Sep 2026 10:30:28 -0700 Subject: [PATCH] Implement initial buildout of core features. --- .env.example | 20 + .gitignore | 2 + .nvmrc | 1 + CLAUDE.md | 18 +- README.md | 24 + apps/server/package.json | 52 + apps/server/src/api-types.ts | 16 + apps/server/src/app.ts | 136 + apps/server/src/auth/accounts.ts | 105 + apps/server/src/auth/identity.ts | 35 + apps/server/src/auth/oauth-client.ts | 98 + apps/server/src/auth/pds.ts | 29 + apps/server/src/auth/roles.ts | 38 + apps/server/src/auth/routes.ts | 153 + apps/server/src/auth/scope.ts | 54 + apps/server/src/auth/web-session.ts | 62 + apps/server/src/blobs/routes.ts | 114 + apps/server/src/blobs/store.ts | 138 + apps/server/src/body.ts | 28 + apps/server/src/config.ts | 300 ++ apps/server/src/db/dialect.ts | 15 + apps/server/src/db/index.ts | 34 + apps/server/src/db/migrate.ts | 20 + .../migrations/0001_plugin_user_settings.ts | 13 + apps/server/src/db/migrations/0002_auth.ts | 31 + apps/server/src/db/migrations/0003_storage.ts | 49 + .../src/db/migrations/0004_providers.ts | 23 + apps/server/src/db/migrations/0005_turns.ts | 21 + apps/server/src/db/migrations/index.ts | 15 + apps/server/src/db/schema.ts | 77 + apps/server/src/env.ts | 6 + apps/server/src/index.ts | 40 + apps/server/src/keys-cli.ts | 6 + apps/server/src/logger.ts | 7 + apps/server/src/migrate-cli.ts | 12 + apps/server/src/plugins/hooks.ts | 94 + apps/server/src/plugins/host.ts | 108 + apps/server/src/plugins/import.ts | 74 + apps/server/src/plugins/registry.ts | 57 + apps/server/src/plugins/routes.ts | 72 + apps/server/src/plugins/user-settings.ts | 96 + apps/server/src/providers/catalog.ts | 163 + apps/server/src/providers/effort.ts | 19 + apps/server/src/providers/generate-text.ts | 22 + apps/server/src/providers/guarded-fetch.ts | 99 + apps/server/src/providers/routes.ts | 130 + apps/server/src/providers/user-credentials.ts | 154 + apps/server/src/secrets.ts | 50 + apps/server/src/server.ts | 188 + apps/server/src/sharing/routes.ts | 79 + apps/server/src/sharing/service.ts | 207 + apps/server/src/storage/chat-service.ts | 300 ++ apps/server/src/storage/local-record-store.ts | 220 + apps/server/src/storage/record-store.ts | 99 + apps/server/src/storage/records.ts | 35 + apps/server/src/storage/routes.ts | 158 + apps/server/src/storage/services.ts | 41 + apps/server/src/storage/space-record-store.ts | 226 + apps/server/src/storage/validate.ts | 45 + apps/server/src/sync/coalescer.ts | 63 + apps/server/src/sync/credentials.ts | 150 + apps/server/src/sync/engine.ts | 313 ++ apps/server/src/sync/events.ts | 23 + apps/server/src/sync/identity.ts | 19 + apps/server/src/sync/recent-writes.ts | 26 + apps/server/src/sync/routes.ts | 119 + apps/server/src/sync/scheduler.ts | 87 + apps/server/src/sync/service-auth.ts | 28 + apps/server/src/turns/accumulator.ts | 137 + apps/server/src/turns/prompt.ts | 177 + apps/server/src/turns/routes.ts | 101 + apps/server/src/turns/runner.ts | 539 +++ apps/server/src/turns/stream-hub.ts | 57 + apps/server/src/web-html.ts | 6 + apps/server/tests/app.test.ts | 80 + apps/server/tests/auth/accounts.test.ts | 78 + apps/server/tests/auth/oauth-client.test.ts | 47 + apps/server/tests/auth/pds.test.ts | 31 + apps/server/tests/auth/roles.test.ts | 26 + apps/server/tests/auth/routes.test.ts | 166 + apps/server/tests/auth/scope.test.ts | 44 + apps/server/tests/auth/web-session.test.ts | 68 + apps/server/tests/blobs/routes.test.ts | 140 + apps/server/tests/blobs/store.test.ts | 97 + apps/server/tests/config.test.ts | 162 + apps/server/tests/db/index.test.ts | 44 + apps/server/tests/db/migrate.test.ts | 67 + .../tests/fixtures/plugins/good/index.ts | 15 + .../tests/fixtures/plugins/good/package.json | 5 + .../tests/fixtures/plugins/noexport/index.ts | 1 + .../fixtures/plugins/noexport/package.json | 5 + apps/server/tests/helpers/auth.ts | 59 + apps/server/tests/helpers/config.ts | 11 + apps/server/tests/helpers/db.ts | 23 + apps/server/tests/helpers/fake-pds.ts | 262 ++ apps/server/tests/helpers/providers.ts | 21 + apps/server/tests/helpers/spaces.ts | 84 + apps/server/tests/helpers/turns.ts | 148 + apps/server/tests/plugins/hooks.test.ts | 75 + apps/server/tests/plugins/host.test.ts | 124 + apps/server/tests/plugins/import.test.ts | 76 + apps/server/tests/plugins/registry.test.ts | 56 + apps/server/tests/plugins/routes.test.ts | 135 + .../tests/plugins/user-settings.test.ts | 109 + apps/server/tests/providers/catalog.test.ts | 188 + apps/server/tests/providers/effort.test.ts | 21 + .../tests/providers/generate-text.test.ts | 50 + .../tests/providers/guarded-fetch.test.ts | 114 + apps/server/tests/providers/routes.test.ts | 145 + .../tests/providers/user-credentials.test.ts | 100 + apps/server/tests/secrets.test.ts | 43 + apps/server/tests/server.test.ts | 82 + apps/server/tests/sharing/sharing.test.ts | 269 ++ .../server/tests/storage/chat-service.test.ts | 212 + .../tests/storage/local-record-store.test.ts | 136 + apps/server/tests/storage/routes.test.ts | 178 + .../tests/storage/space-record-store.test.ts | 96 + apps/server/tests/sync/coalescer.test.ts | 64 + apps/server/tests/sync/credentials.test.ts | 82 + apps/server/tests/sync/engine.test.ts | 215 + apps/server/tests/sync/identity.test.ts | 27 + apps/server/tests/sync/recent-writes.test.ts | 12 + apps/server/tests/sync/routes.test.ts | 195 + apps/server/tests/sync/scheduler.test.ts | 46 + apps/server/tests/turns/accumulator.test.ts | 91 + apps/server/tests/turns/prompt.test.ts | 246 ++ apps/server/tests/turns/routes.test.ts | 124 + apps/server/tests/turns/runner.test.ts | 564 +++ apps/server/tsconfig.json | 4 + apps/web/index.html | 13 + apps/web/package.json | 33 + apps/web/src/App.tsx | 91 + apps/web/src/api.ts | 61 + apps/web/src/app-name.ts | 4 + apps/web/src/components/BusyBanner.tsx | 17 + apps/web/src/components/ChatList.tsx | 35 + apps/web/src/components/Composer.tsx | 156 + apps/web/src/components/ConversationView.tsx | 196 + apps/web/src/components/MessageView.tsx | 125 + apps/web/src/components/SchemaForm.tsx | 84 + apps/web/src/components/ShareControl.tsx | 79 + apps/web/src/components/useAction.ts | 13 + apps/web/src/components/useReplyStream.ts | 66 + apps/web/src/components/useSignOut.ts | 12 + apps/web/src/lib/branch.ts | 48 + apps/web/src/main.tsx | 12 + apps/web/src/pages/ChatPage.tsx | 36 + apps/web/src/pages/LoginPage.tsx | 21 + apps/web/src/pages/SettingsPage.tsx | 374 ++ apps/web/src/pages/SharedPage.tsx | 71 + apps/web/src/session.tsx | 17 + apps/web/src/store/client.ts | 86 + apps/web/src/store/core.ts | 296 ++ apps/web/src/store/errors.ts | 10 + apps/web/src/store/handover.ts | 95 + apps/web/src/store/http-api.ts | 40 + apps/web/src/store/react.tsx | 74 + apps/web/src/store/schema.ts | 34 + apps/web/src/store/sql.ts | 9 + apps/web/src/store/worker.ts | 130 + apps/web/src/styles.css | 18 + apps/web/tests/App.test.tsx | 44 + apps/web/tests/components/ChatList.test.tsx | 52 + apps/web/tests/components/Composer.test.tsx | 67 + .../components/ConversationView.test.tsx | 296 ++ .../web/tests/components/MessageView.test.tsx | 124 + apps/web/tests/components/SchemaForm.test.tsx | 53 + apps/web/tests/lib/branch.test.ts | 54 + apps/web/tests/pages/LoginPage.test.tsx | 20 + apps/web/tests/setup.ts | 11 + apps/web/tests/store/client.test.ts | 31 + apps/web/tests/store/core.test.ts | 277 ++ apps/web/tests/store/handover.test.ts | 121 + apps/web/tests/store/helpers.ts | 65 + apps/web/tests/store/http-api.test.ts | 38 + apps/web/tsconfig.json | 11 + apps/web/vite.config.ts | 53 + biome.json | 11 + config.example.yml | 75 + docs/architecture.md | 46 + docs/plugins.md | 166 + lexicons/README.md | 6 + lexicons/upstream/UPSTREAM.md | 5 + .../upstream/com/atproto/repo/strongRef.json | 15 + .../upstream/com/atproto/repo/uploadBlob.json | 23 + .../com/atproto/server/getServiceAuth.json | 48 + .../atproto/simplespace/checkUserAccess.json | 48 + .../com/atproto/simplespace/createSpace.json | 83 + .../com/atproto/simplespace/defs.json | 47 + .../com/atproto/simplespace/deleteSpace.json | 32 + .../com/atproto/simplespace/getSpace.json | 62 + .../com/atproto/simplespace/listMembers.json | 71 + .../com/atproto/simplespace/putMember.json | 45 + .../com/atproto/simplespace/removeMember.json | 37 + .../com/atproto/simplespace/updateSpace.json | 66 + .../com/atproto/space/applyWrites.json | 163 + .../com/atproto/space/createRecord.json | 79 + lexicons/upstream/com/atproto/space/defs.json | 38 + .../com/atproto/space/deleteRecord.json | 51 + .../upstream/com/atproto/space/getBlob.json | 54 + .../com/atproto/space/getDelegationToken.json | 34 + .../com/atproto/space/getLatestCommit.json | 57 + .../upstream/com/atproto/space/getRecord.json | 76 + .../upstream/com/atproto/space/getRepo.json | 41 + .../com/atproto/space/getSpaceCredential.json | 67 + .../upstream/com/atproto/space/listBlobs.json | 59 + .../com/atproto/space/listRecords.json | 108 + .../com/atproto/space/listRepoOps.json | 111 + .../upstream/com/atproto/space/listRepos.json | 75 + .../com/atproto/space/listSpaces.json | 65 + .../com/atproto/space/notifySpaceDeleted.json | 24 + .../com/atproto/space/notifyWrite.json | 38 + .../upstream/com/atproto/space/putRecord.json | 75 + .../com/atproto/space/registerNotify.json | 49 + .../com/atproto/space/unregisterNotify.json | 29 + package.json | 91 + packages/lexicons/package.json | 27 + packages/lexicons/src/generated/com.ts | 5 + .../lexicons/src/generated/com/atproto.ts | 8 + .../src/generated/com/atproto/repo.ts | 6 + .../com/atproto/repo/strongRef.defs.ts | 48 + .../generated/com/atproto/repo/strongRef.ts | 6 + .../com/atproto/repo/uploadBlob.defs.ts | 41 + .../generated/com/atproto/repo/uploadBlob.ts | 6 + .../src/generated/com/atproto/server.ts | 5 + .../com/atproto/server/getServiceAuth.defs.ts | 40 + .../com/atproto/server/getServiceAuth.ts | 6 + .../src/generated/com/atproto/simplespace.ts | 13 + .../simplespace/checkUserAccess.defs.ts | 41 + .../atproto/simplespace/checkUserAccess.ts | 6 + .../atproto/simplespace/createSpace.defs.ts | 90 + .../com/atproto/simplespace/createSpace.ts | 6 + .../com/atproto/simplespace/defs.defs.ts | 99 + .../generated/com/atproto/simplespace/defs.ts | 5 + .../atproto/simplespace/deleteSpace.defs.ts | 44 + .../com/atproto/simplespace/deleteSpace.ts | 6 + .../com/atproto/simplespace/getSpace.defs.ts | 78 + .../com/atproto/simplespace/getSpace.ts | 6 + .../atproto/simplespace/listMembers.defs.ts | 69 + .../com/atproto/simplespace/listMembers.ts | 6 + .../com/atproto/simplespace/putMember.defs.ts | 47 + .../com/atproto/simplespace/putMember.ts | 6 + .../atproto/simplespace/removeMember.defs.ts | 45 + .../com/atproto/simplespace/removeMember.ts | 6 + .../atproto/simplespace/updateSpace.defs.ts | 92 + .../com/atproto/simplespace/updateSpace.ts | 6 + .../src/generated/com/atproto/space.ts | 24 + .../com/atproto/space/applyWrites.defs.ts | 197 + .../com/atproto/space/applyWrites.ts | 6 + .../com/atproto/space/createRecord.defs.ts | 57 + .../com/atproto/space/createRecord.ts | 6 + .../generated/com/atproto/space/defs.defs.ts | 64 + .../src/generated/com/atproto/space/defs.ts | 5 + .../com/atproto/space/deleteRecord.defs.ts | 46 + .../com/atproto/space/deleteRecord.ts | 6 + .../com/atproto/space/getBlob.defs.ts | 45 + .../generated/com/atproto/space/getBlob.ts | 6 + .../atproto/space/getDelegationToken.defs.ts | 38 + .../com/atproto/space/getDelegationToken.ts | 6 + .../com/atproto/space/getLatestCommit.defs.ts | 48 + .../com/atproto/space/getLatestCommit.ts | 6 + .../com/atproto/space/getRecord.defs.ts | 50 + .../generated/com/atproto/space/getRecord.ts | 6 + .../com/atproto/space/getRepo.defs.ts | 46 + .../generated/com/atproto/space/getRepo.ts | 6 + .../atproto/space/getSpaceCredential.defs.ts | 52 + .../com/atproto/space/getSpaceCredential.ts | 6 + .../com/atproto/space/listBlobs.defs.ts | 54 + .../generated/com/atproto/space/listBlobs.ts | 6 + .../com/atproto/space/listRecords.defs.ts | 89 + .../com/atproto/space/listRecords.ts | 6 + .../com/atproto/space/listRepoOps.defs.ts | 98 + .../com/atproto/space/listRepoOps.ts | 6 + .../com/atproto/space/listRepos.defs.ts | 79 + .../generated/com/atproto/space/listRepos.ts | 6 + .../com/atproto/space/listSpaces.defs.ts | 70 + .../generated/com/atproto/space/listSpaces.ts | 6 + .../atproto/space/notifySpaceDeleted.defs.ts | 41 + .../com/atproto/space/notifySpaceDeleted.ts | 6 + .../com/atproto/space/notifyWrite.defs.ts | 44 + .../com/atproto/space/notifyWrite.ts | 6 + .../com/atproto/space/putRecord.defs.ts | 54 + .../generated/com/atproto/space/putRecord.ts | 6 + .../com/atproto/space/registerNotify.defs.ts | 47 + .../com/atproto/space/registerNotify.ts | 6 + .../atproto/space/unregisterNotify.defs.ts | 44 + .../com/atproto/space/unregisterNotify.ts | 6 + packages/lexicons/src/generated/index.ts | 6 + packages/lexicons/src/generated/network.ts | 5 + .../src/generated/network/sharedcomputer.ts | 5 + .../generated/network/sharedcomputer/chat.ts | 13 + .../sharedcomputer/chat/conversation.defs.ts | 21 + .../sharedcomputer/chat/conversation.ts | 6 + .../chat/conversationRef.defs.ts | 87 + .../sharedcomputer/chat/conversationRef.ts | 6 + .../network/sharedcomputer/chat/defs.defs.ts | 412 ++ .../network/sharedcomputer/chat/defs.ts | 5 + .../network/sharedcomputer/chat/info.defs.ts | 79 + .../network/sharedcomputer/chat/info.ts | 6 + .../sharedcomputer/chat/message.defs.ts | 118 + .../network/sharedcomputer/chat/message.ts | 6 + .../sharedcomputer/chat/permissions.defs.ts | 50 + .../sharedcomputer/chat/permissions.ts | 6 + .../sharedcomputer/chat/preferences.defs.ts | 69 + .../sharedcomputer/chat/preferences.ts | 6 + .../sharedcomputer/chat/requestSync.defs.ts | 45 + .../sharedcomputer/chat/requestSync.ts | 6 + .../sharedcomputer/chat/settings.defs.ts | 24 + .../network/sharedcomputer/chat/settings.ts | 6 + packages/lexicons/src/index.ts | 48 + packages/lexicons/src/nsid.ts | 14 + packages/lexicons/src/publish-cli.ts | 80 + packages/lexicons/src/publish.ts | 100 + packages/lexicons/tests/index.test.ts | 82 + packages/lexicons/tests/lexicons.test.ts | 83 + packages/lexicons/tests/publish.test.ts | 136 + packages/lexicons/tsconfig.json | 4 + packages/plugin-api/package.json | 18 + packages/plugin-api/src/index.ts | 208 + packages/plugin-api/src/testing.ts | 26 + packages/plugin-api/tsconfig.json | 4 + plugins/anthropic/package.json | 15 + plugins/anthropic/src/index.ts | 29 + plugins/anthropic/tests/index.test.ts | 72 + plugins/anthropic/tsconfig.json | 4 + plugins/google/package.json | 15 + plugins/google/src/index.ts | 29 + plugins/google/tests/index.test.ts | 69 + plugins/google/tsconfig.json | 4 + plugins/openai-compatible/package.json | 15 + plugins/openai-compatible/src/index.ts | 72 + plugins/openai-compatible/tests/index.test.ts | 97 + plugins/openai-compatible/tsconfig.json | 4 + plugins/openai/package.json | 15 + plugins/openai/src/index.ts | 30 + plugins/openai/tests/index.test.ts | 77 + plugins/openai/tsconfig.json | 4 + plugins/pdf-text/package.json | 15 + plugins/pdf-text/src/index.ts | 39 + plugins/pdf-text/tests/index.test.ts | 64 + plugins/pdf-text/tsconfig.json | 4 + plugins/titles/package.json | 14 + plugins/titles/src/index.ts | 93 + plugins/titles/tests/index.test.ts | 156 + plugins/titles/tsconfig.json | 4 + pnpm-lock.yaml | 3625 +++++++++++++++++ pnpm-workspace.yaml | 4 + specs/README.md | 14 +- specs/attachments.md | 7 +- specs/auth.md | 39 +- specs/browser-store.md | 8 +- specs/chat-storage.md | 22 +- specs/chat-turns.md | 21 +- specs/foundation.md | 52 +- specs/plugins.md | 78 +- specs/providers.md | 26 +- specs/search.md | 4 - specs/sharing.md | 6 +- specs/titles.md | 8 +- specs/web-ui.md | 7 +- tsconfig.base.json | 20 + vitest.config.ts | 22 + 362 files changed, 26714 insertions(+), 133 deletions(-) create mode 100644 .env.example create mode 100644 .nvmrc create mode 100644 apps/server/package.json create mode 100644 apps/server/src/api-types.ts create mode 100644 apps/server/src/app.ts create mode 100644 apps/server/src/auth/accounts.ts create mode 100644 apps/server/src/auth/identity.ts create mode 100644 apps/server/src/auth/oauth-client.ts create mode 100644 apps/server/src/auth/pds.ts create mode 100644 apps/server/src/auth/roles.ts create mode 100644 apps/server/src/auth/routes.ts create mode 100644 apps/server/src/auth/scope.ts create mode 100644 apps/server/src/auth/web-session.ts create mode 100644 apps/server/src/blobs/routes.ts create mode 100644 apps/server/src/blobs/store.ts create mode 100644 apps/server/src/body.ts create mode 100644 apps/server/src/config.ts create mode 100644 apps/server/src/db/dialect.ts create mode 100644 apps/server/src/db/index.ts create mode 100644 apps/server/src/db/migrate.ts create mode 100644 apps/server/src/db/migrations/0001_plugin_user_settings.ts create mode 100644 apps/server/src/db/migrations/0002_auth.ts create mode 100644 apps/server/src/db/migrations/0003_storage.ts create mode 100644 apps/server/src/db/migrations/0004_providers.ts create mode 100644 apps/server/src/db/migrations/0005_turns.ts create mode 100644 apps/server/src/db/migrations/index.ts create mode 100644 apps/server/src/db/schema.ts create mode 100644 apps/server/src/env.ts create mode 100644 apps/server/src/index.ts create mode 100644 apps/server/src/keys-cli.ts create mode 100644 apps/server/src/logger.ts create mode 100644 apps/server/src/migrate-cli.ts create mode 100644 apps/server/src/plugins/hooks.ts create mode 100644 apps/server/src/plugins/host.ts create mode 100644 apps/server/src/plugins/import.ts create mode 100644 apps/server/src/plugins/registry.ts create mode 100644 apps/server/src/plugins/routes.ts create mode 100644 apps/server/src/plugins/user-settings.ts create mode 100644 apps/server/src/providers/catalog.ts create mode 100644 apps/server/src/providers/effort.ts create mode 100644 apps/server/src/providers/generate-text.ts create mode 100644 apps/server/src/providers/guarded-fetch.ts create mode 100644 apps/server/src/providers/routes.ts create mode 100644 apps/server/src/providers/user-credentials.ts create mode 100644 apps/server/src/secrets.ts create mode 100644 apps/server/src/server.ts create mode 100644 apps/server/src/sharing/routes.ts create mode 100644 apps/server/src/sharing/service.ts create mode 100644 apps/server/src/storage/chat-service.ts create mode 100644 apps/server/src/storage/local-record-store.ts create mode 100644 apps/server/src/storage/record-store.ts create mode 100644 apps/server/src/storage/records.ts create mode 100644 apps/server/src/storage/routes.ts create mode 100644 apps/server/src/storage/services.ts create mode 100644 apps/server/src/storage/space-record-store.ts create mode 100644 apps/server/src/storage/validate.ts create mode 100644 apps/server/src/sync/coalescer.ts create mode 100644 apps/server/src/sync/credentials.ts create mode 100644 apps/server/src/sync/engine.ts create mode 100644 apps/server/src/sync/events.ts create mode 100644 apps/server/src/sync/identity.ts create mode 100644 apps/server/src/sync/recent-writes.ts create mode 100644 apps/server/src/sync/routes.ts create mode 100644 apps/server/src/sync/scheduler.ts create mode 100644 apps/server/src/sync/service-auth.ts create mode 100644 apps/server/src/turns/accumulator.ts create mode 100644 apps/server/src/turns/prompt.ts create mode 100644 apps/server/src/turns/routes.ts create mode 100644 apps/server/src/turns/runner.ts create mode 100644 apps/server/src/turns/stream-hub.ts create mode 100644 apps/server/src/web-html.ts create mode 100644 apps/server/tests/app.test.ts create mode 100644 apps/server/tests/auth/accounts.test.ts create mode 100644 apps/server/tests/auth/oauth-client.test.ts create mode 100644 apps/server/tests/auth/pds.test.ts create mode 100644 apps/server/tests/auth/roles.test.ts create mode 100644 apps/server/tests/auth/routes.test.ts create mode 100644 apps/server/tests/auth/scope.test.ts create mode 100644 apps/server/tests/auth/web-session.test.ts create mode 100644 apps/server/tests/blobs/routes.test.ts create mode 100644 apps/server/tests/blobs/store.test.ts create mode 100644 apps/server/tests/config.test.ts create mode 100644 apps/server/tests/db/index.test.ts create mode 100644 apps/server/tests/db/migrate.test.ts create mode 100644 apps/server/tests/fixtures/plugins/good/index.ts create mode 100644 apps/server/tests/fixtures/plugins/good/package.json create mode 100644 apps/server/tests/fixtures/plugins/noexport/index.ts create mode 100644 apps/server/tests/fixtures/plugins/noexport/package.json create mode 100644 apps/server/tests/helpers/auth.ts create mode 100644 apps/server/tests/helpers/config.ts create mode 100644 apps/server/tests/helpers/db.ts create mode 100644 apps/server/tests/helpers/fake-pds.ts create mode 100644 apps/server/tests/helpers/providers.ts create mode 100644 apps/server/tests/helpers/spaces.ts create mode 100644 apps/server/tests/helpers/turns.ts create mode 100644 apps/server/tests/plugins/hooks.test.ts create mode 100644 apps/server/tests/plugins/host.test.ts create mode 100644 apps/server/tests/plugins/import.test.ts create mode 100644 apps/server/tests/plugins/registry.test.ts create mode 100644 apps/server/tests/plugins/routes.test.ts create mode 100644 apps/server/tests/plugins/user-settings.test.ts create mode 100644 apps/server/tests/providers/catalog.test.ts create mode 100644 apps/server/tests/providers/effort.test.ts create mode 100644 apps/server/tests/providers/generate-text.test.ts create mode 100644 apps/server/tests/providers/guarded-fetch.test.ts create mode 100644 apps/server/tests/providers/routes.test.ts create mode 100644 apps/server/tests/providers/user-credentials.test.ts create mode 100644 apps/server/tests/secrets.test.ts create mode 100644 apps/server/tests/server.test.ts create mode 100644 apps/server/tests/sharing/sharing.test.ts create mode 100644 apps/server/tests/storage/chat-service.test.ts create mode 100644 apps/server/tests/storage/local-record-store.test.ts create mode 100644 apps/server/tests/storage/routes.test.ts create mode 100644 apps/server/tests/storage/space-record-store.test.ts create mode 100644 apps/server/tests/sync/coalescer.test.ts create mode 100644 apps/server/tests/sync/credentials.test.ts create mode 100644 apps/server/tests/sync/engine.test.ts create mode 100644 apps/server/tests/sync/identity.test.ts create mode 100644 apps/server/tests/sync/recent-writes.test.ts create mode 100644 apps/server/tests/sync/routes.test.ts create mode 100644 apps/server/tests/sync/scheduler.test.ts create mode 100644 apps/server/tests/turns/accumulator.test.ts create mode 100644 apps/server/tests/turns/prompt.test.ts create mode 100644 apps/server/tests/turns/routes.test.ts create mode 100644 apps/server/tests/turns/runner.test.ts create mode 100644 apps/server/tsconfig.json create mode 100644 apps/web/index.html create mode 100644 apps/web/package.json create mode 100644 apps/web/src/App.tsx create mode 100644 apps/web/src/api.ts create mode 100644 apps/web/src/app-name.ts create mode 100644 apps/web/src/components/BusyBanner.tsx create mode 100644 apps/web/src/components/ChatList.tsx create mode 100644 apps/web/src/components/Composer.tsx create mode 100644 apps/web/src/components/ConversationView.tsx create mode 100644 apps/web/src/components/MessageView.tsx create mode 100644 apps/web/src/components/SchemaForm.tsx create mode 100644 apps/web/src/components/ShareControl.tsx create mode 100644 apps/web/src/components/useAction.ts create mode 100644 apps/web/src/components/useReplyStream.ts create mode 100644 apps/web/src/components/useSignOut.ts create mode 100644 apps/web/src/lib/branch.ts create mode 100644 apps/web/src/main.tsx create mode 100644 apps/web/src/pages/ChatPage.tsx create mode 100644 apps/web/src/pages/LoginPage.tsx create mode 100644 apps/web/src/pages/SettingsPage.tsx create mode 100644 apps/web/src/pages/SharedPage.tsx create mode 100644 apps/web/src/session.tsx create mode 100644 apps/web/src/store/client.ts create mode 100644 apps/web/src/store/core.ts create mode 100644 apps/web/src/store/errors.ts create mode 100644 apps/web/src/store/handover.ts create mode 100644 apps/web/src/store/http-api.ts create mode 100644 apps/web/src/store/react.tsx create mode 100644 apps/web/src/store/schema.ts create mode 100644 apps/web/src/store/sql.ts create mode 100644 apps/web/src/store/worker.ts create mode 100644 apps/web/src/styles.css create mode 100644 apps/web/tests/App.test.tsx create mode 100644 apps/web/tests/components/ChatList.test.tsx create mode 100644 apps/web/tests/components/Composer.test.tsx create mode 100644 apps/web/tests/components/ConversationView.test.tsx create mode 100644 apps/web/tests/components/MessageView.test.tsx create mode 100644 apps/web/tests/components/SchemaForm.test.tsx create mode 100644 apps/web/tests/lib/branch.test.ts create mode 100644 apps/web/tests/pages/LoginPage.test.tsx create mode 100644 apps/web/tests/setup.ts create mode 100644 apps/web/tests/store/client.test.ts create mode 100644 apps/web/tests/store/core.test.ts create mode 100644 apps/web/tests/store/handover.test.ts create mode 100644 apps/web/tests/store/helpers.ts create mode 100644 apps/web/tests/store/http-api.test.ts create mode 100644 apps/web/tsconfig.json create mode 100644 apps/web/vite.config.ts create mode 100644 biome.json create mode 100644 config.example.yml create mode 100644 docs/architecture.md create mode 100644 docs/plugins.md create mode 100644 lexicons/upstream/UPSTREAM.md create mode 100644 lexicons/upstream/com/atproto/repo/strongRef.json create mode 100644 lexicons/upstream/com/atproto/repo/uploadBlob.json create mode 100644 lexicons/upstream/com/atproto/server/getServiceAuth.json create mode 100644 lexicons/upstream/com/atproto/simplespace/checkUserAccess.json create mode 100644 lexicons/upstream/com/atproto/simplespace/createSpace.json create mode 100644 lexicons/upstream/com/atproto/simplespace/defs.json create mode 100644 lexicons/upstream/com/atproto/simplespace/deleteSpace.json create mode 100644 lexicons/upstream/com/atproto/simplespace/getSpace.json create mode 100644 lexicons/upstream/com/atproto/simplespace/listMembers.json create mode 100644 lexicons/upstream/com/atproto/simplespace/putMember.json create mode 100644 lexicons/upstream/com/atproto/simplespace/removeMember.json create mode 100644 lexicons/upstream/com/atproto/simplespace/updateSpace.json create mode 100644 lexicons/upstream/com/atproto/space/applyWrites.json create mode 100644 lexicons/upstream/com/atproto/space/createRecord.json create mode 100644 lexicons/upstream/com/atproto/space/defs.json create mode 100644 lexicons/upstream/com/atproto/space/deleteRecord.json create mode 100644 lexicons/upstream/com/atproto/space/getBlob.json create mode 100644 lexicons/upstream/com/atproto/space/getDelegationToken.json create mode 100644 lexicons/upstream/com/atproto/space/getLatestCommit.json create mode 100644 lexicons/upstream/com/atproto/space/getRecord.json create mode 100644 lexicons/upstream/com/atproto/space/getRepo.json create mode 100644 lexicons/upstream/com/atproto/space/getSpaceCredential.json create mode 100644 lexicons/upstream/com/atproto/space/listBlobs.json create mode 100644 lexicons/upstream/com/atproto/space/listRecords.json create mode 100644 lexicons/upstream/com/atproto/space/listRepoOps.json create mode 100644 lexicons/upstream/com/atproto/space/listRepos.json create mode 100644 lexicons/upstream/com/atproto/space/listSpaces.json create mode 100644 lexicons/upstream/com/atproto/space/notifySpaceDeleted.json create mode 100644 lexicons/upstream/com/atproto/space/notifyWrite.json create mode 100644 lexicons/upstream/com/atproto/space/putRecord.json create mode 100644 lexicons/upstream/com/atproto/space/registerNotify.json create mode 100644 lexicons/upstream/com/atproto/space/unregisterNotify.json create mode 100644 package.json create mode 100644 packages/lexicons/package.json create mode 100644 packages/lexicons/src/generated/com.ts create mode 100644 packages/lexicons/src/generated/com/atproto.ts create mode 100644 packages/lexicons/src/generated/com/atproto/repo.ts create mode 100644 packages/lexicons/src/generated/com/atproto/repo/strongRef.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/repo/strongRef.ts create mode 100644 packages/lexicons/src/generated/com/atproto/repo/uploadBlob.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/repo/uploadBlob.ts create mode 100644 packages/lexicons/src/generated/com/atproto/server.ts create mode 100644 packages/lexicons/src/generated/com/atproto/server/getServiceAuth.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/server/getServiceAuth.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/checkUserAccess.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/checkUserAccess.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/createSpace.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/createSpace.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/defs.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/deleteSpace.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/deleteSpace.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/getSpace.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/getSpace.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/listMembers.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/listMembers.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/putMember.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/putMember.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/removeMember.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/removeMember.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/updateSpace.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/simplespace/updateSpace.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/applyWrites.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/applyWrites.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/createRecord.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/createRecord.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/defs.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/deleteRecord.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/deleteRecord.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getBlob.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getBlob.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getDelegationToken.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getDelegationToken.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getLatestCommit.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getLatestCommit.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getRecord.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getRecord.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getRepo.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getRepo.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getSpaceCredential.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/getSpaceCredential.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listBlobs.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listBlobs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listRecords.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listRecords.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listRepoOps.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listRepoOps.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listRepos.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listRepos.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listSpaces.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/listSpaces.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/notifySpaceDeleted.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/notifySpaceDeleted.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/notifyWrite.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/notifyWrite.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/putRecord.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/putRecord.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/registerNotify.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/registerNotify.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/unregisterNotify.defs.ts create mode 100644 packages/lexicons/src/generated/com/atproto/space/unregisterNotify.ts create mode 100644 packages/lexicons/src/generated/index.ts create mode 100644 packages/lexicons/src/generated/network.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/conversation.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/conversation.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/conversationRef.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/conversationRef.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/defs.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/info.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/info.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/message.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/message.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/permissions.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/permissions.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/preferences.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/preferences.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/requestSync.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/requestSync.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/settings.defs.ts create mode 100644 packages/lexicons/src/generated/network/sharedcomputer/chat/settings.ts create mode 100644 packages/lexicons/src/index.ts create mode 100644 packages/lexicons/src/nsid.ts create mode 100644 packages/lexicons/src/publish-cli.ts create mode 100644 packages/lexicons/src/publish.ts create mode 100644 packages/lexicons/tests/index.test.ts create mode 100644 packages/lexicons/tests/lexicons.test.ts create mode 100644 packages/lexicons/tests/publish.test.ts create mode 100644 packages/lexicons/tsconfig.json create mode 100644 packages/plugin-api/package.json create mode 100644 packages/plugin-api/src/index.ts create mode 100644 packages/plugin-api/src/testing.ts create mode 100644 packages/plugin-api/tsconfig.json create mode 100644 plugins/anthropic/package.json create mode 100644 plugins/anthropic/src/index.ts create mode 100644 plugins/anthropic/tests/index.test.ts create mode 100644 plugins/anthropic/tsconfig.json create mode 100644 plugins/google/package.json create mode 100644 plugins/google/src/index.ts create mode 100644 plugins/google/tests/index.test.ts create mode 100644 plugins/google/tsconfig.json create mode 100644 plugins/openai-compatible/package.json create mode 100644 plugins/openai-compatible/src/index.ts create mode 100644 plugins/openai-compatible/tests/index.test.ts create mode 100644 plugins/openai-compatible/tsconfig.json create mode 100644 plugins/openai/package.json create mode 100644 plugins/openai/src/index.ts create mode 100644 plugins/openai/tests/index.test.ts create mode 100644 plugins/openai/tsconfig.json create mode 100644 plugins/pdf-text/package.json create mode 100644 plugins/pdf-text/src/index.ts create mode 100644 plugins/pdf-text/tests/index.test.ts create mode 100644 plugins/pdf-text/tsconfig.json create mode 100644 plugins/titles/package.json create mode 100644 plugins/titles/src/index.ts create mode 100644 plugins/titles/tests/index.test.ts create mode 100644 plugins/titles/tsconfig.json create mode 100644 pnpm-lock.yaml create mode 100644 pnpm-workspace.yaml create mode 100644 tsconfig.base.json create mode 100644 vitest.config.ts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b80248c --- /dev/null +++ b/.env.example @@ -0,0 +1,20 @@ +# Secrets for SCN Chat. Public settings live in config.yml. + +NODE_ENV=development +DATABASE_URL=sqlite:./data/scn-chat.sqlite + +# Generate both with: pnpm keys +SECRET_KEY= +OAUTH_PRIVATE_KEYS= +# Retired secret keys, comma-separated, kept only to decrypt older values. +SECRET_KEYS_OLD= + +# Admin API keys, referenced from config.yml. +SCN_API_KEY= +COCORE_API_KEY= +ANTHROPIC_API_KEY= +OPENAI_API_KEY= +GOOGLE_API_KEY= + +# Optional: point at a config file other than ./config.yml. +SCN_CHAT_CONFIG= diff --git a/.gitignore b/.gitignore index 202bddc..0deb3dd 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ coverage/ *.db .DS_Store .claude/settings.local.json +data/ +config.yml diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..a45fd52 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24 diff --git a/CLAUDE.md b/CLAUDE.md index fb835e6..da3048d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,18 @@ # SCN Chat -An AI assistant chat app where each conversation is an atproto permissioned space on the user's own PDS. Only the lexicons exist so far. There is no build, test, or app code yet. +An AI assistant chat app where each conversation is an atproto permissioned space on the user's own PDS. Each feature has a spec in `specs/`, which is the source of truth for its behavior. + +## Layout and commands + +- `lexicons/` holds our lexicon JSON, and `lexicons/upstream/` the vendored `com.atproto` lexicons the server calls. +- `packages/lexicons` is the generated lexicon code and the `nsid` module. `packages/plugin-api` is what plugins build against. +- `apps/server` is the Hono server. `src/server.ts` wires every service together. +- `apps/web` is the React app. `src/store/` is the browser's local SQLite copy. +- `plugins/` holds the shipped plugins. +- `docs/architecture.md` is the human-facing architecture overview, and `docs/plugins.md` the plugin author guide. Update them when a change affects what they describe. +- `pnpm dev` runs the server and Vite together. `pnpm check` runs lint, type checking, and every test. `pnpm codegen` regenerates the lexicon code after a lexicon change, and `pnpm publish-lexicons ` publishes the lexicons from the lexicon authority account. `pnpm keys` prints fresh secrets. +- Local setup: copy `config.example.yml` to `config.yml` for public settings and plugins, and `.env.example` to `.env` for secrets, filled in from `pnpm keys`. Secrets never go in `config.yml`; it references them as `${NAME}`. +- Server tests run on both SQLite and PGlite through `tests/helpers/db.ts`. `tests/helpers/fake-pds.ts` is a fake spaces PDS with real error codes, policies, and signed commits. ## The founding principle @@ -18,7 +30,7 @@ These are settled. Raise a concern before working against any of them. - The browser keeps its own local copy of the user's chats in SQLite compiled to WebAssembly, for the chat list, the open conversation, and search. That copy lives on the user's device, not on the server. - BYO API keys are stored encrypted in the app database, never in records. - Providers, tools, file ingesters, and turn hooks are plugins. Web search, fetch, and image generation are plugins, not core. -- The project is meant to be forked. Code refers to lexicon NSIDs only through one generated module, and app name, URLs, OAuth metadata, and admin models come from environment config. +- The project is meant to be forked. Code refers to lexicon NSIDs only through one generated module, and app name, URLs, OAuth metadata, and admin models come from `config.yml`. ## Planned stack @@ -26,7 +38,7 @@ TypeScript throughout, as a pnpm workspace on Node 24 LTS, run natively with Nod Prefer existing libraries over writing our own, especially the official `@atproto/*` packages for anything protocol related. Dependencies must have licenses compatible with MIT. -Admins, their global models, and roles are configured through environment config and a config file in phase 1. Roles gate which admin models a user may use. There is no admin UI yet. +Admins configure plugins, global models, roles, and every other public setting in `config.yml`, with secrets in `.env`. Roles gate which admin models a user may use. There is no admin UI yet. The web UI must stay barebones with minimal code and styling. A separate designer will build the real interface on top of it. diff --git a/README.md b/README.md index 0f5773c..e456cbb 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,33 @@ Future: - MCP servers as tools - Browser and push notifications, and an installable PWA +## Getting started + +### Requirements: + - Node 22.18 or newer (Node 24 is recommended) + - pnpm 10 + - An ATProto account to sign in with. + - A spaces enabled PDS (currently optional) + +### Setup + 1. Install dependencies: `pnpm install` + 2. `cp .env.example .env` + 3. Run `pnpm keys` to generate secret keys. Copy `OAUTH_PRIVATE_KEYS` and `SECRET_KEY` into your `.env`. + 4. Set your AI inference provider API keys. + 5. Copy config.example.yml to config.yml. The defaults are setup for local development. + 6. Configure your supported AI models. + +### Start the app + + 1. Run `pnpm dev` + 2. Open http://127.0.0.1:5173 and sign in with your ATProto handle. + ## Extending Model providers, tools, file ingesters, and turn hooks are all plugins, so anyone can build their own integration. +See: + - [docs/architecture.md](docs/architecture.md) + - [docs/plugins.md](docs/plugins.md) ## License diff --git a/apps/server/package.json b/apps/server/package.json new file mode 100644 index 0000000..998a3ab --- /dev/null +++ b/apps/server/package.json @@ -0,0 +1,52 @@ +{ + "name": "@scn-chat/server", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/app.ts", + "./api-types": "./src/api-types.ts", + "./web-html": "./src/web-html.ts" + }, + "scripts": { + "dev": "node --watch src/index.ts", + "start": "NODE_ENV=production node src/index.ts", + "migrate": "node src/migrate-cli.ts", + "typecheck": "tsc --noEmit -p .", + "keys": "node src/keys-cli.ts" + }, + "dependencies": { + "@atproto/common-web": "0.0.0-spaces-alpha-20260915165437", + "@atproto/identity": "0.0.0-spaces-alpha-20260915165437", + "@atproto/lex-cbor": "0.0.0-spaces-alpha-20260915165437", + "@atproto/lex-client": "0.0.0-spaces-alpha-20260915165437", + "@atproto/lex-data": "0.0.0-spaces-alpha-20260915165437", + "@atproto/lex-json": "0.0.0-spaces-alpha-20260915165437", + "@atproto/oauth-client-node": "0.0.0-spaces-alpha-20260915165437", + "@atproto/oauth-scopes": "0.0.0-spaces-alpha-20260915165437", + "@atproto/space": "0.0.0-spaces-alpha-20260915165437", + "@atproto/syntax": "0.0.0-spaces-alpha-20260915165437", + "@atproto/xrpc-server": "0.0.0-spaces-alpha-20260915165437", + "@hono/node-server": "2.1.1", + "@scn-chat/lexicons": "workspace:*", + "@scn-chat/plugin-api": "workspace:*", + "ai": "7.0.116", + "better-sqlite3": "13.0.3", + "hono": "4.13.9", + "ipaddr.js": "2.5.0", + "kysely": "0.29.6", + "pg": "8.23.0", + "pino": "10.3.1", + "undici": "8.11.2", + "yaml": "2.9.1", + "zod": "4.6.5" + }, + "devDependencies": { + "@atproto/common": "0.0.0-spaces-alpha-20260915165437", + "@atproto/crypto": "0.0.0-spaces-alpha-20260915165437", + "@electric-sql/pglite": "0.5.8", + "@types/better-sqlite3": "9.6.0", + "@types/pg": "8.23.1", + "kysely-pglite-dialect": "1.3.1" + } +} diff --git a/apps/server/src/api-types.ts b/apps/server/src/api-types.ts new file mode 100644 index 0000000..4db5613 --- /dev/null +++ b/apps/server/src/api-types.ts @@ -0,0 +1,16 @@ +import type { authApiRoutes } from './auth/routes.ts' +import type { blobRoutes } from './blobs/routes.ts' +import type { pluginRoutes } from './plugins/routes.ts' +import type { providerRoutes } from './providers/routes.ts' +import type { sharingRoutes } from './sharing/routes.ts' +import type { storageRoutes } from './storage/routes.ts' +import type { turnRoutes } from './turns/routes.ts' + +/** Route types for the web app's typed client. Every router is mounted under /api. */ +export type AuthApi = ReturnType +export type StorageApi = ReturnType +export type TurnsApi = ReturnType +export type ProvidersApi = ReturnType +export type PluginsApi = ReturnType +export type BlobsApi = ReturnType +export type SharingApi = ReturnType diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts new file mode 100644 index 0000000..dc1f585 --- /dev/null +++ b/apps/server/src/app.ts @@ -0,0 +1,136 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { serveStatic } from '@hono/node-server/serve-static' +import { Hono } from 'hono' +import { sql } from 'kysely' +import { SessionExpired } from './auth/pds.ts' +import { + type AuthDeps, + authApiRoutes, + oauthRoutes, + originCheck, + sessionMiddleware, +} from './auth/routes.ts' +import { type BlobRoutesDeps, blobRoutes } from './blobs/routes.ts' +import { InvalidBody } from './body.ts' +import type { Config } from './config.ts' +import type { Db } from './db/index.ts' +import type { AppEnv } from './env.ts' +import type { Logger } from './logger.ts' +import { type PluginRoutesDeps, pluginRoutes } from './plugins/routes.ts' +import { ModelUnavailable } from './providers/catalog.ts' +import { type ProviderRoutesDeps, providerRoutes } from './providers/routes.ts' +import { sharingRoutes } from './sharing/routes.ts' +import type { SharingService } from './sharing/service.ts' +import { + InvalidCursor, + InvalidRecord, + InvalidStoredRecord, + SpaceNotFound, +} from './storage/record-store.ts' +import { InvalidSpaceUri } from './storage/records.ts' +import { type StorageRoutesDeps, storageRoutes } from './storage/routes.ts' +import { type SyncRoutesDeps, syncRoutes } from './sync/routes.ts' +import { type TurnRoutesDeps, turnRoutes } from './turns/routes.ts' +import { renderIndexHtml } from './web-html.ts' + +export type AppDeps = { + config: Config + db: Db + logger: Logger + /** Built web app to serve, in production. */ + webDist?: string + auth?: AuthDeps + plugins?: PluginRoutesDeps + storage?: StorageRoutesDeps + providers?: ProviderRoutesDeps + turns?: TurnRoutesDeps + blobs?: BlobRoutesDeps + sharing?: SharingService + sync?: SyncRoutesDeps +} + +export function createApp(deps: AppDeps) { + const { config, db, logger } = deps + const app = new Hono() + + app.use('*', async (c, next) => { + const start = performance.now() + await next() + logger.info( + { + method: c.req.method, + path: c.req.path, + status: c.res.status, + ms: Math.round(performance.now() - start), + }, + 'request', + ) + }) + + if (deps.auth) { + app.use('/api/*', sessionMiddleware(deps.auth), originCheck(config)) + app.route('/', oauthRoutes(deps.auth)) + } + + const api = new Hono().get('/health', async (c) => { + try { + await sql`select 1`.execute(db) + return c.json({ status: 'ok' as const, appName: config.appName }) + } catch (err) { + logger.error({ err }, 'health check database query failed') + return c.json({ status: 'error' as const }, 503) + } + }) + + if (deps.auth) api.route('/', authApiRoutes(deps.auth)) + if (deps.plugins) api.route('/plugins', pluginRoutes(deps.plugins)) + if (deps.storage) api.route('/', storageRoutes(deps.storage)) + if (deps.providers) api.route('/', providerRoutes(deps.providers)) + if (deps.turns) api.route('/', turnRoutes(deps.turns)) + if (deps.blobs) api.route('/', blobRoutes(deps.blobs)) + if (deps.sharing) api.route('/', sharingRoutes(deps.sharing)) + api.notFound((c) => c.json({ error: 'NotFound' }, 404)) + + app.route('/api', api) + if (deps.sync) app.route('/', syncRoutes(deps.sync)) + + app.onError((err, c) => { + if (err instanceof SessionExpired) + return c.json({ error: 'SessionExpired', message: err.message }, 401) + if (err instanceof InvalidRecord) + return c.json({ error: 'InvalidRecord', message: err.message }, 400) + if (err instanceof InvalidStoredRecord) { + logger.warn({ err, path: c.req.path }, 'a stored record does not match its lexicon') + return c.json({ error: 'InvalidStoredRecord', message: err.message }, 502) + } + if (err instanceof InvalidBody) + return c.json({ error: 'InvalidRequest', message: err.message }, 400) + if (err instanceof InvalidCursor) + return c.json({ error: 'InvalidRequest', message: err.message }, 400) + if (err instanceof InvalidSpaceUri) + return c.json({ error: 'InvalidRequest', message: err.message }, 400) + if (err instanceof ModelUnavailable) + return c.json({ error: 'ModelUnavailable', message: err.message }, 400) + if (err instanceof SpaceNotFound) + return c.json({ error: 'NotFound', message: err.message }, 404) + logger.error({ err, path: c.req.path }, 'unhandled error') + return c.json({ error: 'InternalServerError' }, 500) + }) + app.all('/api/*', (c) => c.json({ error: 'NotFound' }, 404)) + + if (deps.webDist) { + const index = renderIndexHtml( + readFileSync(join(deps.webDist, 'index.html'), 'utf8'), + config.appName, + ) + app.get('/', (c) => c.html(index)) + app.get('/index.html', (c) => c.html(index)) + app.use('*', serveStatic({ root: deps.webDist })) + app.get('*', (c) => c.html(index)) + } + + return app +} + +export type AppType = ReturnType diff --git a/apps/server/src/auth/accounts.ts b/apps/server/src/auth/accounts.ts new file mode 100644 index 0000000..8004d70 --- /dev/null +++ b/apps/server/src/auth/accounts.ts @@ -0,0 +1,105 @@ +import type { Db } from '../db/index.ts' + +export type StorageMode = 'space' | 'local' + +export type Account = { + did: string + handle: string | null + pdsUrl: string + storageMode: StorageMode + backgroundSync: boolean + lastActiveAt: string +} + +export class SpacesLostError extends Error { + constructor() { + super( + 'Your chats are stored in atproto spaces, but your PDS no longer grants space access. Sign in again once your PDS supports spaces.', + ) + this.name = 'SpacesLostError' + } +} + +type AccountRow = { + did: string + handle: string | null + pds_url: string + storage_mode: StorageMode + background_sync: number + last_active_at: string +} + +function toAccount(row: AccountRow): Account { + return { + did: row.did, + handle: row.handle, + pdsUrl: row.pds_url, + storageMode: row.storage_mode, + backgroundSync: row.background_sync === 1, + lastActiveAt: row.last_active_at, + } +} + +export async function getAccount(db: Db, did: string): Promise { + const row = await db + .selectFrom('account') + .select(['did', 'handle', 'pds_url', 'storage_mode', 'background_sync', 'last_active_at']) + .where('did', '=', did) + .executeTakeFirst() + return row ? toAccount(row) : undefined +} + +/** Record a login, choosing the storage mode for new accounts and keeping it for existing ones. */ +export async function recordLogin( + db: Db, + login: { did: string; handle: string | null; pdsUrl: string; spacesAllowed: boolean }, + now = new Date(), +): Promise { + const existing = await getAccount(db, login.did) + if (existing?.storageMode === 'space' && !login.spacesAllowed) throw new SpacesLostError() + const storageMode: StorageMode = + existing?.storageMode ?? (login.spacesAllowed ? 'space' : 'local') + const at = now.toISOString() + await db + .insertInto('account') + .values({ + did: login.did, + handle: login.handle, + pds_url: login.pdsUrl, + storage_mode: storageMode, + background_sync: 1, + created_at: at, + last_login_at: at, + last_active_at: at, + }) + .onConflict((oc) => + oc.column('did').doUpdateSet({ + handle: login.handle, + pds_url: login.pdsUrl, + last_login_at: at, + last_active_at: at, + }), + ) + .execute() + const account = await getAccount(db, login.did) + if (!account) throw new Error(`Account ${login.did} missing right after upsert`) + return account +} + +/** Mark the user active, at most once a minute. */ +export async function touchActivity(db: Db, account: Account, now = new Date()): Promise { + if (now.getTime() - Date.parse(account.lastActiveAt) < 60_000) return + await db + .updateTable('account') + .set({ last_active_at: now.toISOString() }) + .where('did', '=', account.did) + .execute() +} + +export async function setBackgroundSync(db: Db, did: string, enabled: boolean): Promise { + await db + .updateTable('account') + .set({ background_sync: enabled ? 1 : 0 }) + .where('did', '=', did) + .execute() +} diff --git a/apps/server/src/auth/identity.ts b/apps/server/src/auth/identity.ts new file mode 100644 index 0000000..a2d344f --- /dev/null +++ b/apps/server/src/auth/identity.ts @@ -0,0 +1,35 @@ +import { IdResolver } from '@atproto/identity' + +export type ResolvedIdentity = { did: string; handle: string | null; pdsUrl: string } + +export interface IdentityResolver { + resolve(did: string): Promise + resolveHandle(handle: string): Promise + /** The DID's atproto signing key. */ + resolveSigningKey(did: string, forceRefresh?: boolean): Promise +} + +export class IdentityResolutionError extends Error { + constructor(did: string, reason: string) { + super(`Cannot resolve ${did}: ${reason}`) + this.name = 'IdentityResolutionError' + } +} + +export function createIdentityResolver(plcUrl?: string): IdentityResolver { + const resolver = new IdResolver({ plcUrl }) + return { + async resolve(did) { + const data = await resolver.did.resolveAtprotoData(did) + if (!data.pds) throw new IdentityResolutionError(did, 'no PDS in DID document') + const handleDid = data.handle ? await resolver.handle.resolve(data.handle) : undefined + return { did, handle: handleDid === did ? (data.handle ?? null) : null, pdsUrl: data.pds } + }, + async resolveHandle(handle) { + return (await resolver.handle.resolve(handle)) ?? null + }, + async resolveSigningKey(did, forceRefresh = false) { + return resolver.did.resolveAtprotoKey(did, forceRefresh) + }, + } +} diff --git a/apps/server/src/auth/oauth-client.ts b/apps/server/src/auth/oauth-client.ts new file mode 100644 index 0000000..1d05688 --- /dev/null +++ b/apps/server/src/auth/oauth-client.ts @@ -0,0 +1,98 @@ +import { isIP } from 'node:net' +import { + buildAtprotoLoopbackClientMetadata, + JoseKey, + NodeOAuthClient, + type NodeSavedSession, + type NodeSavedState, + type OAuthSession, + requestLocalLock, +} from '@atproto/oauth-client-node' +import type { Config } from '../config.ts' +import type { Db } from '../db/index.ts' + +/** OAuth client methods we call. Tests pass in a fake oauth client that implements them. */ +export interface OAuthClientLike { + readonly clientMetadata: object + readonly jwks: object + authorize(input: string, options: { scope: string }): Promise + callback(params: URLSearchParams): Promise<{ session: OAuthSession }> + restore(did: string): Promise + revoke(did: string): Promise +} + +export function isLoopbackUrl(url: string): boolean { + const hostname = new URL(url).hostname.replace(/^\[|\]$/g, '') + return ( + hostname === 'localhost' || + hostname === '::1' || + (isIP(hostname) === 4 && hostname.startsWith('127.')) + ) +} + +function dbStore(db: Db, table: 'oauth_state' | 'oauth_session') { + return { + async get(key: string): Promise { + const row = await db + .selectFrom(table) + .select('value') + .where('key', '=', key) + .executeTakeFirst() + return row ? (JSON.parse(row.value) as T) : undefined + }, + async set(key: string, value: T): Promise { + const row = { value: JSON.stringify(value), updated_at: new Date().toISOString() } + await db + .insertInto(table) + .values({ key, ...row }) + .onConflict((oc) => oc.column('key').doUpdateSet(row)) + .execute() + }, + async del(key: string): Promise { + await db.deleteFrom(table).where('key', '=', key).execute() + }, + } +} + +/** Create a confidential client in production, or a loopback public client in development. */ +export async function createOAuthClient( + config: Config, + db: Db, + scope: string, +): Promise { + const redirectUri = `${config.publicUrl}/oauth/callback` as const + const loopback = isLoopbackUrl(config.publicUrl) + const keyset = loopback + ? undefined + : await Promise.all( + config.oauthPrivateKeys.map((jwk) => JoseKey.fromImportable(JSON.stringify(jwk), jwk.kid)), + ) + const clientMetadata = loopback + ? buildAtprotoLoopbackClientMetadata({ redirect_uris: [redirectUri], scope }) + : { + client_id: `${config.publicUrl}/oauth-client-metadata.json`, + client_name: config.appName, + client_uri: config.publicUrl, + redirect_uris: [redirectUri] as [string, ...string[]], + scope, + grant_types: ['authorization_code', 'refresh_token'] as [ + 'authorization_code', + 'refresh_token', + ], + response_types: ['code'] as ['code'], + application_type: 'web' as const, + token_endpoint_auth_method: 'private_key_jwt' as const, + token_endpoint_auth_signing_alg: 'ES256', + jwks_uri: `${config.publicUrl}/oauth/jwks.json`, + dpop_bound_access_tokens: true, + } + return new NodeOAuthClient({ + clientMetadata, + keyset, + allowHttp: loopback, + plcDirectoryUrl: config.plcUrl, + requestLock: requestLocalLock, + stateStore: dbStore(db, 'oauth_state'), + sessionStore: dbStore(db, 'oauth_session'), + }) +} diff --git a/apps/server/src/auth/pds.ts b/apps/server/src/auth/pds.ts new file mode 100644 index 0000000..f48a9f1 --- /dev/null +++ b/apps/server/src/auth/pds.ts @@ -0,0 +1,29 @@ +import { Client } from '@atproto/lex-client' +import { isExpectedSessionError, type OAuthSession } from '@atproto/oauth-client-node' +import type { OAuthClientLike } from './oauth-client.ts' + +export class SessionExpired extends Error { + readonly did: string + + constructor(did: string, cause?: unknown) { + super(`The atproto session for ${did} has expired. Sign in again.`, { cause }) + this.name = 'SessionExpired' + this.did = did + } +} + +export type PdsClientFactory = (did: string) => Promise + +/** Return a function to get the PDS client for a user. */ +export function createPdsClientFactory(oauth: OAuthClientLike): PdsClientFactory { + return async (did) => { + let session: OAuthSession + try { + session = await oauth.restore(did) + } catch (err) { + if (isExpectedSessionError(err)) throw new SessionExpired(did, err) + throw err + } + return new Client(session) + } +} diff --git a/apps/server/src/auth/roles.ts b/apps/server/src/auth/roles.ts new file mode 100644 index 0000000..ba53d27 --- /dev/null +++ b/apps/server/src/auth/roles.ts @@ -0,0 +1,38 @@ +import { isValidDid } from '@atproto/syntax' + +export const IMPLICIT_ROLE = 'user' +const ROLE_PATTERN = /^[a-z][a-z0-9-]*$/ + +export class RolesConfigError extends Error { + constructor(message: string) { + super(message) + this.name = 'RolesConfigError' + } +} + +export type Roles = { rolesFor(did: string): string[]; defined: ReadonlySet } + +/** Validate the config's roles and answer which roles a DID holds. */ +export function createRoles(config: Record = {}): Roles { + for (const [role, dids] of Object.entries(config)) { + if (role === IMPLICIT_ROLE) + throw new RolesConfigError( + `The role name "${IMPLICIT_ROLE}" is implicit and cannot be defined`, + ) + if (!ROLE_PATTERN.test(role)) + throw new RolesConfigError(`Role name "${role}" must match ${ROLE_PATTERN}`) + for (const did of dids) { + if (!isValidDid(did)) + throw new RolesConfigError(`Role "${role}" lists an invalid DID: "${did}"`) + } + } + return { + defined: new Set([IMPLICIT_ROLE, ...Object.keys(config)]), + rolesFor: (did) => [ + IMPLICIT_ROLE, + ...Object.entries(config) + .filter(([, dids]) => dids.includes(did)) + .map(([role]) => role), + ], + } +} diff --git a/apps/server/src/auth/routes.ts b/apps/server/src/auth/routes.ts new file mode 100644 index 0000000..8ef7e85 --- /dev/null +++ b/apps/server/src/auth/routes.ts @@ -0,0 +1,153 @@ +import { Hono, type MiddlewareHandler } from 'hono' +import { deleteCookie, getCookie, setCookie } from 'hono/cookie' +import type { Config } from '../config.ts' +import type { Db } from '../db/index.ts' +import type { AppEnv, AuthUser } from '../env.ts' +import type { Logger } from '../logger.ts' +import { type Account, getAccount, recordLogin, touchActivity } from './accounts.ts' +import type { IdentityResolver } from './identity.ts' +import type { OAuthClientLike } from './oauth-client.ts' +import type { Roles } from './roles.ts' +import { scopeAllowsSpaces } from './scope.ts' +import { + createWebSession, + deleteAllWebSessions, + deleteWebSession, + resolveWebSession, + SESSION_COOKIE, +} from './web-session.ts' + +export type AuthDeps = { + config: Config + db: Db + logger: Logger + oauth: OAuthClientLike + identity: IdentityResolver + roles: Roles + scope: string + /** Runs after each login to set up the user's storage. */ + onLogin?: (account: Account) => Promise +} + +const loginError = (message: string) => `/login?error=${encodeURIComponent(message)}` + +/** Resolve the session cookie to the signed-in user, if any. */ +export function sessionMiddleware(deps: AuthDeps): MiddlewareHandler { + return async (c, next) => { + c.set('user', null) + const token = getCookie(c, SESSION_COOKIE) + if (token) { + const did = await resolveWebSession(deps.db, token, deps.config.sessionTtlDays) + if (did) { + const account = await getAccount(deps.db, did) + if (!account) throw new Error(`A web session names ${did}, which has no account`) + c.set('user', { did: account.did, account, roles: deps.roles.rolesFor(account.did) }) + await touchActivity(deps.db, account) + } + } + await next() + } +} + +/** Refuse state-changing API requests from other origins. */ +export function originCheck(config: Config): MiddlewareHandler { + const allowed = new URL(config.publicUrl).origin + return async (c, next) => { + if (c.req.method !== 'GET' && c.req.method !== 'HEAD' && c.req.header('origin') !== allowed) { + return c.json({ error: 'Forbidden', message: 'Cross-origin request refused' }, 403) + } + await next() + } +} + +export const requireUser: MiddlewareHandler = async (c, next) => { + if (!c.get('user')) return c.json({ error: 'Unauthorized' }, 401) + await next() +} + +export function signedInUser(c: { get(key: 'user'): AuthUser | null }): AuthUser { + const user = c.get('user') + if (!user) throw new Error('signedInUser called on a route without requireUser') + return user +} + +/** Client metadata, keys, and the login and callback redirects. */ +export function oauthRoutes(deps: AuthDeps) { + const secure = deps.config.publicUrl.startsWith('https://') + return new Hono() + .get('/oauth-client-metadata.json', (c) => c.json(deps.oauth.clientMetadata)) + .get('/oauth/jwks.json', (c) => c.json(deps.oauth.jwks)) + .get('/oauth/login', async (c) => { + const identifier = c.req.query('identifier')?.trim() + if (!identifier) return c.redirect(loginError('Enter your handle to sign in.')) + try { + const url = await deps.oauth.authorize(identifier, { scope: deps.scope }) + return c.redirect(url.toString()) + } catch (err) { + deps.logger.warn({ err, identifier }, 'oauth authorize failed') + return c.redirect(loginError(err instanceof Error ? err.message : 'Sign-in failed.')) + } + }) + .get('/oauth/callback', async (c) => { + try { + const { session } = await deps.oauth.callback(new URL(c.req.url).searchParams) + const { scope } = await session.getTokenInfo(false) + const identity = await deps.identity.resolve(session.did) + const account = await recordLogin(deps.db, { + did: session.did, + handle: identity.handle, + pdsUrl: identity.pdsUrl, + spacesAllowed: scopeAllowsSpaces(scope, session.did), + }) + if (account.storageMode === 'local') + deps.logger.info({ did: account.did, scope }, 'signed in with local storage') + if (deps.onLogin) { + await deps + .onLogin(account) + .catch((err) => deps.logger.error({ err, did: account.did }, 'login setup failed')) + } + const token = await createWebSession(deps.db, account.did, deps.config.sessionTtlDays) + setCookie(c, SESSION_COOKIE, token, { + httpOnly: true, + sameSite: 'Lax', + path: '/', + secure, + maxAge: deps.config.sessionTtlDays * 86_400, + }) + return c.redirect('/') + } catch (err) { + deps.logger.warn({ err }, 'oauth callback failed') + return c.redirect(loginError(err instanceof Error ? err.message : 'Sign-in failed.')) + } + }) +} + +/** The signed-in user's details, and logout. */ +export function authApiRoutes(deps: AuthDeps) { + return new Hono() + .get('/me', requireUser, (c) => { + const { did, account, roles } = signedInUser(c) + return c.json({ + did, + handle: account.handle, + storageMode: account.storageMode, + backgroundSync: account.backgroundSync, + roles, + }) + }) + .post('/logout', async (c) => { + const token = getCookie(c, SESSION_COOKIE) + const user = c.get('user') + if (token) await deleteWebSession(deps.db, token) + if (user && c.req.query('everywhere') === '1') { + await deleteAllWebSessions(deps.db, user.did) + try { + await deps.oauth.revoke(user.did) + } catch (err) { + deps.logger.warn({ err, did: user.did }, 'oauth revoke failed') + } + } + deleteCookie(c, SESSION_COOKIE, { path: '/' }) + return c.json({ ok: true }) + }) +} diff --git a/apps/server/src/auth/scope.ts b/apps/server/src/auth/scope.ts new file mode 100644 index 0000000..4960f8a --- /dev/null +++ b/apps/server/src/auth/scope.ts @@ -0,0 +1,54 @@ +import { ScopePermissions, SpacePermission } from '@atproto/oauth-scopes' +import { nsid } from '@scn-chat/lexicons' + +export type ScopeMode = 'permission-set' | 'raw' + +const recordActions = ['read', 'create', 'update', 'delete'].map((action) => `action=${action}`) +const manageOps = ['create', 'update', 'delete'].map((op) => `manage=${op}`) + +function spaceScope(type: string, params: string[]): string { + return `space:${type}?${params.join('&')}` +} + +/** The OAuth scope the app requests, built from the lexicon module. */ +export function buildScope(mode: ScopeMode): string { + if (mode === 'permission-set') return `atproto include:${nsid.permissions}` + return [ + 'atproto', + spaceScope(nsid.conversation, [ + `collection=${nsid.info}`, + `collection=${nsid.message}`, + ...recordActions, + ...manageOps, + ]), + spaceScope(nsid.conversation, ['authority=*', 'action=read']), + spaceScope(nsid.settings, [ + 'skey=self', + `collection=${nsid.preferences}`, + `collection=${nsid.conversationRef}`, + ...recordActions, + ...manageOps, + ]), + 'blob:*/*', + ].join(' ') +} + +/** Does a granted scope let the user create their own conversation spaces? */ +export function scopeAllowsSpaces(scope: string, did: string): boolean { + const resolved = scope + .split(' ') + .filter(Boolean) + .map((part) => { + if (!part.startsWith('space:')) return part + const permission = SpacePermission.fromString(part) + if (!permission) + throw new Error(`The PDS granted a space permission this app cannot read: ${part}`) + return permission.withResolvedAuthority(did as `did:${string}:${string}`).toString() + }) + return new ScopePermissions(resolved).allowsSpace({ + type: nsid.conversation, + authority: did, + skey: 'self', + manage: 'create', + }) +} diff --git a/apps/server/src/auth/web-session.ts b/apps/server/src/auth/web-session.ts new file mode 100644 index 0000000..62df9c7 --- /dev/null +++ b/apps/server/src/auth/web-session.ts @@ -0,0 +1,62 @@ +import { createHash, randomBytes } from 'node:crypto' +import type { Db } from '../db/index.ts' + +export const SESSION_COOKIE = 'scn_session' +const DAY_MS = 86_400_000 + +const hash = (token: string) => createHash('sha256').update(token).digest('hex') + +export async function createWebSession( + db: Db, + did: string, + ttlDays: number, + now = new Date(), +): Promise { + const token = randomBytes(32).toString('base64url') + await db + .insertInto('web_session') + .values({ + token_hash: hash(token), + did, + created_at: now.toISOString(), + expires_at: new Date(now.getTime() + ttlDays * DAY_MS).toISOString(), + }) + .execute() + return token +} + +/** Look up the DID for a session token, extending sessions used in the second half of their life. */ +export async function resolveWebSession( + db: Db, + token: string, + ttlDays: number, + now = new Date(), +): Promise { + const row = await db + .selectFrom('web_session') + .select(['did', 'expires_at']) + .where('token_hash', '=', hash(token)) + .executeTakeFirst() + if (!row) return null + const expires = Date.parse(row.expires_at) + if (expires <= now.getTime()) { + await deleteWebSession(db, token) + return null + } + if (expires - now.getTime() < (ttlDays * DAY_MS) / 2) { + await db + .updateTable('web_session') + .set({ expires_at: new Date(now.getTime() + ttlDays * DAY_MS).toISOString() }) + .where('token_hash', '=', hash(token)) + .execute() + } + return row.did +} + +export async function deleteWebSession(db: Db, token: string): Promise { + await db.deleteFrom('web_session').where('token_hash', '=', hash(token)).execute() +} + +export async function deleteAllWebSessions(db: Db, did: string): Promise { + await db.deleteFrom('web_session').where('did', '=', did).execute() +} diff --git a/apps/server/src/blobs/routes.ts b/apps/server/src/blobs/routes.ts new file mode 100644 index 0000000..b7b1ad0 --- /dev/null +++ b/apps/server/src/blobs/routes.ts @@ -0,0 +1,114 @@ +import { nsid } from '@scn-chat/lexicons' +import { Hono } from 'hono' +import { requireUser, signedInUser } from '../auth/routes.ts' +import type { AppEnv } from '../env.ts' +import type { Logger } from '../logger.ts' +import type { IngesterRegistry } from '../plugins/registry.ts' +import type { ChatServices } from '../storage/services.ts' +import { BlobNotFound, type BlobStore } from './store.ts' + +export const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif']) +export const MAX_IMAGE_BYTES = 20_000_000 +export const MAX_FILE_BYTES = 50_000_000 + +export type BlobRoutesDeps = { + blobsFor: (account: Parameters[0]) => BlobStore + services: ChatServices + ingesters: IngesterRegistry + logger: Logger +} + +/** Uploading attachments, and serving a conversation's blobs to its owner. */ +export function blobRoutes(deps: BlobRoutesDeps) { + return new Hono() + .use(requireUser) + .post('/attachments', async (c) => { + const { account } = signedInUser(c) + const mimeType = + (c.req.header('content-type') ?? '').split(';')[0]?.trim().toLowerCase() ?? '' + const name = decodeURIComponent(c.req.header('x-filename') ?? 'attachment') + const image = IMAGE_TYPES.has(mimeType) + const ingester = image ? undefined : deps.ingesters.match(mimeType) + if (!image && !ingester) + return c.json( + { error: 'UnsupportedMediaType', message: `Files of type ${mimeType} are not supported` }, + 415, + ) + const declared = Number(c.req.header('content-length') ?? 0) + const limit = image ? MAX_IMAGE_BYTES : MAX_FILE_BYTES + if (declared > limit) + return c.json({ error: 'PayloadTooLarge', message: `Limit is ${limit} bytes` }, 413) + const bytes = new Uint8Array(await c.req.arrayBuffer()) + if (bytes.length > limit) + return c.json({ error: 'PayloadTooLarge', message: `Limit is ${limit} bytes` }, 413) + const store = deps.blobsFor(account) + if (image) { + const blob = await store.put(account, bytes, mimeType) + return c.json({ part: { $type: `${nsid.defs}#imagePart`, image: blob } }, 201) + } + let text: string + try { + text = (await (ingester as NonNullable).ingest({ bytes, mimeType, name })) + .text + } catch (err) { + deps.logger.warn({ err, mimeType, name }, 'file ingestion failed') + return c.json( + { + error: 'IngestFailed', + message: err instanceof Error ? err.message : 'Could not read the file', + }, + 422, + ) + } + const file = await store.put(account, bytes, mimeType) + const extracted = await store.put(account, new TextEncoder().encode(text), 'text/plain') + return c.json( + { + part: { + $type: `${nsid.defs}#filePart`, + file, + name, + extracted: { + text: extracted, + method: (ingester as NonNullable).method, + }, + }, + }, + 201, + ) + }) + .get('/conversations/:skey/blobs/:cid', async (c) => { + const { account } = signedInUser(c) + const chats = deps.services.forAccount(account) + try { + const blob = await deps + .blobsFor(account) + .get(account, chats.conversationUri(c.req.param('skey')), c.req.param('cid')) + const mimeType = + c.req.query('type') && IMAGE_TYPES.has(String(c.req.query('type'))) + ? String(c.req.query('type')) + : blob.mimeType + c.header('content-type', mimeType) + c.header('x-content-type-options', 'nosniff') + if (!IMAGE_TYPES.has(mimeType)) c.header('content-disposition', 'attachment') + return c.body(blob.bytes as unknown as ArrayBuffer) + } catch (err) { + if (err instanceof BlobNotFound) return c.json({ error: 'NotFound' }, 404) + throw err + } + }) +} + +/** Blob access for the turn runner, through each account's store. */ +export function turnBlobs(blobsFor: BlobRoutesDeps['blobsFor'], services: ChatServices) { + return { + reader: + (account: Parameters[0], skey: string) => async (cid: string) => + blobsFor(account).get(account, services.forAccount(account).conversationUri(skey), cid), + put: ( + account: Parameters[0], + bytes: Uint8Array, + mimeType: string, + ) => blobsFor(account).put(account, bytes, mimeType), + } +} diff --git a/apps/server/src/blobs/store.ts b/apps/server/src/blobs/store.ts new file mode 100644 index 0000000..901ae84 --- /dev/null +++ b/apps/server/src/blobs/store.ts @@ -0,0 +1,138 @@ +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs' +import { join, resolve } from 'node:path' +import { cidForRawBytes, LexError } from '@atproto/lex-data' +import { atproto } from '@scn-chat/lexicons' +import type { Account } from '../auth/accounts.ts' +import type { PdsClientFactory } from '../auth/pds.ts' +import type { Db } from '../db/index.ts' +import { type JsonRecord, toJson } from '../storage/records.ts' + +// Plain strings don't satisfy the generated methods' branded string types. +type Loose = any + +export type StoredBlob = { bytes: Uint8Array; mimeType: string } + +export class BlobNotFound extends Error { + constructor(cid: string) { + super(`Blob not found: ${cid}`) + this.name = 'BlobNotFound' + } +} + +export interface BlobStore { + /** Store bytes for a user and return the atproto blob reference, in JSON form. */ + put(account: Account, bytes: Uint8Array, mimeType: string): Promise + /** Read a blob referenced from one of the user's conversations. */ + get(account: Account, conversationUri: string, cid: string): Promise +} + +/** Blobs on the user's PDS, uploaded normally and read through the space they belong to. */ +export class SpaceBlobStore implements BlobStore { + private readonly getClient: PdsClientFactory + + constructor(getClient: PdsClientFactory) { + this.getClient = getClient + } + + async put(account: Account, bytes: Uint8Array, mimeType: string): Promise { + const client: Loose = await this.getClient(account.did) + const { blob } = await client.call(atproto.repo.uploadBlob, bytes, { encoding: mimeType }) + return toJson(blob) + } + + async get(account: Account, conversationUri: string, cid: string): Promise { + const client: Loose = await this.getClient(account.did) + try { + const response = await client.xrpc(atproto.space.getBlob, { + params: { space: conversationUri, repo: account.did, cid }, + }) + return { bytes: response.body as Uint8Array, mimeType: response.encoding as string } + } catch (err) { + if ( + err instanceof LexError && + (err.error === 'BlobNotFound' || err.error === 'RecordNotFound') + ) + throw new BlobNotFound(cid) + throw err + } + } +} + +/** Blobs for fallback users, one directory per owner, with the CID a PDS would compute. */ +export class LocalBlobStore implements BlobStore { + private readonly root: string + private readonly db: Db + + constructor(dataDir: string, db: Db) { + this.root = resolve(dataDir, 'blobs') + this.db = db + } + + private dir(did: string): string { + return join(this.root, encodeURIComponent(did)) + } + + async put(account: Account, bytes: Uint8Array, mimeType: string): Promise { + const cid = (await cidForRawBytes(bytes)).toString() + const dir = this.dir(account.did) + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, cid), bytes) + writeFileSync(join(dir, `${cid}.json`), JSON.stringify({ mimeType, size: bytes.length })) + return { $type: 'blob', ref: { $link: cid }, mimeType, size: bytes.length } + } + + async get(account: Account, _conversationUri: string, cid: string): Promise { + if (!/^[a-z0-9]+$/.test(cid)) throw new BlobNotFound(cid) + const path = join(this.dir(account.did), cid) + if (!existsSync(path)) throw new BlobNotFound(cid) + const meta = JSON.parse(readFileSync(`${path}.json`, 'utf8')) as { mimeType: string } + return { bytes: new Uint8Array(readFileSync(path)), mimeType: meta.mimeType } + } + + /** Delete blobs older than a day that no record references. */ + async sweep(now = Date.now()): Promise { + if (!existsSync(this.root)) return 0 + let removed = 0 + for (const owner of readdirSync(this.root)) { + const dir = join(this.root, owner) + for (const name of readdirSync(dir)) { + if (name.endsWith('.json')) continue + const path = join(dir, name) + if (now - statSync(path).mtimeMs < 86_400_000) continue + const referenced = await this.db + .selectFrom('local_record') + .select('rkey') + .where('value_json', 'like', `%${name}%`) + .executeTakeFirst() + if (referenced) continue + rmSync(path, { force: true }) + rmSync(`${path}.json`, { force: true }) + removed++ + } + } + return removed + } +} + +/** The store for an account's storage mode. */ +export function createBlobStores(deps: { + getPdsClient: PdsClientFactory + dataDir: string + db: Db +}) { + const space = new SpaceBlobStore(deps.getPdsClient) + const local = new LocalBlobStore(deps.dataDir, deps.db) + return { + space, + local, + forAccount: (account: Account): BlobStore => (account.storageMode === 'space' ? space : local), + } +} diff --git a/apps/server/src/body.ts b/apps/server/src/body.ts new file mode 100644 index 0000000..a3137f4 --- /dev/null +++ b/apps/server/src/body.ts @@ -0,0 +1,28 @@ +import type { Context } from 'hono' +import type { z } from 'zod' + +export class InvalidBody extends Error { + constructor(details: string) { + super(`Invalid request body: ${details}`) + this.name = 'InvalidBody' + } +} + +/** Parse a JSON request body and validate it against the schema. */ +export async function jsonBody(c: Context, schema: S): Promise> { + let raw: unknown + try { + raw = await c.req.json() + } catch { + throw new InvalidBody('not JSON') + } + const result = schema.safeParse(raw) + if (!result.success) { + throw new InvalidBody( + result.error.issues + .map((issue) => `${issue.path.join('.') || 'body'}: ${issue.message}`) + .join('; '), + ) + } + return result.data +} diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts new file mode 100644 index 0000000..35b8f8f --- /dev/null +++ b/apps/server/src/config.ts @@ -0,0 +1,300 @@ +import { existsSync, readFileSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { parse, YAMLParseError } from 'yaml' +import { z } from 'zod' + +export class ConfigError extends Error { + /** The settings or environment variables at fault. */ + readonly names: string[] + + constructor(names: string[], details: string) { + super(`Invalid configuration for ${names.join(', ')}: ${details}`) + this.name = 'ConfigError' + this.names = names + } +} + +export const DEFAULT_DATABASE_URL = 'sqlite:./data/scn-chat.sqlite' + +const key32 = z + .string() + .refine((value) => Buffer.from(value, 'base64').length === 32, 'must be 32 bytes, base64 encoded') + +const capabilities = z.object({ vision: z.boolean(), reasoning: z.boolean(), tools: z.boolean() }) + +const fileSchema = z + .object({ + app: z + .object({ + name: z.string().min(1).default('SCN Chat'), + publicUrl: z.url().default('http://127.0.0.1:3000'), + port: z.coerce.number().int().min(1).max(65535).default(3000), + dataDir: z.string().min(1).default('./data'), + logLevel: z + .enum(['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent']) + .default('info'), + }) + .prefault({}), + auth: z + .object({ + scopeMode: z.enum(['permission-set', 'raw']).default('permission-set'), + sessionTtlDays: z.coerce.number().int().min(1).default(30), + plcUrl: z.url().default('https://plc.directory'), + }) + .prefault({}), + turns: z + .object({ + ratePerMinute: z.coerce.number().int().min(1).default(10), + maxSteps: z.coerce.number().int().min(1).default(8), + timeoutSeconds: z.coerce.number().int().min(1).default(600), + backfillMinutes: z.coerce.number().int().min(0).default(60), + }) + .prefault({}), + sync: z + .object({ + safetyNet: z + .object({ + enabled: z.boolean().optional(), + intervalMinutes: z.number().positive().optional(), + activeWithinHours: z.number().positive().optional(), + }) + .optional(), + discovery: z + .object({ + intervalMinutes: z.number().positive().optional(), + activeWithinDays: z.number().positive().optional(), + }) + .optional(), + allowUserOptOut: z.boolean().optional(), + }) + .prefault({}), + roles: z.record(z.string(), z.array(z.string())).default({}), + plugins: z + .array( + z.object({ + package: z.string().min(1), + options: z.record(z.string(), z.unknown()).optional(), + }), + ) + .default([]), + models: z + .array( + z.object({ + provider: z.string().min(1), + id: z.string().min(1), + name: z.string().min(1), + capabilities, + roles: z.array(z.string()), + default: z.boolean().optional(), + }), + ) + .default([]), + }) + .strict() + +const envSchema = z.object({ + NODE_ENV: z.enum(['development', 'test', 'production']).default('development'), + DATABASE_URL: z.string().min(1).default(DEFAULT_DATABASE_URL), + SECRET_KEY: key32, + SECRET_KEYS_OLD: z + .string() + .default('') + .transform((value) => + value + .split(',') + .map((key) => key.trim()) + .filter(Boolean), + ) + .pipe(z.array(key32)), + OAUTH_PRIVATE_KEYS: z + .string() + .default('[]') + .transform((value, ctx) => { + try { + return JSON.parse(value) as unknown + } catch { + ctx.addIssue({ code: 'custom', message: 'must be a JSON array of private JWKs' }) + return z.NEVER + } + }) + .pipe(z.array(z.looseObject({ kid: z.string().min(1), kty: z.string(), d: z.string() }))), +}) + +/** Environment variables that override public settings. */ +export const ENV_OVERRIDES: Record = { + APP_NAME: ['app', 'name'], + PUBLIC_URL: ['app', 'publicUrl'], + PORT: ['app', 'port'], + DATA_DIR: ['app', 'dataDir'], + LOG_LEVEL: ['app', 'logLevel'], + OAUTH_SCOPE_MODE: ['auth', 'scopeMode'], + SESSION_TTL_DAYS: ['auth', 'sessionTtlDays'], + PLC_URL: ['auth', 'plcUrl'], + TURN_RATE_PER_MINUTE: ['turns', 'ratePerMinute'], + TURN_MAX_STEPS: ['turns', 'maxSteps'], + TURN_TIMEOUT_SECONDS: ['turns', 'timeoutSeconds'], + TURN_BACKFILL_MINUTES: ['turns', 'backfillMinutes'], +} + +export type FileConfig = z.infer +export type ModelConfig = FileConfig['models'][number] +export type SyncConfig = FileConfig['sync'] +export type PluginEntry = FileConfig['plugins'][number] + +export type Config = Readonly<{ + nodeEnv: 'development' | 'test' | 'production' + port: number + publicUrl: string + appName: string + dataDir: string + databaseUrl: string + logLevel: string + secretKey: Buffer + oldSecretKeys: Buffer[] + oauthPrivateKeys: { kid: string; [key: string]: unknown }[] + oauthScopeMode: 'permission-set' | 'raw' + sessionTtlDays: number + plcUrl: string + turns: { ratePerMinute: number; maxSteps: number; timeoutMs: number; backfillWindowMs: number } + /** The directory holding config.yml, which plugin packages are resolved from. */ + configDir: string + plugins: PluginEntry[] + models: ModelConfig[] + roles: Record + sync: SyncConfig +}> + +const EXACT_REFERENCE = /^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/ +const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g + +/** Replace ${NAME} references with environment values. An exact reference to a missing variable becomes unset. */ +export function interpolate(value: unknown, env: Record): unknown { + if (typeof value === 'string') { + const exact = value.match(EXACT_REFERENCE) + if (exact?.[1]) return env[exact[1]] || undefined + return value.replace(REFERENCE, (_match, name: string) => { + const found = env[name] + if (!found) + throw new ConfigError([name], `referenced in config.yml but not set in the environment`) + return found + }) + } + if (Array.isArray(value)) + return value.map((item) => interpolate(item, env)).filter((item) => item !== undefined) + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value) + .map(([key, item]) => [key, interpolate(item, env)] as const) + .filter(([, item]) => item !== undefined), + ) + } + return value +} + +/** Read and parse config.yml, failing with a pointer to the example when it is missing. */ +export function readConfigFile(path: string): unknown { + if (!existsSync(path)) { + throw new ConfigError( + ['config.yml'], + `no config file at ${path}. Copy config.example.yml to config.yml and edit it, or set SCN_CHAT_CONFIG.`, + ) + } + try { + return parse(readFileSync(path, 'utf8')) ?? {} + } catch (err) { + if (err instanceof YAMLParseError) + throw new ConfigError(['config.yml'], `${resolve(path)} is not valid YAML: ${err.message}`) + throw err + } +} + +export const defaultConfigPath = resolve( + dirname(new URL(import.meta.url).pathname), + '../../../config.yml', +) + +function issuesToError(issues: z.core.$ZodIssue[], prefix = ''): ConfigError { + const names = [ + ...new Set(issues.map((issue) => `${prefix}${issue.path.join('.')}` || 'config.yml')), + ] + const details = issues + .map((issue) => `${prefix}${issue.path.join('.')}: ${issue.message}`) + .join('; ') + return new ConfigError(names, details) +} + +/** Build the config from config.yml's contents and the environment, once at startup. */ +export function loadConfig(options: { + env?: Record + file?: unknown + configPath?: string +}): Config { + const env = options.env ?? process.env + const raw = interpolate(options.file ?? {}, env) as Record> + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) + throw new ConfigError(['config.yml'], 'must be a mapping') + for (const [variable, [section, key]] of Object.entries(ENV_OVERRIDES)) { + const value = env[variable] + if (value !== undefined && value !== '') + raw[section] = { ...(raw[section] ?? {}), [key]: value } + } + const fileResult = fileSchema.safeParse(raw) + const envResult = envSchema.safeParse( + Object.fromEntries(Object.entries(env).filter(([, value]) => value !== '')), + ) + const issues = [ + ...(fileResult.success ? [] : [issuesToError(fileResult.error.issues)]), + ...(envResult.success ? [] : [issuesToError(envResult.error.issues)]), + ] + if (issues.length) { + throw new ConfigError( + issues.flatMap((error) => error.names), + issues + .map((error) => error.message.replace(/^Invalid configuration for [^:]+: /, '')) + .join('; '), + ) + } + const file = fileResult.data as FileConfig + const secrets = envResult.data as z.infer + if (secrets.NODE_ENV === 'production') { + const problems: [string, string][] = [] + if (!file.app.publicUrl.startsWith('https://')) + problems.push(['app.publicUrl', 'must be HTTPS in production']) + if (secrets.OAUTH_PRIVATE_KEYS.length === 0) + problems.push(['OAUTH_PRIVATE_KEYS', 'is required in production']) + if (file.auth.scopeMode === 'raw') + problems.push(['auth.scopeMode', 'raw is for local development only']) + if (problems.length) { + throw new ConfigError( + problems.map(([name]) => name), + problems.map(([name, message]) => `${name}: ${message}`).join('; '), + ) + } + } + return Object.freeze({ + nodeEnv: secrets.NODE_ENV, + port: file.app.port, + publicUrl: file.app.publicUrl.replace(/\/$/, ''), + appName: file.app.name, + dataDir: file.app.dataDir, + databaseUrl: secrets.DATABASE_URL, + logLevel: file.app.logLevel, + secretKey: Buffer.from(secrets.SECRET_KEY, 'base64'), + oldSecretKeys: secrets.SECRET_KEYS_OLD.map((key) => Buffer.from(key, 'base64')), + oauthPrivateKeys: secrets.OAUTH_PRIVATE_KEYS, + oauthScopeMode: file.auth.scopeMode, + sessionTtlDays: file.auth.sessionTtlDays, + plcUrl: file.auth.plcUrl, + turns: { + ratePerMinute: file.turns.ratePerMinute, + maxSteps: file.turns.maxSteps, + timeoutMs: file.turns.timeoutSeconds * 1000, + backfillWindowMs: file.turns.backfillMinutes * 60_000, + }, + configDir: dirname(options.configPath ?? defaultConfigPath), + plugins: file.plugins, + models: file.models, + roles: file.roles, + sync: file.sync, + }) +} diff --git a/apps/server/src/db/dialect.ts b/apps/server/src/db/dialect.ts new file mode 100644 index 0000000..6cd08a5 --- /dev/null +++ b/apps/server/src/db/dialect.ts @@ -0,0 +1,15 @@ +import { type ColumnDefinitionBuilder, type Kysely, PostgresAdapter } from 'kysely' + +export function isPostgres(db: Kysely): boolean { + return db.getExecutor().adapter instanceof PostgresAdapter +} + +/** An auto-incrementing integer primary key on either database. */ +export function autoIncrementKey(db: Kysely) { + const postgres = isPostgres(db) + return { + type: postgres ? ('serial' as const) : ('integer' as const), + build: (col: ColumnDefinitionBuilder) => + postgres ? col.primaryKey() : col.primaryKey().autoIncrement(), + } +} diff --git a/apps/server/src/db/index.ts b/apps/server/src/db/index.ts new file mode 100644 index 0000000..3cf7ace --- /dev/null +++ b/apps/server/src/db/index.ts @@ -0,0 +1,34 @@ +import { mkdirSync } from 'node:fs' +import { dirname } from 'node:path' +import SqliteDatabase from 'better-sqlite3' +import { Kysely, PostgresDialect, SqliteDialect } from 'kysely' +import pg from 'pg' +import type { Database } from './schema.ts' + +export type Db = Kysely + +export class UnsupportedDatabaseError extends Error { + constructor(url: string) { + const scheme = url.split(':')[0] + super(`Unsupported DATABASE_URL scheme "${scheme}:". Use sqlite: or postgres://...`) + this.name = 'UnsupportedDatabaseError' + } +} + +/** Open the database named by a DATABASE_URL. */ +export function createDb(url: string): Db { + if (url.startsWith('sqlite:')) { + const path = url.slice('sqlite:'.length) + if (path !== ':memory:') mkdirSync(dirname(path), { recursive: true }) + const sqlite = new SqliteDatabase(path) + sqlite.pragma('journal_mode = WAL') + sqlite.pragma('foreign_keys = ON') + return new Kysely({ dialect: new SqliteDialect({ database: sqlite }) }) + } + if (url.startsWith('postgres://') || url.startsWith('postgresql://')) { + return new Kysely({ + dialect: new PostgresDialect({ pool: new pg.Pool({ connectionString: url }) }), + }) + } + throw new UnsupportedDatabaseError(url) +} diff --git a/apps/server/src/db/migrate.ts b/apps/server/src/db/migrate.ts new file mode 100644 index 0000000..93a9403 --- /dev/null +++ b/apps/server/src/db/migrate.ts @@ -0,0 +1,20 @@ +import { type Migration, Migrator } from 'kysely/migration' +import type { Db } from './index.ts' +import { migrations as appMigrations } from './migrations/index.ts' + +export type MigrationOutcome = { applied: string[] } + +/** Run every pending migration, failing loudly if any migration fails. */ +export async function migrateToLatest( + db: Db, + migrations: Record = appMigrations, +): Promise { + const migrator = new Migrator({ db, provider: { getMigrations: async () => migrations } }) + const { error, results = [] } = await migrator.migrateToLatest() + if (error) throw error instanceof Error ? error : new Error(String(error)) + return { + applied: results + .filter((result) => result.status === 'Success') + .map((result) => result.migrationName), + } +} diff --git a/apps/server/src/db/migrations/0001_plugin_user_settings.ts b/apps/server/src/db/migrations/0001_plugin_user_settings.ts new file mode 100644 index 0000000..7535c28 --- /dev/null +++ b/apps/server/src/db/migrations/0001_plugin_user_settings.ts @@ -0,0 +1,13 @@ +import type { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('plugin_user_settings') + .addColumn('did', 'text', (col) => col.notNull()) + .addColumn('plugin_id', 'text', (col) => col.notNull()) + .addColumn('values_json', 'text', (col) => col.notNull()) + .addColumn('secrets_encrypted', 'text') + .addColumn('updated_at', 'text', (col) => col.notNull()) + .addPrimaryKeyConstraint('plugin_user_settings_pk', ['did', 'plugin_id']) + .execute() +} diff --git a/apps/server/src/db/migrations/0002_auth.ts b/apps/server/src/db/migrations/0002_auth.ts new file mode 100644 index 0000000..c4bc658 --- /dev/null +++ b/apps/server/src/db/migrations/0002_auth.ts @@ -0,0 +1,31 @@ +import type { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('account') + .addColumn('did', 'text', (col) => col.primaryKey()) + .addColumn('handle', 'text') + .addColumn('pds_url', 'text', (col) => col.notNull()) + .addColumn('storage_mode', 'text', (col) => col.notNull()) + .addColumn('background_sync', 'integer', (col) => col.notNull().defaultTo(1)) + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('last_login_at', 'text', (col) => col.notNull()) + .addColumn('last_active_at', 'text', (col) => col.notNull()) + .execute() + await db.schema + .createTable('web_session') + .addColumn('token_hash', 'text', (col) => col.primaryKey()) + .addColumn('did', 'text', (col) => col.notNull()) + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('expires_at', 'text', (col) => col.notNull()) + .execute() + await db.schema.createIndex('web_session_did').on('web_session').column('did').execute() + for (const table of ['oauth_state', 'oauth_session']) { + await db.schema + .createTable(table) + .addColumn('key', 'text', (col) => col.primaryKey()) + .addColumn('value', 'text', (col) => col.notNull()) + .addColumn('updated_at', 'text', (col) => col.notNull()) + .execute() + } +} diff --git a/apps/server/src/db/migrations/0003_storage.ts b/apps/server/src/db/migrations/0003_storage.ts new file mode 100644 index 0000000..b5029af --- /dev/null +++ b/apps/server/src/db/migrations/0003_storage.ts @@ -0,0 +1,49 @@ +import type { Kysely } from 'kysely' +import { autoIncrementKey } from '../dialect.ts' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('local_space') + .addColumn('uri', 'text', (col) => col.primaryKey()) + .addColumn('owner_did', 'text', (col) => col.notNull()) + .addColumn('type', 'text', (col) => col.notNull()) + .addColumn('skey', 'text', (col) => col.notNull()) + .addColumn('created_at', 'text', (col) => col.notNull()) + .execute() + await db.schema + .createTable('local_record') + .addColumn('space_uri', 'text', (col) => col.notNull()) + .addColumn('collection', 'text', (col) => col.notNull()) + .addColumn('rkey', 'text', (col) => col.notNull()) + .addColumn('value_json', 'text', (col) => col.notNull()) + .addColumn('cid', 'text', (col) => col.notNull()) + .addColumn('updated_at', 'text', (col) => col.notNull()) + .addPrimaryKeyConstraint('local_record_pk', ['space_uri', 'collection', 'rkey']) + .execute() + const seq = autoIncrementKey(db) + await db.schema + .createTable('local_op') + .addColumn('seq', seq.type, seq.build) + .addColumn('space_uri', 'text', (col) => col.notNull()) + .addColumn('collection', 'text', (col) => col.notNull()) + .addColumn('rkey', 'text', (col) => col.notNull()) + .addColumn('cid', 'text') + .addColumn('value_json', 'text') + .addColumn('created_at', 'text', (col) => col.notNull()) + .execute() + await db.schema + .createIndex('local_op_space') + .on('local_op') + .columns(['space_uri', 'seq']) + .execute() + await db.schema + .createTable('sync_state') + .addColumn('space_uri', 'text', (col) => col.primaryKey()) + .addColumn('owner_did', 'text', (col) => col.notNull()) + .addColumn('last_rev', 'text') + .addColumn('registered_until', 'text') + .addColumn('last_synced_at', 'text') + .addColumn('last_error', 'text') + .execute() + await db.schema.createIndex('sync_state_owner').on('sync_state').column('owner_did').execute() +} diff --git a/apps/server/src/db/migrations/0004_providers.ts b/apps/server/src/db/migrations/0004_providers.ts new file mode 100644 index 0000000..0b1de64 --- /dev/null +++ b/apps/server/src/db/migrations/0004_providers.ts @@ -0,0 +1,23 @@ +import type { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('provider_credential') + .addColumn('id', 'text', (col) => col.primaryKey()) + .addColumn('owner_did', 'text', (col) => col.notNull()) + .addColumn('provider_id', 'text', (col) => col.notNull()) + .addColumn('name', 'text') + .addColumn('slug', 'text') + .addColumn('base_url', 'text') + .addColumn('api_key_encrypted', 'text', (col) => col.notNull()) + .addColumn('key_hint', 'text', (col) => col.notNull()) + .addColumn('models_json', 'text', (col) => col.notNull()) + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('updated_at', 'text', (col) => col.notNull()) + .execute() + await db.schema + .createIndex('provider_credential_owner') + .on('provider_credential') + .column('owner_did') + .execute() +} diff --git a/apps/server/src/db/migrations/0005_turns.ts b/apps/server/src/db/migrations/0005_turns.ts new file mode 100644 index 0000000..be1877c --- /dev/null +++ b/apps/server/src/db/migrations/0005_turns.ts @@ -0,0 +1,21 @@ +import type { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('turn_request') + .addColumn('conversation_uri', 'text', (col) => col.notNull()) + .addColumn('message_rkey', 'text', (col) => col.notNull()) + .addColumn('attempt', 'integer', (col) => col.notNull()) + .addColumn('owner_did', 'text', (col) => col.notNull()) + .addColumn('requested_at', 'text', (col) => col.notNull()) + .addPrimaryKeyConstraint('turn_request_pk', ['conversation_uri', 'message_rkey', 'attempt']) + .execute() + await db.schema + .createTable('turn_claim') + .addColumn('conversation_uri', 'text', (col) => col.notNull()) + .addColumn('reply_rkey', 'text', (col) => col.notNull()) + .addColumn('owner_did', 'text', (col) => col.notNull()) + .addColumn('claimed_at', 'text', (col) => col.notNull()) + .addPrimaryKeyConstraint('turn_claim_pk', ['conversation_uri', 'reply_rkey']) + .execute() +} diff --git a/apps/server/src/db/migrations/index.ts b/apps/server/src/db/migrations/index.ts new file mode 100644 index 0000000..4b1a49a --- /dev/null +++ b/apps/server/src/db/migrations/index.ts @@ -0,0 +1,15 @@ +import type { Migration } from 'kysely/migration' +import * as m0001 from './0001_plugin_user_settings.ts' +import * as m0002 from './0002_auth.ts' +import * as m0003 from './0003_storage.ts' +import * as m0004 from './0004_providers.ts' +import * as m0005 from './0005_turns.ts' + +/** App migrations in order. Keys sort in the order they run. */ +export const migrations: Record = { + '0001_plugin_user_settings': m0001, + '0002_auth': m0002, + '0003_storage': m0003, + '0004_providers': m0004, + '0005_turns': m0005, +} diff --git a/apps/server/src/db/schema.ts b/apps/server/src/db/schema.ts new file mode 100644 index 0000000..e206065 --- /dev/null +++ b/apps/server/src/db/schema.ts @@ -0,0 +1,77 @@ +import type { Generated } from 'kysely' + +/** Every table in the app database. */ +export type Database = { + plugin_user_settings: { + did: string + plugin_id: string + values_json: string + secrets_encrypted: string | null + updated_at: string + } + account: { + did: string + handle: string | null + pds_url: string + storage_mode: 'space' | 'local' + background_sync: number + created_at: string + last_login_at: string + last_active_at: string + } + web_session: { token_hash: string; did: string; created_at: string; expires_at: string } + oauth_state: { key: string; value: string; updated_at: string } + oauth_session: { key: string; value: string; updated_at: string } + local_space: { uri: string; owner_did: string; type: string; skey: string; created_at: string } + local_record: { + space_uri: string + collection: string + rkey: string + value_json: string + cid: string + updated_at: string + } + local_op: { + seq: Generated + space_uri: string + collection: string + rkey: string + cid: string | null + value_json: string | null + created_at: string + } + sync_state: { + space_uri: string + owner_did: string + last_rev: string | null + registered_until: string | null + last_synced_at: string | null + last_error: string | null + } + provider_credential: { + id: string + owner_did: string + provider_id: string + name: string | null + slug: string | null + base_url: string | null + api_key_encrypted: string + key_hint: string + models_json: string + created_at: string + updated_at: string + } + turn_request: { + conversation_uri: string + message_rkey: string + attempt: number + owner_did: string + requested_at: string + } + turn_claim: { + conversation_uri: string + reply_rkey: string + owner_did: string + claimed_at: string + } +} diff --git a/apps/server/src/env.ts b/apps/server/src/env.ts new file mode 100644 index 0000000..aa96aba --- /dev/null +++ b/apps/server/src/env.ts @@ -0,0 +1,6 @@ +import type { Account } from './auth/accounts.ts' + +export type AuthUser = { did: string; account: Account; roles: string[] } + +/** Hono context variables shared by every route. */ +export type AppEnv = { Variables: { user: AuthUser | null } } diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts new file mode 100644 index 0000000..dafbddc --- /dev/null +++ b/apps/server/src/index.ts @@ -0,0 +1,40 @@ +import { existsSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { serve } from '@hono/node-server' +import { defaultConfigPath, loadConfig, readConfigFile } from './config.ts' +import { createDb } from './db/index.ts' +import { createLogger } from './logger.ts' +import { importPlugins } from './plugins/import.ts' +import { createServer } from './server.ts' + +if (existsSync('.env')) process.loadEnvFile('.env') +else if (existsSync('../../.env')) process.loadEnvFile('../../.env') + +const configPath = process.env.SCN_CHAT_CONFIG || defaultConfigPath +const config = loadConfig({ env: process.env, file: readConfigFile(configPath), configPath }) +const logger = createLogger(config.logLevel) +const db = createDb(config.databaseUrl) +const plugins = await importPlugins(config.plugins, config.configDir) +const webDist = fileURLToPath(new URL('../../web/dist', import.meta.url)) + +const server = await createServer({ + config, + appConfig: { plugins, models: config.models, roles: config.roles, sync: config.sync }, + db, + logger, + webDist: config.nodeEnv === 'production' ? webDist : undefined, +}) + +const http = serve({ fetch: server.app.fetch, port: config.port }, ({ port }) => { + logger.info({ port, publicUrl: config.publicUrl }, `${config.appName} server listening`) +}) + +const shutdown = async () => { + logger.info('shutting down') + http.close() + await server.close() + await db.destroy() + process.exit(0) +} +process.on('SIGTERM', shutdown) +process.on('SIGINT', shutdown) diff --git a/apps/server/src/keys-cli.ts b/apps/server/src/keys-cli.ts new file mode 100644 index 0000000..c700e34 --- /dev/null +++ b/apps/server/src/keys-cli.ts @@ -0,0 +1,6 @@ +import { randomBytes } from 'node:crypto' +import { JoseKey } from '@atproto/oauth-client-node' + +const key = await JoseKey.generate(['ES256'], `key-${Date.now()}`) +console.log(`OAUTH_PRIVATE_KEYS='${JSON.stringify([key.privateJwk])}'`) +console.log(`SECRET_KEY=${randomBytes(32).toString('base64')}`) diff --git a/apps/server/src/logger.ts b/apps/server/src/logger.ts new file mode 100644 index 0000000..2a82550 --- /dev/null +++ b/apps/server/src/logger.ts @@ -0,0 +1,7 @@ +import pino, { type Logger } from 'pino' + +export type { Logger } + +export function createLogger(level: string): Logger { + return pino({ level }) +} diff --git a/apps/server/src/migrate-cli.ts b/apps/server/src/migrate-cli.ts new file mode 100644 index 0000000..a84e883 --- /dev/null +++ b/apps/server/src/migrate-cli.ts @@ -0,0 +1,12 @@ +import { existsSync } from 'node:fs' +import { DEFAULT_DATABASE_URL } from './config.ts' +import { createDb } from './db/index.ts' +import { migrateToLatest } from './db/migrate.ts' + +if (existsSync('.env')) process.loadEnvFile('.env') +else if (existsSync('../../.env')) process.loadEnvFile('../../.env') + +const db = createDb(process.env.DATABASE_URL || DEFAULT_DATABASE_URL) +const { applied } = await migrateToLatest(db) +console.log(applied.length > 0 ? `Applied: ${applied.join(', ')}` : 'No pending migrations') +await db.destroy() diff --git a/apps/server/src/plugins/hooks.ts b/apps/server/src/plugins/hooks.ts new file mode 100644 index 0000000..7d73472 --- /dev/null +++ b/apps/server/src/plugins/hooks.ts @@ -0,0 +1,94 @@ +import type { + ActionHooks, + FilterHooks, + HookHandler, + HookName, + HookOrder, + Logger, +} from '@scn-chat/plugin-api' + +type Entry = { + pluginId: string + position: number + order: HookOrder + handler: (...args: never[]) => unknown +} + +const orderRank: Record = { pre: 0, normal: 1, post: 2 } + +export class PluginHookError extends Error { + readonly pluginId: string + readonly hook: HookName + + constructor(pluginId: string, hook: HookName, cause: unknown) { + super( + `Plugin "${pluginId}" failed in ${hook}: ${cause instanceof Error ? cause.message : String(cause)}`, + { + cause, + }, + ) + this.name = 'PluginHookError' + this.pluginId = pluginId + this.hook = hook + } +} + +/** Runs hook handlers in order: pre, normal, post, then by the plugin's place in the config. */ +export class HookRunner { + private readonly entries = new Map() + + add( + name: N, + pluginId: string, + position: number, + handler: HookHandler, + order: HookOrder = 'normal', + ) { + const list = this.entries.get(name) ?? [] + list.push({ pluginId, position, order, handler: handler as Entry['handler'] }) + list.sort((a, b) => orderRank[a.order] - orderRank[b.order] || a.position - b.position) + this.entries.set(name, list) + } + + /** Pass a value through every filter, failing on the first handler that throws. */ + async filter( + name: N, + value: FilterHooks[N]['value'], + context: FilterHooks[N]['context'], + ): Promise { + let current = value + for (const entry of this.entries.get(name) ?? []) { + try { + current = await (entry.handler as (v: typeof current, c: typeof context) => typeof current)( + current, + context, + ) + } catch (err) { + throw new PluginHookError(entry.pluginId, name, err) + } + } + return current + } + + /** Run every action, logging failures without stopping the others. */ + async action( + name: N, + payload: ActionHooks[N], + logger: Logger, + ): Promise { + for (const entry of this.entries.get(name) ?? []) { + try { + await (entry.handler as (p: typeof payload) => unknown)(payload) + } catch (err) { + logger.error({ err, pluginId: entry.pluginId, hook: name }, 'plugin action failed') + } + } + } + + /** The resolved handler order for each hook, by plugin ID. */ + describe(): Record { + return Object.fromEntries( + [...this.entries].map(([name, list]) => [name, list.map((entry) => entry.pluginId)]), + ) + } +} diff --git a/apps/server/src/plugins/host.ts b/apps/server/src/plugins/host.ts new file mode 100644 index 0000000..7da1030 --- /dev/null +++ b/apps/server/src/plugins/host.ts @@ -0,0 +1,108 @@ +import { + type GenerateTextRequest, + type InfoPatch, + type Ingester, + type Logger, + type ModelProvider, + PLUGIN_API_VERSION, + type Plugin, + type PluginContext, + type Tool, + type ToolSource, +} from '@scn-chat/plugin-api' +import { HookRunner } from './hooks.ts' +import { IngesterRegistry, Registry } from './registry.ts' + +export class PluginLoadError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options) + this.name = 'PluginLoadError' + } +} + +/** The app services that plugins can call. */ +export type PluginServices = { + generateText(request: GenerateTextRequest): Promise<{ text: string; finishReason: string }> + updateInfo( + user: string, + conversation: string, + patch: InfoPatch, + options?: { unlessUserTitled?: boolean }, + ): Promise + userSettings(plugin: Plugin, user: string): Promise +} + +export type PluginHost = { + plugins: readonly Plugin[] + providers: Registry + tools: Registry> + toolSources: Registry + ingesters: IngesterRegistry + hooks: HookRunner + close(): Promise +} + +const ID_PATTERN = /^[a-z0-9-]+$/ + +/** Check every plugin, then call each setup once, in config order. */ +export async function loadPlugins( + plugins: Plugin[], + deps: { + services: PluginServices + logger: Logger & { child(bindings: object): Logger } + app: { name: string; publicUrl: string } + }, +): Promise { + const seen = new Set() + for (const plugin of plugins) { + if (!ID_PATTERN.test(plugin.id)) + throw new PluginLoadError(`Plugin ID "${plugin.id}" must match ${ID_PATTERN}`) + if (seen.has(plugin.id)) throw new PluginLoadError(`Two plugins use the ID "${plugin.id}"`) + seen.add(plugin.id) + if (plugin.apiVersion !== PLUGIN_API_VERSION) { + throw new PluginLoadError( + `Plugin "${plugin.id}" targets plugin API version ${plugin.apiVersion}, but this server supports version ${PLUGIN_API_VERSION}`, + ) + } + } + + const host: PluginHost = { + plugins, + providers: new Registry('provider', (provider) => provider.id), + tools: new Registry('tool', (tool) => tool.name), + toolSources: new Registry('tool source', (source) => source.id), + ingesters: new IngesterRegistry(), + hooks: new HookRunner(), + close: async () => { + for (const fn of closers.reverse()) await fn() + }, + } + const closers: (() => void | Promise)[] = [] + + for (const [position, plugin] of plugins.entries()) { + const ctx: PluginContext = { + providers: { + register: (provider: ModelProvider) => host.providers.register(provider, plugin.id), + }, + tools: { register: (tool: Tool) => host.tools.register(tool, plugin.id) }, + toolSources: { + register: (source: ToolSource) => host.toolSources.register(source, plugin.id), + }, + ingesters: { register: (ingester: Ingester) => host.ingesters.register(ingester, plugin.id) }, + hooks: { + on: (name, handler, options) => + host.hooks.add(name, plugin.id, position, handler, options?.order), + }, + models: { generateText: (request) => deps.services.generateText(request) }, + conversations: { updateInfo: (...args) => deps.services.updateInfo(...args) }, + userSettings: (user) => deps.services.userSettings(plugin, user), + logger: deps.logger.child({ plugin: plugin.id }), + app: Object.freeze({ ...deps.app }), + onClose: (fn) => closers.push(fn), + } + await plugin.setup(ctx) + } + + deps.logger.info({ hooks: host.hooks.describe() }, 'plugins loaded') + return host +} diff --git a/apps/server/src/plugins/import.ts b/apps/server/src/plugins/import.ts new file mode 100644 index 0000000..4e460a7 --- /dev/null +++ b/apps/server/src/plugins/import.ts @@ -0,0 +1,74 @@ +import { existsSync, readFileSync, statSync } from 'node:fs' +import { createRequire } from 'node:module' +import { isAbsolute, join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import type { Plugin } from '@scn-chat/plugin-api' +import type { z } from 'zod' +import type { PluginEntry } from '../config.ts' +import { PluginLoadError } from './host.ts' + +type PluginModule = { default?: unknown; optionsSchema?: z.ZodType } + +type PackageJson = { main?: string; exports?: string | Record } + +/** A local plugin directory's entry file, from its package.json exports or main. */ +function localEntry(dir: string): string | undefined { + const manifest = join(dir, 'package.json') + if (!existsSync(manifest)) return undefined + const pkg = JSON.parse(readFileSync(manifest, 'utf8')) as PackageJson + const root = typeof pkg.exports === 'object' ? pkg.exports['.'] : pkg.exports + const target = + typeof root === 'string' + ? root + : ((root as Record | undefined)?.import ?? + (root as Record | undefined)?.default) + const entry = target ?? pkg.main ?? 'index.js' + return join(dir, entry) +} + +function resolvePackage(specifier: string, configDir: string): string { + if (specifier.startsWith('.') || isAbsolute(specifier)) { + const path = resolve(configDir, specifier) + if (existsSync(path) && statSync(path).isDirectory()) { + const entry = localEntry(path) + if (entry && existsSync(entry)) return entry + } else if (existsSync(path)) return path + throw new Error(`not found at ${path}`) + } + return createRequire(join(configDir, 'config.yml')).resolve(specifier) +} + +/** Import each configured plugin package, validate its options, and call its factory, in config order. */ +export async function importPlugins(entries: PluginEntry[], configDir: string): Promise { + const plugins: Plugin[] = [] + for (const entry of entries) { + let resolved: string + try { + resolved = resolvePackage(entry.package, configDir) + } catch (err) { + throw new PluginLoadError( + `Cannot load plugin package "${entry.package}" from ${configDir}: ${err instanceof Error ? err.message : String(err)}`, + { cause: err }, + ) + } + const module = (await import(pathToFileURL(resolved).href)) as PluginModule + if (typeof module.default !== 'function') { + throw new PluginLoadError(`Plugin package "${entry.package}" has no default export factory`) + } + let options: unknown = entry.options ?? {} + if (module.optionsSchema) { + const result = module.optionsSchema.safeParse(options) + if (!result.success) { + const problems = result.error.issues.map( + (issue) => `${issue.path.join('.') || 'options'}: ${issue.message}`, + ) + throw new PluginLoadError( + `Plugin "${entry.package}" has invalid options: ${problems.join('; ')}`, + ) + } + options = result.data + } + plugins.push((module.default as (options: unknown) => Plugin)(options)) + } + return plugins +} diff --git a/apps/server/src/plugins/registry.ts b/apps/server/src/plugins/registry.ts new file mode 100644 index 0000000..73d8f96 --- /dev/null +++ b/apps/server/src/plugins/registry.ts @@ -0,0 +1,57 @@ +import type { Ingester } from '@scn-chat/plugin-api' + +export class DuplicateRegistrationError extends Error { + constructor(kind: string, key: string, pluginId: string, existingPluginId: string) { + super( + `Plugin "${pluginId}" registered ${kind} "${key}", which plugin "${existingPluginId}" already registered`, + ) + this.name = 'DuplicateRegistrationError' + } +} + +/** Implementations picked by a unique key. */ +export class Registry { + private readonly items = new Map() + private readonly kind: string + private readonly keyOf: (item: T) => string + + constructor(kind: string, keyOf: (item: T) => string) { + this.kind = kind + this.keyOf = keyOf + } + + register(item: T, pluginId: string): void { + const key = this.keyOf(item) + const existing = this.items.get(key) + if (existing) throw new DuplicateRegistrationError(this.kind, key, pluginId, existing.pluginId) + this.items.set(key, { item, pluginId }) + } + + get(key: string): T | undefined { + return this.items.get(key)?.item + } + + list(): T[] { + return [...this.items.values()].map(({ item }) => item) + } +} + +function accepts(pattern: string, mimeType: string): boolean { + return pattern.endsWith('/*') ? mimeType.startsWith(pattern.slice(0, -1)) : pattern === mimeType +} + +/** Ingesters, matched by MIME type with the highest priority winning. */ +export class IngesterRegistry extends Registry { + constructor() { + super('ingester', (ingester) => ingester.id) + } + + match(mimeType: string): Ingester | undefined { + let best: Ingester | undefined + for (const ingester of this.list()) { + if (!ingester.accepts.some((pattern) => accepts(pattern, mimeType))) continue + if (!best || (ingester.priority ?? 0) > (best.priority ?? 0)) best = ingester + } + return best + } +} diff --git a/apps/server/src/plugins/routes.ts b/apps/server/src/plugins/routes.ts new file mode 100644 index 0000000..cd3f513 --- /dev/null +++ b/apps/server/src/plugins/routes.ts @@ -0,0 +1,72 @@ +import { Hono } from 'hono' +import { z } from 'zod' +import { requireUser, signedInUser } from '../auth/routes.ts' +import type { Db } from '../db/index.ts' +import type { AppEnv } from '../env.ts' +import type { SecretBox } from '../secrets.ts' +import type { PluginHost } from './host.ts' +import { + clearUserSettings, + InvalidPluginSettingsError, + readUserSettings, + secretKeys, + writeUserSettings, +} from './user-settings.ts' + +export type PluginRoutesDeps = { db: Db; box: SecretBox; host: PluginHost } + +/** Per-user plugin settings: forms, saving, and resetting. */ +export function pluginRoutes(deps: PluginRoutesDeps) { + const configurable = () => deps.host.plugins.filter((plugin) => plugin.userSettings) + const find = (id: string) => configurable().find((plugin) => plugin.id === id) + + return new Hono() + .use(requireUser) + .get('/settings', async (c) => { + const { did } = signedInUser(c) + const plugins = await Promise.all( + configurable().map(async (plugin) => { + const schema = plugin.userSettings as z.ZodObject + const secrets = secretKeys(schema) + const base = { + id: plugin.id, + name: plugin.name, + schema: z.toJSONSchema(schema), + secretFields: secrets, + } + try { + const values = (await readUserSettings(deps.db, deps.box, did, plugin)) as Record< + string, + unknown + > + const secretsSet = secrets.filter((key) => Boolean(values[key])) + for (const key of secrets) values[key] = '' + return { ...base, values, secretsSet, error: null } + } catch (err) { + if (!(err instanceof InvalidPluginSettingsError)) throw err + return { ...base, values: {}, secretsSet: [], error: err.message } + } + }), + ) + return c.json({ plugins }) + }) + .put('/:id/settings', async (c) => { + const plugin = find(c.req.param('id')) + if (!plugin) return c.json({ error: 'NotFound' }, 404) + const body = (await c.req.json()) as Record + try { + await writeUserSettings(deps.db, deps.box, signedInUser(c).did, plugin, body) + } catch (err) { + if (err instanceof z.ZodError) + return c.json({ error: 'InvalidSettings', message: err.message }, 400) + throw err + } + return c.json({ ok: true }) + }) + .delete('/:id/settings', async (c) => { + const plugin = find(c.req.param('id')) + if (!plugin) return c.json({ error: 'NotFound' }, 404) + await clearUserSettings(deps.db, signedInUser(c).did, plugin.id) + return c.json({ ok: true }) + }) +} diff --git a/apps/server/src/plugins/user-settings.ts b/apps/server/src/plugins/user-settings.ts new file mode 100644 index 0000000..83c0628 --- /dev/null +++ b/apps/server/src/plugins/user-settings.ts @@ -0,0 +1,96 @@ +import type { Plugin } from '@scn-chat/plugin-api' +import type { z } from 'zod' +import type { Db } from '../db/index.ts' +import type { SecretBox } from '../secrets.ts' + +export class InvalidPluginSettingsError extends Error { + readonly pluginId: string + + constructor(pluginId: string, details: string) { + super(`Stored settings for plugin "${pluginId}" are invalid: ${details}`) + this.name = 'InvalidPluginSettingsError' + this.pluginId = pluginId + } +} + +type SettingsSchema = z.ZodObject + +/** Top-level fields marked with .meta({ secret: true }). */ +export function secretKeys(schema: SettingsSchema): string[] { + return Object.entries(schema.shape) + .filter(([, field]) => (field as z.ZodType).meta()?.secret === true) + .map(([key]) => key) +} + +async function readRaw(db: Db, box: SecretBox, did: string, pluginId: string) { + const row = await db + .selectFrom('plugin_user_settings') + .select(['values_json', 'secrets_encrypted']) + .where('did', '=', did) + .where('plugin_id', '=', pluginId) + .executeTakeFirst() + if (!row) return { values: {}, secrets: {} } + return { + values: JSON.parse(row.values_json) as Record, + secrets: row.secrets_encrypted + ? (JSON.parse(box.decrypt(row.secrets_encrypted)) as Record) + : {}, + } +} + +/** A user's validated settings for a plugin, with defaults when none are stored. */ +export async function readUserSettings( + db: Db, + box: SecretBox, + did: string, + plugin: Plugin, +): Promise { + if (!plugin.userSettings) return {} + const { values, secrets } = await readRaw(db, box, did, plugin.id) + const result = plugin.userSettings.safeParse({ ...values, ...secrets }) + if (!result.success) throw new InvalidPluginSettingsError(plugin.id, result.error.message) + return result.data +} + +/** Validate and store a user's settings. A missing or empty secret keeps the stored one. */ +export async function writeUserSettings( + db: Db, + box: SecretBox, + did: string, + plugin: Plugin, + input: Record, +) { + if (!plugin.userSettings) + throw new InvalidPluginSettingsError(plugin.id, 'plugin has no user settings') + const secretFields = secretKeys(plugin.userSettings) + const stored = await readRaw(db, box, did, plugin.id) + const merged = { ...input } + for (const key of secretFields) { + if (merged[key] === undefined || merged[key] === '') merged[key] = stored.secrets[key] + } + const parsed = plugin.userSettings.parse(merged) as Record + const values = Object.fromEntries( + Object.entries(parsed).filter(([key]) => !secretFields.includes(key)), + ) + const secrets = Object.fromEntries( + Object.entries(parsed).filter(([key]) => secretFields.includes(key)), + ) + const row = { + values_json: JSON.stringify(values), + secrets_encrypted: secretFields.length > 0 ? box.encrypt(JSON.stringify(secrets)) : null, + updated_at: new Date().toISOString(), + } + await db + .insertInto('plugin_user_settings') + .values({ did, plugin_id: plugin.id, ...row }) + .onConflict((oc) => oc.columns(['did', 'plugin_id']).doUpdateSet(row)) + .execute() +} + +export async function clearUserSettings(db: Db, did: string, pluginId: string): Promise { + await db + .deleteFrom('plugin_user_settings') + .where('did', '=', did) + .where('plugin_id', '=', pluginId) + .execute() +} diff --git a/apps/server/src/providers/catalog.ts b/apps/server/src/providers/catalog.ts new file mode 100644 index 0000000..111c076 --- /dev/null +++ b/apps/server/src/providers/catalog.ts @@ -0,0 +1,163 @@ +import type { Capabilities, LanguageModelV4, ModelProvider, ModelRef } from '@scn-chat/plugin-api' +import type { Roles } from '../auth/roles.ts' +import type { ModelConfig } from '../config.ts' +import type { Db } from '../db/index.ts' +import type { Logger } from '../logger.ts' +import type { Registry } from '../plugins/registry.ts' +import type { SecretBox } from '../secrets.ts' +import { credentialFor, listCredentials, USER_PROVIDER_PREFIX } from './user-credentials.ts' + +export class ProviderConfigError extends Error { + constructor(message: string) { + super(message) + this.name = 'ProviderConfigError' + } +} + +export class ModelUnavailable extends Error { + readonly ref: ModelRef + + constructor(ref: ModelRef) { + super(`Model ${ref.provider}/${ref.id} is not available to you`) + this.name = 'ModelUnavailable' + this.ref = ref + } +} + +export type ListedModel = { + provider: string + id: string + name: string + capabilities: Capabilities + source: 'admin' | 'user' + default: boolean +} + +export type ResolvedModel = { + ref: ModelRef + provider: ModelProvider + model: LanguageModelV4 + capabilities: Capabilities +} + +/** Check the admin models against the registered providers and defined roles. */ +export function validateAdminModels( + models: ModelConfig[], + registry: Registry, + roles: Roles, +): ModelConfig[] { + for (const model of models) { + const name = `${model.provider}/${model.id}` + const provider = registry.get(model.provider) + if (!provider) + throw new ProviderConfigError( + `Admin model ${name} names provider "${model.provider}", which no plugin registered`, + ) + if (!provider.hasAdminKey) + throw new ProviderConfigError( + `Admin model ${name} uses provider "${model.provider}", which has no admin key`, + ) + if (!Array.isArray(model.roles) || model.roles.length === 0) { + throw new ProviderConfigError(`Admin model ${name} must list the roles allowed to use it`) + } + const undefinedRole = model.roles.find((role) => !roles.defined.has(role)) + if (undefinedRole) + throw new ProviderConfigError(`Admin model ${name} names undefined role "${undefinedRole}"`) + } + if (models.filter((model) => model.default).length > 1) + throw new ProviderConfigError('More than one admin model is marked default') + return models +} + +export type CatalogDeps = { + db: Db + box: SecretBox + providers: Registry + adminModels: ModelConfig[] + roles: Roles + guardedFetch: typeof fetch + logger: Logger +} + +/** Which models a user can use, and how to reach them. */ +export class ModelCatalog { + private readonly deps: CatalogDeps + + constructor(deps: CatalogDeps) { + this.deps = deps + } + + private adminModelsFor(did: string): ModelConfig[] { + const userRoles = new Set(this.deps.roles.rolesFor(did)) + return this.deps.adminModels.filter((model) => model.roles.some((role) => userRoles.has(role))) + } + + defaultModel(): ModelRef | undefined { + const model = this.deps.adminModels.find((m) => m.default) + return model ? { provider: model.provider, id: model.id } : undefined + } + + async listForUser(did: string): Promise { + const listed = new Map() + for (const model of this.adminModelsFor(did)) { + listed.set(`${model.provider}/${model.id}`, { + provider: model.provider, + id: model.id, + name: model.name, + capabilities: model.capabilities, + source: 'admin', + default: Boolean(model.default), + }) + } + for (const credential of await listCredentials(this.deps.db, did)) { + if (!this.deps.providers.get(credential.providerId)) { + this.deps.logger.warn( + { did, provider: credential.providerId }, + 'a saved key is for a provider no plugin registers', + ) + continue + } + for (const model of credential.models) { + listed.set(`${credential.modelProvider}/${model.id}`, { + provider: credential.modelProvider, + id: model.id, + name: model.name, + capabilities: model.capabilities, + source: 'user', + default: false, + }) + } + } + return [...listed.values()] + } + + /** Resolve a model reference for a user: their own key first, then the admin key if their roles allow it. */ + async resolve(did: string, ref: ModelRef): Promise { + const own = await credentialFor(this.deps.db, this.deps.box, did, ref) + if (own) { + const provider = this.deps.providers.get(own.summary.providerId) + if (!provider) throw new ModelUnavailable(ref) + const endpoint = ref.provider.startsWith(USER_PROVIDER_PREFIX) + const model = provider.createModel({ + modelId: ref.id, + apiKey: own.apiKey, + baseURL: own.summary.baseUrl ?? undefined, + fetch: endpoint && !provider.allowPrivateNetworks ? this.deps.guardedFetch : undefined, + }) + return { ref, provider, model, capabilities: own.info.capabilities } + } + const admin = this.adminModelsFor(did).find( + (m) => m.provider === ref.provider && m.id === ref.id, + ) + if (!admin) throw new ModelUnavailable(ref) + const provider = this.deps.providers.get(admin.provider) + if (!provider) + throw new Error(`Admin model ${admin.provider}/${admin.id} has no registered provider`) + return { + ref, + provider, + model: provider.createModel({ modelId: ref.id }), + capabilities: admin.capabilities, + } + } +} diff --git a/apps/server/src/providers/effort.ts b/apps/server/src/providers/effort.ts new file mode 100644 index 0000000..5d91c54 --- /dev/null +++ b/apps/server/src/providers/effort.ts @@ -0,0 +1,19 @@ +import type { Logger } from '../logger.ts' + +export type SdkReasoning = 'provider-default' | 'none' | 'low' | 'medium' | 'high' | 'xhigh' + +const map: Record = { + none: 'none', + low: 'low', + medium: 'medium', + high: 'high', + max: 'xhigh', +} + +/** The AI SDK reasoning setting for a lexicon effort value. */ +export function mapEffort(effort: string | undefined, logger: Logger): SdkReasoning { + if (effort === undefined) return 'provider-default' + const mapped = map[effort] + if (!mapped) logger.warn({ effort }, 'unknown effort value, using the provider default') + return mapped ?? 'provider-default' +} diff --git a/apps/server/src/providers/generate-text.ts b/apps/server/src/providers/generate-text.ts new file mode 100644 index 0000000..c0a5dcc --- /dev/null +++ b/apps/server/src/providers/generate-text.ts @@ -0,0 +1,22 @@ +import type { GenerateTextRequest } from '@scn-chat/plugin-api' +import { generateText } from 'ai' +import type { Logger } from '../logger.ts' +import type { ModelCatalog } from './catalog.ts' +import { mapEffort } from './effort.ts' + +/** Make one-off model calls for plugins, resolving the model the same way as a turn. */ +export function createGenerateText(catalog: ModelCatalog, logger: Logger) { + return async (request: GenerateTextRequest): Promise<{ text: string; finishReason: string }> => { + const resolved = await catalog.resolve(request.user, request.model) + const result = await generateText({ + model: resolved.model, + instructions: request.system, + prompt: request.prompt, + maxOutputTokens: request.maxOutputTokens, + reasoning: resolved.capabilities.reasoning ? mapEffort(request.effort, logger) : undefined, + abortSignal: request.signal, + providerOptions: resolved.provider.providerOptions as never, + }) + return { text: result.text, finishReason: result.finishReason } + } +} diff --git a/apps/server/src/providers/guarded-fetch.ts b/apps/server/src/providers/guarded-fetch.ts new file mode 100644 index 0000000..60bf385 --- /dev/null +++ b/apps/server/src/providers/guarded-fetch.ts @@ -0,0 +1,99 @@ +import { type LookupAddress, lookup } from 'node:dns' +import { isIP, type Socket } from 'node:net' +import ipaddr from 'ipaddr.js' +import { Agent, buildConnector, fetch as undiciFetch } from 'undici' + +const BLOCKED_RANGES = new Set([ + 'unspecified', + 'broadcast', + 'multicast', + 'linkLocal', + 'loopback', + 'carrierGradeNat', + 'private', + 'reserved', + 'uniqueLocal', + 'ipv4Mapped', + 'rfc6145', + 'rfc6052', + '6to4', + 'teredo', +]) + +export class PrivateNetworkError extends Error { + constructor(address: string) { + super(`Refusing to connect to private network address ${address}`) + this.name = 'PrivateNetworkError' + } +} + +/** Is this an address a user endpoint must not reach? Unparseable addresses count as private. */ +export function isPrivateAddress(address: string): boolean { + const bare = address.replace(/^\[|\]$/g, '') + if (!ipaddr.isValid(bare)) return true + return BLOCKED_RANGES.has(ipaddr.process(bare).range()) +} + +type LookupCallback = ( + err: NodeJS.ErrnoException | null, + address: string | LookupAddress[], + family?: number, +) => void + +type IsBlocked = (address: string) => boolean + +/** DNS lookup that fails when any resolved address is private. */ +export const guardedLookup = + (isBlocked: IsBlocked = isPrivateAddress) => + (hostname: string, options: { all?: boolean; family?: number }, callback: LookupCallback) => { + lookup(hostname, { ...options, all: true }, (err, addresses) => { + if (err) return callback(err, []) + const blocked = addresses.find((entry) => isBlocked(entry.address)) + if (blocked) + return callback(new PrivateNetworkError(blocked.address) as NodeJS.ErrnoException, []) + const first = addresses[0] + if (!first) + return callback(new Error(`No addresses for ${hostname}`) as NodeJS.ErrnoException, []) + if (options.all) return callback(null, addresses) + callback(null, first.address, first.family) + }) + } + +type Connector = buildConnector.connector + +/** Wrap a connector so the socket's real remote address is checked before anything is sent. */ +export function guardConnector( + connector: Connector, + isBlocked: IsBlocked = isPrivateAddress, +): Connector { + return (options, callback) => + connector(options, (err, socket) => { + if (err || !socket) return callback(err ?? new Error('connection failed'), null) + const address = (socket as Socket).remoteAddress + if (!address || isBlocked(address)) { + socket.destroy() + return callback(new PrivateNetworkError(address ?? 'unknown'), null) + } + callback(null, socket) + }) +} + +/** A fetch for user-supplied endpoints that cannot reach private networks or follow redirects. */ +export function createGuardedFetch(isBlocked: IsBlocked = isPrivateAddress): typeof fetch { + const agent = new Agent({ + connect: guardConnector( + buildConnector({ lookup: guardedLookup(isBlocked) as never }), + isBlocked, + ), + }) + return (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)) + const host = url.hostname.replace(/^\[|\]$/g, '') + if (isIP(host) && isBlocked(host)) throw new PrivateNetworkError(host) + return undiciFetch(input as never, { + ...(init as object), + redirect: 'error', + dispatcher: agent, + }) + }) as unknown as typeof fetch +} diff --git a/apps/server/src/providers/routes.ts b/apps/server/src/providers/routes.ts new file mode 100644 index 0000000..b6cd44b --- /dev/null +++ b/apps/server/src/providers/routes.ts @@ -0,0 +1,130 @@ +import type { ModelProvider } from '@scn-chat/plugin-api' +import { Hono } from 'hono' +import { z } from 'zod' +import { requireUser, signedInUser } from '../auth/routes.ts' +import { jsonBody } from '../body.ts' +import type { Db } from '../db/index.ts' +import type { AppEnv } from '../env.ts' +import type { Logger } from '../logger.ts' +import type { Registry } from '../plugins/registry.ts' +import type { SecretBox } from '../secrets.ts' +import type { ModelCatalog } from './catalog.ts' +import { + CredentialInputError, + deleteCredential, + listCredentials, + saveCredential, +} from './user-credentials.ts' + +export type ProviderRoutesDeps = { + db: Db + box: SecretBox + catalog: ModelCatalog + providers: Registry + guardedFetch: typeof fetch + logger: Logger +} + +const capabilities = z.object({ vision: z.boolean(), reasoning: z.boolean(), tools: z.boolean() }) + +const newCredential = z.object({ + providerId: z.string().min(1), + apiKey: z.string(), + name: z.string().optional(), + slug: z.string().optional(), + baseUrl: z.string().optional(), + models: z + .array(z.object({ id: z.string().min(1), name: z.string().min(1), capabilities })) + .min(1, 'Add at least one model'), +}) + +const listModelsRequest = z.object({ + apiKey: z.string().min(1, 'An API key is required'), + baseUrl: z.string().optional(), +}) + +/** Models, providers that accept user keys, and the user's own credentials. */ +export function providerRoutes(deps: ProviderRoutesDeps) { + const userProvider = (id: string) => { + const provider = deps.providers.get(id) + return provider?.userKeys ? provider : undefined + } + + return new Hono() + .use(requireUser) + .get('/models', async (c) => { + const { did } = signedInUser(c) + return c.json({ + models: await deps.catalog.listForUser(did), + defaultModel: deps.catalog.defaultModel() ?? null, + }) + }) + .get('/providers', (c) => + c.json({ + providers: deps.providers + .list() + .filter((provider) => provider.userKeys) + .map((provider) => ({ + id: provider.id, + name: provider.name, + userEndpoints: Boolean(provider.userEndpoints), + listsModels: Boolean(provider.listModels), + })), + }), + ) + .post('/providers/:id/list-models', async (c) => { + const provider = userProvider(c.req.param('id')) + if (!provider?.listModels) return c.json({ error: 'NotFound' }, 404) + const body = await jsonBody(c, listModelsRequest) + if (body.baseUrl && !provider.userEndpoints) + return c.json({ error: 'InvalidRequest', message: 'No user endpoints' }, 400) + try { + const models = await provider.listModels({ + apiKey: body.apiKey, + baseURL: body.baseUrl, + fetch: body.baseUrl && !provider.allowPrivateNetworks ? deps.guardedFetch : undefined, + }) + return c.json({ models }) + } catch (err) { + deps.logger.warn({ err, provider: provider.id }, 'listing models failed') + return c.json( + { + error: 'UpstreamFailure', + message: err instanceof Error ? err.message : 'Listing models failed', + }, + 502, + ) + } + }) + .get('/credentials', async (c) => + c.json({ credentials: await listCredentials(deps.db, signedInUser(c).did) }), + ) + .post('/credentials', async (c) => { + const body = await jsonBody(c, newCredential) + const provider = userProvider(body.providerId) + if (!provider) + return c.json( + { error: 'InvalidRequest', message: `Unknown provider ${body.providerId}` }, + 400, + ) + try { + const credential = await saveCredential(deps.db, deps.box, signedInUser(c).did, provider, { + apiKey: body.apiKey, + name: body.name, + slug: body.slug, + baseUrl: body.baseUrl, + models: body.models, + }) + return c.json({ credential }, 201) + } catch (err) { + if (err instanceof CredentialInputError) { + return c.json({ error: 'InvalidRequest', message: err.message }, 400) + } + throw err + } + }) + .delete('/credentials/:id', async (c) => { + const deleted = await deleteCredential(deps.db, signedInUser(c).did, c.req.param('id')) + return deleted ? c.json({ ok: true }) : c.json({ error: 'NotFound' }, 404) + }) +} diff --git a/apps/server/src/providers/user-credentials.ts b/apps/server/src/providers/user-credentials.ts new file mode 100644 index 0000000..a95b260 --- /dev/null +++ b/apps/server/src/providers/user-credentials.ts @@ -0,0 +1,154 @@ +import type { ModelInfo, ModelProvider } from '@scn-chat/plugin-api' +import type { Db } from '../db/index.ts' +import type { SecretBox } from '../secrets.ts' +import { newTid } from '../storage/records.ts' + +export const USER_PROVIDER_PREFIX = 'user:' +const SLUG_PATTERN = /^[a-z0-9-]+$/ + +export class CredentialInputError extends Error { + constructor(message: string) { + super(message) + this.name = 'CredentialInputError' + } +} + +export type CredentialSummary = { + id: string + providerId: string + /** The provider value model references use: the provider ID, or user: for a user endpoint. */ + modelProvider: string + name: string | null + slug: string | null + baseUrl: string | null + keyHint: string + models: ModelInfo[] +} + +type Row = { + id: string + provider_id: string + name: string | null + slug: string | null + base_url: string | null + key_hint: string + models_json: string + api_key_encrypted: string +} + +const summarize = (row: Row): CredentialSummary => ({ + id: row.id, + providerId: row.provider_id, + modelProvider: row.slug ? `${USER_PROVIDER_PREFIX}${row.slug}` : row.provider_id, + name: row.name, + slug: row.slug, + baseUrl: row.base_url, + keyHint: row.key_hint, + models: JSON.parse(row.models_json) as ModelInfo[], +}) + +async function rows(db: Db, did: string): Promise { + return db + .selectFrom('provider_credential') + .select([ + 'id', + 'provider_id', + 'name', + 'slug', + 'base_url', + 'key_hint', + 'models_json', + 'api_key_encrypted', + ]) + .where('owner_did', '=', did) + .orderBy('created_at') + .execute() +} + +export async function listCredentials(db: Db, did: string): Promise { + return (await rows(db, did)).map(summarize) +} + +export type NewCredential = { + apiKey: string + name?: string + slug?: string + baseUrl?: string + models: ModelInfo[] +} + +/** Store a user's key for a provider, replacing their earlier key unless it is a separate endpoint. */ +export async function saveCredential( + db: Db, + box: SecretBox, + did: string, + provider: ModelProvider, + input: NewCredential, +): Promise { + if (!provider.userKeys) + throw new CredentialInputError(`${provider.name} does not accept user keys`) + if (!input.apiKey.trim()) throw new CredentialInputError('An API key is required') + const endpoint = Boolean(input.baseUrl) + if (endpoint && !provider.userEndpoints) + throw new CredentialInputError(`${provider.name} does not accept user endpoints`) + if (endpoint && !(input.slug && SLUG_PATTERN.test(input.slug))) { + throw new CredentialInputError(`An endpoint needs a slug matching ${SLUG_PATTERN}`) + } + if (endpoint && !URL.canParse(input.baseUrl as string)) + throw new CredentialInputError('The base URL is not a valid URL') + const existing = await rows(db, did) + if (endpoint && existing.some((row) => row.slug === input.slug)) { + throw new CredentialInputError(`You already have an endpoint named "${input.slug}"`) + } + const now = new Date().toISOString() + const row = { + provider_id: provider.id, + name: input.name ?? null, + slug: endpoint ? (input.slug as string) : null, + base_url: endpoint ? (input.baseUrl as string) : null, + api_key_encrypted: box.encrypt(input.apiKey), + key_hint: input.apiKey.slice(-4), + models_json: JSON.stringify(input.models), + updated_at: now, + } + const replace = endpoint + ? undefined + : existing.find((r) => r.provider_id === provider.id && !r.slug) + if (replace) { + await db.updateTable('provider_credential').set(row).where('id', '=', replace.id).execute() + return summarize({ ...replace, ...row }) + } + const id = newTid() + await db + .insertInto('provider_credential') + .values({ id, owner_did: did, created_at: now, ...row }) + .execute() + return summarize({ id, ...row }) +} + +export async function deleteCredential(db: Db, did: string, id: string): Promise { + const result = await db + .deleteFrom('provider_credential') + .where('owner_did', '=', did) + .where('id', '=', id) + .executeTakeFirst() + return Number(result.numDeletedRows) > 0 +} + +/** The user's credential that covers a model reference, with its decrypted key. */ +export async function credentialFor( + db: Db, + box: SecretBox, + did: string, + ref: { provider: string; id: string }, +) { + const all = await rows(db, did) + const row = ref.provider.startsWith(USER_PROVIDER_PREFIX) + ? all.find((r) => r.slug === ref.provider.slice(USER_PROVIDER_PREFIX.length)) + : all.find((r) => r.provider_id === ref.provider && !r.slug) + if (!row) return undefined + const summary = summarize(row) + const info = summary.models.find((model) => model.id === ref.id) + if (!info) return undefined + return { summary, info, apiKey: box.decrypt(row.api_key_encrypted) } +} diff --git a/apps/server/src/secrets.ts b/apps/server/src/secrets.ts new file mode 100644 index 0000000..16a10dd --- /dev/null +++ b/apps/server/src/secrets.ts @@ -0,0 +1,50 @@ +import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto' + +const VERSION = 'v1' + +export class SecretDecryptError extends Error { + constructor(reason: string) { + super(`Cannot decrypt secret: ${reason}`) + this.name = 'SecretDecryptError' + } +} + +function keyId(key: Buffer): string { + return createHash('sha256').update(key).digest('hex').slice(0, 8) +} + +/** Encrypt secrets with the current key, and decrypt with the current or any retired key. */ +export class SecretBox { + private readonly current: { id: string; key: Buffer } + private readonly keys: Map + + constructor(currentKey: Buffer, oldKeys: Buffer[] = []) { + this.current = { id: keyId(currentKey), key: currentKey } + this.keys = new Map([currentKey, ...oldKeys].map((key) => [keyId(key), key])) + } + + encrypt(plaintext: string): string { + const iv = randomBytes(12) + const cipher = createCipheriv('aes-256-gcm', this.current.key, iv) + const ciphertext = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]) + const tag = cipher.getAuthTag() + return [VERSION, this.current.id, iv, tag, ciphertext] + .map((part) => (typeof part === 'string' ? part : part.toString('base64url'))) + .join(':') + } + + decrypt(value: string): string { + const [version, id, iv, tag, ciphertext] = value.split(':') + if (version !== VERSION || !id || !iv || !tag || ciphertext === undefined) { + throw new SecretDecryptError('unrecognized format') + } + const key = this.keys.get(id) + if (!key) throw new SecretDecryptError(`no key with id ${id}`) + const decipher = createDecipheriv('aes-256-gcm', key, Buffer.from(iv, 'base64url')) + decipher.setAuthTag(Buffer.from(tag, 'base64url')) + return Buffer.concat([ + decipher.update(Buffer.from(ciphertext, 'base64url')), + decipher.final(), + ]).toString('utf8') + } +} diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts new file mode 100644 index 0000000..4c5513c --- /dev/null +++ b/apps/server/src/server.ts @@ -0,0 +1,188 @@ +import type { Plugin } from '@scn-chat/plugin-api' +import { createApp } from './app.ts' +import { type Account, getAccount } from './auth/accounts.ts' +import { createIdentityResolver, type IdentityResolver } from './auth/identity.ts' +import { createOAuthClient, type OAuthClientLike } from './auth/oauth-client.ts' +import { createPdsClientFactory } from './auth/pds.ts' +import { createRoles } from './auth/roles.ts' +import { buildScope } from './auth/scope.ts' +import { turnBlobs } from './blobs/routes.ts' +import { createBlobStores } from './blobs/store.ts' +import type { Config, ModelConfig, SyncConfig } from './config.ts' +import type { Db } from './db/index.ts' +import { migrateToLatest } from './db/migrate.ts' +import type { Logger } from './logger.ts' +import { loadPlugins, type PluginServices } from './plugins/host.ts' +import { readUserSettings } from './plugins/user-settings.ts' +import { ModelCatalog, validateAdminModels } from './providers/catalog.ts' +import { createGenerateText } from './providers/generate-text.ts' +import { createGuardedFetch } from './providers/guarded-fetch.ts' +import { SecretBox } from './secrets.ts' +import { SharingService } from './sharing/service.ts' +import { parseSpaceUri } from './storage/records.ts' +import { createChatServices } from './storage/services.ts' +import { CredentialCache, mintSpaceCredential } from './sync/credentials.ts' +import { SyncEngine } from './sync/engine.ts' +import { SyncEventBus } from './sync/events.ts' +import { RecentWrites } from './sync/recent-writes.ts' +import { resolveSyncConfig, startSyncScheduler } from './sync/scheduler.ts' +import { TurnRunner } from './turns/runner.ts' +import { StreamHub } from './turns/stream-hub.ts' + +export type AppConfig = { + plugins: Plugin[] + models?: ModelConfig[] + roles?: Record + sync?: SyncConfig +} + +export type ServerDeps = { + config: Config + appConfig: AppConfig + db: Db + logger: Logger + webDist?: string + /** Replacements for network-facing pieces, for tests. */ + oauth?: OAuthClientLike + identity?: IdentityResolver + startBackgroundJobs?: boolean +} + +/** Build every service and the HTTP app from config. */ +export async function createServer(deps: ServerDeps) { + const { config, appConfig, db, logger } = deps + await migrateToLatest(db) + const roles = createRoles(appConfig.roles) + const box = new SecretBox(config.secretKey, config.oldSecretKeys) + const scope = buildScope(config.oauthScopeMode) + const oauth = deps.oauth ?? (await createOAuthClient(config, db, scope)) + const getPdsClient = createPdsClientFactory(oauth) + const identity = deps.identity ?? createIdentityResolver(config.plcUrl) + const resolvePds = async (did: string) => (await identity.resolve(did)).pdsUrl + const events = new SyncEventBus() + const recentWrites = new RecentWrites() + const services = createChatServices({ db, getPdsClient, logger, events, recentWrites }) + const blobStores = createBlobStores({ getPdsClient, dataDir: config.dataDir, db }) + const guardedFetch = createGuardedFetch() + + const accountFor = async (did: string): Promise => { + const account = await getAccount(db, did) + if (!account) throw new Error(`No account for ${did}`) + return account + } + let catalog: ModelCatalog | undefined + const pluginServices: PluginServices = { + generateText: (request) => { + if (!catalog) throw new Error('Models are not ready yet') + return createGenerateText(catalog, logger)(request) + }, + updateInfo: async (user, conversation, patch, options) => { + const chats = services.forAccount(await accountFor(user)) + await chats.updateInfo(parseSpaceUri(conversation).skey, patch, options) + }, + userSettings: (plugin, user) => readUserSettings(db, box, user, plugin), + } + const host = await loadPlugins(appConfig.plugins, { + services: pluginServices, + logger, + app: { name: config.appName, publicUrl: config.publicUrl }, + }) + catalog = new ModelCatalog({ + db, + box, + providers: host.providers, + adminModels: validateAdminModels(appConfig.models ?? [], host.providers, roles), + roles, + guardedFetch, + logger, + }) + + const credentials = new CredentialCache({ getPdsClient, resolvePds }) + const engine = new SyncEngine({ + db, + services, + events, + recentWrites, + credentials, + resolveSigningKey: (did) => identity.resolveSigningKey(did), + publicUrl: config.publicUrl, + logger, + backfillWindowMs: config.turns.backfillWindowMs, + }) + const syncConfig = resolveSyncConfig(appConfig.sync) + const hub = new StreamHub() + const blobsFor = blobStores.forAccount + const runner = new TurnRunner({ + db, + services, + catalog, + host, + hub, + blobs: turnBlobs(blobsFor, services), + config: config.turns, + logger, + }) + runner.attach(events) + const sharing = new SharingService({ + getPdsClient, + identity, + mintCredential: (viewer, space) => + mintSpaceCredential({ getPdsClient, resolvePds }, viewer, space), + logger, + }) + + const onLogin = async (account: Account) => { + await services.forAccount(account).ensureSettingsSpace() + if (account.storageMode !== 'space') return + const did = account.did + const background = async () => { + await engine.registerIndex(did) + await engine.discover(did) + await engine.syncIndex(did) + } + background().catch((err) => logger.warn({ err, did }, 'post-login sync failed')) + } + + const app = createApp({ + config, + db, + logger, + webDist: deps.webDist, + auth: { config, db, logger, oauth, identity, roles, scope, onLogin }, + plugins: { db, box, host }, + storage: { db, services, events, engine, hooks: host.hooks, syncConfig, logger }, + providers: { db, box, catalog, providers: host.providers, guardedFetch, logger }, + turns: { services, runner, hub }, + blobs: { blobsFor, services, ingesters: host.ingesters, logger }, + sharing, + sync: { + db, + engine, + events, + publicUrl: config.publicUrl, + resolveSigningKey: identity.resolveSigningKey, + logger, + }, + }) + + const stops: (() => void)[] = [] + if (deps.startBackgroundJobs !== false) { + await runner.recover() + stops.push(startSyncScheduler({ engine, db, config: syncConfig, logger })) + const sweep = setInterval(() => { + blobStores.local.sweep().catch((err) => logger.warn({ err }, 'blob sweep failed')) + }, 86_400_000) + stops.push(() => clearInterval(sweep)) + } + + return { + app, + runner, + engine, + async close() { + for (const stop of stops) stop() + await runner.idle() + await host.close() + }, + } +} diff --git a/apps/server/src/sharing/routes.ts b/apps/server/src/sharing/routes.ts new file mode 100644 index 0000000..d693b0a --- /dev/null +++ b/apps/server/src/sharing/routes.ts @@ -0,0 +1,79 @@ +import { Hono } from 'hono' +import { requireUser, signedInUser } from '../auth/routes.ts' +import { IMAGE_TYPES } from '../blobs/routes.ts' +import type { AppEnv } from '../env.ts' +import { SharedNotFound, ShareInputError, type ShareMode, type SharingService } from './service.ts' + +const MODES = new Set(['private', 'people', 'public']) +const NEEDS_SPACES = 'Sharing needs a PDS that supports atproto spaces.' + +/** Share settings for the owner, and read-only shared views for viewers. */ +export function sharingRoutes(sharing: SharingService) { + return new Hono() + .use(requireUser) + .get('/conversations/:skey/sharing', async (c) => { + const { account } = signedInUser(c) + if (account.storageMode !== 'space') + return c.json({ error: 'Forbidden', message: NEEDS_SPACES }, 403) + return c.json(await sharing.getSettings(account, c.req.param('skey'))) + }) + .put('/conversations/:skey/sharing', async (c) => { + const { account } = signedInUser(c) + if (account.storageMode !== 'space') + return c.json({ error: 'Forbidden', message: NEEDS_SPACES }, 403) + const body = (await c.req.json()) as { mode?: ShareMode; members?: string[] } + if (!body.mode || !MODES.has(body.mode)) + return c.json( + { error: 'InvalidRequest', message: 'mode must be private, people, or public' }, + 400, + ) + try { + await sharing.setSettings(account, c.req.param('skey'), body.mode, body.members ?? []) + } catch (err) { + if (err instanceof ShareInputError) + return c.json({ error: 'InvalidRequest', message: err.message }, 400) + throw err + } + return c.json({ ok: true }) + }) + .get('/shared/:ownerDid/:skey', async (c) => { + const { account } = signedInUser(c) + if (account.storageMode !== 'space') { + return c.json( + { + error: 'Forbidden', + message: 'Viewing shared chats needs a PDS that supports atproto spaces.', + }, + 403, + ) + } + try { + return c.json(await sharing.view(account, c.req.param('ownerDid'), c.req.param('skey'))) + } catch (err) { + if (err instanceof SharedNotFound) return c.json({ error: 'NotFound' }, 404) + throw err + } + }) + .get('/shared/:ownerDid/:skey/blobs/:cid', async (c) => { + const { account } = signedInUser(c) + if (account.storageMode !== 'space') + return c.json({ error: 'Forbidden', message: NEEDS_SPACES }, 403) + try { + const bytes = await sharing.blob( + account, + c.req.param('ownerDid'), + c.req.param('skey'), + c.req.param('cid'), + ) + const type = c.req.query('type') + const mimeType = type && IMAGE_TYPES.has(type) ? type : 'application/octet-stream' + c.header('content-type', mimeType) + c.header('x-content-type-options', 'nosniff') + if (!IMAGE_TYPES.has(mimeType)) c.header('content-disposition', 'attachment') + return c.body(bytes as unknown as ArrayBuffer) + } catch (err) { + if (err instanceof SharedNotFound) return c.json({ error: 'NotFound' }, 404) + throw err + } + }) +} diff --git a/apps/server/src/sharing/service.ts b/apps/server/src/sharing/service.ts new file mode 100644 index 0000000..214ed28 --- /dev/null +++ b/apps/server/src/sharing/service.ts @@ -0,0 +1,207 @@ +import { LexError } from '@atproto/lex-data' +import { atproto, nsid } from '@scn-chat/lexicons' +import type { Account } from '../auth/accounts.ts' +import type { IdentityResolver } from '../auth/identity.ts' +import type { PdsClientFactory } from '../auth/pds.ts' +import type { Logger } from '../logger.ts' +import { type JsonRecord, spaceUri, toJson } from '../storage/records.ts' +import { assertValidStored, keepValid } from '../storage/validate.ts' +import { CredentialError } from '../sync/credentials.ts' + +// Plain strings don't satisfy the generated methods' branded string types. +type Loose = any + +export type ShareMode = 'private' | 'people' | 'public' +export type ShareSettings = { mode: ShareMode; members: { did: string; handle: string | null }[] } + +const PUBLIC = { $type: 'com.atproto.simplespace.defs#publicPolicy' } +const MEMBER_LIST = { $type: 'com.atproto.simplespace.defs#memberListPolicy' } + +export class ShareInputError extends Error { + constructor(message: string) { + super(message) + this.name = 'ShareInputError' + } +} + +/** The conversation is missing, private, deleted, or the viewer is not allowed: all look the same. */ +export class SharedNotFound extends Error { + constructor() { + super('Conversation not found') + this.name = 'SharedNotFound' + } +} + +type CredentialClient = { client(service: string): Loose } + +export type SharingDeps = { + getPdsClient: PdsClientFactory + identity: IdentityResolver + /** Get a fresh, uncached space credential acting for the viewer. */ + mintCredential: (viewerDid: string, space: string) => Promise + logger: Logger +} + +const DENIED = new Set([ + 'UserNotAuthorized', + 'AppNotAuthorized', + 'NotAuthorized', + 'SpaceNotFound', + 'SpaceDeleted', + 'RecordNotFound', + 'BlobNotFound', + 'RepoNotFound', +]) + +/** Share settings for an owner's conversations, and shared views read with the viewer's credential. */ +export class SharingService { + private readonly deps: SharingDeps + + constructor(deps: SharingDeps) { + this.deps = deps + } + + private async owner(account: Account): Promise { + return this.deps.getPdsClient(account.did) + } + + private async members(client: Loose, space: string): Promise { + const dids: string[] = [] + let cursor: string | undefined + do { + const out = await client.call(atproto.simplespace.listMembers, { space, cursor, limit: 100 }) + dids.push( + ...out.members.filter((m: { read: boolean }) => m.read).map((m: { did: string }) => m.did), + ) + cursor = out.cursor + } while (cursor) + return dids + } + + async getSettings(account: Account, skey: string): Promise { + const client = await this.owner(account) + const space = spaceUri(account.did, nsid.conversation, skey) + const [config, dids] = await Promise.all([ + client.call(atproto.simplespace.getSpace, { space }), + this.members(client, space), + ]) + const isPublic = (config.readPolicy.$type as string).endsWith('#publicPolicy') + const members = await Promise.all( + dids.map(async (did) => ({ + did, + handle: await this.deps.identity.resolve(did).then( + (identity) => identity.handle, + (err) => { + this.deps.logger.warn({ err, did }, 'could not resolve a shared member handle') + return null + }, + ), + })), + ) + return { mode: isPublic ? 'public' : members.length ? 'people' : 'private', members } + } + + private async toDids(members: string[]): Promise { + return Promise.all( + members.map(async (member) => { + if (member.startsWith('did:')) return member + const did = await this.deps.identity.resolveHandle(member.replace(/^@/, '')) + if (!did) throw new ShareInputError(`Cannot find the handle "${member}"`) + return did + }), + ) + } + + async setSettings( + account: Account, + skey: string, + mode: ShareMode, + members: string[] = [], + ): Promise { + const client = await this.owner(account) + const space = spaceUri(account.did, nsid.conversation, skey) + const current = await this.members(client, space) + if (mode === 'public') { + await client.call(atproto.simplespace.updateSpace, { space, readPolicy: PUBLIC }) + return + } + await client.call(atproto.simplespace.updateSpace, { space, readPolicy: MEMBER_LIST }) + const wanted = mode === 'people' ? await this.toDids(members) : [] + for (const did of current.filter((d) => !wanted.includes(d))) { + await client.call(atproto.simplespace.removeMember, { space, did }) + } + for (const did of wanted.filter((d) => !current.includes(d))) { + await client.call(atproto.simplespace.putMember, { space, did, read: true, write: false }) + } + } + + private async viewerClient(viewer: Account, ownerDid: string, skey: string) { + const space = spaceUri(ownerDid, nsid.conversation, skey) + try { + const [credential, owner] = await Promise.all([ + this.deps.mintCredential(viewer.did, space), + this.deps.identity.resolve(ownerDid), + ]) + return { space, client: credential.client(owner.pdsUrl), ownerHandle: owner.handle } + } catch (err) { + if ( + (err instanceof CredentialError && DENIED.has(err.code)) || + (err instanceof LexError && DENIED.has(err.error)) + ) { + throw new SharedNotFound() + } + throw err + } + } + + private async read(fn: () => Promise): Promise { + try { + return await fn() + } catch (err) { + if (err instanceof LexError && DENIED.has(err.error)) throw new SharedNotFound() + throw err + } + } + + /** A conversation shared with the viewer, without its system prompt or the owner's preferences. */ + async view(viewer: Account, ownerDid: string, skey: string) { + const { space, client, ownerHandle } = await this.viewerClient(viewer, ownerDid, skey) + const info = await this.read(() => + client.call(atproto.space.getRecord, { + space, + repo: ownerDid, + collection: nsid.info, + rkey: 'self', + }), + ) + const messages: { rkey: string; value: JsonRecord; cid: string }[] = [] + let cursor: string | undefined + do { + const out = await this.read(() => + client.call(atproto.space.listRecords, { + space, + repo: ownerDid, + collection: nsid.message, + cursor, + limit: 100, + }), + ) + for (const record of out.records) { + messages.push({ rkey: record.rkey, value: toJson(record.value), cid: record.cid }) + } + cursor = out.cursor + } while (cursor) + const infoValue = toJson(info.value) + assertValidStored(space, nsid.info, 'self', infoValue) + return { + owner: { did: ownerDid, handle: ownerHandle }, + title: (infoValue.title as string | undefined) ?? null, + messages: keepValid(space, nsid.message, messages, this.deps.logger), + } + } + + async blob(viewer: Account, ownerDid: string, skey: string, cid: string): Promise { + const { space, client } = await this.viewerClient(viewer, ownerDid, skey) + return this.read(() => client.call(atproto.space.getBlob, { space, repo: ownerDid, cid })) + } +} diff --git a/apps/server/src/storage/chat-service.ts b/apps/server/src/storage/chat-service.ts new file mode 100644 index 0000000..e4abad2 --- /dev/null +++ b/apps/server/src/storage/chat-service.ts @@ -0,0 +1,300 @@ +import { nsid } from '@scn-chat/lexicons' +import type { InfoPatch } from '@scn-chat/plugin-api' +import type { Logger } from '../logger.ts' +import { + type Op, + type OpsPage, + type RecordKey, + type RecordStore, + SpaceExists, + type StoredRecord, +} from './record-store.ts' +import { type JsonRecord, newTid, parseSpaceUri, spaceUri } from './records.ts' +import { assertValidStored, keepValid } from './validate.ts' + +export const SETTINGS_SKEY = 'self' + +/** Get the record written by a key's newest op. */ +function currentRecord(op: Op): StoredRecord { + if (!op.cid || !op.value) + throw new Error(`The newest op for ${op.collection}/${op.rkey} has no record value`) + return { rkey: op.rkey, value: op.value, cid: op.cid } +} + +export type ConversationSummary = { + skey: string + uri: string + title: string | null + tags: string[] + updatedAt: string + cid: string +} + +export type ConversationChanges = { + info: StoredRecord | null + messages: StoredRecord[] + deleted: { collection: string; rkey: string }[] + rev: string | null + full: boolean +} + +export type IndexChanges = { + conversations: ConversationSummary[] + deleted: string[] + rev: string | null + full: boolean +} + +/** Chat operations for one user, keeping their chat index in step with every write. */ +export class ChatService { + readonly did: string + readonly store: RecordStore + readonly settingsUri: string + private readonly logger: Logger + private readonly onWrite: (space: string, cid: string) => void + + constructor( + store: RecordStore, + logger: Logger, + onWrite: (space: string, cid: string) => void = () => {}, + ) { + this.did = store.did + this.store = store + this.logger = logger + this.onWrite = onWrite + this.settingsUri = spaceUri(store.did, nsid.settings, SETTINGS_SKEY) + } + + conversationUri(skey: string): string { + return spaceUri(this.did, nsid.conversation, skey) + } + + async ensureSettingsSpace(): Promise { + try { + await this.store.createSpace(nsid.settings, SETTINGS_SKEY) + } catch (err) { + if (!(err instanceof SpaceExists)) throw err + } + } + + private async put( + space: string, + collection: string, + rkey: string, + value: JsonRecord, + create = false, + ) { + const { cid } = create + ? await this.store.createRecord(space, collection, rkey, value) + : await this.store.putRecord(space, collection, rkey, value) + this.onWrite(space, cid) + return cid + } + + async createConversation(input: { systemPrompt?: string; tags?: string[] } = {}) { + const skey = newTid() + const uri = await this.store.createSpace(nsid.conversation, skey) + const now = new Date().toISOString() + const info: JsonRecord = { $type: nsid.info, createdAt: now } + if (input.systemPrompt) info.systemPrompt = input.systemPrompt + await this.put(uri, nsid.info, 'self', info) + await this.ensureSettingsSpace() + await this.writeRef(skey, { tags: input.tags ?? [] }) + return { skey, uri } + } + + private async readRef(skey: string): Promise { + return (await this.store.getRecord(this.settingsUri, nsid.conversationRef, skey))?.value + } + + private async writeRef( + skey: string, + changes: { title?: string | null; tags?: string[] }, + ): Promise { + const existing = (await this.readRef(skey)) ?? {} + const ref: JsonRecord = { + ...existing, + $type: nsid.conversationRef, + conversation: this.conversationUri(skey), + updatedAt: new Date().toISOString(), + } + if (changes.title === null) delete ref.title + else if (changes.title !== undefined) ref.title = changes.title + if (changes.tags !== undefined) ref.tags = changes.tags + await this.put(this.settingsUri, nsid.conversationRef, skey, ref) + } + + /** Bump the index entry, logging failures for discovery to repair. */ + private async touchRef(skey: string): Promise { + try { + await this.writeRef(skey, {}) + } catch (err) { + this.logger.warn( + { err, conversation: this.conversationUri(skey) }, + 'chat index update failed', + ) + } + } + + async createMessage(skey: string, rkey: string, record: JsonRecord): Promise { + const cid = await this.put(this.conversationUri(skey), nsid.message, rkey, record, true) + await this.touchRef(skey) + return cid + } + + async putMessage(skey: string, rkey: string, record: JsonRecord): Promise { + const cid = await this.put(this.conversationUri(skey), nsid.message, rkey, record) + await this.touchRef(skey) + return cid + } + + /** Read a record and check it against its lexicon. */ + private async getValid(space: string, collection: string, rkey: string) { + const record = await this.store.getRecord(space, collection, rkey) + if (record) assertValidStored(space, collection, rkey, record.value) + return record + } + + async getInfo(skey: string): Promise { + return (await this.getValid(this.conversationUri(skey), nsid.info, 'self'))?.value ?? null + } + + async updateInfo( + skey: string, + patch: InfoPatch, + options: { unlessUserTitled?: boolean } = {}, + ): Promise { + const info = await this.getInfo(skey) + if (!info) return false + if (options.unlessUserTitled && info.titleSource === 'user') return false + const next: JsonRecord = { ...info, ...patch, $type: nsid.info } + await this.put(this.conversationUri(skey), nsid.info, 'self', next) + if (patch.title !== undefined) await this.writeRef(skey, { title: patch.title }) + else await this.touchRef(skey) + return true + } + + /** Add a missing index entry, with the title from the conversation's info record. */ + async repairRef(skey: string): Promise { + const title = (await this.getInfo(skey))?.title as string | undefined + await this.writeRef(skey, title === undefined ? {} : { title }) + } + + /** Copy the info title into the index entry when the two differ. */ + async syncRefTitle(skey: string, title: string | undefined): Promise { + const ref = await this.readRef(skey) + if (ref && ref.title !== title) await this.writeRef(skey, { title: title ?? null }) + } + + async setTags(skey: string, tags: string[]): Promise { + await this.writeRef(skey, { tags }) + } + + async deleteConversation(skey: string): Promise { + await this.store.deleteSpace(this.conversationUri(skey)) + await this.store.deleteRecord(this.settingsUri, nsid.conversationRef, skey) + } + + async getPreferences(): Promise { + return (await this.getValid(this.settingsUri, nsid.preferences, 'self'))?.value ?? null + } + + async putPreferences(record: JsonRecord): Promise { + await this.ensureSettingsSpace() + await this.put(this.settingsUri, nsid.preferences, 'self', { + ...record, + $type: nsid.preferences, + updatedAt: new Date().toISOString(), + }) + } + + private summary(record: StoredRecord): ConversationSummary { + const value = record.value as { + conversation: string + title?: string + tags?: string[] + updatedAt: string + } + return { + skey: record.rkey, + uri: value.conversation, + title: value.title ?? null, + tags: value.tags ?? [], + updatedAt: value.updatedAt, + cid: record.cid, + } + } + + /** Get the chat index, newest first, or its changes since a revision. */ + async listConversations(since?: string | null): Promise { + if (since) { + const page = await this.store.listOps(this.settingsUri, since) + const refs = page.ops.filter((op) => op.collection === nsid.conversationRef) + const latest = [...new Map(refs.map((op) => [op.rkey, op])).values()] + const changed = latest.filter((op) => op.cid).map(currentRecord) + return { + conversations: keepValid(this.settingsUri, nsid.conversationRef, changed, this.logger).map( + (record) => this.summary(record), + ), + deleted: latest.filter((op) => !op.cid).map((op) => op.rkey), + rev: page.rev, + full: false, + } + } + const [records, rev] = await Promise.all([ + this.store.listRecords(this.settingsUri, nsid.conversationRef), + this.store.headRev(this.settingsUri), + ]) + const conversations = keepValid( + this.settingsUri, + nsid.conversationRef, + records, + this.logger, + ).map((record) => this.summary(record)) + conversations.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)) + return { conversations, deleted: [], rev, full: true } + } + + /** Get a conversation's info and messages, or their changes since a revision. */ + async getConversation(skey: string, since?: string | null): Promise { + const uri = this.conversationUri(skey) + if (since) { + const page: OpsPage = await this.store.listOps(uri, since) + const latest = [ + ...new Map(page.ops.map((op) => [`${op.collection}/${op.rkey}`, op])).values(), + ] + const info = latest.find((op) => op.collection === nsid.info && op.cid) + if (info) assertValidStored(uri, nsid.info, info.rkey, currentRecord(info).value) + const messages = latest.filter((op) => op.collection === nsid.message && op.cid) + return { + info: info ? currentRecord(info) : null, + messages: keepValid(uri, nsid.message, messages.map(currentRecord), this.logger), + deleted: latest + .filter((op) => !op.cid) + .map((op) => ({ collection: op.collection, rkey: op.rkey })), + rev: page.rev, + full: false, + } + } + const [info, messages, rev] = await Promise.all([ + this.getValid(uri, nsid.info, 'self'), + this.store.listRecords(uri, nsid.message), + this.store.headRev(uri), + ]) + return { + info, + messages: keepValid(uri, nsid.message, messages, this.logger), + deleted: [], + rev, + full: true, + } + } + + listKeys(skey?: string): Promise { + return this.store.listKeys(skey ? this.conversationUri(skey) : this.settingsUri) + } + + skeyOf(uri: string): string { + return parseSpaceUri(uri).skey + } +} diff --git a/apps/server/src/storage/local-record-store.ts b/apps/server/src/storage/local-record-store.ts new file mode 100644 index 0000000..1047b77 --- /dev/null +++ b/apps/server/src/storage/local-record-store.ts @@ -0,0 +1,220 @@ +import type { Db } from '../db/index.ts' +import { + InvalidCursor, + type OpsPage, + RecordExists, + type RecordKey, + type RecordStore, + SpaceExists, + SpaceNotFound, + type StoredRecord, +} from './record-store.ts' +import { type JsonRecord, recordCid, spaceUri } from './records.ts' +import { assertValidRecord } from './validate.ts' + +const rev = (seq: number) => String(seq).padStart(16, '0') + +/** A stand-in for a PDS in the app database, for users whose PDS lacks spaces. */ +export class LocalRecordStore implements RecordStore { + readonly did: string + private readonly db: Db + + constructor(did: string, db: Db) { + this.did = did + this.db = db + } + + private async assertSpace(space: string): Promise { + const row = await this.db + .selectFrom('local_space') + .select('uri') + .where('uri', '=', space) + .executeTakeFirst() + if (!row) throw new SpaceNotFound(space) + } + + async createSpace(type: string, skey: string): Promise { + const uri = spaceUri(this.did, type, skey) + const existing = await this.db + .selectFrom('local_space') + .select('uri') + .where('uri', '=', uri) + .executeTakeFirst() + if (existing) throw new SpaceExists(uri) + await this.db + .insertInto('local_space') + .values({ uri, owner_did: this.did, type, skey, created_at: new Date().toISOString() }) + .execute() + return uri + } + + async deleteSpace(space: string): Promise { + await this.assertSpace(space) + await this.db.transaction().execute(async (tx) => { + await tx.deleteFrom('local_op').where('space_uri', '=', space).execute() + await tx.deleteFrom('local_record').where('space_uri', '=', space).execute() + await tx.deleteFrom('local_space').where('uri', '=', space).execute() + }) + } + + async getRecord(space: string, collection: string, rkey: string): Promise { + await this.assertSpace(space) + const row = await this.db + .selectFrom('local_record') + .select(['value_json', 'cid']) + .where('space_uri', '=', space) + .where('collection', '=', collection) + .where('rkey', '=', rkey) + .executeTakeFirst() + return row ? { rkey, value: JSON.parse(row.value_json) as JsonRecord, cid: row.cid } : null + } + + async listRecords(space: string, collection: string): Promise { + await this.assertSpace(space) + const rows = await this.db + .selectFrom('local_record') + .select(['rkey', 'value_json', 'cid']) + .where('space_uri', '=', space) + .where('collection', '=', collection) + .orderBy('rkey') + .execute() + return rows.map((row) => ({ + rkey: row.rkey, + value: JSON.parse(row.value_json) as JsonRecord, + cid: row.cid, + })) + } + + async listKeys(space: string): Promise { + await this.assertSpace(space) + return this.db + .selectFrom('local_record') + .select(['collection', 'rkey', 'cid']) + .where('space_uri', '=', space) + .orderBy('collection') + .orderBy('rkey') + .execute() + } + + private async write( + space: string, + collection: string, + rkey: string, + value: JsonRecord, + mode: 'create' | 'put', + ) { + assertValidRecord(collection, value) + await this.assertSpace(space) + const cid = await recordCid(value) + const now = new Date().toISOString() + const valueJson = JSON.stringify(value) + await this.db.transaction().execute(async (tx) => { + const existing = await tx + .selectFrom('local_record') + .select('cid') + .where('space_uri', '=', space) + .where('collection', '=', collection) + .where('rkey', '=', rkey) + .executeTakeFirst() + if (existing && mode === 'create') throw new RecordExists(space, collection, rkey) + if (existing) { + await tx + .updateTable('local_record') + .set({ value_json: valueJson, cid, updated_at: now }) + .where('space_uri', '=', space) + .where('collection', '=', collection) + .where('rkey', '=', rkey) + .execute() + } else { + await tx + .insertInto('local_record') + .values({ + space_uri: space, + collection, + rkey, + value_json: valueJson, + cid, + updated_at: now, + }) + .execute() + } + await tx + .insertInto('local_op') + .values({ space_uri: space, collection, rkey, cid, value_json: valueJson, created_at: now }) + .execute() + }) + return { cid } + } + + createRecord(space: string, collection: string, rkey: string, value: JsonRecord) { + return this.write(space, collection, rkey, value, 'create') + } + + putRecord(space: string, collection: string, rkey: string, value: JsonRecord) { + return this.write(space, collection, rkey, value, 'put') + } + + async deleteRecord(space: string, collection: string, rkey: string): Promise { + await this.assertSpace(space) + await this.db.transaction().execute(async (tx) => { + const result = await tx + .deleteFrom('local_record') + .where('space_uri', '=', space) + .where('collection', '=', collection) + .where('rkey', '=', rkey) + .executeTakeFirst() + if (Number(result.numDeletedRows) === 0) return + await tx + .insertInto('local_op') + .values({ + space_uri: space, + collection, + rkey, + cid: null, + value_json: null, + created_at: new Date().toISOString(), + }) + .execute() + }) + } + + async listOps(space: string, since?: string | null): Promise { + await this.assertSpace(space) + if (since && !/^\d+$/.test(since)) throw new InvalidCursor(since) + const rows = await this.db + .selectFrom('local_op') + .select(['seq', 'collection', 'rkey', 'cid', 'value_json']) + .where('space_uri', '=', space) + .where('seq', '>', since ? Number(since) : 0) + .orderBy('seq') + .execute() + const ops = rows.map((row) => ({ + rev: rev(row.seq), + collection: row.collection, + rkey: row.rkey, + cid: row.cid, + value: row.value_json ? (JSON.parse(row.value_json) as JsonRecord) : undefined, + })) + return { ops, rev: ops.at(-1)?.rev ?? since ?? null } + } + + async headRev(space: string): Promise { + await this.assertSpace(space) + const row = await this.db + .selectFrom('local_op') + .select((eb) => eb.fn.max('seq').as('seq')) + .where('space_uri', '=', space) + .executeTakeFirst() + return row?.seq ? rev(Number(row.seq)) : null + } + + async listSpaces(type: string): Promise { + const rows = await this.db + .selectFrom('local_space') + .select('uri') + .where('owner_did', '=', this.did) + .where('type', '=', type) + .execute() + return rows.map((row) => row.uri) + } +} diff --git a/apps/server/src/storage/record-store.ts b/apps/server/src/storage/record-store.ts new file mode 100644 index 0000000..524c530 --- /dev/null +++ b/apps/server/src/storage/record-store.ts @@ -0,0 +1,99 @@ +import type { JsonRecord } from './records.ts' + +export type StoredRecord = { rkey: string; value: JsonRecord; cid: string } +export type RecordKey = { collection: string; rkey: string; cid: string } +export type Op = { + rev: string + collection: string + rkey: string + cid: string | null + /** The previous CID, null for a create. Absent when the backend does not track it. */ + prev?: string | null + value?: JsonRecord +} +export type SignedCommit = Record +export type OpsPage = { + ops: Op[] + rev: string | null + /** The signed commit at the head, missing only when the account has no repo in the space. */ + commit?: SignedCommit +} + +export type SpacePolicy = { read: 'memberList' | 'public' } + +/** A PDS-shaped store of one user's records, backed by their PDS or by the local fallback. */ +export interface RecordStore { + readonly did: string + createSpace(type: string, skey: string): Promise + deleteSpace(space: string): Promise + getRecord(space: string, collection: string, rkey: string): Promise + listRecords(space: string, collection: string): Promise + listKeys(space: string): Promise + createRecord( + space: string, + collection: string, + rkey: string, + value: JsonRecord, + ): Promise<{ cid: string }> + putRecord( + space: string, + collection: string, + rkey: string, + value: JsonRecord, + ): Promise<{ cid: string }> + deleteRecord(space: string, collection: string, rkey: string): Promise + /** Every op after `since`, and the revision to pass next time. */ + listOps(space: string, since?: string | null): Promise + /** The newest revision, without reading the op log. */ + headRev(space: string): Promise + listSpaces(type: string): Promise +} + +export class RecordExists extends Error { + constructor(space: string, collection: string, rkey: string) { + super(`A ${collection} record already exists at ${rkey} in ${space}`) + this.name = 'RecordExists' + } +} + +export class SpaceExists extends Error { + constructor(space: string) { + super(`Space already exists: ${space}`) + this.name = 'SpaceExists' + } +} + +export class InvalidCursor extends Error { + constructor(since: string) { + super(`Not a revision from this store: ${since}`) + this.name = 'InvalidCursor' + } +} + +export class SpaceNotFound extends Error { + constructor(space: string) { + super(`Space not found: ${space}`) + this.name = 'SpaceNotFound' + } +} + +/** A record on the PDS that doesn't match its lexicon. */ +export class InvalidStoredRecord extends Error { + readonly collection: string + + constructor(space: string, collection: string, rkey: string, details: string) { + super(`The ${collection} record ${rkey} in ${space} does not match its lexicon: ${details}`) + this.name = 'InvalidStoredRecord' + this.collection = collection + } +} + +export class InvalidRecord extends Error { + readonly collection: string + + constructor(collection: string, details: string) { + super(`Invalid ${collection} record: ${details}`) + this.name = 'InvalidRecord' + this.collection = collection + } +} diff --git a/apps/server/src/storage/records.ts b/apps/server/src/storage/records.ts new file mode 100644 index 0000000..bab525c --- /dev/null +++ b/apps/server/src/storage/records.ts @@ -0,0 +1,35 @@ +import { TID } from '@atproto/common-web' +import { cidForLex } from '@atproto/lex-cbor' +import type { LexValue } from '@atproto/lex-data' +import { jsonToLex, lexToJson } from '@atproto/lex-json' + +/** A record in its atproto JSON form, the only form records take inside the server. */ +export type JsonRecord = Record + +export const newTid = (): string => TID.nextStr() + +export const toLex = (value: JsonRecord): LexValue => + jsonToLex(value as Parameters[0]) +export const toJson = (value: unknown): JsonRecord => lexToJson(value as LexValue) as JsonRecord + +/** The record's CID, computed the way a PDS computes it. */ +export async function recordCid(value: JsonRecord): Promise { + return (await cidForLex(toLex(value))).toString() +} + +export function spaceUri(did: string, type: string, skey: string): string { + return `at://${did}/space/${type}/${skey}` +} + +export class InvalidSpaceUri extends Error { + constructor(uri: string) { + super(`Not a space URI: ${uri}`) + this.name = 'InvalidSpaceUri' + } +} + +export function parseSpaceUri(uri: string): { did: string; type: string; skey: string } { + const match = uri.match(/^at:\/\/(did:[^/]+)\/space\/([^/]+)\/([^/]+)$/) + if (!match?.[1] || !match[2] || !match[3]) throw new InvalidSpaceUri(uri) + return { did: match[1], type: match[2], skey: match[3] } +} diff --git a/apps/server/src/storage/routes.ts b/apps/server/src/storage/routes.ts new file mode 100644 index 0000000..364f8cd --- /dev/null +++ b/apps/server/src/storage/routes.ts @@ -0,0 +1,158 @@ +import { Hono } from 'hono' +import { streamSSE } from 'hono/streaming' +import { setBackgroundSync } from '../auth/accounts.ts' +import { requireUser, signedInUser } from '../auth/routes.ts' +import type { Db } from '../db/index.ts' +import type { AppEnv } from '../env.ts' +import type { Logger } from '../logger.ts' +import type { HookRunner } from '../plugins/hooks.ts' +import type { SyncEngine } from '../sync/engine.ts' +import type { SyncEventBus, SyncEvents } from '../sync/events.ts' +import type { ResolvedSyncConfig } from '../sync/scheduler.ts' +import type { JsonRecord } from './records.ts' +import type { ChatServices } from './services.ts' + +export type StorageRoutesDeps = { + db: Db + services: ChatServices + events: SyncEventBus + engine?: SyncEngine + hooks?: HookRunner + syncConfig: ResolvedSyncConfig + logger: Logger +} + +const OPEN_SYNC_WAIT_MS = 2_000 + +/** The chat API used by the web app. Every read goes to the user's record store. */ +export function storageRoutes(deps: StorageRoutesDeps) { + const chatsFor = (c: { get(key: 'user'): ReturnType | null }) => + deps.services.forAccount(signedInUser(c).account) + + /** Sync a spaces conversation before reading it, answering from the PDS after two seconds regardless. */ + const syncBeforeRead = async (did: string, skey: string) => { + if (!deps.engine) return + const sync = deps.engine + .syncConversation(did, skey) + .catch((err) => deps.logger.warn({ err, did, skey }, 'sync on open failed')) + await Promise.race([sync, new Promise((resolve) => setTimeout(resolve, OPEN_SYNC_WAIT_MS))]) + } + + return new Hono() + .use(requireUser) + .get('/conversations', async (c) => + c.json(await chatsFor(c).listConversations(c.req.query('since'))), + ) + .post('/conversations', async (c) => { + const body = (await c.req.json().catch(() => ({}))) as { + systemPrompt?: string + tags?: string[] + } + const chats = chatsFor(c) + const created = await chats.createConversation({ + systemPrompt: body.systemPrompt, + tags: body.tags, + }) + await deps.hooks?.action( + 'conversation:created', + { user: chats.did, conversation: created.uri }, + deps.logger, + ) + return c.json(created, 201) + }) + .get('/conversations/keys', async (c) => c.json({ keys: await chatsFor(c).listKeys() })) + .get('/conversations/:skey', async (c) => { + const user = signedInUser(c) + const skey = c.req.param('skey') + if (user.account.storageMode === 'space') await syncBeforeRead(user.did, skey) + const result = await chatsFor(c).getConversation(skey, c.req.query('since')) + if (result.full && !result.info) return c.json({ error: 'NotFound' }, 404) + return c.json(result) + }) + .get('/conversations/:skey/keys', async (c) => + c.json({ keys: await chatsFor(c).listKeys(c.req.param('skey')) }), + ) + .patch('/conversations/:skey', async (c) => { + const skey = c.req.param('skey') + const body = (await c.req.json()) as { + title?: string + tags?: string[] + systemPrompt?: string + } + const chats = chatsFor(c) + const patch: Record = {} + if (body.title !== undefined) Object.assign(patch, { title: body.title, titleSource: 'user' }) + if (body.systemPrompt !== undefined) patch.systemPrompt = body.systemPrompt + if (Object.keys(patch).length > 0 && !(await chats.updateInfo(skey, patch))) { + return c.json({ error: 'NotFound' }, 404) + } + if (body.tags !== undefined) await chats.setTags(skey, body.tags) + return c.json({ ok: true }) + }) + .delete('/conversations/:skey', async (c) => { + const chats = chatsFor(c) + const skey = c.req.param('skey') + await chats.deleteConversation(skey) + await deps.hooks?.action( + 'conversation:deleted', + { user: chats.did, conversation: chats.conversationUri(skey) }, + deps.logger, + ) + return c.json({ ok: true }) + }) + .post('/conversations/:skey/sync', async (c) => { + const user = signedInUser(c) + if (user.account.storageMode === 'space' && deps.engine) + await deps.engine.syncConversation(user.did, c.req.param('skey')) + return c.json({ ok: true }) + }) + .get('/preferences', async (c) => c.json({ preferences: await chatsFor(c).getPreferences() })) + .put('/preferences', async (c) => { + await chatsFor(c).putPreferences((await c.req.json()) as JsonRecord) + return c.json({ ok: true }) + }) + .get('/account', (c) => { + const { account } = signedInUser(c) + return c.json({ + backgroundSync: account.backgroundSync, + allowUserOptOut: deps.syncConfig.allowUserOptOut, + }) + }) + .put('/account', async (c) => { + const body = (await c.req.json()) as { backgroundSync?: boolean } + if (body.backgroundSync !== undefined) { + if (!deps.syncConfig.allowUserOptOut) { + return c.json( + { error: 'Forbidden', message: 'This app does not allow turning off background sync' }, + 403, + ) + } + await setBackgroundSync(deps.db, signedInUser(c).did, body.backgroundSync) + } + return c.json({ ok: true }) + }) + .get('/events', (c) => { + const { did } = signedInUser(c) + return streamSSE(c, async (stream) => { + let writes = Promise.resolve() + const send = (event: string, data: object) => { + writes = writes.then(() => stream.writeSSE({ event, data: JSON.stringify(data) })) + } + const onConversation = (e: SyncEvents['conversation:changed'][0]) => + e.did === did && send('conversation-changed', { skey: e.skey }) + const onDeleted = (e: SyncEvents['conversation:deleted'][0]) => + e.did === did && send('conversation-deleted', { skey: e.skey }) + const onIndex = (e: SyncEvents['index:changed'][0]) => + e.did === did && send('index-changed', {}) + deps.events.on('conversation:changed', onConversation) + deps.events.on('conversation:deleted', onDeleted) + deps.events.on('index:changed', onIndex) + const heartbeat = setInterval(() => send('ping', {}), 25_000) + await new Promise((resolve) => stream.onAbort(resolve)) + clearInterval(heartbeat) + deps.events.off('conversation:changed', onConversation) + deps.events.off('conversation:deleted', onDeleted) + deps.events.off('index:changed', onIndex) + }) + }) +} diff --git a/apps/server/src/storage/services.ts b/apps/server/src/storage/services.ts new file mode 100644 index 0000000..490b5bf --- /dev/null +++ b/apps/server/src/storage/services.ts @@ -0,0 +1,41 @@ +import type { Account } from '../auth/accounts.ts' +import type { PdsClientFactory } from '../auth/pds.ts' +import type { Db } from '../db/index.ts' +import type { Logger } from '../logger.ts' +import type { SyncEventBus } from '../sync/events.ts' +import type { RecentWrites } from '../sync/recent-writes.ts' +import { ChatService } from './chat-service.ts' +import { LocalRecordStore } from './local-record-store.ts' +import type { RecordStore } from './record-store.ts' +import { SpaceRecordStore } from './space-record-store.ts' + +export type ChatServices = { + forAccount(account: Account): ChatService + storeFor(account: Account): RecordStore +} + +/** Build each user's ChatService over the backend for their storage mode. */ +export function createChatServices(deps: { + db: Db + getPdsClient: PdsClientFactory + logger: Logger + events: SyncEventBus + recentWrites: RecentWrites +}): ChatServices { + const storeFor = (account: Account): RecordStore => + account.storageMode === 'space' + ? new SpaceRecordStore(account.did, deps.getPdsClient) + : new LocalRecordStore(account.did, deps.db) + return { + storeFor, + forAccount(account) { + const chats: ChatService = new ChatService(storeFor(account), deps.logger, (space, cid) => { + deps.recentWrites.add(cid) + if (space === chats.settingsUri) deps.events.emit('index:changed', { did: account.did }) + else + deps.events.emit('conversation:changed', { did: account.did, skey: chats.skeyOf(space) }) + }) + return chats + }, + } +} diff --git a/apps/server/src/storage/space-record-store.ts b/apps/server/src/storage/space-record-store.ts new file mode 100644 index 0000000..45bb950 --- /dev/null +++ b/apps/server/src/storage/space-record-store.ts @@ -0,0 +1,226 @@ +import type { Client } from '@atproto/lex-client' +import { LexError } from '@atproto/lex-data' +import { atproto } from '@scn-chat/lexicons' +import type { PdsClientFactory } from '../auth/pds.ts' +import { + type OpsPage, + RecordExists, + type RecordKey, + type RecordStore, + SpaceExists, + SpaceNotFound, + type StoredRecord, +} from './record-store.ts' +import { type JsonRecord, toJson, toLex } from './records.ts' +import { assertValidRecord } from './validate.ts' + +const MEMBER_LIST = { $type: 'com.atproto.simplespace.defs#memberListPolicy' } as const +const OPEN = { $type: 'com.atproto.simplespace.defs#open' } as const +const PAGE = 100 + +// Plain strings don't satisfy the generated methods' branded string types. +type Loose = any + +const errorCode = (err: unknown) => (err instanceof LexError ? err.error : undefined) + +/** A user's records on their spaces-enabled PDS. */ +export class SpaceRecordStore implements RecordStore { + readonly did: string + private readonly getClient: PdsClientFactory + + constructor(did: string, getClient: PdsClientFactory) { + this.did = did + this.getClient = getClient + } + + private async client(): Promise { + return this.getClient(this.did) + } + + private async rawCall(method: Loose, input: Loose): Promise { + const client: Loose = await this.client() + return client.call(method, input) + } + + private async call(method: Loose, input: Loose, space: string): Promise { + try { + return await this.rawCall(method, input) + } catch (err) { + if (errorCode(err) === 'SpaceNotFound') throw new SpaceNotFound(space) + throw err + } + } + + async createSpace(type: string, skey: string): Promise { + try { + const { uri } = await this.rawCall(atproto.simplespace.createSpace, { + type, + skey, + readPolicy: MEMBER_LIST, + writePolicy: MEMBER_LIST, + appAccess: OPEN, + }) + return uri as string + } catch (err) { + if (errorCode(err) === 'SpaceAlreadyExists') + throw new SpaceExists(`at://${this.did}/space/${type}/${skey}`) + throw err + } + } + + async deleteSpace(space: string): Promise { + await this.call(atproto.simplespace.deleteSpace, { space }, space) + } + + async getRecord(space: string, collection: string, rkey: string): Promise { + try { + const out = await this.rawCall(atproto.space.getRecord, { + space, + repo: this.did, + collection, + rkey, + }) + return { rkey, value: toJson(out.value), cid: out.cid } + } catch (err) { + if (errorCode(err) === 'RecordNotFound' || errorCode(err) === 'RepoNotFound') return null + if (errorCode(err) === 'SpaceNotFound') throw new SpaceNotFound(space) + throw err + } + } + + private async listAll(space: string, params: Record): Promise { + const records: Loose[] = [] + let cursor: string | undefined + do { + let out: Loose + try { + out = await this.call( + atproto.space.listRecords, + { space, repo: this.did, limit: PAGE, cursor, ...params }, + space, + ) + } catch (err) { + if (errorCode(err) === 'RepoNotFound') return records + throw err + } + records.push(...out.records) + cursor = out.cursor + } while (cursor) + return records + } + + async listRecords(space: string, collection: string): Promise { + const records = await this.listAll(space, { collection }) + return records.map((record) => ({ + rkey: record.rkey, + value: toJson(record.value), + cid: record.cid, + })) + } + + async listKeys(space: string): Promise { + const records = await this.listAll(space, { excludeValues: true }) + return records.map((record) => ({ + collection: record.collection, + rkey: record.rkey, + cid: record.cid, + })) + } + + async createRecord( + space: string, + collection: string, + rkey: string, + value: JsonRecord, + ): Promise<{ cid: string }> { + assertValidRecord(collection, value) + try { + const out = await this.rawCall(atproto.space.createRecord, { + space, + repo: this.did, + collection, + rkey, + validate: false, + record: toLex(value), + }) + return { cid: out.cid } + } catch (err) { + if (errorCode(err) === 'RecordAlreadyExists') throw new RecordExists(space, collection, rkey) + if (errorCode(err) === 'SpaceNotFound') throw new SpaceNotFound(space) + throw err + } + } + + async putRecord( + space: string, + collection: string, + rkey: string, + value: JsonRecord, + ): Promise<{ cid: string }> { + assertValidRecord(collection, value) + const out = await this.call( + atproto.space.putRecord, + { space, repo: this.did, collection, rkey, validate: false, record: toLex(value) }, + space, + ) + return { cid: out.cid } + } + + async deleteRecord(space: string, collection: string, rkey: string): Promise { + await this.call(atproto.space.deleteRecord, { space, repo: this.did, collection, rkey }, space) + } + + async listOps(space: string, since?: string | null): Promise { + const page: OpsPage = { ops: [], rev: since ?? null } + let cursor: string | undefined + do { + let out: Loose + try { + out = await this.call( + atproto.space.listRepoOps, + { space, repo: this.did, since: since ?? undefined, cursor, limit: PAGE }, + space, + ) + } catch (err) { + if (errorCode(err) === 'RepoNotFound') return page + throw err + } + for (const op of out.ops) { + page.ops.push({ + rev: op.rev, + collection: op.collection, + rkey: op.rkey, + cid: op.cid, + prev: op.prev, + value: op.value === undefined ? undefined : toJson(op.value), + }) + page.rev = op.rev + } + cursor = out.cursor + if (!cursor) page.commit = out.commit + } while (cursor) + if (!page.commit) throw new Error(`listRepoOps reached the head of ${space} without a commit`) + return page + } + + async headRev(space: string): Promise { + try { + const out = await this.call(atproto.space.getLatestCommit, { space, repo: this.did }, space) + return out.commit.rev + } catch (err) { + if (errorCode(err) === 'RepoNotFound') return null + throw err + } + } + + async listSpaces(type: string): Promise { + const uris: string[] = [] + let cursor: string | undefined + do { + const out = await this.rawCall(atproto.space.listSpaces, { type, cursor, limit: PAGE }) + uris.push(...out.spaces.map((space: { uri: string }) => space.uri)) + cursor = out.cursor + } while (cursor) + return uris + } +} diff --git a/apps/server/src/storage/validate.ts b/apps/server/src/storage/validate.ts new file mode 100644 index 0000000..aa4fa27 --- /dev/null +++ b/apps/server/src/storage/validate.ts @@ -0,0 +1,45 @@ +import { isRecordNsid, validateRecord } from '@scn-chat/lexicons' +import type { Logger } from '../logger.ts' +import { InvalidRecord, InvalidStoredRecord } from './record-store.ts' +import type { JsonRecord } from './records.ts' + +/** Refuse to write one of our records unless it matches its lexicon. */ +export function assertValidRecord(collection: string, value: JsonRecord): void { + if (!isRecordNsid(collection)) return + const result = validateRecord(collection, value) + if (!result.success) throw new InvalidRecord(collection, result.error) +} + +/** Check a record read from the PDS against its lexicon. */ +export function assertValidStored( + space: string, + collection: string, + rkey: string, + value: JsonRecord, +): void { + if (!isRecordNsid(collection)) throw new Error(`No lexicon to check ${collection} against`) + const result = validateRecord(collection, value) + if (!result.success) throw new InvalidStoredRecord(space, collection, rkey, result.error) +} + +/** Drop and log any records that don't match their lexicon. */ +export function keepValid( + space: string, + collection: string, + records: T[], + logger: Logger, +): T[] { + return records.filter((record) => { + try { + assertValidStored(space, collection, record.rkey, record.value) + return true + } catch (err) { + if (!(err instanceof InvalidStoredRecord)) throw err + logger.warn( + { err, space, rkey: record.rkey }, + 'skipping a record that does not match its lexicon', + ) + return false + } + }) +} diff --git a/apps/server/src/sync/coalescer.ts b/apps/server/src/sync/coalescer.ts new file mode 100644 index 0000000..4c86a25 --- /dev/null +++ b/apps/server/src/sync/coalescer.ts @@ -0,0 +1,63 @@ +type State = { + running: Promise | null + dirty: boolean + lastRun: number + timer: NodeJS.Timeout | null +} + +/** Runs work per key one at a time, at most once per interval. */ +export class Coalescer { + private readonly states = new Map() + private readonly intervalMs: number + + constructor(intervalMs = 2_000) { + this.intervalMs = intervalMs + } + + /** Request a run, resolving once a run that started after this request finishes. */ + request(key: string, work: () => Promise): Promise { + const state = this.states.get(key) ?? { running: null, dirty: false, lastRun: 0, timer: null } + this.states.set(key, state) + return new Promise((resolve, reject) => { + const waiters = this.waiters.get(key) ?? [] + waiters.push({ resolve, reject }) + this.waiters.set(key, waiters) + if (state.running) { + state.dirty = true + return + } + this.schedule(key, work, state) + }) + } + + private readonly waiters = new Map< + string, + { resolve: () => void; reject: (err: unknown) => void }[] + >() + + private schedule(key: string, work: () => Promise, state: State) { + if (state.timer) return + const wait = Math.max(0, state.lastRun + this.intervalMs - Date.now()) + state.timer = setTimeout(() => { + state.timer = null + const waiters = this.waiters.get(key) ?? [] + this.waiters.set(key, []) + state.dirty = false + state.lastRun = Date.now() + state.running = work() + .then( + () => { + for (const waiter of waiters) waiter.resolve() + }, + (err) => { + for (const waiter of waiters) waiter.reject(err) + }, + ) + .finally(() => { + state.running = null + if (state.dirty || (this.waiters.get(key)?.length ?? 0) > 0) + this.schedule(key, work, state) + }) + }, wait) + } +} diff --git a/apps/server/src/sync/credentials.ts b/apps/server/src/sync/credentials.ts new file mode 100644 index 0000000..97f87b7 --- /dev/null +++ b/apps/server/src/sync/credentials.ts @@ -0,0 +1,150 @@ +import { Client } from '@atproto/lex-client' +import { LexError } from '@atproto/lex-data' +import { JoseKey } from '@atproto/oauth-client-node' +import { createDpopProof } from '@atproto/space' +import { atproto } from '@scn-chat/lexicons' +import type { PdsClientFactory } from '../auth/pds.ts' + +// Plain strings don't satisfy the generated methods' branded string types. +type Loose = any + +const CREDENTIAL_TTL_MS = 2 * 60 * 60_000 +const REFRESH_MARGIN_MS = 5 * 60_000 + +/** A DPoP-bound space credential, usable against any repo host in its space. */ +export class SpaceCredential { + readonly token: string + readonly expiresAt: number + private readonly key: JoseKey + private readonly fetchImpl: typeof fetch + + constructor(token: string, key: JoseKey, expiresAt: number, fetchImpl: typeof fetch = fetch) { + this.token = token + this.key = key + this.expiresAt = expiresAt + this.fetchImpl = fetchImpl + } + + fetch = async (input: string | URL | Request, init?: RequestInit): Promise => { + const request = new Request(input, { ...init, redirect: 'error' }) + request.headers.set('authorization', `DPoP ${this.token}`) + request.headers.set( + 'dpop', + await createDpopProof(this.key, { + htm: request.method, + htu: request.url, + credential: this.token, + }), + ) + return this.fetchImpl(request) + } + + client(service: string): Client { + return new Client({ service, fetch: this.fetch }) + } +} + +export class CredentialError extends Error { + readonly code: string + + constructor(code: string, message: string) { + super(message) + this.name = 'CredentialError' + this.code = code + } +} + +export type CredentialDeps = { + /** A client for the user asking, acting through their own OAuth session. */ + getPdsClient: PdsClientFactory + /** The PDS answering for a space authority. */ + resolvePds: (did: string) => Promise + fetch?: typeof fetch + now?: () => number +} + +const authorityOf = (space: string) => { + const did = space.match(/^at:\/\/(did:[^/]+)\/space\//)?.[1] + if (!did) throw new CredentialError('InvalidSpace', `Not a space URI: ${space}`) + return did +} + +/** Get a space credential for a space, acting for a user, without caching. */ +export async function mintSpaceCredential( + deps: CredentialDeps, + actorDid: string, + space: string, +): Promise { + const now = deps.now ?? Date.now + const fetchImpl = deps.fetch ?? fetch + const actor: Loose = await deps.getPdsClient(actorDid) + const { token } = await actor.call(atproto.space.getDelegationToken, { space }) + const authorityPds = await deps.resolvePds(authorityOf(space)) + const key = await JoseKey.generate(['ES256']) + const url = new URL('/xrpc/com.atproto.space.getSpaceCredential', authorityPds) + const request = new Request(url, { + method: 'POST', + redirect: 'error', + headers: { + accept: 'application/json', + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ space }), + }) + request.headers.set('dpop', await createDpopProof(key, { htm: request.method, htu: request.url })) + const response = await fetchImpl(request) + const body = (await response.json().catch(() => ({}))) as { + credential?: string + error?: string + message?: string + } + if (!response.ok || !body.credential) { + throw new CredentialError( + body.error ?? 'InvalidResponse', + body.message ?? `getSpaceCredential returned ${response.status}`, + ) + } + return new SpaceCredential(body.credential, key, now() + CREDENTIAL_TTL_MS, fetchImpl) +} + +/** Space credentials cached per space and actor for their two-hour life. */ +export class CredentialCache { + private readonly cache = new Map() + private readonly deps: CredentialDeps + + constructor(deps: CredentialDeps) { + this.deps = deps + } + + async get(actorDid: string, space: string): Promise { + const key = `${actorDid} ${space}` + const cached = this.cache.get(key) + if (cached && cached.expiresAt - REFRESH_MARGIN_MS > (this.deps.now ?? Date.now)()) + return cached + const fresh = await mintSpaceCredential(this.deps, actorDid, space) + this.cache.set(key, fresh) + return fresh + } + + discard(actorDid: string, space: string): void { + this.cache.delete(`${actorDid} ${space}`) + } + + /** Run a call with a cached credential, retrying once with a fresh one after an auth failure. */ + async withCredential( + actorDid: string, + space: string, + fn: (credential: SpaceCredential) => Promise, + ): Promise { + try { + return await fn(await this.get(actorDid, space)) + } catch (err) { + const code = err instanceof LexError ? err.error : undefined + if (code !== 'AuthenticationRequired' && code !== 'InvalidToken' && code !== 'ExpiredToken') + throw err + this.discard(actorDid, space) + return fn(await this.get(actorDid, space)) + } + } +} diff --git a/apps/server/src/sync/engine.ts b/apps/server/src/sync/engine.ts new file mode 100644 index 0000000..f4b8702 --- /dev/null +++ b/apps/server/src/sync/engine.ts @@ -0,0 +1,313 @@ +import { parseCid } from '@atproto/lex-data' +import { RepoCommit, verifyCommit } from '@atproto/space' +import { atproto, isRecordNsid, nsid, validateRecord } from '@scn-chat/lexicons' +import { type Account, getAccount } from '../auth/accounts.ts' +import type { Db } from '../db/index.ts' +import type { Logger } from '../logger.ts' +import type { ChatService } from '../storage/chat-service.ts' +import { type OpsPage, type RecordStore, SpaceNotFound } from '../storage/record-store.ts' +import type { JsonRecord } from '../storage/records.ts' +import type { ChatServices } from '../storage/services.ts' +import { Coalescer } from './coalescer.ts' +import type { CredentialCache } from './credentials.ts' +import type { SyncEventBus } from './events.ts' +import { serviceId } from './identity.ts' +import type { RecentWrites } from './recent-writes.ts' + +// Plain strings don't satisfy the generated methods' branded string types. +type Loose = any + +export type SyncEngineDeps = { + db: Db + services: ChatServices + events: SyncEventBus + recentWrites: RecentWrites + credentials: CredentialCache + resolveSigningKey: (did: string) => Promise + publicUrl: string + logger: Logger + backfillWindowMs: number + now?: () => number +} + +type Context = { account: Account; chats: ChatService; store: RecordStore } + +/** Keeps sync cursors for spaces users and turns their PDS changes into events. It stores no content. */ +export class SyncEngine { + private readonly deps: SyncEngineDeps + private readonly coalescer = new Coalescer() + private readonly locks = new Map>() + + constructor(deps: SyncEngineDeps) { + this.deps = deps + } + + private now(): number { + return (this.deps.now ?? Date.now)() + } + + private async context(did: string): Promise { + const account = await getAccount(this.deps.db, did) + if (account?.storageMode !== 'space') return null + const chats = this.deps.services.forAccount(account) + return { account, chats, store: chats.store } + } + + /** Serialize work per key. */ + private lock(key: string, fn: () => Promise): Promise { + const previous = this.locks.get(key) ?? Promise.resolve() + const next = previous.catch(() => {}).then(fn) + this.locks.set(key, next) + return next.finally(() => { + if (this.locks.get(key) === next) this.locks.delete(key) + }) + } + + private async state(space: string) { + return this.deps.db + .selectFrom('sync_state') + .selectAll() + .where('space_uri', '=', space) + .executeTakeFirst() + } + + private async saveState( + space: string, + owner: string, + changes: { + last_rev?: string | null + registered_until?: string | null + last_error?: string | null + }, + ) { + const row = { ...changes, last_synced_at: new Date(this.now()).toISOString() } + await this.deps.db + .insertInto('sync_state') + .values({ + space_uri: space, + owner_did: owner, + last_rev: null, + registered_until: null, + last_error: null, + ...row, + }) + .onConflict((oc) => oc.column('space_uri').doUpdateSet(row)) + .execute() + } + + private async dropState(space: string) { + await this.deps.db.deleteFrom('sync_state').where('space_uri', '=', space).execute() + } + + /** Sync a user's chat index, coalescing bursts of requests. */ + syncIndex(did: string): Promise { + return this.coalescer.request(`index ${did}`, () => this.runIndexSync(did, false)) + } + + /** Sync one conversation now. */ + async syncConversation(did: string, skey: string): Promise { + const ctx = await this.context(did) + if (ctx) await this.runConversationSync(ctx, skey, false) + } + + private runIndexSync(did: string, verify: boolean): Promise { + return this.lock(`index ${did}`, async () => { + const ctx = await this.context(did) + if (!ctx) return + const space = ctx.chats.settingsUri + const state = await this.state(space) + if (!state?.last_rev) return this.firstIndexSync(ctx) + const page = await ctx.store.listOps(space, state.last_rev) + for (const op of page.ops) { + if (op.collection !== nsid.conversationRef) continue + if (!op.cid) { + await this.dropState(ctx.chats.conversationUri(op.rkey)) + this.deps.events.emit('conversation:deleted', { did, skey: op.rkey }) + } else if (!this.deps.recentWrites.has(op.cid)) { + await this.runConversationSync(ctx, op.rkey, false) + } + } + if (page.ops.some((op) => !op.cid || !this.deps.recentWrites.has(op.cid))) { + this.deps.events.emit('index:changed', { did }) + } + if (verify && !(await this.commitMatches(ctx, space, page))) { + this.deps.logger.warn({ did, space }, 'chat index set hash mismatch, resetting cursor') + return this.firstIndexSync(ctx) + } + await this.saveState(space, did, { last_rev: page.rev ?? state.last_rev, last_error: null }) + }) + } + + /** Start at the head of the op log, syncing only conversations changed within the backfill window. */ + private async firstIndexSync(ctx: Context): Promise { + const space = ctx.chats.settingsUri + const head = await ctx.store.headRev(space) + const { conversations } = await ctx.chats.listConversations() + const cutoff = this.now() - this.deps.backfillWindowMs + for (const conversation of conversations) { + if (Date.parse(conversation.updatedAt) >= cutoff) + await this.runConversationSync(ctx, conversation.skey, false) + } + await this.saveState(space, ctx.account.did, { last_rev: head, last_error: null }) + } + + private runConversationSync(ctx: Context, skey: string, verify: boolean): Promise { + const space = ctx.chats.conversationUri(skey) + return this.lock(`conversation ${space}`, async () => { + const did = ctx.account.did + const state = await this.state(space) + const first = !state?.last_rev + let page: OpsPage + try { + page = await ctx.store.listOps(space, state?.last_rev) + } catch (err) { + if (!(err instanceof SpaceNotFound)) throw err + await this.dropState(space) + this.deps.events.emit('conversation:deleted', { did, skey }) + return + } + let external = false + for (const op of page.ops) { + if (!op.cid) { + external = true + continue + } + // Skip records replaced by a later op in this page. + if (!op.value) continue + if (this.deps.recentWrites.has(op.cid)) continue + external = true + if (!isRecordNsid(op.collection)) continue + const valid = validateRecord(op.collection, op.value) + if (!valid.success) { + this.deps.logger.warn( + { space, rkey: op.rkey, error: valid.error }, + 'synced record failed validation', + ) + if (op.collection === nsid.message) { + this.deps.events.emit('message:invalid', { + did, + skey, + rkey: op.rkey, + raw: op.value, + error: valid.error, + }) + } + continue + } + if (op.collection === nsid.info) { + await ctx.chats.syncRefTitle(skey, op.value.title as string | undefined) + } + if (op.collection !== nsid.message) continue + this.deps.events.emit('message:changed', { + did, + skey, + rkey: op.rkey, + record: op.value as JsonRecord, + cid: op.cid, + kind: op.prev ? 'update' : 'create', + live: !first, + }) + } + if (external) this.deps.events.emit('conversation:changed', { did, skey }) + if ((verify || first) && !(await this.commitMatches(ctx, space, page))) { + this.deps.logger.warn({ space }, 'conversation set hash mismatch, clearing cursor') + await this.saveState(space, did, { last_rev: null }) + return + } + await this.saveState(space, did, { + last_rev: page.rev ?? state?.last_rev ?? null, + last_error: null, + }) + }) + } + + /** Check the op log's signed commit against the set hash of the space's current records. */ + async commitMatches(ctx: Context, space: string, page: OpsPage): Promise { + const commit = page.commit as Loose + // No repo in the space yet, nothing to verify. + if (!commit) return true + const keys = await ctx.store.listKeys(space) + const local = RepoCommit.fromRecords( + keys.map((key) => ({ + collection: key.collection as Loose, + rkey: key.rkey as Loose, + cid: parseCid(key.cid), + })), + ) + if (!local.matches(commit)) return false + const didKey = await this.deps.resolveSigningKey(ctx.account.did) + return verifyCommit( + commit, + { space: space as Loose, author: ctx.account.did as Loose, rev: commit.rev }, + didKey, + ) + } + + /** Register for write notifications on the user's settings space. */ + async registerIndex(did: string): Promise { + const ctx = await this.context(did) + if (!ctx) return + const space = ctx.chats.settingsUri + try { + const { expiresAt } = await this.deps.credentials.withCredential( + did, + space, + async (credential) => { + const client: Loose = credential.client(ctx.account.pdsUrl) + return client.call(atproto.space.registerNotify, { + space, + service: serviceId(this.deps.publicUrl), + }) + }, + ) + await this.saveState(space, did, { registered_until: expiresAt, last_error: null }) + } catch (err) { + this.deps.logger.warn({ err, did }, 'registerNotify failed') + await this.saveState(space, did, { + last_error: err instanceof Error ? err.message : String(err), + }) + } + } + + /** Renew registrations with less than an hour left. */ + async renewRegistrations(): Promise { + const soon = new Date(this.now() + 60 * 60_000).toISOString() + const rows = await this.deps.db + .selectFrom('sync_state') + .innerJoin('account', 'account.did', 'sync_state.owner_did') + .select('sync_state.owner_did') + .where('account.storage_mode', '=', 'space') + .where('sync_state.space_uri', 'like', `%/space/${nsid.settings}/self`) + .where((eb) => + eb.or([ + eb('sync_state.registered_until', 'is', null), + eb('sync_state.registered_until', '<', soon), + ]), + ) + .execute() + for (const row of rows) await this.registerIndex(row.owner_did) + } + + /** Find conversations missing from the index, add their entries, and sync them. */ + async discover(did: string): Promise { + const ctx = await this.context(did) + if (!ctx) return + const [uris, { conversations }] = await Promise.all([ + ctx.store.listSpaces(nsid.conversation), + ctx.chats.listConversations(), + ]) + const known = new Set(conversations.map((conversation) => conversation.uri)) + for (const uri of uris) { + if (known.has(uri)) continue + const skey = ctx.chats.skeyOf(uri) + await ctx.chats.repairRef(skey) + await this.runConversationSync(ctx, skey, false) + this.deps.events.emit('index:changed', { did }) + } + } + + /** Sync and verify a user's index, as the safety net does. */ + safetyNet(did: string): Promise { + return this.runIndexSync(did, true) + } +} diff --git a/apps/server/src/sync/events.ts b/apps/server/src/sync/events.ts new file mode 100644 index 0000000..fc7d56e --- /dev/null +++ b/apps/server/src/sync/events.ts @@ -0,0 +1,23 @@ +import { EventEmitter } from 'node:events' +import type { JsonRecord } from '../storage/records.ts' + +export type SyncEvents = { + 'message:changed': [ + { + did: string + skey: string + rkey: string + record: JsonRecord + cid: string + kind: 'create' | 'update' + live: boolean + }, + ] + 'message:invalid': [{ did: string; skey: string; rkey: string; raw: JsonRecord; error: string }] + 'conversation:changed': [{ did: string; skey: string }] + 'conversation:deleted': [{ did: string; skey: string }] + 'index:changed': [{ did: string }] +} + +/** In-process events about chat changes, for turns and for open browser tabs. */ +export class SyncEventBus extends EventEmitter {} diff --git a/apps/server/src/sync/identity.ts b/apps/server/src/sync/identity.ts new file mode 100644 index 0000000..8b74652 --- /dev/null +++ b/apps/server/src/sync/identity.ts @@ -0,0 +1,19 @@ +/** The app's did:web, derived from its public URL. */ +export function appDid(publicUrl: string): string { + const url = new URL(publicUrl) + return `did:web:${encodeURIComponent(url.host)}` +} + +export const SERVICE_FRAGMENT = 'scn_chat' + +export const serviceId = (publicUrl: string) => `${appDid(publicUrl)}#${SERVICE_FRAGMENT}` + +export function didDocument(publicUrl: string) { + return { + '@context': ['https://www.w3.org/ns/did/v1'], + id: appDid(publicUrl), + service: [ + { id: `#${SERVICE_FRAGMENT}`, type: 'AtprotoSpaceService', serviceEndpoint: publicUrl }, + ], + } +} diff --git a/apps/server/src/sync/recent-writes.ts b/apps/server/src/sync/recent-writes.ts new file mode 100644 index 0000000..e9a8187 --- /dev/null +++ b/apps/server/src/sync/recent-writes.ts @@ -0,0 +1,26 @@ +/** CIDs of records the server wrote recently, so sync can skip its own changes. */ +export class RecentWrites { + private readonly entries = new Map() + private readonly ttlMs: number + + constructor(ttlMs = 10 * 60_000) { + this.ttlMs = ttlMs + } + + add(cid: string, now = Date.now()): void { + this.entries.set(cid, now + this.ttlMs) + if (this.entries.size > 10_000) this.prune(now) + } + + has(cid: string, now = Date.now()): boolean { + const expires = this.entries.get(cid) + if (expires === undefined) return false + if (expires > now) return true + this.entries.delete(cid) + return false + } + + private prune(now: number): void { + for (const [cid, expires] of this.entries) if (expires <= now) this.entries.delete(cid) + } +} diff --git a/apps/server/src/sync/routes.ts b/apps/server/src/sync/routes.ts new file mode 100644 index 0000000..fd70372 --- /dev/null +++ b/apps/server/src/sync/routes.ts @@ -0,0 +1,119 @@ +import { nsid } from '@scn-chat/lexicons' +import { Hono } from 'hono' +import { z } from 'zod' +import { getAccount } from '../auth/accounts.ts' +import { jsonBody } from '../body.ts' +import type { Db } from '../db/index.ts' +import type { Logger } from '../logger.ts' +import { InvalidSpaceUri, parseSpaceUri } from '../storage/records.ts' +import type { SyncEngine } from './engine.ts' +import type { SyncEventBus } from './events.ts' +import { appDid, didDocument, serviceId } from './identity.ts' +import { ServiceAuthError, verifyServiceAuth } from './service-auth.ts' + +export type SyncRoutesDeps = { + db: Db + engine: SyncEngine + events: SyncEventBus + publicUrl: string + resolveSigningKey: (did: string, forceRefresh: boolean) => Promise + logger: Logger +} + +const authorityOf = (space: string) => parseSpaceUri(space) + +const notification = z.object({ space: z.string() }) +const syncRequest = z.object({ conversation: z.string().optional() }) + +/** The app's did:web document and the XRPC endpoints PDSes and clients call. */ +export function syncRoutes(deps: SyncRoutesDeps) { + const background = (work: Promise, context: object) => + work.catch((err) => deps.logger.warn({ err, ...context }, 'background sync failed')) + + /** Check that a space notification came from its space authority, returning the refusal if not. */ + const authorize = async ( + authorization: string | undefined, + method: string, + authority: string, + ) => { + try { + const issuer = await verifyServiceAuth( + authorization, + serviceId(deps.publicUrl), + `com.atproto.space.${method}`, + deps.resolveSigningKey, + ) + if (issuer !== authority) throw new ServiceAuthError('Issuer is not the space authority') + return null + } catch (err) { + if (!(err instanceof ServiceAuthError)) throw err + deps.logger.warn({ err, method }, 'rejected a space notification') + return { error: 'AuthenticationRequired', message: err.message } + } + } + + return new Hono() + .get('/.well-known/did.json', (c) => c.json(didDocument(deps.publicUrl))) + .post('/xrpc/com.atproto.space.notifyWrite', async (c) => { + const { space } = await jsonBody(c, notification) + const { did, type, skey } = authorityOf(space) + const refusal = await authorize(c.req.header('authorization'), 'notifyWrite', did) + if (refusal) return c.json(refusal, 401) + if (type === nsid.settings) background(deps.engine.syncIndex(did), { did }) + else if (type === nsid.conversation) + background(deps.engine.syncConversation(did, skey), { did, skey }) + return c.body(null, 200) + }) + .post('/xrpc/com.atproto.space.notifySpaceDeleted', async (c) => { + const { space } = await jsonBody(c, notification) + const { did, type, skey } = authorityOf(space) + const refusal = await authorize(c.req.header('authorization'), 'notifySpaceDeleted', did) + if (refusal) return c.json(refusal, 401) + await deps.db.deleteFrom('sync_state').where('space_uri', '=', space).execute() + if (type === nsid.conversation) deps.events.emit('conversation:deleted', { did, skey }) + return c.body(null, 200) + }) + .post(`/xrpc/${nsid.requestSync}`, async (c) => { + let issuer: string + try { + issuer = await verifyServiceAuth( + c.req.header('authorization'), + appDid(deps.publicUrl), + nsid.requestSync, + deps.resolveSigningKey, + ) + } catch (err) { + return c.json({ error: 'AuthenticationRequired', message: (err as Error).message }, 401) + } + const account = await getAccount(deps.db, issuer) + if (account?.storageMode !== 'space') { + return c.json( + { + error: 'AuthenticationRequired', + message: 'Caller is not a spaces account on this app', + }, + 401, + ) + } + const body = await jsonBody(c, syncRequest) + if (!body.conversation) { + await deps.engine.syncIndex(issuer) + return c.json({}) + } + let target: ReturnType + try { + target = authorityOf(body.conversation) + } catch (err) { + if (!(err instanceof InvalidSpaceUri)) throw err + return c.json({ error: 'UnknownConversation', message: 'Not a conversation URI' }, 400) + } + if (target.did !== issuer || target.type !== nsid.conversation) { + return c.json( + { error: 'UnknownConversation', message: 'Conversation does not belong to the caller' }, + 400, + ) + } + await deps.engine.syncConversation(issuer, target.skey) + return c.json({}) + }) +} diff --git a/apps/server/src/sync/scheduler.ts b/apps/server/src/sync/scheduler.ts new file mode 100644 index 0000000..41e6f08 --- /dev/null +++ b/apps/server/src/sync/scheduler.ts @@ -0,0 +1,87 @@ +import type { SyncConfig } from '../config.ts' +import type { Db } from '../db/index.ts' +import type { Logger } from '../logger.ts' +import type { SyncEngine } from './engine.ts' + +export type ResolvedSyncConfig = { + safetyNet: { enabled: boolean; intervalMinutes: number; activeWithinHours: number } + discovery: { intervalMinutes: number; activeWithinDays: number } + allowUserOptOut: boolean +} + +export function resolveSyncConfig(config: SyncConfig = {}): ResolvedSyncConfig { + return { + safetyNet: { + enabled: config.safetyNet?.enabled ?? true, + intervalMinutes: config.safetyNet?.intervalMinutes ?? 15, + activeWithinHours: config.safetyNet?.activeWithinHours ?? 24, + }, + discovery: { + intervalMinutes: config.discovery?.intervalMinutes ?? 60, + activeWithinDays: config.discovery?.activeWithinDays ?? 30, + }, + allowUserOptOut: config.allowUserOptOut ?? true, + } +} + +/** Spaces users active since the cutoff who have not turned off background sync. */ +export async function eligibleUsers( + db: Db, + config: ResolvedSyncConfig, + activeSince: Date, +): Promise { + let query = db + .selectFrom('account') + .select('did') + .where('storage_mode', '=', 'space') + .where('last_active_at', '>=', activeSince.toISOString()) + if (config.allowUserOptOut) query = query.where('background_sync', '=', 1) + return (await query.execute()).map((row) => row.did) +} + +/** Timers for registration renewal, the safety net, and discovery. */ +export function startSyncScheduler(deps: { + engine: SyncEngine + db: Db + config: ResolvedSyncConfig + logger: Logger +}) { + const { engine, db, config, logger } = deps + const run = (name: string, work: () => Promise) => () => { + work().catch((err) => logger.error({ err, job: name }, 'sync job failed')) + } + const forEach = async (users: string[], fn: (did: string) => Promise) => { + for (const did of users) { + await fn(did).catch((err) => logger.warn({ err, did }, 'sync job failed for user')) + } + } + const timers = [ + setInterval( + run('renew', () => engine.renewRegistrations()), + 5 * 60_000, + ), + ] + if (config.safetyNet.enabled) { + timers.push( + setInterval( + run('safety-net', async () => { + const since = new Date(Date.now() - config.safetyNet.activeWithinHours * 3_600_000) + await forEach(await eligibleUsers(db, config, since), (did) => engine.safetyNet(did)) + }), + config.safetyNet.intervalMinutes * 60_000, + ), + ) + } + timers.push( + setInterval( + run('discovery', async () => { + const since = new Date(Date.now() - config.discovery.activeWithinDays * 86_400_000) + await forEach(await eligibleUsers(db, config, since), (did) => engine.discover(did)) + }), + config.discovery.intervalMinutes * 60_000, + ), + ) + return () => { + for (const timer of timers) clearInterval(timer) + } +} diff --git a/apps/server/src/sync/service-auth.ts b/apps/server/src/sync/service-auth.ts new file mode 100644 index 0000000..2ddfbe5 --- /dev/null +++ b/apps/server/src/sync/service-auth.ts @@ -0,0 +1,28 @@ +import { AuthRequiredError, verifyJwt } from '@atproto/xrpc-server' + +export class ServiceAuthError extends Error { + constructor(message: string) { + super(message) + this.name = 'ServiceAuthError' + } +} + +/** Verify an inbound service auth token and return the issuing DID. */ +export async function verifyServiceAuth( + authorization: string | undefined, + audience: string, + lxm: string, + resolveSigningKey: (did: string, forceRefresh: boolean) => Promise, +): Promise { + const token = authorization?.match(/^Bearer (.+)$/)?.[1] + if (!token) throw new ServiceAuthError('Missing service auth') + try { + const payload = await verifyJwt(token, audience, lxm, (iss, force) => + resolveSigningKey(iss.split('#')[0] ?? iss, force), + ) + return payload.iss.split('#')[0] ?? payload.iss + } catch (err) { + if (err instanceof AuthRequiredError) throw new ServiceAuthError(err.message) + throw err + } +} diff --git a/apps/server/src/turns/accumulator.ts b/apps/server/src/turns/accumulator.ts new file mode 100644 index 0000000..4334349 --- /dev/null +++ b/apps/server/src/turns/accumulator.ts @@ -0,0 +1,137 @@ +import { nsid } from '@scn-chat/lexicons' +import type { TextStreamPart, ToolSet } from 'ai' + +type Part = Record + +export type StreamEvent = + | { type: 'part-start'; index: number; partType: string } + | { type: 'delta'; index: number; text: string } + | { type: 'part'; index: number; part: Part } + +const defs = (name: string) => `${nsid.defs}#${name}` +const encode = (value: unknown) => (typeof value === 'string' ? value : JSON.stringify(value)) +const providerData = (metadata: object | undefined) => + metadata && Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : undefined + +/** Builds lexicon parts from an AI SDK stream, in the order the model produced them. */ +export class PartAccumulator { + readonly parts: Part[] = [] + usage: { inputTokens?: number; outputTokens?: number; reasoningTokens?: number } = {} + private readonly open = new Map() + private readonly emit: (event: StreamEvent) => void + + constructor(emit: (event: StreamEvent) => void = () => {}) { + this.emit = emit + } + + private start(key: string, type: string, part: Part): number { + const index = this.parts.push(part) - 1 + this.open.set(key, index) + this.emit({ type: 'part-start', index, partType: type }) + return index + } + + private add(part: Part): void { + const index = this.parts.push(part) - 1 + this.emit({ type: 'part', index, part }) + } + + /** Feed one stream part, returning an error when the part ends the turn. */ + push(chunk: TextStreamPart): Error | undefined { + switch (chunk.type) { + case 'text-start': + this.start(`text:${chunk.id}`, 'textPart', { $type: defs('textPart'), text: '' }) + break + case 'reasoning-start': + this.start(`reasoning:${chunk.id}`, 'reasoningPart', { + $type: defs('reasoningPart'), + text: '', + }) + break + case 'text-delta': + case 'reasoning-delta': { + const kind = chunk.type === 'text-delta' ? 'text' : 'reasoning' + const key = `${kind}:${chunk.id}` + const index = + this.open.get(key) ?? + this.start(key, `${kind}Part`, { $type: defs(`${kind}Part`), text: '' }) + const part = this.parts[index] as Part + part.text = `${part.text as string}${chunk.text}` + this.emit({ type: 'delta', index, text: chunk.text }) + break + } + case 'text-end': + case 'reasoning-end': { + const key = `${chunk.type === 'text-end' ? 'text' : 'reasoning'}:${chunk.id}` + const index = this.open.get(key) + const data = providerData(chunk.providerMetadata) + if (index !== undefined && data) (this.parts[index] as Part).providerData = data + this.open.delete(key) + break + } + case 'tool-call': { + const data = providerData(chunk.providerMetadata) + this.add({ + $type: defs('toolCallPart'), + callId: chunk.toolCallId, + tool: chunk.toolName, + // Encode unparsable input too, which arrives as the raw string. + input: JSON.stringify(chunk.input), + ...(data ? { providerData: data } : {}), + }) + break + } + case 'tool-result': + this.add({ + $type: defs('toolResultPart'), + callId: chunk.toolCallId, + output: encode(chunk.output), + }) + break + case 'tool-error': + this.add({ + $type: defs('toolResultPart'), + callId: chunk.toolCallId, + output: chunk.error instanceof Error ? chunk.error.message : encode(chunk.error), + isError: true, + }) + break + case 'source': + // Only URL sources are in the lexicon. + if (chunk.sourceType === 'url') { + this.add({ + $type: defs('sourcePart'), + url: chunk.url, + ...(chunk.title ? { title: chunk.title } : {}), + }) + } + break + case 'finish': + this.usage = { + inputTokens: chunk.totalUsage.inputTokens, + outputTokens: chunk.totalUsage.outputTokens, + reasoningTokens: chunk.totalUsage.outputTokenDetails.reasoningTokens, + } + break + case 'error': + return chunk.error instanceof Error ? chunk.error : new Error(String(chunk.error)) + case 'file': + case 'reasoning-file': + return new Error('The model returned a file, which SCN Chat cannot store yet') + case 'tool-output-denied': + case 'tool-approval-request': + case 'tool-approval-response': + return new Error(`Unexpected ${chunk.type} part: SCN Chat does not use tool approvals`) + } + return undefined + } + + /** Get the usage fields that are whole, non-negative numbers. */ + lexiconUsage(): Record | undefined { + const entries = Object.entries(this.usage).filter( + (entry): entry is [string, number] => + typeof entry[1] === 'number' && Number.isInteger(entry[1]) && entry[1] >= 0, + ) + return entries.length ? Object.fromEntries(entries) : undefined + } +} diff --git a/apps/server/src/turns/prompt.ts b/apps/server/src/turns/prompt.ts new file mode 100644 index 0000000..6890435 --- /dev/null +++ b/apps/server/src/turns/prompt.ts @@ -0,0 +1,177 @@ +import type { BranchMessage, Capabilities, ModelProvider, ModelRef } from '@scn-chat/plugin-api' +import type { JSONValue, ModelMessage } from 'ai' +import type { JsonRecord } from '../storage/records.ts' + +type Part = Record & { $type: string } +type BlobRef = { ref: { $link: string }; mimeType: string } + +export class TurnInputError extends Error { + constructor(message: string) { + super(message) + this.name = 'TurnInputError' + } +} + +/** Reads a blob referenced from the conversation, for attachments. */ +export type BlobReader = (cid: string) => Promise<{ bytes: Uint8Array; mimeType: string }> + +const kind = (part: Part) => part.$type.split('#')[1] +const partsOf = (record: JsonRecord): Part[] => { + const content = record.content as { $type: string; parts?: Part[] } + if (!content.$type.endsWith('#plainContent')) + throw new TurnInputError( + 'This conversation has encrypted messages, which SCN Chat cannot read yet.', + ) + return content.parts as Part[] +} + +/** Follow parent keys from the given message back to the root, returning the chain in order. */ +export function branchTo(messages: Map, rkey: string): BranchMessage[] { + const chain: BranchMessage[] = [] + const seen = new Set() + let current: string | undefined = rkey + while (current && !seen.has(current)) { + seen.add(current) + const message = messages.get(current) + if (!message) break + chain.unshift(message) + current = message.record.parent + } + return chain +} + +/** Get the model of the nearest completed reply on the branch. */ +export function lastReplyModel(branch: BranchMessage[]): ModelRef | undefined { + for (let i = branch.length - 1; i >= 0; i--) { + const record = branch[i]?.record as unknown as JsonRecord | undefined + if (record?.role === 'assistant' && record.status === 'complete' && record.model) + return record.model as ModelRef + } + return undefined +} + +export function buildInstructions(preferences: JsonRecord | null, info: JsonRecord | null): string { + return [preferences?.customInstructions, info?.systemPrompt] + .filter((value): value is string => typeof value === 'string' && value.trim() !== '') + .join('\n\n') +} + +type ProviderOptions = Record> + +const parseProviderData = (value: unknown): ProviderOptions | undefined => + value === undefined ? undefined : (JSON.parse(value as string) as ProviderOptions) + +async function userContent(parts: Part[], capabilities: Capabilities, readBlob: BlobReader) { + const content: Exclude['content'], string> = [] + for (const part of parts) { + const type = kind(part) + if (type === 'textPart') content.push({ type: 'text', text: part.text as string }) + else if (type === 'imagePart') { + if (!capabilities.vision) + throw new TurnInputError('This model cannot read images. Choose a model with vision.') + const image = part.image as BlobRef + const blob = await readBlob(image.ref.$link) + content.push({ type: 'file', data: blob.bytes, mediaType: image.mimeType }) + } else if (type === 'filePart') { + const name = (part.name as string | undefined) ?? 'Attached file' + const extracted = part.extracted as { text: BlobRef } | undefined + if (!extracted) { + content.push({ + type: 'text', + text: `A file named "${name}" was attached, but it could not be read.`, + }) + continue + } + const text = new TextDecoder().decode((await readBlob(extracted.text.ref.$link)).bytes) + content.push({ type: 'text', text: `# ${name}\n\n${text}` }) + } + } + return content +} + +/** Convert assistant parts to AI SDK messages, splitting at tool results. */ +function assistantMessages(parts: Part[], replay: boolean): ModelMessage[] { + const messages: ModelMessage[] = [] + let assistant: Exclude['content'], string> = [] + let tool: Extract['content'] = [] + const toolNames = new Map() + const flushAssistant = () => { + if (assistant.length) messages.push({ role: 'assistant', content: assistant }) + assistant = [] + } + const flushTool = () => { + if (tool.length) messages.push({ role: 'tool', content: tool }) + tool = [] + } + for (const part of parts) { + const type = kind(part) + const providerOptions = replay ? parseProviderData(part.providerData) : undefined + if (type === 'toolResultPart') { + flushAssistant() + const callId = part.callId as string + const toolName = toolNames.get(callId) + if (!toolName) throw new Error(`Tool result ${callId} has no tool call before it`) + const output = part.output as string + tool.push({ + type: 'tool-result', + toolCallId: callId, + toolName, + output: part.isError + ? { type: 'error-text', value: output } + : { type: 'text', value: output }, + }) + continue + } + flushTool() + if (type === 'textPart') + assistant.push({ + type: 'text', + text: part.text as string, + ...(providerOptions ? { providerOptions } : {}), + }) + else if (type === 'reasoningPart' && replay) { + assistant.push({ + type: 'reasoning', + text: (part.text as string | undefined) ?? '', + ...(providerOptions ? { providerOptions } : {}), + }) + } else if (type === 'toolCallPart') { + toolNames.set(part.callId as string, part.tool as string) + assistant.push({ + type: 'tool-call', + toolCallId: part.callId as string, + toolName: part.tool as string, + input: JSON.parse(part.input as string), + ...(providerOptions ? { providerOptions } : {}), + }) + } + } + flushAssistant() + flushTool() + return messages +} + +/** Convert the branch to AI SDK messages, leaving out errored and pending replies. */ +export async function toModelMessages( + branch: BranchMessage[], + options: { provider: ModelProvider; capabilities: Capabilities; readBlob: BlobReader }, +): Promise { + const messages: ModelMessage[] = [] + for (const { record } of branch) { + const json = record as unknown as JsonRecord + const parts = partsOf(json) + if (json.role === 'user') { + const content = await userContent(parts, options.capabilities, options.readBlob) + if (content.length) messages.push({ role: 'user', content }) + } else if ( + json.role === 'assistant' && + (json.status === 'complete' || json.status === 'cancelled') + ) { + const sameProvider = (json.model as ModelRef | undefined)?.provider === options.provider.id + messages.push( + ...assistantMessages(parts, sameProvider && options.provider.replay === 'replay'), + ) + } + } + return messages +} diff --git a/apps/server/src/turns/routes.ts b/apps/server/src/turns/routes.ts new file mode 100644 index 0000000..8652c5f --- /dev/null +++ b/apps/server/src/turns/routes.ts @@ -0,0 +1,101 @@ +import { nsid } from '@scn-chat/lexicons' +import type { ModelRef } from '@scn-chat/plugin-api' +import { Hono } from 'hono' +import { streamSSE } from 'hono/streaming' +import { requireUser, signedInUser } from '../auth/routes.ts' +import type { AppEnv } from '../env.ts' +import type { JsonRecord } from '../storage/records.ts' +import { newTid } from '../storage/records.ts' +import type { ChatServices } from '../storage/services.ts' +import type { TurnRunner } from './runner.ts' +import type { HubEvent, StreamHub } from './stream-hub.ts' + +export type TurnRoutesDeps = { services: ChatServices; runner: TurnRunner; hub: StreamHub } + +const USER_PARTS = new Set(['textPart', 'imagePart', 'filePart']) + +type SendBody = { + parent?: string + parts: JsonRecord[] + generation?: { model?: ModelRef; effort?: string; tools?: string[] } +} + +/** Sending messages, regenerating, cancelling, and streaming replies. */ +export function turnRoutes(deps: TurnRoutesDeps) { + return new Hono() + .use(requireUser) + .post('/conversations/:skey/messages', async (c) => { + const user = signedInUser(c) + const skey = c.req.param('skey') + const body = (await c.req.json()) as SendBody + const parts = Array.isArray(body.parts) ? body.parts : [] + const invalid = parts.find((part) => !USER_PARTS.has(String(part.$type).split('#')[1] ?? '')) + if (invalid || parts.length === 0) { + return c.json( + { error: 'InvalidRequest', message: 'A message needs text or attachment parts only' }, + 400, + ) + } + const chats = deps.services.forAccount(user.account) + const rkey = newTid() + const record: JsonRecord = { + $type: nsid.message, + role: 'user', + content: { $type: `${nsid.defs}#plainContent`, parts }, + createdAt: new Date().toISOString(), + } + if (body.parent) record.parent = body.parent + if (body.generation) record.generation = { ...body.generation, attempt: 0 } + await chats.createMessage(skey, rkey, record) + const started = body.generation ? await deps.runner.start(user.did, skey, rkey) : undefined + return c.json( + { rkey, replyRkey: started?.replyRkey ?? null, status: started?.status ?? null }, + 201, + ) + }) + .post('/conversations/:skey/messages/:rkey/regenerate', async (c) => { + const user = signedInUser(c) + const skey = c.req.param('skey') + const rkey = c.req.param('rkey') + const body = (await c.req.json().catch(() => ({}))) as { model?: ModelRef; effort?: string } + const chats = deps.services.forAccount(user.account) + const existing = await chats.store.getRecord(chats.conversationUri(skey), nsid.message, rkey) + if (!existing || existing.value.role !== 'user') return c.json({ error: 'NotFound' }, 404) + const previous = (existing.value.generation ?? {}) as { attempt?: number } & JsonRecord + const generation: JsonRecord = { ...previous, attempt: (previous.attempt ?? 0) + 1 } + if (body.model) generation.model = body.model + if (body.effort) generation.effort = body.effort + await chats.putMessage(skey, rkey, { ...existing.value, generation }) + const started = await deps.runner.start(user.did, skey, rkey) + return c.json({ replyRkey: started.replyRkey ?? null, status: started.status }) + }) + .post('/conversations/:skey/messages/:rkey/cancel', (c) => { + const user = signedInUser(c) + const cancelled = deps.runner.cancel(user.did, c.req.param('skey'), c.req.param('rkey')) + return c.json({ cancelled }) + }) + .get('/conversations/:skey/messages/:rkey/stream', (c) => { + const user = signedInUser(c) + const key = deps.runner.streamKey(user.did, c.req.param('skey'), c.req.param('rkey')) + return streamSSE(c, async (stream) => { + if (!deps.hub.has(key)) { + await stream.writeSSE({ event: 'status', data: JSON.stringify({ status: 'unknown' }) }) + return + } + let writes = Promise.resolve() + await new Promise((resolve) => { + const send = (event: HubEvent) => { + const { type, ...data } = event + writes = writes.then(() => stream.writeSSE({ event: type, data: JSON.stringify(data) })) + if (type === 'status') resolve() + } + const unsubscribe = deps.hub.subscribe(key, send) + stream.onAbort(() => { + unsubscribe() + resolve() + }) + }) + await writes + }) + }) +} diff --git a/apps/server/src/turns/runner.ts b/apps/server/src/turns/runner.ts new file mode 100644 index 0000000..c5e0730 --- /dev/null +++ b/apps/server/src/turns/runner.ts @@ -0,0 +1,539 @@ +import { nsid } from '@scn-chat/lexicons' +import type { BranchMessage, InfoRecord, ModelRef, TurnContext } from '@scn-chat/plugin-api' +import { isStepCount, jsonSchema, streamText, type ToolSet, tool } from 'ai' +import { type Account, getAccount } from '../auth/accounts.ts' +import type { Db } from '../db/index.ts' +import type { Logger } from '../logger.ts' +import type { PluginHost } from '../plugins/host.ts' +import type { ModelCatalog } from '../providers/catalog.ts' +import { mapEffort } from '../providers/effort.ts' +import type { ChatService } from '../storage/chat-service.ts' +import { RecordExists } from '../storage/record-store.ts' +import type { JsonRecord } from '../storage/records.ts' +import type { ChatServices } from '../storage/services.ts' +import type { SyncEventBus, SyncEvents } from '../sync/events.ts' +import { PartAccumulator } from './accumulator.ts' +import { + type BlobReader, + branchTo, + buildInstructions, + lastReplyModel, + TurnInputError, + toModelMessages, +} from './prompt.ts' +import type { StreamHub } from './stream-hub.ts' + +export type TurnBlobs = { + reader(account: Account, skey: string): BlobReader + put(account: Account, bytes: Uint8Array, mimeType: string): Promise +} + +export type TurnRunnerDeps = { + db: Db + services: ChatServices + catalog: ModelCatalog + host?: PluginHost + hub: StreamHub + blobs: TurnBlobs + config: { ratePerMinute: number; maxSteps: number; timeoutMs: number; backfillWindowMs: number } + logger: Logger + now?: () => number +} + +export type StartResult = { + status: 'claimed' | 'queued' | 'exists' | 'skipped' + replyRkey?: string +} + +const MAX_RECORD_BYTES = 900_000 +const defs = (name: string) => `${nsid.defs}#${name}` +const replyKey = (userRkey: string, attempt: number) => `${userRkey}.r${attempt}` + +/** Shorten an error message and remove anything that looks like a secret. */ +export function safeErrorMessage(err: unknown): string { + const raw = err instanceof Error ? err.message : String(err) + return raw + .replace(/sk-[A-Za-z0-9_-]{8,}/g, '[redacted]') + .replace(/AIza[0-9A-Za-z_-]{20,}/g, '[redacted]') + .replace(/(bearer\s+)[A-Za-z0-9._~+/-]+=*/gi, '$1[redacted]') + .slice(0, 500) +} + +type Loaded = { + account: Account + chats: ChatService + info: JsonRecord | null + preferences: JsonRecord | null + messages: Map +} + +/** Runs every turn, whether the web UI or a direct write asked for it. */ +export class TurnRunner { + private readonly deps: TurnRunnerDeps + private readonly running = new Map() + private readonly startTimes = new Map() + private readonly queued = new Map() + private readonly drainTimers = new Map() + private readonly background = new Set>() + + constructor(deps: TurnRunnerDeps) { + this.deps = deps + } + + private now(): number { + return (this.deps.now ?? Date.now)() + } + + /** Resolves once every turn started so far has finished, for tests and shutdown. */ + async idle(): Promise { + while (this.background.size) await Promise.allSettled([...this.background]) + } + + streamKey(did: string, skey: string, replyRkey: string): string { + return `${did}/${skey}/${replyRkey}` + } + + private async load(did: string, skey: string): Promise { + const account = await getAccount(this.deps.db, did) + if (!account) throw new Error(`No account for ${did}`) + const chats = this.deps.services.forAccount(account) + const [conversation, preferences] = await Promise.all([ + chats.getConversation(skey), + chats.getPreferences(), + ]) + const messages = new Map() + for (const message of conversation.messages) { + messages.set(message.rkey, { + rkey: message.rkey, + author: did, + record: message.value as never, + }) + } + return { account, chats, info: conversation.info?.value ?? null, preferences, messages } + } + + /** Get the user's turn start times from the last minute, oldest first. */ + private recentStarts(did: string): number[] { + let times = this.startTimes.get(did) + if (!times) { + times = [] + this.startTimes.set(did, times) + } + const cutoff = this.now() - 60_000 + while (times.length && (times[0] as number) <= cutoff) times.shift() + return times + } + + private scheduleDrain(did: string): void { + if (this.drainTimers.has(did)) return + const [oldest] = this.recentStarts(did) + if (oldest === undefined) throw new Error(`Queued a turn for ${did} with no recent starts`) + const wait = Math.max(50, oldest + 60_000 - this.now()) + const timer = setTimeout(() => { + this.drainTimers.delete(did) + const pending = this.queued.get(did) ?? [] + this.queued.set(did, []) + for (const turn of pending) void this.start(did, turn.skey, turn.userRkey) + }, wait) + timer.unref() + this.drainTimers.set(did, timer) + } + + /** Start the turn a user message asks for, if it still needs one. */ + async start(did: string, skey: string, userRkey: string): Promise { + const loaded = await this.load(did, skey) + const userMessage = loaded.messages.get(userRkey) + const record = userMessage?.record as unknown as JsonRecord | undefined + const generation = record?.generation as + | { model?: ModelRef; effort?: string; attempt?: number; tools?: string[] } + | undefined + if (!record || record.role !== 'user' || !generation) return { status: 'skipped' } + const attempt = generation.attempt ?? 0 + const replyRkey = replyKey(userRkey, attempt) + if (loaded.messages.has(replyRkey)) return { status: 'exists', replyRkey } + const conversationUri = loaded.chats.conversationUri(skey) + + await this.deps.db + .insertInto('turn_request') + .values({ + conversation_uri: conversationUri, + message_rkey: userRkey, + attempt, + owner_did: did, + requested_at: new Date(this.now()).toISOString(), + }) + .onConflict((oc) => oc.columns(['conversation_uri', 'message_rkey', 'attempt']).doNothing()) + .execute() + + const starts = this.recentStarts(did) + if (starts.length >= this.deps.config.ratePerMinute) { + const queue = this.queued.get(did) ?? [] + if (!queue.some((turn) => turn.skey === skey && turn.userRkey === userRkey)) + queue.push({ skey, userRkey }) + this.queued.set(did, queue) + this.deps.hub.publish(this.streamKey(did, skey, replyRkey), { type: 'queued' }) + this.scheduleDrain(did) + return { status: 'queued', replyRkey } + } + + const branch = branchTo(loaded.messages, userRkey) + const model = + generation.model ?? + lastReplyModel(branch) ?? + (loaded.preferences?.defaultModel as ModelRef | undefined) ?? + this.deps.catalog.defaultModel() + const effort = generation.effort ?? (loaded.preferences?.defaultEffort as string | undefined) + const placeholder: JsonRecord = { + $type: nsid.message, + role: 'assistant', + parent: userRkey, + content: { $type: defs('plainContent'), parts: [] }, + status: 'pending', + createdAt: new Date(this.now()).toISOString(), + ...(model ? { model } : {}), + ...(effort ? { effort } : {}), + } + try { + await loaded.chats.createMessage(skey, replyRkey, placeholder) + } catch (err) { + if (err instanceof RecordExists) { + await this.clearRequest(conversationUri, userRkey, attempt) + return { status: 'exists', replyRkey } + } + throw err + } + starts.push(this.now()) + this.deps.hub.open(this.streamKey(did, skey, replyRkey)) + await this.deps.db + .insertInto('turn_claim') + .values({ + conversation_uri: conversationUri, + reply_rkey: replyRkey, + owner_did: did, + claimed_at: new Date(this.now()).toISOString(), + }) + .onConflict((oc) => oc.columns(['conversation_uri', 'reply_rkey']).doNothing()) + .execute() + await this.clearRequest(conversationUri, userRkey, attempt) + + const work = this.generate( + loaded, + skey, + userMessage as BranchMessage, + replyRkey, + placeholder, + model, + effort, + generation, + ) + .catch((err) => + this.deps.logger.error({ err, did, skey, replyRkey }, 'turn failed outside generation'), + ) + .finally(() => this.background.delete(work)) + this.background.add(work) + return { status: 'claimed', replyRkey } + } + + private async clearRequest(conversationUri: string, userRkey: string, attempt: number) { + await this.deps.db + .deleteFrom('turn_request') + .where('conversation_uri', '=', conversationUri) + .where('message_rkey', '=', userRkey) + .where('attempt', '=', attempt) + .execute() + } + + cancel(did: string, skey: string, replyRkey: string): boolean { + const controller = this.running.get(this.streamKey(did, skey, replyRkey)) + controller?.abort(new Error('cancelled')) + return Boolean(controller) + } + + private async tools( + requested: string[] | undefined, + context: { user: string; conversation: string; signal: AbortSignal }, + ) { + if (!requested?.length) return undefined + const host = this.deps.host + const set: ToolSet = {} + for (const registered of host?.tools.list() ?? []) { + if (!requested.includes(registered.name)) continue + set[registered.name] = tool({ + description: registered.description, + inputSchema: registered.inputSchema as never, + execute: async (input: never) => registered.run(input, context), + } as never) + } + for (const source of host?.toolSources.list() ?? []) { + for (const definition of await source.list(context.user)) { + const name = `${source.id}_${definition.name}` + if (!requested.includes(name)) continue + set[name] = tool({ + description: definition.description, + inputSchema: jsonSchema(definition.inputSchema as never), + execute: async (input: unknown) => + source.call(context.user, definition.name, input, context), + } as never) + } + } + const missing = requested.filter((name) => !(name in set)) + if (missing.length) + throw new TurnInputError(`These tools are not available: ${missing.join(', ')}`) + return set + } + + /** Move the largest tool outputs into blobs until the record fits a PDS write. */ + private async fitRecord(account: Account, record: JsonRecord): Promise { + const content = record.content as { parts: Record[] } + const size = () => new TextEncoder().encode(JSON.stringify(record)).length + const results = content.parts + .filter( + (part) => + part.$type === defs('toolResultPart') && + typeof part.output === 'string' && + !part.outputBlob, + ) + .sort((a, b) => String(b.output).length - String(a.output).length) + for (const part of results) { + if (size() <= MAX_RECORD_BYTES) break + const bytes = new TextEncoder().encode(String(part.output)) + part.outputBlob = await this.deps.blobs.put(account, bytes, 'text/plain') + part.output = `Output stored as a blob (${bytes.length} bytes).` + } + if (size() > MAX_RECORD_BYTES) throw new Error('The reply is too large to store') + return record + } + + private async generate( + loaded: Loaded, + skey: string, + userMessage: BranchMessage, + replyRkey: string, + placeholder: JsonRecord, + model: ModelRef | undefined, + effort: string | undefined, + generation: { tools?: string[] }, + ): Promise { + const { account, chats } = loaded + const did = account.did + const conversationUri = chats.conversationUri(skey) + const key = this.streamKey(did, skey, replyRkey) + const controller = new AbortController() + this.running.set(key, controller) + const timeout = setTimeout( + () => controller.abort(new Error('timeout')), + this.deps.config.timeoutMs, + ) + const accumulator = new PartAccumulator((event) => this.deps.hub.publish(key, event)) + const context: TurnContext = { + user: did, + conversation: { + uri: conversationUri, + ...(loaded.info ? { info: loaded.info as unknown as InfoRecord } : {}), + }, + preferences: (loaded.preferences ?? undefined) as never, + userMessage: { rkey: userMessage.rkey, record: userMessage.record }, + reply: { rkey: replyRkey, record: placeholder as never }, + ...(model ? { model } : {}), + effort: effort as never, + } + let status: 'complete' | 'error' | 'cancelled' = 'complete' + let error: string | undefined + try { + if (!model) + throw new Error('No model selected. Choose a model or set a default in your preferences.') + const resolved = await this.deps.catalog.resolve(did, model) + const hooks = this.deps.host?.hooks + const branch = branchTo(loaded.messages, userMessage.rkey) + const initial = { + instructions: buildInstructions(loaded.preferences, loaded.info), + messages: branch, + } + const prompt = hooks ? await hooks.filter('messages:beforeModel', initial, context) : initial + const messages = await toModelMessages(prompt.messages, { + provider: resolved.provider, + capabilities: resolved.capabilities, + readBlob: this.deps.blobs.reader(account, skey), + }) + const result = streamText({ + model: resolved.model, + instructions: prompt.instructions || undefined, + messages, + tools: await this.tools(generation.tools, { + user: did, + conversation: conversationUri, + signal: controller.signal, + }), + stopWhen: isStepCount(this.deps.config.maxSteps), + reasoning: resolved.capabilities.reasoning + ? mapEffort(effort, this.deps.logger) + : undefined, + abortSignal: controller.signal, + providerOptions: resolved.provider.providerOptions as never, + }) + for await (const chunk of result.stream) { + if (chunk.type === 'abort') status = 'cancelled' + const failure = accumulator.push(chunk) + if (failure && status !== 'cancelled') { + status = 'error' + error = safeErrorMessage(failure) + } + } + if (controller.signal.aborted && status === 'complete') status = 'cancelled' + } catch (err) { + if (controller.signal.aborted) status = 'cancelled' + else { + status = 'error' + error = safeErrorMessage(err) + this.deps.logger.warn({ err, did, skey, replyRkey }, 'turn ended with an error') + } + } finally { + clearTimeout(timeout) + this.running.delete(key) + } + + let content: JsonRecord = { $type: defs('plainContent'), parts: accumulator.parts } + if (status === 'complete' && this.deps.host) { + try { + content = (await this.deps.host.hooks.filter( + 'message:afterModel', + content as never, + context, + )) as never + } catch (err) { + this.deps.logger.warn({ err, did, skey, replyRkey }, 'message:afterModel filter failed') + content = { $type: defs('plainContent'), parts: [] } + status = 'error' + error = safeErrorMessage(err) + } + } + const usage = accumulator.lexiconUsage() + let final: JsonRecord = { + ...placeholder, + content, + status, + ...(usage ? { usage } : {}), + ...(error ? { error } : {}), + } + try { + final = await this.fitRecord(account, final) + await chats.putMessage(skey, replyRkey, final) + } catch (err) { + this.deps.logger.error({ err, did, skey, replyRkey }, 'could not write the finished reply') + status = 'error' + error = safeErrorMessage(err) + final = { ...placeholder, status, error } + try { + await chats.putMessage(skey, replyRkey, final) + } catch (fallbackErr) { + // Leave the claim for recovery to mark the reply interrupted after a restart. + this.deps.logger.error( + { err: fallbackErr, did, skey, replyRkey }, + 'could not write the error reply either', + ) + this.deps.hub.publish(key, { type: 'status', status, error }) + return + } + } + this.deps.hub.publish(key, { type: 'status', status, ...(error ? { error } : {}) }) + await this.deps.db + .deleteFrom('turn_claim') + .where('conversation_uri', '=', conversationUri) + .where('reply_rkey', '=', replyRkey) + .execute() + if (this.deps.host) { + await this.deps.host.hooks.action( + 'turn:after', + { ...context, reply: { rkey: replyRkey, record: final as never } }, + this.deps.logger, + ) + } + } + + /** Reply with an error to a user message that failed validation. */ + async answerInvalid(event: SyncEvents['message:invalid'][0]): Promise { + const raw = event.raw + if (raw.role !== 'user' || !raw.generation || typeof raw.generation !== 'object') return + const attempt = Number((raw.generation as { attempt?: unknown }).attempt ?? 0) + const account = await getAccount(this.deps.db, event.did) + if (!account || !Number.isInteger(attempt)) return + const chats = this.deps.services.forAccount(account) + try { + await chats.createMessage(event.skey, replyKey(event.rkey, attempt), { + $type: nsid.message, + role: 'assistant', + parent: event.rkey, + content: { $type: defs('plainContent'), parts: [] }, + status: 'error', + error: `Your message is not a valid ${nsid.message} record: ${event.error}`.slice(0, 3000), + createdAt: new Date(this.now()).toISOString(), + }) + } catch (err) { + if (!(err instanceof RecordExists)) throw err + } + } + + /** Start turns for direct writes that ask for a reply. */ + attach(events: SyncEventBus): void { + events.on('message:changed', (event) => { + const record = event.record + if (record.role !== 'user' || !record.generation) return + const recent = + Date.parse(String(record.createdAt)) >= this.now() - this.deps.config.backfillWindowMs + if (!event.live && !recent) return + this.start(event.did, event.skey, event.rkey).catch((err) => + this.deps.logger.error( + { err, did: event.did, skey: event.skey }, + 'could not start turn from sync', + ), + ) + }) + events.on('message:invalid', (event) => { + this.answerInvalid(event).catch((err) => + this.deps.logger.error({ err, did: event.did }, 'could not answer invalid message'), + ) + }) + } + + /** After a restart, mark this server's interrupted replies and retry recent queued turns. */ + async recover(): Promise { + const claims = await this.deps.db.selectFrom('turn_claim').selectAll().execute() + for (const claim of claims) { + try { + const account = await getAccount(this.deps.db, claim.owner_did) + if (account) { + const chats = this.deps.services.forAccount(account) + const skey = chats.skeyOf(claim.conversation_uri) + const reply = await chats.store.getRecord( + claim.conversation_uri, + nsid.message, + claim.reply_rkey, + ) + if (reply?.value.status === 'pending') { + await chats.putMessage(skey, claim.reply_rkey, { + ...reply.value, + status: 'error', + error: 'interrupted', + }) + } + } + } catch (err) { + this.deps.logger.warn({ err, claim }, 'could not mark an interrupted reply') + } + await this.deps.db + .deleteFrom('turn_claim') + .where('conversation_uri', '=', claim.conversation_uri) + .where('reply_rkey', '=', claim.reply_rkey) + .execute() + } + const cutoff = new Date(this.now() - this.deps.config.backfillWindowMs).toISOString() + await this.deps.db.deleteFrom('turn_request').where('requested_at', '<', cutoff).execute() + const requests = await this.deps.db.selectFrom('turn_request').selectAll().execute() + for (const request of requests) { + const skey = request.conversation_uri.slice(request.conversation_uri.lastIndexOf('/') + 1) + await this.start(request.owner_did, skey, request.message_rkey).catch((err) => + this.deps.logger.warn({ err, request }, 'could not retry a queued turn'), + ) + } + } +} diff --git a/apps/server/src/turns/stream-hub.ts b/apps/server/src/turns/stream-hub.ts new file mode 100644 index 0000000..3f1d844 --- /dev/null +++ b/apps/server/src/turns/stream-hub.ts @@ -0,0 +1,57 @@ +import type { StreamEvent } from './accumulator.ts' + +export type HubEvent = + | StreamEvent + | { type: 'queued' } + | { type: 'status'; status: string; error?: string } + +type Channel = { events: HubEvent[]; listeners: Set<(event: HubEvent) => void>; done: boolean } + +/** Buffers each running turn's events so a late subscriber can catch up, then follow live. */ +export class StreamHub { + private readonly channels = new Map() + private readonly retainMs: number + + constructor(retainMs = 60_000) { + this.retainMs = retainMs + } + + private channel(key: string): Channel { + let channel = this.channels.get(key) + if (!channel) { + channel = { events: [], listeners: new Set(), done: false } + this.channels.set(key, channel) + } + return channel + } + + /** Open a turn's channel as soon as it's claimed. */ + open(key: string): void { + this.channel(key) + } + + publish(key: string, event: HubEvent): void { + const channel = this.channel(key) + channel.events.push(event) + for (const listener of channel.listeners) listener(event) + if (event.type === 'status') { + channel.done = true + setTimeout(() => { + if (this.channels.get(key) === channel) this.channels.delete(key) + }, this.retainMs).unref() + } + } + + has(key: string): boolean { + return this.channels.has(key) + } + + /** Replay buffered events, then deliver live ones until the turn ends. */ + subscribe(key: string, listener: (event: HubEvent) => void): () => void { + const channel = this.channel(key) + for (const event of channel.events) listener(event) + if (channel.done) return () => {} + channel.listeners.add(listener) + return () => channel.listeners.delete(listener) + } +} diff --git a/apps/server/src/web-html.ts b/apps/server/src/web-html.ts new file mode 100644 index 0000000..66ffedd --- /dev/null +++ b/apps/server/src/web-html.ts @@ -0,0 +1,6 @@ +const escapeHtml = (text: string) => text.replace(/[&<>"']/g, (ch) => `&#${ch.charCodeAt(0)};`) + +/** Fill the web app's index.html with the configured app name. */ +export function renderIndexHtml(html: string, appName: string): string { + return html.replaceAll('__APP_NAME__', escapeHtml(appName)) +} diff --git a/apps/server/tests/app.test.ts b/apps/server/tests/app.test.ts new file mode 100644 index 0000000..ac0fa65 --- /dev/null +++ b/apps/server/tests/app.test.ts @@ -0,0 +1,80 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import SqliteDatabase from 'better-sqlite3' +import { Kysely, SqliteDialect } from 'kysely' +import pino from 'pino' +import { afterAll, describe, expect, it } from 'vitest' +import { createApp } from '../src/app.ts' +import type { Db } from '../src/db/index.ts' +import type { Database } from '../src/db/schema.ts' +import { testConfig } from './helpers/config.ts' +import { createSqliteDb, dialects } from './helpers/db.ts' + +const config = testConfig() +const logger = pino({ level: 'silent' }) + +describe.each(dialects)('GET /api/health on $name', ({ create }) => { + it('returns 200 with the app name when the database is reachable', async () => { + const db = create() + const res = await createApp({ config, db, logger }).request('/api/health') + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ status: 'ok', appName: 'Test Chat' }) + await db.destroy() + }) +}) + +describe('GET /api/health', () => { + it('returns 503 when the database query fails', async () => { + const sqlite = new SqliteDatabase(':memory:') + const db: Db = new Kysely({ dialect: new SqliteDialect({ database: sqlite }) }) + sqlite.close() + const res = await createApp({ config, db, logger }).request('/api/health') + expect(res.status).toBe(503) + expect(await res.json()).toEqual({ status: 'error' }) + }) +}) + +describe('production static serving', () => { + const webDist = mkdtempSync(join(tmpdir(), 'scn-web-')) + writeFileSync(join(webDist, 'index.html'), 'index') + writeFileSync(join(webDist, 'app.js'), 'console.log(1)') + afterAll(() => rmSync(webDist, { recursive: true, force: true })) + const db = createSqliteDb() + const app = createApp({ config, db, logger, webDist }) + + it('serves built files', async () => { + const res = await app.request('/app.js') + expect(res.status).toBe(200) + expect(await res.text()).toBe('console.log(1)') + }) + + it('falls back to index.html for unknown non-API paths', async () => { + const res = await app.request('/c/some-chat') + expect(res.status).toBe(200) + expect(await res.text()).toContain('index') + }) + + it('fills the app name into index.html, escaped', async () => { + const dist = mkdtempSync(join(tmpdir(), 'scn-web-')) + writeFileSync(join(dist, 'index.html'), '__APP_NAME__') + const named = createApp({ + config: { ...config, appName: 'Chat & ' }, + db, + logger, + webDist: dist, + }) + for (const path of ['/', '/index.html', '/c/some-chat']) { + expect(await (await named.request(path)).text()).toBe( + 'Chat & <Co>', + ) + } + rmSync(dist, { recursive: true, force: true }) + }) + + it('returns 404 for unknown API paths', async () => { + const res = await app.request('/api/nope') + expect(res.status).toBe(404) + expect(await res.json()).toEqual({ error: 'NotFound' }) + }) +}) diff --git a/apps/server/tests/auth/accounts.test.ts b/apps/server/tests/auth/accounts.test.ts new file mode 100644 index 0000000..369d49e --- /dev/null +++ b/apps/server/tests/auth/accounts.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import { + getAccount, + recordLogin, + SpacesLostError, + setBackgroundSync, + touchActivity, +} from '../../src/auth/accounts.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { dialects } from '../helpers/db.ts' + +const login = { did: 'did:plc:alice', handle: 'alice.test', pdsUrl: 'https://pds.test' } + +describe.each(dialects)('accounts on $name', ({ create }) => { + const setup = async () => { + const db = create() + await migrateToLatest(db) + return db + } + + it('gives a new account space storage when its scope allows spaces', async () => { + const db = await setup() + expect((await recordLogin(db, { ...login, spacesAllowed: true })).storageMode).toBe('space') + await db.destroy() + }) + + it('gives a new account local storage when its scope lacks spaces', async () => { + const db = await setup() + expect((await recordLogin(db, { ...login, spacesAllowed: false })).storageMode).toBe('local') + await db.destroy() + }) + + it('fails the login of a space account whose scope no longer allows spaces', async () => { + const db = await setup() + await recordLogin(db, { ...login, spacesAllowed: true }) + await expect(recordLogin(db, { ...login, spacesAllowed: false })).rejects.toBeInstanceOf( + SpacesLostError, + ) + await db.destroy() + }) + + it('keeps a local account local when its scope later allows spaces', async () => { + const db = await setup() + await recordLogin(db, { ...login, spacesAllowed: false }) + expect((await recordLogin(db, { ...login, spacesAllowed: true })).storageMode).toBe('local') + await db.destroy() + }) + + it('updates the handle on each login', async () => { + const db = await setup() + await recordLogin(db, { ...login, spacesAllowed: true }) + await recordLogin(db, { ...login, handle: 'alice.new', spacesAllowed: true }) + expect((await getAccount(db, login.did))?.handle).toBe('alice.new') + await db.destroy() + }) + + it('updates activity at most once a minute', async () => { + const db = await setup() + const start = new Date('2026-09-26T12:00:00Z') + const account = await recordLogin(db, { ...login, spacesAllowed: true }, start) + await touchActivity(db, account, new Date(start.getTime() + 30_000)) + expect((await getAccount(db, login.did))?.lastActiveAt).toBe(start.toISOString()) + await touchActivity(db, account, new Date(start.getTime() + 90_000)) + expect((await getAccount(db, login.did))?.lastActiveAt).toBe( + new Date(start.getTime() + 90_000).toISOString(), + ) + await db.destroy() + }) + + it('stores the background sync setting', async () => { + const db = await setup() + await recordLogin(db, { ...login, spacesAllowed: true }) + expect((await getAccount(db, login.did))?.backgroundSync).toBe(true) + await setBackgroundSync(db, login.did, false) + expect((await getAccount(db, login.did))?.backgroundSync).toBe(false) + await db.destroy() + }) +}) diff --git a/apps/server/tests/auth/oauth-client.test.ts b/apps/server/tests/auth/oauth-client.test.ts new file mode 100644 index 0000000..23f12a5 --- /dev/null +++ b/apps/server/tests/auth/oauth-client.test.ts @@ -0,0 +1,47 @@ +import { JoseKey } from '@atproto/oauth-client-node' +import { describe, expect, it } from 'vitest' +import { createOAuthClient, isLoopbackUrl } from '../../src/auth/oauth-client.ts' +import { buildScope } from '../../src/auth/scope.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { testConfig } from '../helpers/config.ts' +import { createSqliteDb } from '../helpers/db.ts' + +describe('isLoopbackUrl', () => { + it.each([ + ['http://127.0.0.1:3000', true], + ['http://localhost:3000', true], + ['http://[::1]:3000', true], + ['https://chat.example.com', false], + ])('%s is loopback: %s', (url, expected) => { + expect(isLoopbackUrl(url)).toBe(expected) + }) +}) + +describe('createOAuthClient', () => { + it('uses loopback client metadata for a loopback public URL', async () => { + const db = createSqliteDb() + await migrateToLatest(db) + const client = await createOAuthClient(testConfig(), db, buildScope('raw')) + expect(String(client.clientMetadata.client_id)).toMatch(/^http:\/\/localhost/) + expect(client.clientMetadata.token_endpoint_auth_method).toBe('none') + }) + + it('is a confidential client with published keys for an HTTPS public URL', async () => { + const db = createSqliteDb() + await migrateToLatest(db) + const key = await JoseKey.generate(['ES256'], 'test-key') + const config = testConfig({ + PUBLIC_URL: 'https://chat.example.com', + OAUTH_PRIVATE_KEYS: JSON.stringify([key.privateJwk]), + }) + const client = await createOAuthClient(config, db, buildScope('permission-set')) + expect(client.clientMetadata).toMatchObject({ + client_id: 'https://chat.example.com/oauth-client-metadata.json', + token_endpoint_auth_method: 'private_key_jwt', + jwks_uri: 'https://chat.example.com/oauth/jwks.json', + dpop_bound_access_tokens: true, + }) + expect(JSON.stringify(client.jwks)).toContain('test-key') + expect(JSON.stringify(client.jwks)).not.toContain('"d"') + }) +}) diff --git a/apps/server/tests/auth/pds.test.ts b/apps/server/tests/auth/pds.test.ts new file mode 100644 index 0000000..29f5224 --- /dev/null +++ b/apps/server/tests/auth/pds.test.ts @@ -0,0 +1,31 @@ +import { TokenRefreshError } from '@atproto/oauth-client-node' +import { describe, expect, it, vi } from 'vitest' +import { createPdsClientFactory, SessionExpired } from '../../src/auth/pds.ts' +import { fakeOAuth } from '../helpers/auth.ts' + +describe('createPdsClientFactory', () => { + it('returns a client for a restorable session', async () => { + const getClient = createPdsClientFactory(fakeOAuth()) + await expect(getClient('did:plc:alice')).resolves.toBeDefined() + }) + + it('throws SessionExpired when the session has expired or been revoked', async () => { + const oauth = fakeOAuth({ + restore: vi.fn(async () => { + throw new TokenRefreshError('did:plc:alice', 'refresh failed') + }), + }) + const promise = createPdsClientFactory(oauth)('did:plc:alice') + await expect(promise).rejects.toBeInstanceOf(SessionExpired) + }) + + it('passes other restore failures through unchanged', async () => { + const outage = new Error('fetch failed') + const oauth = fakeOAuth({ + restore: vi.fn(async () => { + throw outage + }), + }) + await expect(createPdsClientFactory(oauth)('did:plc:alice')).rejects.toBe(outage) + }) +}) diff --git a/apps/server/tests/auth/roles.test.ts b/apps/server/tests/auth/roles.test.ts new file mode 100644 index 0000000..9c98ec2 --- /dev/null +++ b/apps/server/tests/auth/roles.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' +import { createRoles, RolesConfigError } from '../../src/auth/roles.ts' + +describe('createRoles', () => { + it('gives every user the implicit user role plus each configured role listing them', () => { + const roles = createRoles({ staff: ['did:plc:alice'], beta: ['did:plc:alice', 'did:plc:bob'] }) + expect(roles.rolesFor('did:plc:alice')).toEqual(['user', 'staff', 'beta']) + expect(roles.rolesFor('did:plc:carol')).toEqual(['user']) + }) + + it('refuses a role named user', () => { + expect(() => createRoles({ user: ['did:plc:alice'] })).toThrow(RolesConfigError) + }) + + it('refuses an invalid role name, naming it', () => { + expect(() => createRoles({ 'Bad Role': [] })).toThrow(/Bad Role/) + }) + + it('refuses an invalid DID, naming the role and entry', () => { + expect(() => createRoles({ staff: ['alice'] })).toThrow(/staff.*"alice"/) + }) + + it('lists the defined roles', () => { + expect([...createRoles({ staff: [] }).defined]).toEqual(['user', 'staff']) + }) +}) diff --git a/apps/server/tests/auth/routes.test.ts b/apps/server/tests/auth/routes.test.ts new file mode 100644 index 0000000..fa2302a --- /dev/null +++ b/apps/server/tests/auth/routes.test.ts @@ -0,0 +1,166 @@ +import pino from 'pino' +import { describe, expect, it, vi } from 'vitest' +import { createApp } from '../../src/app.ts' +import { SpacesLostError } from '../../src/auth/accounts.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { authDeps, fakeOAuth, fakeSession, ORIGIN, sessionCookie } from '../helpers/auth.ts' +import { testConfig } from '../helpers/config.ts' +import { createSqliteDb } from '../helpers/db.ts' + +async function setup(overrides: Parameters[1] = {}) { + const db = createSqliteDb() + await migrateToLatest(db) + const auth = authDeps(db, overrides) + const app = createApp({ config: testConfig(), db, logger: pino({ level: 'silent' }), auth }) + return { db, auth, app } +} + +async function signIn(app: Awaited>['app']) { + const res = await app.request('/oauth/callback?code=abc&state=xyz') + return sessionCookie(res) +} + +describe('client metadata and keys', () => { + it('serves the client metadata and public keys', async () => { + const { app } = await setup() + expect(await (await app.request('/oauth-client-metadata.json')).json()).toEqual({ + client_id: 'http://localhost', + }) + expect(await (await app.request('/oauth/jwks.json')).json()).toEqual({ keys: [] }) + }) +}) + +describe('GET /oauth/login', () => { + it('redirects to the authorization URL with the app scope', async () => { + const { app, auth } = await setup() + const res = await app.request('/oauth/login?identifier=alice.test') + expect(res.status).toBe(302) + expect(res.headers.get('location')).toBe('https://pds.test/oauth/authorize?request_uri=x') + expect(auth.oauth.authorize).toHaveBeenCalledWith('alice.test', { scope: auth.scope }) + }) + + it('sends the user back to the login page without an identifier', async () => { + const { app } = await setup() + const res = await app.request('/oauth/login') + expect(res.headers.get('location')).toMatch(/^\/login\?error=/) + }) + + it('shows an invalid_scope error from the PDS on the login page', async () => { + const oauth = fakeOAuth({ + authorize: vi.fn(async () => { + throw new Error('invalid_scope: permission set not found') + }), + }) + const { app } = await setup({ oauth }) + const res = await app.request('/oauth/login?identifier=alice.test') + expect(decodeURIComponent(res.headers.get('location') ?? '')).toContain('invalid_scope') + }) +}) + +describe('GET /oauth/callback', () => { + it('sets an HttpOnly, SameSite=Lax session cookie and redirects home', async () => { + const { app } = await setup() + const res = await app.request('/oauth/callback?code=abc&state=xyz') + expect(res.status).toBe(302) + expect(res.headers.get('location')).toBe('/') + const cookie = res.headers.get('set-cookie') ?? '' + expect(cookie).toMatch(/HttpOnly/) + expect(cookie).toMatch(/SameSite=Lax/) + }) + + it('creates a space account when the granted scope allows spaces', async () => { + const { app } = await setup() + const cookie = await signIn(app) + const me = (await (await app.request('/api/me', { headers: { cookie } })).json()) as { + storageMode: string + } + expect(me).toMatchObject({ did: 'did:plc:alice', handle: 'alice.test', storageMode: 'space' }) + }) + + it('creates a local account when the PDS dropped the space permissions', async () => { + const oauth = fakeOAuth({ + callback: vi.fn(async () => ({ session: fakeSession('did:plc:alice', 'atproto') })), + }) + const { app } = await setup({ oauth }) + const cookie = await signIn(app) + const me = (await (await app.request('/api/me', { headers: { cookie } })).json()) as { + storageMode: string + } + expect(me.storageMode).toBe('local') + }) + + it('explains the failure when a space account loses spaces support', async () => { + const { app, db } = await setup() + await signIn(app) + const oauth = fakeOAuth({ + callback: vi.fn(async () => ({ session: fakeSession('did:plc:alice', 'atproto') })), + }) + const again = createApp({ + config: testConfig(), + db, + logger: pino({ level: 'silent' }), + auth: authDeps(db, { oauth }), + }) + const res = await again.request('/oauth/callback?code=abc&state=xyz') + expect(decodeURIComponent(res.headers.get('location') ?? '')).toContain( + new SpacesLostError().message, + ) + }) +}) + +describe('API session handling', () => { + it('returns 401 from /api/me without a session', async () => { + const { app } = await setup() + expect((await app.request('/api/me')).status).toBe(401) + }) + + it('includes the user roles in /api/me', async () => { + const { app } = await setup() + const cookie = await signIn(app) + const me = (await (await app.request('/api/me', { headers: { cookie } })).json()) as { + roles: string[] + } + expect(me.roles).toEqual(['user', 'staff']) + }) + + it('refuses a non-GET API request with a foreign Origin', async () => { + const { app } = await setup() + const cookie = await signIn(app) + const res = await app.request('/api/logout', { + method: 'POST', + headers: { cookie, origin: 'https://evil.test' }, + }) + expect(res.status).toBe(403) + }) + + it('refuses a non-GET API request with no Origin', async () => { + const { app } = await setup() + expect((await app.request('/api/logout', { method: 'POST' })).status).toBe(403) + }) +}) + +describe('POST /api/logout', () => { + it('clears the web session and keeps the OAuth session', async () => { + const { app, auth } = await setup() + const cookie = await signIn(app) + const res = await app.request('/api/logout', { + method: 'POST', + headers: { cookie, origin: ORIGIN }, + }) + expect(res.status).toBe(200) + expect((await app.request('/api/me', { headers: { cookie } })).status).toBe(401) + expect(auth.oauth.revoke).not.toHaveBeenCalled() + }) + + it('revokes the OAuth session and every web session when signing out everywhere', async () => { + const { app, auth } = await setup() + const first = await signIn(app) + const second = await signIn(app) + await app.request('/api/logout?everywhere=1', { + method: 'POST', + headers: { cookie: first, origin: ORIGIN }, + }) + expect(auth.oauth.revoke).toHaveBeenCalledWith('did:plc:alice') + expect((await app.request('/api/me', { headers: { cookie: second } })).status).toBe(401) + }) +}) diff --git a/apps/server/tests/auth/scope.test.ts b/apps/server/tests/auth/scope.test.ts new file mode 100644 index 0000000..c74398c --- /dev/null +++ b/apps/server/tests/auth/scope.test.ts @@ -0,0 +1,44 @@ +import { SpacePermission } from '@atproto/oauth-scopes' +import { nsid } from '@scn-chat/lexicons' +import { describe, expect, it } from 'vitest' +import { buildScope, scopeAllowsSpaces } from '../../src/auth/scope.ts' + +const did = 'did:plc:alice' + +describe('buildScope', () => { + it('includes the permission set by default, with its NSID from the lexicon module', () => { + expect(buildScope('permission-set')).toBe(`atproto include:${nsid.permissions}`) + }) + + it('builds raw space and blob scopes that parse as valid permissions', () => { + const parts = buildScope('raw').split(' ') + expect(parts[0]).toBe('atproto') + expect(parts).toContain('blob:*/*') + const spaces = parts.filter((part) => part.startsWith('space:')) + expect(spaces).toHaveLength(3) + for (const scope of spaces) expect(SpacePermission.fromString(scope), scope).not.toBeNull() + }) +}) + +describe('scopeAllowsSpaces', () => { + it('is true for the raw scope on the user own DID', () => { + expect(scopeAllowsSpaces(buildScope('raw'), did)).toBe(true) + }) + + it('is false when the PDS dropped the space permissions', () => { + expect(scopeAllowsSpaces('atproto blob:*/*', did)).toBe(false) + }) + + it('is false for a conversation permission scoped to someone else', () => { + expect( + scopeAllowsSpaces( + `atproto space:${nsid.conversation}?authority=did:plc:bob&manage=create`, + did, + ), + ).toBe(false) + }) + + it('throws on a space permission it cannot parse', () => { + expect(() => scopeAllowsSpaces('atproto space:not a permission', did)).toThrow(/cannot read/) + }) +}) diff --git a/apps/server/tests/auth/web-session.test.ts b/apps/server/tests/auth/web-session.test.ts new file mode 100644 index 0000000..892d00a --- /dev/null +++ b/apps/server/tests/auth/web-session.test.ts @@ -0,0 +1,68 @@ +import { sql } from 'kysely' +import { describe, expect, it } from 'vitest' +import { + createWebSession, + deleteAllWebSessions, + resolveWebSession, +} from '../../src/auth/web-session.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { dialects } from '../helpers/db.ts' + +const DAY = 86_400_000 + +describe.each(dialects)('web sessions on $name', ({ create }) => { + const setup = async () => { + const db = create() + await migrateToLatest(db) + return db + } + + it('resolves a new session token to its DID and stores only its hash', async () => { + const db = await setup() + const token = await createWebSession(db, 'did:plc:alice', 30) + expect(await resolveWebSession(db, token, 30)).toBe('did:plc:alice') + const { rows } = await sql<{ token_hash: string }>`select token_hash from web_session`.execute( + db, + ) + expect(rows[0]?.token_hash).not.toBe(token) + await db.destroy() + }) + + it('rejects an expired session', async () => { + const db = await setup() + const start = new Date('2026-09-01T00:00:00Z') + const token = await createWebSession(db, 'did:plc:alice', 30, start) + expect(await resolveWebSession(db, token, 30, new Date(start.getTime() + 31 * DAY))).toBeNull() + await db.destroy() + }) + + it('extends a session used in the second half of its life', async () => { + const db = await setup() + const start = new Date('2026-09-01T00:00:00Z') + const token = await createWebSession(db, 'did:plc:alice', 30, start) + await resolveWebSession(db, token, 30, new Date(start.getTime() + 20 * DAY)) + expect(await resolveWebSession(db, token, 30, new Date(start.getTime() + 45 * DAY))).toBe( + 'did:plc:alice', + ) + await db.destroy() + }) + + it('does not extend a session used in the first half of its life', async () => { + const db = await setup() + const start = new Date('2026-09-01T00:00:00Z') + const token = await createWebSession(db, 'did:plc:alice', 30, start) + await resolveWebSession(db, token, 30, new Date(start.getTime() + 5 * DAY)) + expect(await resolveWebSession(db, token, 30, new Date(start.getTime() + 31 * DAY))).toBeNull() + await db.destroy() + }) + + it('deletes every session for a user', async () => { + const db = await setup() + const a = await createWebSession(db, 'did:plc:alice', 30) + const b = await createWebSession(db, 'did:plc:alice', 30) + await deleteAllWebSessions(db, 'did:plc:alice') + expect(await resolveWebSession(db, a, 30)).toBeNull() + expect(await resolveWebSession(db, b, 30)).toBeNull() + await db.destroy() + }) +}) diff --git a/apps/server/tests/blobs/routes.test.ts b/apps/server/tests/blobs/routes.test.ts new file mode 100644 index 0000000..ec6d622 --- /dev/null +++ b/apps/server/tests/blobs/routes.test.ts @@ -0,0 +1,140 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { createApp } from '../../src/app.ts' +import { buildScope } from '../../src/auth/scope.ts' +import { createBlobStores } from '../../src/blobs/store.ts' +import { IngesterRegistry } from '../../src/plugins/registry.ts' +import { authDeps, fakeOAuth, fakeSession, ORIGIN, sessionCookie } from '../helpers/auth.ts' +import { testConfig } from '../helpers/config.ts' +import { spacesHarness } from '../helpers/spaces.ts' + +async function setup(storageMode: 'space' | 'local') { + const h = await spacesHarness({ storageMode }) + const stores = createBlobStores({ + getPdsClient: async (did) => h.pds.client(did) as never, + dataDir: mkdtempSync(join(tmpdir(), 'scn-')), + db: h.db, + }) + const ingesters = new IngesterRegistry() + ingesters.register( + { + id: 'pdf-text', + accepts: ['application/pdf'], + method: 'text', + ingest: async ({ bytes }) => { + const text = new TextDecoder().decode(bytes) + if (text.includes('SCAN')) throw new Error('This PDF has no text layer') + return { text: `extracted: ${text}` } + }, + }, + 'test', + ) + const scope = storageMode === 'space' ? buildScope('raw') : 'atproto' + const oauth = fakeOAuth({ + callback: vi.fn(async () => ({ session: fakeSession('did:plc:alice', scope) })), + }) + const app = createApp({ + config: testConfig(), + db: h.db, + logger: h.logger, + auth: authDeps(h.db, { oauth }), + blobs: { blobsFor: stores.forAccount, services: h.services, ingesters, logger: h.logger }, + }) + const cookie = sessionCookie(await app.request('/oauth/callback?code=a&state=b')) + const upload = async (bytes: Uint8Array, type: string, name = 'file') => { + const res = await app.request('/api/attachments', { + method: 'POST', + headers: { cookie, origin: ORIGIN, 'content-type': type, 'x-filename': name }, + body: bytes, + }) + return { + status: res.status, + body: (await res.json()) as { part?: Record; message?: string }, + } + } + return { ...h, app, cookie, upload } +} + +describe.each(['space', 'local'] as const)('attachments for a %s account', (mode) => { + it('turns an uploaded image into an image part whose blob the conversation can serve once referenced', async () => { + const { upload, chats, app, cookie } = await setup(mode) + const { status, body } = await upload(new Uint8Array([137, 80, 78, 71]), 'image/png', 'pic.png') + expect(status).toBe(201) + expect(body.part).toMatchObject({ + $type: 'network.sharedcomputer.chat.defs#imagePart', + image: { $type: 'blob', mimeType: 'image/png' }, + }) + const { skey } = await chats.createConversation() + await chats.createMessage(skey, '3mmmmmmmmmmm1', { + $type: 'network.sharedcomputer.chat.message', + role: 'user', + content: { $type: 'network.sharedcomputer.chat.defs#plainContent', parts: [body.part] }, + createdAt: new Date().toISOString(), + } as never) + const cid = (body.part?.image as { ref: { $link: string } }).ref.$link + const res = await app.request(`/api/conversations/${skey}/blobs/${cid}?type=image/png`, { + headers: { cookie }, + }) + expect(res.status).toBe(200) + expect(res.headers.get('x-content-type-options')).toBe('nosniff') + expect(new Uint8Array(await res.arrayBuffer())).toEqual(new Uint8Array([137, 80, 78, 71])) + }) + + it('extracts text from a PDF and stores it as a text/plain blob with method text', async () => { + const { upload } = await setup(mode) + const { status, body } = await upload( + new TextEncoder().encode('%PDF body'), + 'application/pdf', + 'report.pdf', + ) + expect(status).toBe(201) + expect(body.part).toMatchObject({ + $type: 'network.sharedcomputer.chat.defs#filePart', + name: 'report.pdf', + file: { mimeType: 'application/pdf' }, + extracted: { method: 'text', text: { mimeType: 'text/plain' } }, + }) + }) + + it('fails with a clear message for a PDF without text', async () => { + const { upload } = await setup(mode) + const { status, body } = await upload(new TextEncoder().encode('SCAN'), 'application/pdf') + expect(status).toBe(422) + expect(body.message).toMatch(/no text layer/) + }) + + it('refuses an unsupported type with 415', async () => { + const { upload } = await setup(mode) + expect((await upload(new Uint8Array([1]), 'text/csv')).status).toBe(415) + }) + + it('refuses an image over 20 MB with 413', async () => { + const { upload } = await setup(mode) + expect((await upload(new Uint8Array(20_000_001), 'image/png')).status).toBe(413) + }) +}) + +describe('blob route', () => { + it('returns 404 for a blob not referenced in the conversation', async () => { + const { upload, chats, app, cookie } = await setup('space') + const { body } = await upload(new Uint8Array([1, 2]), 'image/png') + const { skey } = await chats.createConversation() + const cid = (body.part?.image as { ref: { $link: string } }).ref.$link + expect( + (await app.request(`/api/conversations/${skey}/blobs/${cid}`, { headers: { cookie } })) + .status, + ).toBe(404) + }) + + it('serves non-image blobs as downloads', async () => { + const { upload, app, cookie } = await setup('local') + const { body } = await upload(new TextEncoder().encode('%PDF body'), 'application/pdf') + const cid = (body.part?.file as { ref: { $link: string } }).ref.$link + const res = await app.request(`/api/conversations/3aaaaaaaaaaaa/blobs/${cid}`, { + headers: { cookie }, + }) + expect(res.headers.get('content-disposition')).toBe('attachment') + }) +}) diff --git a/apps/server/tests/blobs/store.test.ts b/apps/server/tests/blobs/store.test.ts new file mode 100644 index 0000000..d62de46 --- /dev/null +++ b/apps/server/tests/blobs/store.test.ts @@ -0,0 +1,97 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, utimesSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { sha256RawToCid } from '@atproto/common' +import { describe, expect, it } from 'vitest' +import { BlobNotFound, LocalBlobStore, SpaceBlobStore } from '../../src/blobs/store.ts' +import { spacesHarness, userMessage } from '../helpers/spaces.ts' + +describe('LocalBlobStore', () => { + const setup = async () => { + const h = await spacesHarness({ storageMode: 'local' }) + const dir = mkdtempSync(join(tmpdir(), 'scn-blobs-')) + return { ...h, store: new LocalBlobStore(dir, h.db), dir } + } + + it('gives blobs the CID a PDS computes for the same bytes', async () => { + const { store, account } = await setup() + const bytes = new TextEncoder().encode('hello blob') + const ref = await store.put(account, bytes, 'text/plain') + const expected = sha256RawToCid(createHash('sha256').update(bytes).digest()).toString() + expect(ref).toEqual({ + $type: 'blob', + ref: { $link: expected }, + mimeType: 'text/plain', + size: bytes.length, + }) + }) + + it('reads a stored blob back with its type', async () => { + const { store, account } = await setup() + const ref = (await store.put(account, new Uint8Array([1, 2, 3]), 'image/png')) as { + ref: { $link: string } + } + expect(await store.get(account, 'at://x', ref.ref.$link)).toEqual({ + bytes: new Uint8Array([1, 2, 3]), + mimeType: 'image/png', + }) + }) + + it('keeps each owner blobs separate', async () => { + const { store, account } = await setup() + const ref = (await store.put(account, new Uint8Array([9]), 'image/png')) as { + ref: { $link: string } + } + await expect( + store.get({ ...account, did: 'did:plc:bob' }, 'at://x', ref.ref.$link), + ).rejects.toBeInstanceOf(BlobNotFound) + }) + + it('sweeps unreferenced blobs older than a day and keeps referenced ones', async () => { + const { store, account, chats, dir } = await setup() + const orphan = (await store.put(account, new Uint8Array([1]), 'image/png')) as { + ref: { $link: string } + } + const kept = (await store.put(account, new Uint8Array([2]), 'image/png')) as { + ref: { $link: string } + } + const { skey } = await chats.createConversation() + await chats.createMessage(skey, '3mmmmmmmmmmm1', { + ...userMessage('see image'), + content: { + $type: 'network.sharedcomputer.chat.defs#plainContent', + parts: [{ $type: 'network.sharedcomputer.chat.defs#imagePart', image: kept }], + }, + } as never) + const old = (Date.now() - 2 * 86_400_000) / 1000 + for (const cid of [orphan.ref.$link, kept.ref.$link]) + utimesSync(join(dir, 'blobs', encodeURIComponent(account.did), cid), old, old) + expect(await store.sweep()).toBe(1) + await expect(store.get(account, 'at://x', orphan.ref.$link)).rejects.toBeInstanceOf( + BlobNotFound, + ) + await expect(store.get(account, 'at://x', kept.ref.$link)).resolves.toBeDefined() + }) +}) + +describe('SpaceBlobStore', () => { + it('reads a blob back through its conversation, with the type the PDS reports', async () => { + const h = await spacesHarness() + const store = new SpaceBlobStore(async (did) => h.pds.client(did) as never) + const image = await store.put(h.account, new Uint8Array([1, 2, 3]), 'image/png') + const { skey, uri } = await h.chats.createConversation() + await h.chats.createMessage(skey, '3mmmmmmmmmmm1', { + ...userMessage('see image'), + content: { + $type: 'network.sharedcomputer.chat.defs#plainContent', + parts: [{ $type: 'network.sharedcomputer.chat.defs#imagePart', image }], + }, + } as never) + const cid = (image as { ref: { $link: string } }).ref.$link + expect(await store.get(h.account, uri, cid)).toEqual({ + bytes: new Uint8Array([1, 2, 3]), + mimeType: 'image/png', + }) + }) +}) diff --git a/apps/server/tests/config.test.ts b/apps/server/tests/config.test.ts new file mode 100644 index 0000000..2f1e158 --- /dev/null +++ b/apps/server/tests/config.test.ts @@ -0,0 +1,162 @@ +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { ConfigError, interpolate, loadConfig, readConfigFile } from '../src/config.ts' + +const SECRET_KEY = Buffer.alloc(32, 1).toString('base64') +const env = { SECRET_KEY } + +describe('loadConfig', () => { + it('returns typed defaults when config.yml sets nothing', () => { + const config = loadConfig({ env, file: {} }) + expect(config).toMatchObject({ + nodeEnv: 'development', + port: 3000, + appName: 'SCN Chat', + oauthScopeMode: 'permission-set', + }) + expect(config.turns).toEqual({ + ratePerMinute: 10, + maxSteps: 8, + timeoutMs: 600_000, + backfillWindowMs: 3_600_000, + }) + }) + + it('reads public settings from config.yml', () => { + const config = loadConfig({ + env, + file: { + app: { name: 'My Chat', publicUrl: 'https://chat.example.com/', port: 8080 }, + turns: { maxSteps: 3 }, + roles: { staff: ['did:plc:a'] }, + }, + }) + expect(config).toMatchObject({ + appName: 'My Chat', + publicUrl: 'https://chat.example.com', + port: 8080, + roles: { staff: ['did:plc:a'] }, + }) + expect(config.turns.maxSteps).toBe(3) + }) + + it('lets environment variables override public settings', () => { + const config = loadConfig({ + env: { ...env, PORT: '9000', OAUTH_SCOPE_MODE: 'raw', TURN_MAX_STEPS: '2' }, + file: { app: { port: 8080 } }, + }) + expect(config).toMatchObject({ port: 9000, oauthScopeMode: 'raw' }) + expect(config.turns.maxSteps).toBe(2) + }) + + it('fails naming the setting when a value in config.yml is invalid', () => { + expect(() => loadConfig({ env, file: { app: { port: 'not-a-port' } } })).toThrow(/app\.port/) + }) + + it('refuses unknown top-level sections', () => { + expect(() => loadConfig({ env, file: { aap: {} } })).toThrow(ConfigError) + }) + + it('reports every invalid setting and secret at once', () => { + try { + loadConfig({ env: {}, file: { app: { port: 0, publicUrl: 'nope' } } }) + expect.unreachable() + } catch (err) { + expect((err as ConfigError).names.sort()).toEqual(['SECRET_KEY', 'app.port', 'app.publicUrl']) + } + }) + + it('fails naming SECRET_KEY when it is not 32 bytes', () => { + expect(() => + loadConfig({ env: { SECRET_KEY: Buffer.alloc(16).toString('base64') }, file: {} }), + ).toThrow(/SECRET_KEY/) + }) + + it('parses retired keys from a comma-separated list', () => { + const old = Buffer.alloc(32, 2).toString('base64') + expect( + loadConfig({ env: { ...env, SECRET_KEYS_OLD: `${old}, ${old}` }, file: {} }).oldSecretKeys, + ).toHaveLength(2) + }) + + it('treats empty values in .env as unset', () => { + const config = loadConfig({ + env: { ...env, OAUTH_PRIVATE_KEYS: '', SECRET_KEYS_OLD: '', DATABASE_URL: '' }, + file: {}, + }) + expect(config).toMatchObject({ + oauthPrivateKeys: [], + oldSecretKeys: [], + databaseUrl: 'sqlite:./data/scn-chat.sqlite', + }) + }) + + it('fails naming OAUTH_PRIVATE_KEYS when it is not a JSON array of keys', () => { + expect(() => loadConfig({ env: { ...env, OAUTH_PRIVATE_KEYS: 'not json' }, file: {} })).toThrow( + /OAUTH_PRIVATE_KEYS/, + ) + }) + + it('refuses an HTTP public URL, missing keys, and raw scopes in production', () => { + try { + loadConfig({ + env: { ...env, NODE_ENV: 'production' }, + file: { app: { publicUrl: 'http://chat.example.com' }, auth: { scopeMode: 'raw' } }, + }) + expect.unreachable() + } catch (err) { + expect((err as ConfigError).names.sort()).toEqual([ + 'OAUTH_PRIVATE_KEYS', + 'app.publicUrl', + 'auth.scopeMode', + ]) + } + }) + + it('returns a frozen object with the config directory for resolving plugins', () => { + const config = loadConfig({ env, file: {}, configPath: '/srv/scn/config.yml' }) + expect(Object.isFrozen(config)).toBe(true) + expect(config.configDir).toBe('/srv/scn') + }) +}) + +describe('interpolate', () => { + it('replaces references with environment values', () => { + expect(interpolate({ a: '${KEY}', b: 'Bearer ${KEY}' }, { KEY: 'sk-1' })).toEqual({ + a: 'sk-1', + b: 'Bearer sk-1', + }) + }) + + it('leaves out a value that is exactly a reference to a missing variable', () => { + expect(interpolate({ options: { apiKey: '${MISSING}', name: 'x' } }, {})).toEqual({ + options: { name: 'x' }, + }) + }) + + it('fails naming the variable when a longer string references a missing one', () => { + expect(() => interpolate({ url: 'https://${HOST}/v1' }, {})).toThrow(/HOST/) + }) + + it('reaches plugin options inside lists', () => { + const file = { plugins: [{ package: 'p', options: { apiKey: '${KEY}' } }] } + expect(interpolate(file, { KEY: 'k' })).toEqual({ + plugins: [{ package: 'p', options: { apiKey: 'k' } }], + }) + }) +}) + +describe('readConfigFile', () => { + it('fails with a message naming config.example.yml when the file is missing', () => { + expect(() => readConfigFile('/nonexistent/config.yml')).toThrow(/config\.example\.yml/) + }) + + it('reports invalid YAML as a config error with its line', () => { + const path = join(mkdtempSync(join(tmpdir(), 'scn-config-')), 'config.yml') + writeFileSync(path, 'models: []\n - provider: x\n') + expect(() => readConfigFile(path)).toThrow(ConfigError) + expect(() => readConfigFile(path)).toThrow(/not valid YAML.*line 2/s) + }) +}) diff --git a/apps/server/tests/db/index.test.ts b/apps/server/tests/db/index.test.ts new file mode 100644 index 0000000..c3bda09 --- /dev/null +++ b/apps/server/tests/db/index.test.ts @@ -0,0 +1,44 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { sql } from 'kysely' +import { afterEach, describe, expect, it } from 'vitest' +import { createDb, UnsupportedDatabaseError } from '../../src/db/index.ts' + +const cleanup: (() => Promise | void)[] = [] +afterEach(async () => { + for (const fn of cleanup.splice(0)) await fn() +}) + +describe('createDb', () => { + it('opens a working SQLite database for a sqlite: URL, creating its directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'scn-db-')) + cleanup.push(() => rmSync(dir, { recursive: true, force: true })) + const db = createDb(`sqlite:${join(dir, 'nested', 'app.sqlite')}`) + cleanup.push(() => db.destroy()) + const { rows } = await sql<{ one: number }>`select 1 as one`.execute(db) + expect(rows[0]?.one).toBe(1) + }) + + it('turns on WAL mode for file databases', async () => { + const dir = mkdtempSync(join(tmpdir(), 'scn-db-')) + cleanup.push(() => rmSync(dir, { recursive: true, force: true })) + const db = createDb(`sqlite:${join(dir, 'app.sqlite')}`) + cleanup.push(() => db.destroy()) + const { rows } = await sql<{ journal_mode: string }>`pragma journal_mode`.execute(db) + expect(rows[0]?.journal_mode).toBe('wal') + }) + + it('creates a Postgres database for a postgres:// URL without connecting', () => { + const db = createDb('postgres://user:pass@localhost:5432/scn') + cleanup.push(() => db.destroy()) + expect(db).toBeDefined() + }) + + it('rejects an unsupported scheme with a clear message', () => { + expect(() => createDb('mysql://localhost/scn')).toThrow(UnsupportedDatabaseError) + expect(() => createDb('mysql://localhost/scn')).toThrow( + /Unsupported DATABASE_URL scheme "mysql:"/, + ) + }) +}) diff --git a/apps/server/tests/db/migrate.test.ts b/apps/server/tests/db/migrate.test.ts new file mode 100644 index 0000000..fc574da --- /dev/null +++ b/apps/server/tests/db/migrate.test.ts @@ -0,0 +1,67 @@ +import { type Kysely, sql } from 'kysely' +import type { Migration } from 'kysely/migration' +import { afterEach, describe, expect, it } from 'vitest' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { dialects } from '../helpers/db.ts' + +const order: string[] = [] + +const fixtures: Record = { + '0001_first': { + up: async (db: Kysely) => { + order.push('0001_first') + await db.schema + .createTable('fixture') + .addColumn('id', 'text', (col) => col.primaryKey()) + .execute() + }, + }, + '0002_second': { + up: async (db: Kysely) => { + order.push('0002_second') + await db.schema.alterTable('fixture').addColumn('value', 'text').execute() + }, + }, +} + +afterEach(() => { + order.length = 0 +}) + +describe.each(dialects)('migrateToLatest on $name', ({ create }) => { + it('applies pending migrations in order', async () => { + const db = create() + const { applied } = await migrateToLatest(db, fixtures) + expect(applied).toEqual(['0001_first', '0002_second']) + expect(order).toEqual(['0001_first', '0002_second']) + await sql`insert into fixture (id, value) values ('a', 'b')`.execute(db) + await db.destroy() + }) + + it('applies nothing when run again', async () => { + const db = create() + await migrateToLatest(db, fixtures) + const { applied } = await migrateToLatest(db, fixtures) + expect(applied).toEqual([]) + await db.destroy() + }) + + it('fails loudly when a migration throws', async () => { + const db = create() + const broken: Record = { + '0001_broken': { + up: async () => { + throw new Error('boom') + }, + }, + } + await expect(migrateToLatest(db, broken)).rejects.toThrow('boom') + await db.destroy() + }) + + it('runs the app migrations cleanly', async () => { + const db = create() + await expect(migrateToLatest(db)).resolves.toBeDefined() + await db.destroy() + }) +}) diff --git a/apps/server/tests/fixtures/plugins/good/index.ts b/apps/server/tests/fixtures/plugins/good/index.ts new file mode 100644 index 0000000..7f8ddcc --- /dev/null +++ b/apps/server/tests/fixtures/plugins/good/index.ts @@ -0,0 +1,15 @@ +import { definePlugin } from '@scn-chat/plugin-api' +import { z } from 'zod' + +export const optionsSchema = z + .object({ greeting: z.string().default('hello'), count: z.number().optional() }) + .strict() + +export default function good(options: z.infer) { + return definePlugin({ + id: `good-${options.greeting}`, + name: 'Good', + apiVersion: 1, + setup: () => {}, + }) +} diff --git a/apps/server/tests/fixtures/plugins/good/package.json b/apps/server/tests/fixtures/plugins/good/package.json new file mode 100644 index 0000000..d30ffe4 --- /dev/null +++ b/apps/server/tests/fixtures/plugins/good/package.json @@ -0,0 +1,5 @@ +{ + "name": "good-plugin", + "type": "module", + "exports": "./index.ts" +} diff --git a/apps/server/tests/fixtures/plugins/noexport/index.ts b/apps/server/tests/fixtures/plugins/noexport/index.ts new file mode 100644 index 0000000..2eb7bc7 --- /dev/null +++ b/apps/server/tests/fixtures/plugins/noexport/index.ts @@ -0,0 +1 @@ +export const notAFactory = true diff --git a/apps/server/tests/fixtures/plugins/noexport/package.json b/apps/server/tests/fixtures/plugins/noexport/package.json new file mode 100644 index 0000000..9dac50f --- /dev/null +++ b/apps/server/tests/fixtures/plugins/noexport/package.json @@ -0,0 +1,5 @@ +{ + "name": "noexport-plugin", + "type": "module", + "exports": "./index.ts" +} diff --git a/apps/server/tests/helpers/auth.ts b/apps/server/tests/helpers/auth.ts new file mode 100644 index 0000000..f1b5325 --- /dev/null +++ b/apps/server/tests/helpers/auth.ts @@ -0,0 +1,59 @@ +import type { OAuthSession } from '@atproto/oauth-client-node' +import pino from 'pino' +import { vi } from 'vitest' +import type { IdentityResolver } from '../../src/auth/identity.ts' +import type { OAuthClientLike } from '../../src/auth/oauth-client.ts' +import { createRoles } from '../../src/auth/roles.ts' +import type { AuthDeps } from '../../src/auth/routes.ts' +import { buildScope } from '../../src/auth/scope.ts' +import type { Db } from '../../src/db/index.ts' +import { testConfig } from './config.ts' + +export const ORIGIN = 'http://127.0.0.1:3000' + +export function fakeSession(did: string, scope: string) { + return { did, getTokenInfo: vi.fn(async () => ({ scope })) } as unknown as OAuthSession +} + +export function fakeOAuth(overrides: Partial = {}): OAuthClientLike { + return { + clientMetadata: { client_id: 'http://localhost' }, + jwks: { keys: [] }, + authorize: vi.fn(async () => new URL('https://pds.test/oauth/authorize?request_uri=x')), + callback: vi.fn(async () => ({ session: fakeSession('did:plc:alice', buildScope('raw')) })), + restore: vi.fn(async (did: string) => fakeSession(did, buildScope('raw'))), + revoke: vi.fn(async () => {}), + ...overrides, + } +} + +export const fakeIdentity: IdentityResolver = { + resolve: vi.fn(async (did: string) => ({ + did, + handle: 'alice.test', + pdsUrl: 'https://pds.test', + })), + resolveHandle: vi.fn(async () => 'did:plc:alice'), + resolveSigningKey: vi.fn(async () => 'did:key:zQ3shtest'), +} + +export function authDeps(db: Db, overrides: Partial = {}): AuthDeps { + return { + config: testConfig(), + db, + logger: pino({ level: 'silent' }), + oauth: fakeOAuth(), + identity: fakeIdentity, + roles: createRoles({ staff: ['did:plc:alice'] }), + scope: buildScope('raw'), + ...overrides, + } +} + +/** The session cookie from a response, ready to send back. */ +export function sessionCookie(res: Response): string { + const header = res.headers.get('set-cookie') ?? '' + const match = header.match(/scn_session=([^;]+)/) + if (!match) throw new Error(`No session cookie in: ${header}`) + return `scn_session=${match[1]}` +} diff --git a/apps/server/tests/helpers/config.ts b/apps/server/tests/helpers/config.ts new file mode 100644 index 0000000..b0f5a3e --- /dev/null +++ b/apps/server/tests/helpers/config.ts @@ -0,0 +1,11 @@ +import { loadConfig } from '../../src/config.ts' + +export const TEST_SECRET_KEY = Buffer.alloc(32, 1).toString('base64') + +/** A test config: env holds secrets and overrides, file holds config.yml's contents. */ +export function testConfig(env: Record = {}, file: Record = {}) { + return loadConfig({ + env: { NODE_ENV: 'test', SECRET_KEY: TEST_SECRET_KEY, ...env }, + file: { app: { name: 'Test Chat' }, ...file }, + }) +} diff --git a/apps/server/tests/helpers/db.ts b/apps/server/tests/helpers/db.ts new file mode 100644 index 0000000..ae2a84d --- /dev/null +++ b/apps/server/tests/helpers/db.ts @@ -0,0 +1,23 @@ +import { PGlite } from '@electric-sql/pglite' +import SqliteDatabase from 'better-sqlite3' +import { Kysely, SqliteDialect } from 'kysely' +import { PGliteDialect } from 'kysely-pglite-dialect' +import type { Db } from '../../src/db/index.ts' +import type { Database } from '../../src/db/schema.ts' + +export type Dialect = { name: 'sqlite' | 'postgres'; create: () => Db } + +export function createSqliteDb(): Db { + const sqlite = new SqliteDatabase(':memory:') + sqlite.pragma('foreign_keys = ON') + return new Kysely({ dialect: new SqliteDialect({ database: sqlite }) }) +} + +/** Both databases, for tests that must pass on each. */ +export const dialects: Dialect[] = [ + { name: 'sqlite', create: createSqliteDb }, + { + name: 'postgres', + create: () => new Kysely({ dialect: new PGliteDialect(new PGlite()) }), + }, +] diff --git a/apps/server/tests/helpers/fake-pds.ts b/apps/server/tests/helpers/fake-pds.ts new file mode 100644 index 0000000..0ed7741 --- /dev/null +++ b/apps/server/tests/helpers/fake-pds.ts @@ -0,0 +1,262 @@ +import { TID } from '@atproto/common-web' +import { type Keypair, Secp256k1Keypair } from '@atproto/crypto' +import { cidForRawBytes, LexError, parseCid } from '@atproto/lex-data' +import { RepoCommit } from '@atproto/space' +import { atproto } from '@scn-chat/lexicons' +import type { Db } from '../../src/db/index.ts' +import { LocalRecordStore } from '../../src/storage/local-record-store.ts' +import { RecordExists, SpaceExists, SpaceNotFound } from '../../src/storage/record-store.ts' +import { parseSpaceUri, toJson, toLex } from '../../src/storage/records.ts' + +type Call = { nsid: string; input: Record } + +// Map the local store's counter revs onto TIDs, keeping their order. +const TID_BASE = 1_700_000_000_000_000 +const toTid = (rev: string) => TID.fromTime(TID_BASE + Number(rev), 0).toString() +const fromTid = (tid: string) => String(TID.fromStr(tid).timestamp() - TID_BASE) + +type OutputSchema = { safeParse(value: unknown): { success: boolean; reason?: unknown } } + +/** Fail the test when the fake answers in a shape the method's lexicon does not allow. */ +function assertOutput(method: { $nsid: string }, output: T): T { + const schema = (method as { main?: { output?: { schema?: OutputSchema } } }).main?.output?.schema + if (!schema) return output + const result = schema.safeParse(output) + if (!result.success) + throw new Error(`FakePds answered ${method.$nsid} off-lexicon: ${String(result.reason)}`) + return output +} + +/** A fake spaces PDS answering the space XRPC methods with a PDS's semantics and error codes. */ +export class FakePds { + readonly calls: Call[] = [] + private readonly db: Db + failNext: { nsid: string; error: string } | null = null + /** Signs commits, as a real PDS signs with the account key. */ + readonly keypair: Keypair + /** Sign commits over a set that includes a record the repo does not have. */ + tamperCommits = false + readonly blobs = new Map() + readonly policies = new Map< + string, + { readPolicy: string; members: Map } + >() + + private policy(space: string) { + let policy = this.policies.get(space) + if (!policy) { + policy = { readPolicy: 'com.atproto.simplespace.defs#memberListPolicy', members: new Map() } + this.policies.set(space, policy) + } + return policy + } + + /** Would the space authority issue this viewer a credential? */ + mayRead(space: string, viewer: string): boolean { + if (parseSpaceUri(space).did === viewer) return true + const policy = this.policy(space) + return policy.readPolicy.endsWith('#publicPolicy') || policy.members.get(viewer)?.read === true + } + + constructor(db: Db, keypair: Keypair) { + this.db = db + this.keypair = keypair + } + + static async create(db: Db): Promise { + return new FakePds(db, await Secp256k1Keypair.create()) + } + + client(did: string) { + const store = new LocalRecordStore(did, this.db) + const call = async ( + method: { $nsid: string }, + input: Record, + options?: { encoding?: string }, + ) => { + this.calls.push({ nsid: method.$nsid, input }) + if (method.$nsid === atproto.repo.uploadBlob.$nsid) { + const bytes = input as unknown as Uint8Array + const cid = await cidForRawBytes(bytes) + this.blobs.set(cid.toString(), { + bytes, + mimeType: options?.encoding ?? 'application/octet-stream', + }) + return { + blob: { $type: 'blob', ref: cid, mimeType: options?.encoding, size: bytes.length }, + } + } + if (this.failNext?.nsid === method.$nsid) { + const { error } = this.failNext + this.failNext = null + throw new LexError(error, `fake ${error}`) + } + try { + return assertOutput(method, await this.dispatch(store, method.$nsid, input)) + } catch (err) { + if (err instanceof RecordExists) throw new LexError('RecordAlreadyExists', err.message) + if (err instanceof SpaceExists) throw new LexError('SpaceAlreadyExists', err.message) + if (err instanceof SpaceNotFound) throw new LexError('SpaceNotFound', err.message) + throw err + } + } + return { + call, + /** Only getBlob, whose response encoding the server reads. */ + xrpc: async (method: { $nsid: string }, options: { params: Record }) => { + if (method.$nsid !== atproto.space.getBlob.$nsid) + throw new Error(`FakePds.xrpc does not implement ${method.$nsid}`) + const body = await call(method, options.params) + const blob = this.blobs.get(options.params.cid as string) + return { body, encoding: blob?.mimeType } + }, + } + } + + private async dispatch(store: LocalRecordStore, nsid: string, input: Record) { + const space = input.space as string + const uri = (collection: string, rkey: string) => `${space}/${store.did}/${collection}/${rkey}` + switch (nsid) { + case atproto.simplespace.createSpace.$nsid: + return { uri: await store.createSpace(input.type as string, input.skey as string) } + case atproto.simplespace.deleteSpace.$nsid: + return store.deleteSpace(space) + case atproto.space.getRecord.$nsid: { + const record = await store.getRecord( + space, + input.collection as string, + input.rkey as string, + ) + if (!record) throw new LexError('RecordNotFound', 'not found') + return { + uri: uri(input.collection as string, record.rkey), + cid: record.cid, + value: toLex(record.value), + } + } + case atproto.space.listRecords.$nsid: { + const collections = input.collection + ? [input.collection as string] + : [...new Set((await store.listKeys(space)).map((key) => key.collection))] + const records = [] + for (const collection of collections) { + for (const record of await store.listRecords(space, collection)) { + records.push({ + collection, + rkey: record.rkey, + cid: record.cid, + ...(input.excludeValues ? {} : { value: toLex(record.value) }), + }) + } + } + return { records } + } + case atproto.space.createRecord.$nsid: + case atproto.space.putRecord.$nsid: { + const write = nsid === atproto.space.createRecord.$nsid ? 'createRecord' : 'putRecord' + const collection = input.collection as string + const rkey = input.rkey as string + const { cid } = await store[write](space, collection, rkey, toJson(input.record)) + return { uri: uri(collection, rkey), cid } + } + case atproto.space.deleteRecord.$nsid: + await store.deleteRecord(space, input.collection as string, input.rkey as string) + return {} + case atproto.space.listRepoOps.$nsid: { + const head = await store.headRev(space) + if (!head) throw new LexError('RepoNotFound', 'no repo in space') + const since = input.since ? fromTid(input.since as string) : undefined + const page = await store.listOps(space, since) + const prevs = await this.prevCids(store, space) + const ops = page.ops.map((op) => ({ + ...op, + prev: prevs.get(`${op.rev}/${op.collection}/${op.rkey}`) ?? null, + rev: toTid(op.rev), + value: op.value ? toLex(op.value) : undefined, + })) + return { ops, commit: await this.commit(store, space, toTid(head)) } + } + case atproto.space.getLatestCommit.$nsid: { + const rev = await store.headRev(space) + if (!rev) throw new LexError('RepoNotFound', 'no repo in space') + return { commit: await this.commit(store, space, toTid(rev)) } + } + case atproto.space.getBlob.$nsid: { + const cid = input.cid as string + const referenced = await this.db + .selectFrom('local_record') + .select('rkey') + .where('space_uri', '=', space) + .where('value_json', 'like', `%${cid}%`) + .executeTakeFirst() + const blob = this.blobs.get(cid) + if (!referenced || !blob) throw new LexError('BlobNotFound', 'blob not found in space') + return blob.bytes + } + case atproto.simplespace.getSpace.$nsid: + return { + uri: space, + readPolicy: { $type: this.policy(space).readPolicy }, + writePolicy: { $type: 'com.atproto.simplespace.defs#memberListPolicy' }, + appAccess: { $type: 'com.atproto.simplespace.defs#open' }, + } + case atproto.simplespace.updateSpace.$nsid: + if (input.readPolicy) + this.policy(space).readPolicy = (input.readPolicy as { $type: string }).$type + return undefined + case atproto.simplespace.listMembers.$nsid: + return { + members: [...this.policy(space).members].map(([did, access]) => ({ did, ...access })), + } + case atproto.simplespace.putMember.$nsid: + this.policy(space).members.set(input.did as string, { + read: input.read as boolean, + write: input.write as boolean, + }) + return undefined + case atproto.simplespace.removeMember.$nsid: + this.policy(space).members.delete(input.did as string) + return undefined + case atproto.space.listSpaces.$nsid: + return { spaces: (await store.listSpaces(input.type as string)).map((u) => ({ uri: u })) } + default: + throw new Error(`FakePds does not implement ${nsid}`) + } + } + + /** Each op's previous CID for its record, as a PDS reports it, keyed by rev and record. */ + private async prevCids(store: LocalRecordStore, space: string) { + const prevs = new Map() + const latest = new Map() + for (const op of (await store.listOps(space)).ops) { + const key = `${op.collection}/${op.rkey}` + prevs.set(`${op.rev}/${key}`, latest.get(key) ?? null) + latest.set(key, op.cid) + } + return prevs + } + + private async commit(store: LocalRecordStore, space: string, rev: string) { + const keys = await store.listKeys(space) + const records = keys.map((key) => ({ + collection: key.collection, + rkey: key.rkey, + cid: parseCid(key.cid), + })) + if (this.tamperCommits) { + records.push({ + collection: 'network.sharedcomputer.chat.message', + rkey: 'phantom', + cid: parseCid(keys[0]?.cid ?? ''), + }) + } + return RepoCommit.fromRecords(records as any).sign( + { space, author: store.did, rev } as any, + this.keypair, + ) + } + + ownerOf(space: string): string { + return parseSpaceUri(space).did + } +} diff --git a/apps/server/tests/helpers/providers.ts b/apps/server/tests/helpers/providers.ts new file mode 100644 index 0000000..0d6cba6 --- /dev/null +++ b/apps/server/tests/helpers/providers.ts @@ -0,0 +1,21 @@ +import type { ModelProvider } from '@scn-chat/plugin-api' +import { MockLanguageModelV4 } from 'ai/test' +import { vi } from 'vitest' + +/** A provider whose models record how they were created. */ +export function fakeProvider(overrides: Partial = {}) { + const created: { modelId: string; apiKey?: string; baseURL?: string; fetch?: unknown }[] = [] + const provider: ModelProvider = { + id: 'fake', + name: 'Fake', + hasAdminKey: true, + userKeys: true, + replay: 'replay', + createModel: vi.fn((access) => { + created.push(access) + return new MockLanguageModelV4({ provider: overrides.id ?? 'fake', modelId: access.modelId }) + }), + ...overrides, + } + return { provider, created } +} diff --git a/apps/server/tests/helpers/spaces.ts b/apps/server/tests/helpers/spaces.ts new file mode 100644 index 0000000..82d5485 --- /dev/null +++ b/apps/server/tests/helpers/spaces.ts @@ -0,0 +1,84 @@ +import { nsid } from '@scn-chat/lexicons' +import pino from 'pino' +import { vi } from 'vitest' +import { recordLogin } from '../../src/auth/accounts.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { createChatServices } from '../../src/storage/services.ts' +import type { CredentialCache } from '../../src/sync/credentials.ts' +import { SyncEngine } from '../../src/sync/engine.ts' +import { SyncEventBus } from '../../src/sync/events.ts' +import { RecentWrites } from '../../src/sync/recent-writes.ts' +import { createSqliteDb } from './db.ts' +import { FakePds } from './fake-pds.ts' + +export const ALICE = 'did:plc:alice' + +export const textContent = (text: string) => ({ + $type: `${nsid.defs}#plainContent`, + parts: [{ $type: `${nsid.defs}#textPart`, text }], +}) + +export const userMessage = (text: string, extra: Record = {}) => ({ + $type: nsid.message, + role: 'user', + content: textContent(text), + createdAt: new Date().toISOString(), + ...extra, +}) + +/** A spaces account on a fake PDS, with the services and sync engine the server would build. */ +export async function spacesHarness( + options: { backfillWindowMs?: number; storageMode?: 'space' | 'local' } = {}, +) { + const db = createSqliteDb() + await migrateToLatest(db) + const logger = pino({ level: 'silent' }) + const pds = await FakePds.create(db) + const events = new SyncEventBus() + const recentWrites = new RecentWrites() + const services = createChatServices({ + db, + getPdsClient: async (did) => pds.client(did) as never, + logger, + events, + recentWrites, + }) + const account = await recordLogin(db, { + did: ALICE, + handle: 'alice.test', + pdsUrl: 'https://pds.test', + spacesAllowed: (options.storageMode ?? 'space') === 'space', + }) + const credentials = { + withCredential: vi.fn( + async (_did: string, _space: string, fn: (credential: unknown) => Promise) => + fn({ + client: () => ({ + call: async () => ({ expiresAt: new Date(Date.now() + 86_400_000).toISOString() }), + }), + }), + ), + } as unknown as CredentialCache + const engine = new SyncEngine({ + db, + services, + events, + recentWrites, + credentials, + resolveSigningKey: async () => pds.keypair.did(), + publicUrl: 'https://chat.example.com', + logger, + backfillWindowMs: options.backfillWindowMs ?? 60 * 60_000, + }) + /** Write as another client would, straight to the PDS, bypassing the server's services. */ + const external = services.storeFor(account) + const chats = services.forAccount(account) + await chats.ensureSettingsSpace() + return { db, pds, events, recentWrites, services, account, chats, engine, external, logger } +} + +export function collect(events: SyncEventBus, name: E) { + const seen: unknown[] = [] + events.on(name as never, ((payload: unknown) => seen.push(payload)) as never) + return seen as never[] +} diff --git a/apps/server/tests/helpers/turns.ts b/apps/server/tests/helpers/turns.ts new file mode 100644 index 0000000..3c385b1 --- /dev/null +++ b/apps/server/tests/helpers/turns.ts @@ -0,0 +1,148 @@ +import type { LanguageModelV4, ModelProvider } from '@scn-chat/plugin-api' +import { MockLanguageModelV4, simulateReadableStream } from 'ai/test' +import { createRoles } from '../../src/auth/roles.ts' +import type { ModelConfig } from '../../src/config.ts' +import type { PluginHost } from '../../src/plugins/host.ts' +import { Registry } from '../../src/plugins/registry.ts' +import { ModelCatalog } from '../../src/providers/catalog.ts' +import { SecretBox } from '../../src/secrets.ts' +import { type TurnBlobs, TurnRunner } from '../../src/turns/runner.ts' +import { StreamHub } from '../../src/turns/stream-hub.ts' +import { spacesHarness } from './spaces.ts' + +const usage = { + inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 5, text: 5, reasoning: 0 }, +} + +/** Stream parts for a plain text reply. */ +export function textReply(text: string) { + return [ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: 't1' }, + ...[...text].map((char) => ({ type: 'text-delta', id: 't1', delta: char })), + { type: 'text-end', id: 't1' }, + { type: 'finish', usage, finishReason: { unified: 'stop', raw: 'stop' } }, + ] +} + +/** A model that streams the given parts, recording each prompt it was sent. */ +export function scriptedModel(parts: unknown[], options: { delayMs?: number; fail?: Error } = {}) { + const prompts: unknown[] = [] + const model = new MockLanguageModelV4({ + doStream: async (call) => { + prompts.push(call.prompt) + if (options.fail) throw options.fail + return { + stream: simulateReadableStream({ + chunks: parts as never[], + chunkDelayInMs: options.delayMs ?? 0, + }), + } + }, + }) + return { model, prompts } +} + +/** A model that answers each call with the next response in the list. */ +export function sequenceModel(responses: unknown[][]) { + let call = 0 + const prompts: unknown[] = [] + const model = new MockLanguageModelV4({ + doStream: async (options) => { + prompts.push(options.prompt) + const parts = responses[Math.min(call++, responses.length - 1)] ?? [] + return { stream: simulateReadableStream({ chunks: parts as never[], chunkDelayInMs: 0 }) } + }, + }) + return { model, prompts } +} + +export async function turnsHarness( + options: { + storageMode?: 'space' | 'local' + model?: () => LanguageModelV4 + adminModels?: Partial[] + host?: PluginHost + ratePerMinute?: number + timeoutMs?: number + } = {}, +) { + const h = await spacesHarness({ storageMode: options.storageMode }) + let modelFactory = options.model ?? (() => scriptedModel(textReply('Hello')).model) + const created: string[] = [] + const provider: ModelProvider = { + id: 'fake', + name: 'Fake', + hasAdminKey: true, + userKeys: true, + replay: 'replay', + createModel: ({ modelId }) => { + created.push(modelId) + return modelFactory() + }, + } + const providers = new Registry('provider', (p) => p.id) + providers.register(provider, 'test') + const caps = { vision: true, reasoning: true, tools: true } + const adminModels = (options.adminModels ?? [{ id: 'default-model', default: true }]).map( + (m) => ({ + provider: 'fake', + id: 'm', + name: 'M', + capabilities: caps, + roles: ['user'], + ...m, + }), + ) as ModelConfig[] + const catalog = new ModelCatalog({ + db: h.db, + box: new SecretBox(Buffer.alloc(32, 8)), + providers, + adminModels, + roles: createRoles(), + guardedFetch: fetch, + logger: h.logger, + }) + const stored = new Map() + const blobs: TurnBlobs = { + reader: () => async (cid) => ({ + bytes: stored.get(cid) ?? new Uint8Array(), + mimeType: 'text/plain', + }), + put: async (_account, bytes, mimeType) => { + const cid = `bafkrei${stored.size}fake` + stored.set(cid, bytes) + return { + $type: 'blob', + ref: { $link: 'bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy' }, + mimeType, + size: bytes.length, + } + }, + } + const hub = new StreamHub() + const runner = new TurnRunner({ + db: h.db, + services: h.services, + catalog, + host: options.host, + hub, + blobs, + config: { + ratePerMinute: options.ratePerMinute ?? 100, + maxSteps: 4, + timeoutMs: options.timeoutMs ?? 10_000, + backfillWindowMs: 60 * 60_000, + }, + logger: h.logger, + }) + const setModel = (factory: () => LanguageModelV4) => { + modelFactory = factory + } + const messages = async (skey: string) => { + const { messages } = await h.chats.getConversation(skey) + return new Map(messages.map((m) => [m.rkey, m.value])) + } + return { ...h, runner, hub, catalog, created, setModel, messages, blobs, stored } +} diff --git a/apps/server/tests/plugins/hooks.test.ts b/apps/server/tests/plugins/hooks.test.ts new file mode 100644 index 0000000..773138c --- /dev/null +++ b/apps/server/tests/plugins/hooks.test.ts @@ -0,0 +1,75 @@ +import type { PlainContent, TurnContext } from '@scn-chat/plugin-api' +import { describe, expect, it, vi } from 'vitest' +import { HookRunner, PluginHookError } from '../../src/plugins/hooks.ts' + +const context = {} as TurnContext +const content = (text: string): PlainContent => ({ + parts: [{ $type: 'network.sharedcomputer.chat.defs#textPart', text }], +}) +const textOf = (value: PlainContent) => (value.parts[0] as { text: string }).text +const logger = { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() } + +describe('HookRunner.filter', () => { + it('runs pre, then normal, then post, with config order inside each group', async () => { + const hooks = new HookRunner() + const append = (tag: string) => (value: PlainContent) => content(`${textOf(value)}${tag}`) + hooks.add('message:afterModel', 'b', 1, append('[b-normal]')) + hooks.add('message:afterModel', 'c', 2, append('[c-pre]'), 'pre') + hooks.add('message:afterModel', 'a', 0, append('[a-post]'), 'post') + hooks.add('message:afterModel', 'a', 0, append('[a-normal]')) + const result = await hooks.filter('message:afterModel', content(''), context) + expect(textOf(result)).toBe('[c-pre][a-normal][b-normal][a-post]') + }) + + it('passes each filter the previous filter return value', async () => { + const hooks = new HookRunner() + const seen: string[] = [] + hooks.add('message:afterModel', 'a', 0, () => content('from a')) + hooks.add('message:afterModel', 'b', 1, (value) => { + seen.push(textOf(value)) + return value + }) + await hooks.filter('message:afterModel', content('original'), context) + expect(seen).toEqual(['from a']) + }) + + it('fails with an error naming the plugin when a filter throws', async () => { + const hooks = new HookRunner() + hooks.add('message:afterModel', 'broken', 0, () => { + throw new Error('boom') + }) + const promise = hooks.filter('message:afterModel', content('x'), context) + await expect(promise).rejects.toBeInstanceOf(PluginHookError) + await expect(promise).rejects.toThrow(/broken/) + }) +}) + +describe('HookRunner.action', () => { + it('logs a throwing action and still runs later actions', async () => { + const hooks = new HookRunner() + const later = vi.fn() + hooks.add('conversation:created', 'broken', 0, () => { + throw new Error('boom') + }) + hooks.add('conversation:created', 'fine', 1, later) + await hooks.action( + 'conversation:created', + { user: 'did:plc:a', conversation: 'at://x' }, + logger, + ) + expect(later).toHaveBeenCalledOnce() + expect(logger.error).toHaveBeenCalledWith( + expect.objectContaining({ pluginId: 'broken' }), + 'plugin action failed', + ) + }) +}) + +describe('HookRunner.describe', () => { + it('reports the resolved handler order', () => { + const hooks = new HookRunner() + hooks.add('turn:after', 'late', 1, () => {}, 'post') + hooks.add('turn:after', 'early', 0, () => {}) + expect(hooks.describe()).toEqual({ 'turn:after': ['early', 'late'] }) + }) +}) diff --git a/apps/server/tests/plugins/host.test.ts b/apps/server/tests/plugins/host.test.ts new file mode 100644 index 0000000..dd0aec7 --- /dev/null +++ b/apps/server/tests/plugins/host.test.ts @@ -0,0 +1,124 @@ +import { definePlugin, type ModelProvider, type Plugin } from '@scn-chat/plugin-api' +import pino from 'pino' +import { describe, expect, it, vi } from 'vitest' +import { loadPlugins, PluginLoadError, type PluginServices } from '../../src/plugins/host.ts' + +const logger = pino({ level: 'silent' }) +const app = { name: 'Test', publicUrl: 'http://127.0.0.1:3000' } +const services: PluginServices = { + generateText: vi.fn(async () => ({ text: 'generated', finishReason: 'stop' })), + updateInfo: vi.fn(async () => {}), + userSettings: vi.fn(async () => ({ enabled: true })), +} + +const plugin = (id: string, setup: Plugin['setup'] = () => {}, apiVersion = 1) => + definePlugin({ id, name: id, apiVersion, setup }) + +const provider = (id: string) => ({ id }) as ModelProvider + +describe('loadPlugins', () => { + it('calls each setup once, in config order', async () => { + const calls: string[] = [] + await loadPlugins( + [ + plugin('first', () => void calls.push('first')), + plugin('second', () => void calls.push('second')), + ], + { services, logger, app }, + ) + expect(calls).toEqual(['first', 'second']) + }) + + it('waits for async setups before loading the next plugin', async () => { + const calls: string[] = [] + const slow = plugin('slow', async () => { + await new Promise((resolve) => setTimeout(resolve, 10)) + calls.push('slow') + }) + await loadPlugins([slow, plugin('fast', () => void calls.push('fast'))], { + services, + logger, + app, + }) + expect(calls).toEqual(['slow', 'fast']) + }) + + it('fails when two plugins share an ID, naming it', async () => { + await expect( + loadPlugins([plugin('dup'), plugin('dup')], { services, logger, app }), + ).rejects.toThrow(/Two plugins use the ID "dup"/) + }) + + it('fails for an unsupported apiVersion, naming the plugin and version', async () => { + const promise = loadPlugins([plugin('future', () => {}, 2)], { services, logger, app }) + await expect(promise).rejects.toBeInstanceOf(PluginLoadError) + await expect(promise).rejects.toThrow(/"future".*version 2/) + }) + + it('fails for an invalid plugin ID', async () => { + await expect(loadPlugins([plugin('Bad ID')], { services, logger, app })).rejects.toThrow( + /must match/, + ) + }) + + it('fails when two plugins register the same provider ID', async () => { + const a = plugin('a', (ctx) => ctx.providers.register(provider('anthropic'))) + const b = plugin('b', (ctx) => ctx.providers.register(provider('anthropic'))) + await expect(loadPlugins([a, b], { services, logger, app })).rejects.toThrow(/anthropic/) + }) + + it('fails when two plugins register the same tool name or ingester ID', async () => { + const tool = { + name: 'search', + description: '', + inputSchema: {} as never, + run: async () => null, + } + const toolA = plugin('a', (ctx) => ctx.tools.register(tool)) + const toolB = plugin('b', (ctx) => ctx.tools.register(tool)) + await expect(loadPlugins([toolA, toolB], { services, logger, app })).rejects.toThrow(/search/) + const ingester = { + id: 'pdf', + accepts: ['application/pdf'], + method: 'text' as const, + ingest: async () => ({ text: '' }), + } + const ingA = plugin('a', (ctx) => ctx.ingesters.register(ingester)) + const ingB = plugin('b', (ctx) => ctx.ingesters.register(ingester)) + await expect(loadPlugins([ingA, ingB], { services, logger, app })).rejects.toThrow(/pdf/) + }) + + it('gives plugins the services through their context', async () => { + let seen: unknown + const reader = plugin('reader', async (ctx) => { + seen = await ctx.userSettings('did:plc:a') + await ctx.conversations.updateInfo( + 'did:plc:a', + 'at://x', + { title: 'T' }, + { unlessUserTitled: true }, + ) + }) + await loadPlugins([reader], { services, logger, app }) + expect(seen).toEqual({ enabled: true }) + expect(services.updateInfo).toHaveBeenCalledWith( + 'did:plc:a', + 'at://x', + { title: 'T' }, + { unlessUserTitled: true }, + ) + }) + + it('runs registered cleanup at close, newest first', async () => { + const order: string[] = [] + const host = await loadPlugins( + [ + plugin('a', (ctx) => ctx.onClose(() => void order.push('a'))), + plugin('b', (ctx) => ctx.onClose(() => void order.push('b'))), + ], + { services, logger, app }, + ) + await host.close() + expect(order).toEqual(['b', 'a']) + }) +}) diff --git a/apps/server/tests/plugins/import.test.ts b/apps/server/tests/plugins/import.test.ts new file mode 100644 index 0000000..921efc8 --- /dev/null +++ b/apps/server/tests/plugins/import.test.ts @@ -0,0 +1,76 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { parse } from 'yaml' +import { loadConfig } from '../../src/config.ts' +import { importPlugins } from '../../src/plugins/import.ts' + +const fixtures = fileURLToPath(new URL('../fixtures/plugins', import.meta.url)) +const repoRoot = fileURLToPath(new URL('../../../..', import.meta.url)) + +describe('importPlugins', () => { + it('loads a local plugin by path, validating and defaulting its options', async () => { + const plugins = await importPlugins( + [{ package: './good' }, { package: './good', options: { greeting: 'hi' } }], + fixtures, + ) + expect(plugins.map((plugin) => plugin.id)).toEqual(['good-hello', 'good-hi']) + }) + + it('stops with an error naming the package and option when options fail the schema', async () => { + await expect( + importPlugins([{ package: './good', options: { count: 'many' } }], fixtures), + ).rejects.toThrow(/"\.\/good".*count/) + await expect( + importPlugins([{ package: './good', options: { typo: true } }], fixtures), + ).rejects.toThrow(/"\.\/good"/) + }) + + it('stops with an error naming a package that cannot be found, and why', async () => { + await expect(importPlugins([{ package: '@scn-chat/plugin-nope' }], fixtures)).rejects.toThrow( + /Cannot load plugin package "@scn-chat\/plugin-nope".*Cannot find/, + ) + }) + + it('reports a broken package.json in a local plugin as a parse error', async () => { + const dir = mkdtempSync(join(tmpdir(), 'scn-plugin-')) + writeFileSync(join(dir, 'package.json'), '{ not json') + await expect(importPlugins([{ package: dir }], fixtures)).rejects.toThrow(/JSON/) + rmSync(dir, { recursive: true, force: true }) + }) + + it('stops when a package has no factory', async () => { + await expect(importPlugins([{ package: './noexport' }], fixtures)).rejects.toThrow( + /no default export factory/, + ) + }) + + it('resolves npm packages from the config directory', async () => { + const [plugin] = await importPlugins([{ package: '@scn-chat/plugin-titles' }], repoRoot) + expect(plugin?.id).toBe('titles') + }) +}) + +describe('config.example.yml', () => { + it('loads with every shipped plugin, with no secrets set', async () => { + const file = parse(readFileSync(`${repoRoot}/config.example.yml`, 'utf8')) + const config = loadConfig({ + env: { SECRET_KEY: Buffer.alloc(32, 1).toString('base64') }, + file, + configPath: `${repoRoot}/config.example.yml`, + }) + const plugins = await importPlugins(config.plugins, config.configDir) + expect(plugins.map((plugin) => plugin.id)).toEqual([ + 'openai-compatible-scn', + 'openai-compatible-cocore', + 'anthropic', + 'openai', + 'google', + 'openai-compatible-custom', + 'pdf-text', + 'titles', + ]) + }) +}) diff --git a/apps/server/tests/plugins/registry.test.ts b/apps/server/tests/plugins/registry.test.ts new file mode 100644 index 0000000..1079fe4 --- /dev/null +++ b/apps/server/tests/plugins/registry.test.ts @@ -0,0 +1,56 @@ +import type { Ingester } from '@scn-chat/plugin-api' +import { describe, expect, it } from 'vitest' +import { + DuplicateRegistrationError, + IngesterRegistry, + Registry, +} from '../../src/plugins/registry.ts' + +const ingester = (id: string, accepts: string[], priority?: number): Ingester => ({ + id, + accepts, + priority, + method: 'text', + ingest: async () => ({ text: id }), +}) + +describe('Registry', () => { + it('returns registered items by key', () => { + const registry = new Registry<{ id: string }>('provider', (item) => item.id) + registry.register({ id: 'anthropic' }, 'plugin-a') + expect(registry.get('anthropic')).toEqual({ id: 'anthropic' }) + expect(registry.list()).toHaveLength(1) + }) + + it('refuses a duplicate key, naming both plugins', () => { + const registry = new Registry<{ id: string }>('provider', (item) => item.id) + registry.register({ id: 'anthropic' }, 'plugin-a') + expect(() => registry.register({ id: 'anthropic' }, 'plugin-b')).toThrow( + DuplicateRegistrationError, + ) + expect(() => registry.register({ id: 'anthropic' }, 'plugin-b')).toThrow( + /plugin-b.*anthropic.*plugin-a/, + ) + }) +}) + +describe('IngesterRegistry.match', () => { + it('picks the highest priority ingester that accepts the type', () => { + const registry = new IngesterRegistry() + registry.register(ingester('pdf-text', ['application/pdf']), 'a') + registry.register(ingester('ocr', ['application/pdf', 'image/*'], 10), 'b') + expect(registry.match('application/pdf')?.id).toBe('ocr') + }) + + it('matches wildcard types', () => { + const registry = new IngesterRegistry() + registry.register(ingester('ocr', ['image/*']), 'a') + expect(registry.match('image/png')?.id).toBe('ocr') + }) + + it('returns nothing when no ingester accepts the type', () => { + const registry = new IngesterRegistry() + registry.register(ingester('pdf-text', ['application/pdf']), 'a') + expect(registry.match('text/csv')).toBeUndefined() + }) +}) diff --git a/apps/server/tests/plugins/routes.test.ts b/apps/server/tests/plugins/routes.test.ts new file mode 100644 index 0000000..5a5d400 --- /dev/null +++ b/apps/server/tests/plugins/routes.test.ts @@ -0,0 +1,135 @@ +import { definePlugin } from '@scn-chat/plugin-api' +import pino from 'pino' +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { createApp } from '../../src/app.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { loadPlugins } from '../../src/plugins/host.ts' +import { SecretBox } from '../../src/secrets.ts' +import { authDeps, ORIGIN, sessionCookie } from '../helpers/auth.ts' +import { testConfig } from '../helpers/config.ts' +import { createSqliteDb } from '../helpers/db.ts' + +const logger = pino({ level: 'silent' }) +const search = definePlugin({ + id: 'search', + name: 'Search', + apiVersion: 1, + userSettings: z.object({ + engine: z.string().default('duckduckgo'), + apiKey: z.string().default('').meta({ secret: true }), + }), + setup: () => {}, +}) + +async function setup() { + const db = createSqliteDb() + await migrateToLatest(db) + const box = new SecretBox(Buffer.alloc(32, 3)) + const host = await loadPlugins( + [search, definePlugin({ id: 'plain', name: 'Plain', apiVersion: 1, setup: () => {} })], + { + services: {} as never, + logger, + app: { name: 'Test', publicUrl: ORIGIN }, + }, + ) + const app = createApp({ + config: testConfig(), + db, + logger, + auth: authDeps(db), + plugins: { db, box, host }, + }) + const cookie = sessionCookie(await app.request('/oauth/callback?code=a&state=b')) + return { app, db, cookie } +} + +type Settings = { + plugins: { + id: string + values: Record + secretsSet: string[] + error: string | null + schema: object + }[] +} + +describe('plugin settings API', () => { + it('requires a signed-in user', async () => { + const { app } = await setup() + expect((await app.request('/api/plugins/settings')).status).toBe(401) + }) + + it('lists only plugins with user settings, with a JSON schema for the form', async () => { + const { app, cookie } = await setup() + const body = (await ( + await app.request('/api/plugins/settings', { headers: { cookie } }) + ).json()) as Settings + expect(body.plugins.map((plugin) => plugin.id)).toEqual(['search']) + expect(body.plugins[0]?.schema).toMatchObject({ type: 'object' }) + }) + + it('saves settings and never returns secret values', async () => { + const { app, cookie } = await setup() + const put = await app.request('/api/plugins/search/settings', { + method: 'PUT', + headers: { cookie, origin: ORIGIN, 'content-type': 'application/json' }, + body: JSON.stringify({ engine: 'kagi', apiKey: 'sk-secret' }), + }) + expect(put.status).toBe(200) + const body = (await ( + await app.request('/api/plugins/settings', { headers: { cookie } }) + ).json()) as Settings + expect(body.plugins[0]).toMatchObject({ + values: { engine: 'kagi', apiKey: '' }, + secretsSet: ['apiKey'], + }) + expect(JSON.stringify(body)).not.toContain('sk-secret') + }) + + it('rejects invalid settings with 400', async () => { + const { app, cookie } = await setup() + const res = await app.request('/api/plugins/search/settings', { + method: 'PUT', + headers: { cookie, origin: ORIGIN, 'content-type': 'application/json' }, + body: JSON.stringify({ engine: 42 }), + }) + expect(res.status).toBe(400) + }) + + it('reports invalid stored settings with an error instead of failing, and resets them on delete', async () => { + const { app, cookie, db } = await setup() + await db + .insertInto('plugin_user_settings') + .values({ + did: 'did:plc:alice', + plugin_id: 'search', + values_json: '{"engine":42}', + secrets_encrypted: null, + updated_at: 'now', + }) + .execute() + const broken = (await ( + await app.request('/api/plugins/settings', { headers: { cookie } }) + ).json()) as Settings + expect(broken.plugins[0]?.error).toMatch(/invalid/) + await app.request('/api/plugins/search/settings', { + method: 'DELETE', + headers: { cookie, origin: ORIGIN }, + }) + const reset = (await ( + await app.request('/api/plugins/settings', { headers: { cookie } }) + ).json()) as Settings + expect(reset.plugins[0]).toMatchObject({ error: null, values: { engine: 'duckduckgo' } }) + }) + + it('returns 404 for a plugin without user settings', async () => { + const { app, cookie } = await setup() + const res = await app.request('/api/plugins/plain/settings', { + method: 'DELETE', + headers: { cookie, origin: ORIGIN }, + }) + expect(res.status).toBe(404) + }) +}) diff --git a/apps/server/tests/plugins/user-settings.test.ts b/apps/server/tests/plugins/user-settings.test.ts new file mode 100644 index 0000000..2d98868 --- /dev/null +++ b/apps/server/tests/plugins/user-settings.test.ts @@ -0,0 +1,109 @@ +import { definePlugin } from '@scn-chat/plugin-api' +import { sql } from 'kysely' +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { + clearUserSettings, + InvalidPluginSettingsError, + readUserSettings, + secretKeys, + writeUserSettings, +} from '../../src/plugins/user-settings.ts' +import { SecretBox } from '../../src/secrets.ts' +import { dialects } from '../helpers/db.ts' + +const box = new SecretBox(Buffer.alloc(32, 7)) +const did = 'did:plc:alice' +const schema = z.object({ + engine: z.string().default('duckduckgo'), + apiKey: z.string().default('').meta({ secret: true }), +}) +const plugin = definePlugin({ + id: 'search', + name: 'Search', + apiVersion: 1, + userSettings: schema, + setup: () => {}, +}) + +describe('secretKeys', () => { + it('finds fields marked secret', () => { + expect(secretKeys(schema)).toEqual(['apiKey']) + }) +}) + +describe.each(dialects)('plugin user settings on $name', ({ create }) => { + const setup = async () => { + const db = create() + await migrateToLatest(db) + return db + } + + it('returns schema defaults when nothing is stored', async () => { + const db = await setup() + expect(await readUserSettings(db, box, did, plugin)).toEqual({ + engine: 'duckduckgo', + apiKey: '', + }) + await db.destroy() + }) + + it('round-trips settings, storing secret fields encrypted', async () => { + const db = await setup() + await writeUserSettings(db, box, did, plugin, { engine: 'kagi', apiKey: 'sk-secret' }) + expect(await readUserSettings(db, box, did, plugin)).toEqual({ + engine: 'kagi', + apiKey: 'sk-secret', + }) + const { rows } = await sql<{ + values_json: string + secrets_encrypted: string + }>`select * from plugin_user_settings`.execute(db) + expect(rows[0]?.values_json).not.toContain('sk-secret') + expect(rows[0]?.secrets_encrypted).not.toContain('sk-secret') + await db.destroy() + }) + + it('keeps the stored secret when a write leaves it empty', async () => { + const db = await setup() + await writeUserSettings(db, box, did, plugin, { engine: 'kagi', apiKey: 'sk-secret' }) + await writeUserSettings(db, box, did, plugin, { engine: 'brave', apiKey: '' }) + expect(await readUserSettings(db, box, did, plugin)).toEqual({ + engine: 'brave', + apiKey: 'sk-secret', + }) + await db.destroy() + }) + + it('raises instead of resetting when stored values fail validation', async () => { + const db = await setup() + await sql`insert into plugin_user_settings (did, plugin_id, values_json, secrets_encrypted, updated_at) + values (${did}, 'search', ${JSON.stringify({ engine: 42 })}, null, 'now')`.execute(db) + await expect(readUserSettings(db, box, did, plugin)).rejects.toBeInstanceOf( + InvalidPluginSettingsError, + ) + await db.destroy() + }) + + it('clears stored values so the defaults apply', async () => { + const db = await setup() + await writeUserSettings(db, box, did, plugin, { engine: 'kagi', apiKey: 'sk-secret' }) + await clearUserSettings(db, did, 'search') + expect(await readUserSettings(db, box, did, plugin)).toEqual({ + engine: 'duckduckgo', + apiKey: '', + }) + await db.destroy() + }) + + it('keeps each user separate', async () => { + const db = await setup() + await writeUserSettings(db, box, did, plugin, { engine: 'kagi' }) + expect(await readUserSettings(db, box, 'did:plc:bob', plugin)).toEqual({ + engine: 'duckduckgo', + apiKey: '', + }) + await db.destroy() + }) +}) diff --git a/apps/server/tests/providers/catalog.test.ts b/apps/server/tests/providers/catalog.test.ts new file mode 100644 index 0000000..8a4a6ac --- /dev/null +++ b/apps/server/tests/providers/catalog.test.ts @@ -0,0 +1,188 @@ +import pino from 'pino' +import { describe, expect, it } from 'vitest' +import { createRoles } from '../../src/auth/roles.ts' +import type { ModelConfig } from '../../src/config.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { Registry } from '../../src/plugins/registry.ts' +import { + ModelCatalog, + ModelUnavailable, + ProviderConfigError, + validateAdminModels, +} from '../../src/providers/catalog.ts' +import { saveCredential } from '../../src/providers/user-credentials.ts' +import { SecretBox } from '../../src/secrets.ts' +import { createSqliteDb } from '../helpers/db.ts' +import { fakeProvider } from '../helpers/providers.ts' + +const caps = { vision: true, reasoning: true, tools: true } +const model = (overrides: Partial = {}): ModelConfig => ({ + provider: 'fake', + id: 'big', + name: 'Big', + capabilities: caps, + roles: ['user'], + ...overrides, +}) +const roles = createRoles({ staff: ['did:plc:staff'] }) +const guardedFetch = (async () => new Response()) as unknown as typeof fetch + +function registryWith(...providers: ReturnType['provider'][]) { + const registry = new Registry<(typeof providers)[number]>('provider', (p) => p.id) + for (const provider of providers) registry.register(provider, 'test') + return registry +} + +describe('validateAdminModels', () => { + const registry = registryWith( + fakeProvider().provider, + fakeProvider({ id: 'keyless', hasAdminKey: false }).provider, + ) + + it('accepts valid models', () => { + expect(validateAdminModels([model()], registry, roles)).toHaveLength(1) + }) + + it.each([ + ['an unregistered provider', model({ provider: 'nope' }), /no plugin registered/], + ['a provider without an admin key', model({ provider: 'keyless' }), /no admin key/], + ['missing roles', model({ roles: undefined as never }), /must list the roles/], + ['empty roles', model({ roles: [] }), /must list the roles/], + ['an undefined role', model({ roles: ['vip'] }), /undefined role "vip"/], + ])('fails for %s', (_name, bad, message) => { + expect(() => validateAdminModels([bad], registry, roles)).toThrow(message) + }) + + it('fails when more than one model is marked default', () => { + expect(() => + validateAdminModels( + [model({ default: true }), model({ id: 'b', default: true })], + registry, + roles, + ), + ).toThrow(ProviderConfigError) + }) +}) + +async function catalogWith( + adminModels: ModelConfig[], + ...providers: ReturnType['provider'][] +) { + const db = createSqliteDb() + await migrateToLatest(db) + const box = new SecretBox(Buffer.alloc(32, 4)) + const registry = registryWith(...providers) + return { + db, + box, + catalog: new ModelCatalog({ + db, + box, + providers: registry, + adminModels, + roles, + guardedFetch, + logger: pino({ level: 'silent' }), + }), + } +} + +describe('ModelCatalog.resolve', () => { + it('uses the user key when the user has one for the model', async () => { + const { provider, created } = fakeProvider() + const { db, box, catalog } = await catalogWith([model()], provider) + await saveCredential(db, box, 'did:plc:alice', provider, { + apiKey: 'sk-user', + models: [{ id: 'big', name: 'Big', capabilities: caps }], + }) + await catalog.resolve('did:plc:alice', { provider: 'fake', id: 'big' }) + expect(created[0]).toMatchObject({ modelId: 'big', apiKey: 'sk-user' }) + }) + + it('falls back to the admin key when a user role allows the model', async () => { + const { provider, created } = fakeProvider() + const { catalog } = await catalogWith([model()], provider) + const resolved = await catalog.resolve('did:plc:alice', { provider: 'fake', id: 'big' }) + expect(created[0]?.apiKey).toBeUndefined() + expect(resolved.capabilities).toEqual(caps) + }) + + it('refuses an admin model the user roles do not allow, but allows the user own key for it', async () => { + const { provider } = fakeProvider() + const { db, box, catalog } = await catalogWith([model({ roles: ['staff'] })], provider) + await expect( + catalog.resolve('did:plc:alice', { provider: 'fake', id: 'big' }), + ).rejects.toBeInstanceOf(ModelUnavailable) + await expect( + catalog.resolve('did:plc:staff', { provider: 'fake', id: 'big' }), + ).resolves.toBeDefined() + await saveCredential(db, box, 'did:plc:alice', provider, { + apiKey: 'sk-user', + models: [{ id: 'big', name: 'Big', capabilities: caps }], + }) + await expect( + catalog.resolve('did:plc:alice', { provider: 'fake', id: 'big' }), + ).resolves.toBeDefined() + }) + + it('fails with ModelUnavailable naming the model', async () => { + const { catalog } = await catalogWith([], fakeProvider().provider) + await expect( + catalog.resolve('did:plc:alice', { provider: 'fake', id: 'ghost' }), + ).rejects.toThrow(/fake\/ghost/) + }) + + it('reaches a user endpoint by its slug through the guarded fetch', async () => { + const { provider, created } = fakeProvider({ + id: 'custom', + userEndpoints: true, + hasAdminKey: false, + }) + const { db, box, catalog } = await catalogWith([], provider) + await saveCredential(db, box, 'did:plc:alice', provider, { + apiKey: 'sk-mine', + slug: 'my-ollama', + baseUrl: 'https://ollama.example.com/v1', + models: [{ id: 'llama', name: 'Llama', capabilities: caps }], + }) + await catalog.resolve('did:plc:alice', { provider: 'user:my-ollama', id: 'llama' }) + expect(created[0]).toMatchObject({ + baseURL: 'https://ollama.example.com/v1', + fetch: guardedFetch, + }) + }) + + it('skips the guard for providers that allow private networks', async () => { + const { provider, created } = fakeProvider({ + id: 'local', + userEndpoints: true, + allowPrivateNetworks: true, + }) + const { db, box, catalog } = await catalogWith([], provider) + await saveCredential(db, box, 'did:plc:alice', provider, { + apiKey: 'k', + slug: 'home', + baseUrl: 'http://192.168.1.5:11434/v1', + models: [{ id: 'qwen', name: 'Qwen', capabilities: caps }], + }) + await catalog.resolve('did:plc:alice', { provider: 'user:home', id: 'qwen' }) + expect(created[0]?.fetch).toBeUndefined() + }) +}) + +describe('ModelCatalog.listForUser', () => { + it('lists allowed admin models and the user own models', async () => { + const { provider } = fakeProvider() + const { db, box, catalog } = await catalogWith( + [model({ default: true }), model({ id: 'secret', roles: ['staff'] })], + provider, + ) + await saveCredential(db, box, 'did:plc:alice', provider, { + apiKey: 'k', + models: [{ id: 'mine', name: 'Mine', capabilities: caps }], + }) + const models = await catalog.listForUser('did:plc:alice') + expect(models.map((m) => `${m.source}:${m.id}`)).toEqual(['admin:big', 'user:mine']) + expect(catalog.defaultModel()).toEqual({ provider: 'fake', id: 'big' }) + }) +}) diff --git a/apps/server/tests/providers/effort.test.ts b/apps/server/tests/providers/effort.test.ts new file mode 100644 index 0000000..538d310 --- /dev/null +++ b/apps/server/tests/providers/effort.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it, vi } from 'vitest' +import { mapEffort } from '../../src/providers/effort.ts' + +describe('mapEffort', () => { + it.each([ + [undefined, 'provider-default'], + ['none', 'none'], + ['low', 'low'], + ['medium', 'medium'], + ['high', 'high'], + ['max', 'xhigh'], + ])('maps %s to %s', (effort, expected) => { + expect(mapEffort(effort, { warn: vi.fn() } as never)).toBe(expected) + }) + + it('maps an unknown value to the provider default and logs a warning', () => { + const logger = { warn: vi.fn() } + expect(mapEffort('ludicrous', logger as never)).toBe('provider-default') + expect(logger.warn).toHaveBeenCalled() + }) +}) diff --git a/apps/server/tests/providers/generate-text.test.ts b/apps/server/tests/providers/generate-text.test.ts new file mode 100644 index 0000000..e9abb37 --- /dev/null +++ b/apps/server/tests/providers/generate-text.test.ts @@ -0,0 +1,50 @@ +import { MockLanguageModelV4 } from 'ai/test' +import { describe, expect, it, vi } from 'vitest' +import type { ModelCatalog } from '../../src/providers/catalog.ts' +import { createGenerateText } from '../../src/providers/generate-text.ts' + +function setup(reasoning: boolean) { + const model = new MockLanguageModelV4({ + doGenerate: async () => + ({ + content: [{ type: 'text', text: 'A short title' }], + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { inputTokens: { total: 1 }, outputTokens: { total: 3 } }, + warnings: [], + }) as never, + }) + const catalog = { + resolve: vi.fn(async () => ({ + model, + provider: { providerOptions: undefined }, + capabilities: { reasoning }, + })), + } as unknown as ModelCatalog + return { model, catalog, generate: createGenerateText(catalog, { warn: vi.fn() } as never) } +} + +const request = { + user: 'did:plc:alice', + model: { provider: 'fake', id: 'm' }, + prompt: 'Title this', +} + +describe('createGenerateText', () => { + it('resolves the model for the user and returns the text and finish reason', async () => { + const { catalog, generate } = setup(false) + expect(await generate(request)).toEqual({ text: 'A short title', finishReason: 'stop' }) + expect(catalog.resolve).toHaveBeenCalledWith('did:plc:alice', { provider: 'fake', id: 'm' }) + }) + + it('passes the requested effort to a reasoning model', async () => { + const { model, generate } = setup(true) + await generate({ ...request, effort: 'none' }) + expect(model.doGenerateCalls[0]?.reasoning).toBe('none') + }) + + it('sends no reasoning setting to a model without reasoning', async () => { + const { model, generate } = setup(false) + await generate({ ...request, effort: 'none' }) + expect(model.doGenerateCalls[0]?.reasoning).toBeUndefined() + }) +}) diff --git a/apps/server/tests/providers/guarded-fetch.test.ts b/apps/server/tests/providers/guarded-fetch.test.ts new file mode 100644 index 0000000..c8c00ed --- /dev/null +++ b/apps/server/tests/providers/guarded-fetch.test.ts @@ -0,0 +1,114 @@ +import { createServer, type Server } from 'node:http' +import type { AddressInfo, Socket } from 'node:net' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { + createGuardedFetch, + guardConnector, + isPrivateAddress, + PrivateNetworkError, +} from '../../src/providers/guarded-fetch.ts' + +let server: Server +let port: number + +beforeAll(async () => { + server = createServer((req, res) => { + if (req.url === '/redirect') { + res.writeHead(302, { location: '/ok' }) + return res.end() + } + res.end('ok') + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + port = (server.address() as AddressInfo).port +}) +afterAll(() => new Promise((resolve) => server.close(() => resolve()))) + +const rootCause = (err: unknown): unknown => { + let current = err + while (current instanceof Error && current.cause) current = current.cause + return current +} + +describe('isPrivateAddress', () => { + it.each([ + ['127.0.0.1', true], + ['10.1.2.3', true], + ['192.168.1.1', true], + ['172.16.0.1', true], + ['169.254.169.254', true], + ['100.64.0.1', true], + ['0.0.0.0', true], + ['::1', true], + ['fc00::1', true], + ['fe80::1', true], + ['::ffff:127.0.0.1', true], + ['::ffff:10.0.0.1', true], + ['not an address', true], + ['93.184.216.34', false], + ['2606:4700:4700::1111', false], + ])('%s is private: %s', (address, expected) => { + expect(isPrivateAddress(address)).toBe(expected) + }) +}) + +describe('createGuardedFetch', () => { + it('refuses a private IP literal before connecting', async () => { + await expect(createGuardedFetch()(`http://127.0.0.1:${port}/ok`)).rejects.toBeInstanceOf( + PrivateNetworkError, + ) + }) + + it('refuses an IPv4-mapped IPv6 literal', async () => { + await expect( + createGuardedFetch()(`http://[::ffff:127.0.0.1]:${port}/ok`), + ).rejects.toBeInstanceOf(PrivateNetworkError) + }) + + it('refuses a hostname that resolves to a private address', async () => { + const error = await createGuardedFetch()(`http://localhost:${port}/ok`).catch( + (err: unknown) => err, + ) + expect(rootCause(error)).toBeInstanceOf(PrivateNetworkError) + }) + + it('does not follow redirects', async () => { + const allowAll = createGuardedFetch(() => false) + expect(await (await allowAll(`http://localhost:${port}/ok`)).text()).toBe('ok') + await expect(allowAll(`http://localhost:${port}/redirect`)).rejects.toThrow() + }) +}) + +describe('guardConnector', () => { + it('refuses a socket whose real remote address is private, even if the lookup passed', async () => { + let destroyed = false + const socket = { + remoteAddress: '10.0.0.5', + destroy: () => void (destroyed = true), + } as unknown as Socket + const connector = guardConnector((( + _options: unknown, + callback: (err: Error | null, socket: Socket) => void, + ) => callback(null, socket)) as never) + const result = await new Promise<{ err: Error | null }>((resolve) => + connector({} as never, ((err: Error | null) => resolve({ err })) as never), + ) + expect(result.err).toBeInstanceOf(PrivateNetworkError) + expect(destroyed).toBe(true) + }) + + it('passes a socket with a public remote address', async () => { + const socket = { remoteAddress: '93.184.216.34', destroy: () => {} } as unknown as Socket + const connector = guardConnector((( + _options: unknown, + callback: (err: Error | null, socket: Socket) => void, + ) => callback(null, socket)) as never) + const result = await new Promise<{ err: Error | null; socket: unknown }>((resolve) => + connector( + {} as never, + ((err: Error | null, s: unknown) => resolve({ err, socket: s })) as never, + ), + ) + expect(result).toEqual({ err: null, socket }) + }) +}) diff --git a/apps/server/tests/providers/routes.test.ts b/apps/server/tests/providers/routes.test.ts new file mode 100644 index 0000000..0046dc4 --- /dev/null +++ b/apps/server/tests/providers/routes.test.ts @@ -0,0 +1,145 @@ +import pino from 'pino' +import { describe, expect, it, vi } from 'vitest' +import { createApp } from '../../src/app.ts' +import { createRoles } from '../../src/auth/roles.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { Registry } from '../../src/plugins/registry.ts' +import { ModelCatalog } from '../../src/providers/catalog.ts' +import { SecretBox } from '../../src/secrets.ts' +import { authDeps, ORIGIN, sessionCookie } from '../helpers/auth.ts' +import { testConfig } from '../helpers/config.ts' +import { createSqliteDb } from '../helpers/db.ts' +import { fakeProvider } from '../helpers/providers.ts' + +const caps = { vision: false, reasoning: false, tools: false } + +async function setup() { + const db = createSqliteDb() + await migrateToLatest(db) + const box = new SecretBox(Buffer.alloc(32, 6)) + const logger = pino({ level: 'silent' }) + const providers = new Registry['provider']>( + 'provider', + (p) => p.id, + ) + providers.register( + fakeProvider({ + listModels: vi.fn(async () => [{ id: 'listed', name: 'Listed', capabilities: caps }]), + }).provider, + 't', + ) + providers.register(fakeProvider({ id: 'adminonly', userKeys: false }).provider, 't') + const guardedFetch = (async () => new Response()) as unknown as typeof fetch + const catalog = new ModelCatalog({ + db, + box, + providers, + adminModels: [ + { + provider: 'fake', + id: 'big', + name: 'Big', + capabilities: caps, + roles: ['user'], + default: true, + }, + ], + roles: createRoles(), + guardedFetch, + logger, + }) + const app = createApp({ + config: testConfig(), + db, + logger, + auth: authDeps(db), + providers: { db, box, catalog, providers, guardedFetch, logger }, + }) + const cookie = sessionCookie(await app.request('/oauth/callback?code=a&state=b')) + const call = async (method: string, path: string, body?: object) => { + const res = await app.request(`/api${path}`, { + method, + headers: { cookie, origin: ORIGIN, 'content-type': 'application/json' }, + body: body ? JSON.stringify(body) : undefined, + }) + return { status: res.status, body: (await res.json()) as Record } + } + return { call, app, cookie } +} + +describe('provider routes', () => { + it('lists the models the user may use and the default model', async () => { + const { call } = await setup() + const { body } = await call('GET', '/models') + expect(body).toMatchObject({ + models: [{ provider: 'fake', id: 'big', source: 'admin' }], + defaultModel: { provider: 'fake', id: 'big' }, + }) + }) + + it('lists only providers that accept user keys', async () => { + const { call } = await setup() + const { body } = await call('GET', '/providers') + expect(body.providers).toMatchObject([{ id: 'fake', listsModels: true }]) + }) + + it('adds a credential, shows its models, and never returns the key', async () => { + const { call } = await setup() + const created = await call('POST', '/credentials', { + providerId: 'fake', + apiKey: 'sk-secret-7777', + models: [{ id: 'mine', name: 'Mine', capabilities: caps }], + }) + expect(created.status).toBe(201) + const list = await call('GET', '/credentials') + expect(list.body.credentials).toMatchObject([{ keyHint: '7777' }]) + expect(JSON.stringify(list.body)).not.toContain('sk-secret') + expect( + ((await call('GET', '/models')).body.models as { id: string }[]).map((m) => m.id), + ).toContain('mine') + }) + + it('refuses a credential for a provider without user keys', async () => { + const { call } = await setup() + expect( + (await call('POST', '/credentials', { providerId: 'adminonly', apiKey: 'k' })).status, + ).toBe(400) + }) + + it('rejects a key that covers no models', async () => { + const { call } = await setup() + const res = await call('POST', '/credentials', { providerId: 'fake', apiKey: 'k', models: [] }) + expect(res.status).toBe(400) + expect(res.body.message).toMatch(/at least one model/) + }) + + it('rejects a credential body that is not JSON', async () => { + const { app, cookie } = await setup() + const res = await app.request('/api/credentials', { + method: 'POST', + headers: { cookie, origin: ORIGIN, 'content-type': 'application/json' }, + body: '{', + }) + expect(res.status).toBe(400) + }) + + it('deletes a credential', async () => { + const { call } = await setup() + const { body } = await call('POST', '/credentials', { + providerId: 'fake', + apiKey: 'k', + models: [ + { id: 'm', name: 'M', capabilities: { vision: false, reasoning: false, tools: false } }, + ], + }) + const id = (body.credential as { id: string }).id + expect((await call('DELETE', `/credentials/${id}`)).status).toBe(200) + expect((await call('GET', '/credentials')).body.credentials).toEqual([]) + }) + + it('lists a provider models with a pasted key', async () => { + const { call } = await setup() + const { body } = await call('POST', '/providers/fake/list-models', { apiKey: 'k' }) + expect(body.models).toMatchObject([{ id: 'listed' }]) + }) +}) diff --git a/apps/server/tests/providers/user-credentials.test.ts b/apps/server/tests/providers/user-credentials.test.ts new file mode 100644 index 0000000..bdb3c3f --- /dev/null +++ b/apps/server/tests/providers/user-credentials.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { + CredentialInputError, + credentialFor, + deleteCredential, + listCredentials, + saveCredential, +} from '../../src/providers/user-credentials.ts' +import { SecretBox } from '../../src/secrets.ts' +import { dialects } from '../helpers/db.ts' +import { fakeProvider } from '../helpers/providers.ts' + +const caps = { vision: false, reasoning: false, tools: false } +const box = new SecretBox(Buffer.alloc(32, 5)) +const did = 'did:plc:alice' + +describe.each(dialects)('user credentials on $name', ({ create }) => { + const setup = async () => { + const db = create() + await migrateToLatest(db) + return db + } + + it('stores the key encrypted and lists only its last four characters', async () => { + const db = await setup() + await saveCredential(db, box, did, fakeProvider().provider, { + apiKey: 'sk-live-abcd1234', + models: [], + }) + const [credential] = await listCredentials(db, did) + expect(credential).toMatchObject({ providerId: 'fake', modelProvider: 'fake', keyHint: '1234' }) + expect(JSON.stringify(credential)).not.toContain('sk-live') + const row = await db + .selectFrom('provider_credential') + .select('api_key_encrypted') + .executeTakeFirstOrThrow() + expect(row.api_key_encrypted).not.toContain('sk-live') + await db.destroy() + }) + + it('replaces the earlier key for the same provider', async () => { + const db = await setup() + const { provider } = fakeProvider() + await saveCredential(db, box, did, provider, { apiKey: 'old-key-1111', models: [] }) + await saveCredential(db, box, did, provider, { apiKey: 'new-key-2222', models: [] }) + expect((await listCredentials(db, did)).map((c) => c.keyHint)).toEqual(['2222']) + await db.destroy() + }) + + it('refuses a key for a provider that does not accept user keys', async () => { + const db = await setup() + const { provider } = fakeProvider({ userKeys: false }) + await expect( + saveCredential(db, box, did, provider, { apiKey: 'k', models: [] }), + ).rejects.toBeInstanceOf(CredentialInputError) + await db.destroy() + }) + + it('requires user endpoints to be allowed and to have a valid, unique slug', async () => { + const db = await setup() + const endpoint = { apiKey: 'k', baseUrl: 'https://x.example/v1', models: [] } + await expect( + saveCredential(db, box, did, fakeProvider().provider, { ...endpoint, slug: 'x' }), + ).rejects.toThrow(/user endpoints/) + const { provider } = fakeProvider({ userEndpoints: true }) + await expect( + saveCredential(db, box, did, provider, { ...endpoint, slug: 'Bad Slug' }), + ).rejects.toThrow(/slug/) + const saved = await saveCredential(db, box, did, provider, { ...endpoint, slug: 'mine' }) + expect(saved.modelProvider).toBe('user:mine') + await expect( + saveCredential(db, box, did, provider, { ...endpoint, slug: 'mine' }), + ).rejects.toThrow(/already/) + await db.destroy() + }) + + it('finds the credential covering a model reference, with its decrypted key', async () => { + const db = await setup() + await saveCredential(db, box, did, fakeProvider().provider, { + apiKey: 'sk-9', + models: [{ id: 'm', name: 'M', capabilities: caps }], + }) + expect((await credentialFor(db, box, did, { provider: 'fake', id: 'm' }))?.apiKey).toBe('sk-9') + expect(await credentialFor(db, box, did, { provider: 'fake', id: 'other' })).toBeUndefined() + await db.destroy() + }) + + it('deletes only the owner credential', async () => { + const db = await setup() + const saved = await saveCredential(db, box, did, fakeProvider().provider, { + apiKey: 'k', + models: [], + }) + expect(await deleteCredential(db, 'did:plc:bob', saved.id)).toBe(false) + expect(await deleteCredential(db, did, saved.id)).toBe(true) + expect(await listCredentials(db, did)).toEqual([]) + await db.destroy() + }) +}) diff --git a/apps/server/tests/secrets.test.ts b/apps/server/tests/secrets.test.ts new file mode 100644 index 0000000..fab9ef3 --- /dev/null +++ b/apps/server/tests/secrets.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { SecretBox, SecretDecryptError } from '../src/secrets.ts' + +const keyA = Buffer.alloc(32, 1) +const keyB = Buffer.alloc(32, 2) + +describe('SecretBox', () => { + it('round-trips a secret', () => { + const box = new SecretBox(keyA) + expect(box.decrypt(box.encrypt('sk-live-123'))).toBe('sk-live-123') + }) + + it('never stores the plaintext and uses a fresh IV each time', () => { + const box = new SecretBox(keyA) + const first = box.encrypt('sk-live-123') + expect(first).not.toContain('sk-live-123') + expect(box.encrypt('sk-live-123')).not.toBe(first) + }) + + it('writes the version and key ID into the value', () => { + expect(new SecretBox(keyA).encrypt('x')).toMatch(/^v1:[0-9a-f]{8}:/) + }) + + it('decrypts values from a retired key and encrypts new values with the current key', () => { + const old = new SecretBox(keyA).encrypt('sk-old') + const rotated = new SecretBox(keyB, [keyA]) + expect(rotated.decrypt(old)).toBe('sk-old') + expect(rotated.encrypt('sk-new').split(':')[1]).toBe(rotated.encrypt('x').split(':')[1]) + expect(new SecretBox(keyB).decrypt(rotated.encrypt('sk-new'))).toBe('sk-new') + }) + + it('fails loudly for a value from an unknown key', () => { + const value = new SecretBox(keyA).encrypt('x') + expect(() => new SecretBox(keyB).decrypt(value)).toThrow(SecretDecryptError) + }) + + it('fails loudly for a tampered value', () => { + const box = new SecretBox(keyA) + const parts = box.encrypt('secret').split(':') + parts[4] = Buffer.from('tampered').toString('base64url') + expect(() => box.decrypt(parts.join(':'))).toThrow() + }) +}) diff --git a/apps/server/tests/server.test.ts b/apps/server/tests/server.test.ts new file mode 100644 index 0000000..fa5abfb --- /dev/null +++ b/apps/server/tests/server.test.ts @@ -0,0 +1,82 @@ +import { definePlugin } from '@scn-chat/plugin-api' +import pino from 'pino' +import { describe, expect, it, vi } from 'vitest' +import { createServer } from '../src/server.ts' +import { fakeIdentity, fakeOAuth, fakeSession, ORIGIN, sessionCookie } from './helpers/auth.ts' +import { testConfig } from './helpers/config.ts' +import { createSqliteDb } from './helpers/db.ts' +import { fakeProvider } from './helpers/providers.ts' + +const caps = { vision: false, reasoning: false, tools: false } + +async function build() { + const { provider } = fakeProvider() + const server = await createServer({ + config: testConfig(), + appConfig: { + plugins: [ + definePlugin({ + id: 'fake', + name: 'Fake', + apiVersion: 1, + setup: (ctx) => ctx.providers.register(provider), + }), + ], + models: [ + { + provider: 'fake', + id: 'm', + name: 'M', + capabilities: caps, + roles: ['user'], + default: true, + }, + ], + }, + db: createSqliteDb(), + logger: pino({ level: 'silent' }), + oauth: fakeOAuth({ + callback: vi.fn(async () => ({ session: fakeSession('did:plc:alice', 'atproto') })), + }), + identity: fakeIdentity, + startBackgroundJobs: false, + }) + return server +} + +describe('createServer', () => { + it('wires every route: health, identity, OAuth, and the signed-in API', async () => { + const { app, close } = await build() + expect((await app.request('/api/health')).status).toBe(200) + expect((await app.request('/.well-known/did.json')).status).toBe(200) + expect((await app.request('/oauth-client-metadata.json')).status).toBe(200) + const cookie = sessionCookie(await app.request('/oauth/callback?code=a&state=b')) + const models = (await (await app.request('/api/models', { headers: { cookie } })).json()) as { + models: { id: string }[] + } + expect(models.models.map((m) => m.id)).toEqual(['m']) + const created = await app.request('/api/conversations', { + method: 'POST', + headers: { cookie, origin: ORIGIN }, + }) + expect(created.status).toBe(201) + await close() + }) + + it('fails to start when an admin model names an unregistered provider', async () => { + await expect( + createServer({ + config: testConfig(), + appConfig: { + plugins: [], + models: [{ provider: 'ghost', id: 'm', name: 'M', capabilities: caps, roles: ['user'] }], + }, + db: createSqliteDb(), + logger: pino({ level: 'silent' }), + oauth: fakeOAuth(), + identity: fakeIdentity, + startBackgroundJobs: false, + }), + ).rejects.toThrow(/ghost/) + }) +}) diff --git a/apps/server/tests/sharing/sharing.test.ts b/apps/server/tests/sharing/sharing.test.ts new file mode 100644 index 0000000..de3c9a3 --- /dev/null +++ b/apps/server/tests/sharing/sharing.test.ts @@ -0,0 +1,269 @@ +import { nsid } from '@scn-chat/lexicons' +import { sql } from 'kysely' +import { describe, expect, it, vi } from 'vitest' +import { createApp } from '../../src/app.ts' +import { recordLogin } from '../../src/auth/accounts.ts' +import { buildScope } from '../../src/auth/scope.ts' +import { SharingService } from '../../src/sharing/service.ts' +import { recordCid } from '../../src/storage/records.ts' +import { CredentialError } from '../../src/sync/credentials.ts' +import { authDeps, fakeOAuth, fakeSession, ORIGIN, sessionCookie } from '../helpers/auth.ts' +import { testConfig } from '../helpers/config.ts' +import { spacesHarness, textContent } from '../helpers/spaces.ts' + +const BOB = 'did:plc:bob' +const CAROL = 'did:plc:carol' +const LOCAL = 'did:plc:local' + +async function setup() { + const h = await spacesHarness() + for (const did of [BOB, CAROL]) + await recordLogin(h.db, { did, handle: null, pdsUrl: 'https://pds.test', spacesAllowed: true }) + await recordLogin(h.db, { + did: LOCAL, + handle: null, + pdsUrl: 'https://pds.test', + spacesAllowed: false, + }) + const handles: Record = { 'bob.test': BOB, 'carol.test': CAROL } + const identity = { + resolve: vi.fn(async (did: string) => ({ + did, + handle: Object.keys(handles).find((h) => handles[h] === did) ?? 'alice.test', + pdsUrl: 'https://pds.test', + })), + resolveHandle: vi.fn(async (handle: string) => handles[handle] ?? null), + resolveSigningKey: vi.fn(async () => 'did:key:x'), + } + const mintCredential = vi.fn(async (viewer: string, space: string) => { + if (!h.pds.mayRead(space, viewer)) + throw new CredentialError('UserNotAuthorized', 'not a member') + return { client: () => h.pds.client(h.pds.ownerOf(space)) } + }) + const sharing = new SharingService({ + getPdsClient: async (did) => h.pds.client(did) as never, + identity, + mintCredential, + logger: h.logger, + }) + const appFor = async (did: string) => { + const scope = did === LOCAL ? 'atproto' : buildScope('raw') + const oauth = fakeOAuth({ callback: vi.fn(async () => ({ session: fakeSession(did, scope) })) }) + const app = createApp({ + config: testConfig(), + db: h.db, + logger: h.logger, + auth: authDeps(h.db, { oauth, identity }), + sharing, + }) + const cookie = sessionCookie(await app.request('/oauth/callback?code=a&state=b')) + return { + raw: (path: string) => app.request(`/api${path}`, { headers: { cookie } }), + get: async (path: string) => { + const res = await app.request(`/api${path}`, { headers: { cookie } }) + return { status: res.status, body: (await res.json()) as Record } + }, + put: async (path: string, body: object) => { + const res = await app.request(`/api${path}`, { + method: 'PUT', + headers: { cookie, origin: ORIGIN, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) + return { status: res.status, body: (await res.json()) as Record } + }, + } + } + const { skey, uri } = await h.chats.createConversation({ systemPrompt: 'secret system prompt' }) + await h.chats.updateInfo(skey, { title: 'Tile quotes', titleSource: 'user' }) + await h.chats.createMessage(skey, '3mmmmmmmmmmm1', { + $type: nsid.message, + role: 'user', + content: textContent('hi'), + createdAt: new Date().toISOString(), + } as never) + await h.chats.createMessage(skey, '3mmmmmmmmmmm1.r0', { + $type: nsid.message, + role: 'assistant', + parent: '3mmmmmmmmmmm1', + status: 'complete', + content: textContent('hello'), + createdAt: new Date().toISOString(), + } as never) + return { ...h, appFor, skey, uri, mintCredential } +} + +describe('share settings', () => { + it('shares with people as read-only members and removes dropped members', async () => { + const { appFor, skey, uri, pds } = await setup() + const alice = await appFor('did:plc:alice') + await alice.put(`/conversations/${skey}/sharing`, { + mode: 'people', + members: ['bob.test', CAROL], + }) + expect(pds.policies.get(uri)?.members.get(BOB)).toEqual({ read: true, write: false }) + await alice.put(`/conversations/${skey}/sharing`, { mode: 'people', members: [CAROL] }) + expect(pds.policies.get(uri)?.members.has(BOB)).toBe(false) + expect((await alice.get(`/conversations/${skey}/sharing`)).body).toEqual({ + mode: 'people', + members: [{ did: CAROL, handle: 'carol.test' }], + }) + }) + + it('makes a conversation public while keeping the member list, and private again removes everyone', async () => { + const { appFor, skey, uri, pds } = await setup() + const alice = await appFor('did:plc:alice') + await alice.put(`/conversations/${skey}/sharing`, { mode: 'people', members: [BOB] }) + await alice.put(`/conversations/${skey}/sharing`, { mode: 'public' }) + expect(pds.policies.get(uri)?.readPolicy).toMatch(/publicPolicy/) + expect(pds.policies.get(uri)?.members.has(BOB)).toBe(true) + await alice.put(`/conversations/${skey}/sharing`, { mode: 'private' }) + expect((await alice.get(`/conversations/${skey}/sharing`)).body).toEqual({ + mode: 'private', + members: [], + }) + }) + + it('fails with 400 naming an unresolvable handle', async () => { + const { appFor, skey } = await setup() + const res = await (await appFor('did:plc:alice')).put(`/conversations/${skey}/sharing`, { + mode: 'people', + members: ['nobody.test'], + }) + expect(res.status).toBe(400) + expect(res.body.message).toContain('nobody.test') + }) + + it('reflects changes made outside the app', async () => { + const { appFor, skey, uri, pds } = await setup() + pds.policies.set(uri, { + readPolicy: 'com.atproto.simplespace.defs#publicPolicy', + members: new Map(), + }) + expect( + (await (await appFor('did:plc:alice')).get(`/conversations/${skey}/sharing`)).body.mode, + ).toBe('public') + }) + + it('refuses fallback owners', async () => { + const { appFor, skey } = await setup() + const local = await appFor(LOCAL) + expect((await local.get(`/conversations/${skey}/sharing`)).status).toBe(403) + expect((await local.put(`/conversations/${skey}/sharing`, { mode: 'public' })).status).toBe(403) + }) +}) + +describe('shared view', () => { + it('lets a member read a people-shared conversation through their own credential, without the system prompt', async () => { + const { appFor, skey, mintCredential } = await setup() + await (await appFor('did:plc:alice')).put(`/conversations/${skey}/sharing`, { + mode: 'people', + members: [BOB], + }) + const view = await (await appFor(BOB)).get(`/shared/did:plc:alice/${skey}`) + expect(view.status).toBe(200) + expect(view.body).toMatchObject({ title: 'Tile quotes', owner: { did: 'did:plc:alice' } }) + expect((view.body.messages as { rkey: string }[]).map((m) => m.rkey)).toEqual([ + '3mmmmmmmmmmm1', + '3mmmmmmmmmmm1.r0', + ]) + expect(JSON.stringify(view.body)).not.toContain('secret system prompt') + expect(mintCredential).toHaveBeenCalledWith(BOB, expect.stringContaining(skey)) + }) + + it('leaves out a message another client wrote that does not match the lexicon', async () => { + const { appFor, skey, uri, db } = await setup() + await (await appFor('did:plc:alice')).put(`/conversations/${skey}/sharing`, { mode: 'public' }) + const value = { $type: nsid.message, role: 'user', createdAt: new Date().toISOString() } + const bad = JSON.stringify(value) + const cid = await recordCid(value) + await sql`insert into local_record values (${uri}, ${nsid.message}, 'bad', ${bad}, ${cid}, 'now')`.execute( + db, + ) + const view = await (await appFor(BOB)).get(`/shared/did:plc:alice/${skey}`) + expect((view.body.messages as { rkey: string }[]).map((m) => m.rkey)).not.toContain('bad') + }) + + it('returns 404 to a non-member, and to anyone for a private conversation', async () => { + const { appFor, skey } = await setup() + expect((await (await appFor(CAROL)).get(`/shared/did:plc:alice/${skey}`)).status).toBe(404) + await (await appFor('did:plc:alice')).put(`/conversations/${skey}/sharing`, { + mode: 'people', + members: [BOB], + }) + expect((await (await appFor(CAROL)).get(`/shared/did:plc:alice/${skey}`)).status).toBe(404) + }) + + it('lets any spaces user read a public conversation', async () => { + const { appFor, skey } = await setup() + await (await appFor('did:plc:alice')).put(`/conversations/${skey}/sharing`, { mode: 'public' }) + expect((await (await appFor(CAROL)).get(`/shared/did:plc:alice/${skey}`)).status).toBe(200) + }) + + it('returns 404 for a missing conversation', async () => { + const { appFor } = await setup() + expect((await (await appFor(BOB)).get('/shared/did:plc:alice/3zzzzzzzzzzzz')).status).toBe(404) + }) + + it('takes a revocation into account on the next request', async () => { + const { appFor, skey } = await setup() + const alice = await appFor('did:plc:alice') + const bob = await appFor(BOB) + await alice.put(`/conversations/${skey}/sharing`, { mode: 'people', members: [BOB] }) + expect((await bob.get(`/shared/did:plc:alice/${skey}`)).status).toBe(200) + await alice.put(`/conversations/${skey}/sharing`, { mode: 'private' }) + expect((await bob.get(`/shared/did:plc:alice/${skey}`)).status).toBe(404) + }) + + it('refuses fallback viewers with 403', async () => { + const { appFor, skey } = await setup() + await (await appFor('did:plc:alice')).put(`/conversations/${skey}/sharing`, { mode: 'public' }) + expect((await (await appFor(LOCAL)).get(`/shared/did:plc:alice/${skey}`)).status).toBe(403) + }) + + it('loads an attachment in a shared conversation through the viewer credential', async () => { + const h = await setup() + const upload = (await h.pds + .client('did:plc:alice') + .call({ $nsid: 'com.atproto.repo.uploadBlob' }, new Uint8Array([7, 7]) as never, { + encoding: 'image/png', + })) as { blob: { ref: { toString(): string } } } + const cid = upload.blob.ref.toString() + await h.chats.createMessage(h.skey, '3mmmmmmmmmmm2', { + $type: nsid.message, + role: 'user', + content: { + $type: `${nsid.defs}#plainContent`, + parts: [ + { + $type: `${nsid.defs}#imagePart`, + image: { $type: 'blob', ref: { $link: cid }, mimeType: 'image/png', size: 2 }, + }, + ], + }, + createdAt: new Date().toISOString(), + } as never) + const carol = await h.appFor(CAROL) + expect((await carol.raw(`/shared/did:plc:alice/${h.skey}/blobs/${cid}`)).status).toBe(404) + await (await h.appFor('did:plc:alice')).put(`/conversations/${h.skey}/sharing`, { + mode: 'people', + members: [BOB], + }) + const res = await (await h.appFor(BOB)).raw( + `/shared/did:plc:alice/${h.skey}/blobs/${cid}?type=image/png`, + ) + expect(res.status).toBe(200) + expect(res.headers.get('content-type')).toBe('image/png') + expect(new Uint8Array(await res.arrayBuffer())).toEqual(new Uint8Array([7, 7])) + }) + + it('returns 401 when signed out', async () => { + const h = await spacesHarness() + const app = createApp({ + config: testConfig(), + db: h.db, + logger: h.logger, + sharing: {} as never, + }) + expect((await app.request('/api/shared/did:plc:alice/3aaaaaaaaaaaa')).status).toBe(401) + }) +}) diff --git a/apps/server/tests/storage/chat-service.test.ts b/apps/server/tests/storage/chat-service.test.ts new file mode 100644 index 0000000..f626e1a --- /dev/null +++ b/apps/server/tests/storage/chat-service.test.ts @@ -0,0 +1,212 @@ +import { nsid } from '@scn-chat/lexicons' +import { sql } from 'kysely' +import pino from 'pino' +import { describe, expect, it } from 'vitest' +import type { Db } from '../../src/db/index.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { ChatService } from '../../src/storage/chat-service.ts' +import { LocalRecordStore } from '../../src/storage/local-record-store.ts' +import { InvalidStoredRecord } from '../../src/storage/record-store.ts' +import { recordCid } from '../../src/storage/records.ts' +import { dialects } from '../helpers/db.ts' + +const did = 'did:plc:alice' +const logger = pino({ level: 'silent' }) +const text = (value: string) => ({ + $type: `${nsid.defs}#plainContent`, + parts: [{ $type: `${nsid.defs}#textPart`, text: value }], +}) +const userMessage = (value: string) => ({ + $type: nsid.message, + role: 'user', + content: text(value), + createdAt: new Date().toISOString(), +}) + +describe.each(dialects)('ChatService on $name', ({ create }) => { + const setup = async () => { + const db = create() + await migrateToLatest(db) + const writes: string[] = [] + const chats = new ChatService(new LocalRecordStore(did, db), logger, (_space, cid) => + writes.push(cid), + ) + return { db, chats, writes } + } + + it('creates the space, its info record, and its index entry', async () => { + const { chats, db } = await setup() + const { skey, uri } = await chats.createConversation({ + systemPrompt: 'Be brief', + tags: ['work'], + }) + expect(uri).toBe(`at://${did}/space/${nsid.conversation}/${skey}`) + expect((await chats.getInfo(skey))?.systemPrompt).toBe('Be brief') + const { conversations } = await chats.listConversations() + expect(conversations).toMatchObject([{ skey, uri, tags: ['work'] }]) + await db.destroy() + }) + + it('bumps the index entry when a message is written', async () => { + const { chats, db } = await setup() + const { skey } = await chats.createConversation() + const before = (await chats.listConversations()).conversations[0]?.updatedAt ?? '' + await new Promise((resolve) => setTimeout(resolve, 5)) + await chats.createMessage(skey, '3mmmmmmmmmmmm', userMessage('hi')) + const after = (await chats.listConversations()).conversations[0]?.updatedAt ?? '' + expect(after > before).toBe(true) + await db.destroy() + }) + + it('copies a title change into the index entry', async () => { + const { chats, db } = await setup() + const { skey } = await chats.createConversation() + await chats.updateInfo(skey, { title: 'Tile quotes', titleSource: 'user' }) + expect((await chats.listConversations()).conversations[0]?.title).toBe('Tile quotes') + await db.destroy() + }) + + it('writes tags only to the index entry', async () => { + const { chats, db } = await setup() + const { skey } = await chats.createConversation() + await chats.setTags(skey, ['reno']) + expect((await chats.listConversations()).conversations[0]?.tags).toEqual(['reno']) + expect((await chats.getInfo(skey))?.tags).toBeUndefined() + await db.destroy() + }) + + it('skips an info update when the user titled the conversation and unlessUserTitled is set', async () => { + const { chats, db } = await setup() + const { skey } = await chats.createConversation() + await chats.updateInfo(skey, { title: 'Mine', titleSource: 'user' }) + expect( + await chats.updateInfo( + skey, + { title: 'Generated', titleSource: 'generated' }, + { unlessUserTitled: true }, + ), + ).toBe(false) + expect((await chats.getInfo(skey))?.title).toBe('Mine') + await db.destroy() + }) + + it('deletes the space and its index entry', async () => { + const { chats, db } = await setup() + const { skey } = await chats.createConversation() + await chats.deleteConversation(skey) + expect((await chats.listConversations()).conversations).toEqual([]) + await db.destroy() + }) + + it('returns only index changes since a revision, including deletions', async () => { + const { chats, db } = await setup() + const a = await chats.createConversation() + const { rev } = await chats.listConversations() + const b = await chats.createConversation() + await chats.deleteConversation(a.skey) + const changes = await chats.listConversations(rev) + expect(changes.conversations.map((c) => c.skey)).toEqual([b.skey]) + expect(changes.deleted).toEqual([a.skey]) + expect(changes.full).toBe(false) + await db.destroy() + }) + + it('returns a conversation in full, or only its changes since a revision', async () => { + const { chats, db } = await setup() + const { skey } = await chats.createConversation() + await chats.createMessage(skey, '3mmmmmmmmmmm1', userMessage('one')) + const full = await chats.getConversation(skey) + expect(full.messages).toHaveLength(1) + expect(full.info).not.toBeNull() + await chats.createMessage(skey, '3mmmmmmmmmmm2', userMessage('two')) + const changes = await chats.getConversation(skey, full.rev) + expect(changes.messages.map((m) => m.rkey)).toEqual(['3mmmmmmmmmmm2']) + await db.destroy() + }) + + it('stores preferences in the settings space', async () => { + const { chats, db } = await setup() + await chats.putPreferences({ customInstructions: 'Use metric units', generateTitles: false }) + expect(await chats.getPreferences()).toMatchObject({ + customInstructions: 'Use metric units', + generateTitles: false, + }) + await db.destroy() + }) + + it('creates the settings space idempotently', async () => { + const { chats, db } = await setup() + await chats.ensureSettingsSpace() + await expect(chats.ensureSettingsSpace()).resolves.toBeUndefined() + await db.destroy() + }) + + it('reports each write so sync can ignore its own changes', async () => { + const { chats, writes, db } = await setup() + const { skey } = await chats.createConversation() + const cid = await chats.createMessage(skey, '3mmmmmmmmmmmm', userMessage('hi')) + expect(writes).toContain(cid) + await db.destroy() + }) + + /** Write a record as another client might, skipping the store's validation. */ + const writeRaw = async ( + db: Db, + space: string, + collection: string, + rkey: string, + value: Record, + ) => { + const json = JSON.stringify(value) + const cid = await recordCid(value) + await sql`insert into local_record values (${space}, ${collection}, ${rkey}, ${json}, ${cid}, 'now')`.execute( + db, + ) + await sql`insert into local_op (space_uri, collection, rkey, cid, value_json, created_at) values (${space}, ${collection}, ${rkey}, ${cid}, ${json}, 'now')`.execute( + db, + ) + } + + it('leaves out messages that do not match the lexicon, in full and incremental reads', async () => { + const { chats, db } = await setup() + const { skey, uri } = await chats.createConversation() + const before = (await chats.getConversation(skey)).rev + await chats.createMessage(skey, '3mmmmmmmmmmm1', userMessage('hi')) + await writeRaw(db, uri, nsid.message, 'bad', { $type: nsid.message, role: 'user' }) + const rkeys = (changes: { messages: { rkey: string }[] }) => changes.messages.map((m) => m.rkey) + expect(rkeys(await chats.getConversation(skey))).toEqual(['3mmmmmmmmmmm1']) + expect(rkeys(await chats.getConversation(skey, before))).toEqual(['3mmmmmmmmmmm1']) + await db.destroy() + }) + + it('fails loudly on an info record that does not match the lexicon', async () => { + const { chats, db } = await setup() + const { skey, uri } = await chats.createConversation() + await sql`delete from local_record where space_uri = ${uri}`.execute(db) + await writeRaw(db, uri, nsid.info, 'self', { $type: nsid.info, title: 7 }) + await expect(chats.getConversation(skey)).rejects.toBeInstanceOf(InvalidStoredRecord) + await db.destroy() + }) + + it('leaves invalid index entries out of the chat list', async () => { + const { chats, db } = await setup() + const { skey } = await chats.createConversation() + await writeRaw(db, chats.settingsUri, nsid.conversationRef, 'bad', { + $type: nsid.conversationRef, + title: 'no conversation or updatedAt', + }) + const { conversations } = await chats.listConversations() + expect(conversations.map((c) => c.skey)).toEqual([skey]) + await db.destroy() + }) + + it('removes the index title when the info record loses its title', async () => { + const { chats, db } = await setup() + const { skey } = await chats.createConversation() + await chats.updateInfo(skey, { title: 'Old' }) + await chats.syncRefTitle(skey, undefined) + const { conversations } = await chats.listConversations() + expect(conversations[0]?.title).toBeNull() + await db.destroy() + }) +}) diff --git a/apps/server/tests/storage/local-record-store.test.ts b/apps/server/tests/storage/local-record-store.test.ts new file mode 100644 index 0000000..40e76aa --- /dev/null +++ b/apps/server/tests/storage/local-record-store.test.ts @@ -0,0 +1,136 @@ +import { cidForCbor } from '@atproto/common' +import { nsid } from '@scn-chat/lexicons' +import { describe, expect, it } from 'vitest' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { LocalRecordStore } from '../../src/storage/local-record-store.ts' +import { + InvalidCursor, + InvalidRecord, + RecordExists, + SpaceExists, + SpaceNotFound, +} from '../../src/storage/record-store.ts' +import { spaceUri, toLex } from '../../src/storage/records.ts' +import { dialects } from '../helpers/db.ts' + +const did = 'did:plc:alice' +const info = (title?: string) => ({ + $type: nsid.info, + createdAt: '2026-09-26T00:00:00.000Z', + ...(title ? { title } : {}), +}) + +describe.each(dialects)('LocalRecordStore on $name', ({ create }) => { + const setup = async () => { + const db = create() + await migrateToLatest(db) + const store = new LocalRecordStore(did, db) + const space = await store.createSpace(nsid.conversation, '3aaaaaaaaaaaa') + return { db, store, space } + } + + it('creates spaces with the PDS URI format and refuses duplicates', async () => { + const { store, space, db } = await setup() + expect(space).toBe(spaceUri(did, nsid.conversation, '3aaaaaaaaaaaa')) + await expect(store.createSpace(nsid.conversation, '3aaaaaaaaaaaa')).rejects.toBeInstanceOf( + SpaceExists, + ) + await db.destroy() + }) + + it('creates, reads, updates, lists, and deletes records', async () => { + const { store, space, db } = await setup() + await store.createRecord(space, nsid.info, 'self', info('a')) + await store.putRecord(space, nsid.info, 'self', info('b')) + expect((await store.getRecord(space, nsid.info, 'self'))?.value.title).toBe('b') + expect(await store.listRecords(space, nsid.info)).toHaveLength(1) + await store.deleteRecord(space, nsid.info, 'self') + expect(await store.getRecord(space, nsid.info, 'self')).toBeNull() + await db.destroy() + }) + + it('throws RecordExists when creating over an existing key', async () => { + const { store, space, db } = await setup() + await store.createRecord(space, nsid.info, 'self', info()) + await expect(store.createRecord(space, nsid.info, 'self', info())).rejects.toBeInstanceOf( + RecordExists, + ) + await db.destroy() + }) + + it('rejects an invalid record before writing', async () => { + const { store, space, db } = await setup() + await expect( + store.putRecord(space, nsid.info, 'self', { $type: nsid.info }), + ).rejects.toBeInstanceOf(InvalidRecord) + expect(await store.listKeys(space)).toEqual([]) + await db.destroy() + }) + + it('throws SpaceNotFound for a missing space', async () => { + const { store, db } = await setup() + const missing = spaceUri(did, nsid.conversation, '3bbbbbbbbbbbb') + await expect(store.putRecord(missing, nsid.info, 'self', info())).rejects.toBeInstanceOf( + SpaceNotFound, + ) + await db.destroy() + }) + + it('computes the same CID an independent atproto CBOR encoder does', async () => { + const { store, space, db } = await setup() + const record = info('CID check') + const { cid } = await store.createRecord(space, nsid.info, 'self', record) + expect(cid).toBe((await cidForCbor(toLex(record))).toString()) + await db.destroy() + }) + + it('lists ops after a revision, including deletes, and reports the head revision', async () => { + const { store, space, db } = await setup() + await store.createRecord(space, nsid.info, 'self', info('a')) + const { rev: first } = await store.listOps(space) + await store.putRecord(space, nsid.info, 'self', info('b')) + await store.deleteRecord(space, nsid.info, 'self') + const page = await store.listOps(space, first) + expect(page.ops.map((op) => op.cid === null)).toEqual([false, true]) + expect(page.ops[0]?.value?.title).toBe('b') + expect(await store.headRev(space)).toBe(page.rev) + await db.destroy() + }) + + it('returns keys and CIDs without values', async () => { + const { store, space, db } = await setup() + const { cid } = await store.createRecord(space, nsid.info, 'self', info()) + expect(await store.listKeys(space)).toEqual([{ collection: nsid.info, rkey: 'self', cid }]) + await db.destroy() + }) + + it('deletes a space with its records and ops', async () => { + const { store, space, db } = await setup() + await store.createRecord(space, nsid.info, 'self', info()) + await store.deleteSpace(space) + expect(await store.listSpaces(nsid.conversation)).toEqual([]) + await expect(store.listOps(space)).rejects.toBeInstanceOf(SpaceNotFound) + const rows = await db + .selectFrom('local_op') + .select('seq') + .where('space_uri', '=', space) + .execute() + expect(rows).toEqual([]) + await db.destroy() + }) + + it('raises SpaceNotFound when reading a space that does not exist, as a PDS does', async () => { + const { store, db } = await setup() + const missing = 'at://did:plc:alice/space/network.sharedcomputer.chat.conversation/nope' + await expect(store.getRecord(missing, nsid.info, 'self')).rejects.toBeInstanceOf(SpaceNotFound) + await expect(store.listRecords(missing, nsid.message)).rejects.toBeInstanceOf(SpaceNotFound) + await expect(store.headRev(missing)).rejects.toBeInstanceOf(SpaceNotFound) + await db.destroy() + }) + + it('rejects a revision cursor that did not come from this store', async () => { + const { store, space, db } = await setup() + await expect(store.listOps(space, '3mwhpfnmcvc26')).rejects.toBeInstanceOf(InvalidCursor) + await db.destroy() + }) +}) diff --git a/apps/server/tests/storage/routes.test.ts b/apps/server/tests/storage/routes.test.ts new file mode 100644 index 0000000..1a127b5 --- /dev/null +++ b/apps/server/tests/storage/routes.test.ts @@ -0,0 +1,178 @@ +import { nsid } from '@scn-chat/lexicons' +import { sql } from 'kysely' +import { describe, expect, it, vi } from 'vitest' +import { createApp } from '../../src/app.ts' +import { buildScope } from '../../src/auth/scope.ts' +import { resolveSyncConfig } from '../../src/sync/scheduler.ts' +import { authDeps, fakeOAuth, fakeSession, ORIGIN, sessionCookie } from '../helpers/auth.ts' +import { testConfig } from '../helpers/config.ts' +import { spacesHarness } from '../helpers/spaces.ts' + +type Json = Record + +async function setup(storageMode: 'space' | 'local', syncOptions = {}) { + const h = await spacesHarness({ storageMode }) + const scope = storageMode === 'space' ? buildScope('raw') : 'atproto' + const oauth = fakeOAuth({ + callback: vi.fn(async () => ({ session: fakeSession('did:plc:alice', scope) })), + }) + const engine = { syncConversation: vi.fn(async () => {}) } + const app = createApp({ + config: testConfig(), + db: h.db, + logger: h.logger, + auth: authDeps(h.db, { oauth }), + storage: { + db: h.db, + services: h.services, + events: h.events, + engine: engine as never, + syncConfig: resolveSyncConfig(syncOptions), + logger: h.logger, + }, + }) + const cookie = sessionCookie(await app.request('/oauth/callback?code=a&state=b')) + const get = async (path: string) => { + const res = await app.request(`/api${path}`, { headers: { cookie } }) + return { status: res.status, body: (await res.json()) as Json } + } + const send = async (method: string, path: string, body?: object) => { + const res = await app.request(`/api${path}`, { + method, + headers: { cookie, origin: ORIGIN, 'content-type': 'application/json' }, + body: body ? JSON.stringify(body) : undefined, + }) + return { status: res.status, body: (await res.json()) as Json } + } + return { ...h, app, cookie, get, send, engine } +} + +describe.each(['space', 'local'] as const)('chat API for a %s account', (mode) => { + it('creates a conversation and lists it in the index', async () => { + const { get, send } = await setup(mode) + const created = await send('POST', '/conversations', { systemPrompt: 'Be brief' }) + expect(created.status).toBe(201) + const list = await get('/conversations') + expect(list.body.conversations).toMatchObject([{ skey: created.body.skey }]) + expect(list.body.rev).toBeTruthy() + }) + + it('returns a conversation with its info, and 404 for a missing one', async () => { + const { get, send } = await setup(mode) + const { body } = await send('POST', '/conversations', { systemPrompt: 'Be brief' }) + const conversation = await get(`/conversations/${body.skey}`) + expect((conversation.body.info as Json).value).toMatchObject({ systemPrompt: 'Be brief' }) + expect((await get('/conversations/3zzzzzzzzzzzz')).status).toBe(404) + }) + + it('reports a stored info record that does not match its lexicon as a 502', async () => { + const { get, send, db } = await setup(mode) + const { body } = await send('POST', '/conversations') + const bad = JSON.stringify({ $type: nsid.info, createdAt: 12 }) + await sql`update local_record set value_json = ${bad} where collection = ${nsid.info}`.execute( + db, + ) + const res = await get(`/conversations/${body.skey}`) + expect(res.status).toBe(502) + expect(res.body.error).toBe('InvalidStoredRecord') + }) + + it('returns only index changes since a revision', async () => { + const { get, send } = await setup(mode) + await send('POST', '/conversations') + const { body: first } = await get('/conversations') + const { body: second } = await send('POST', '/conversations') + const changes = await get(`/conversations?since=${first.rev}`) + expect(changes.body.conversations).toMatchObject([{ skey: second.skey }]) + expect(changes.body.full).toBe(false) + }) + + it('renames a conversation as the user, copying the title into the index', async () => { + const { get, send } = await setup(mode) + const { body } = await send('POST', '/conversations') + await send('PATCH', `/conversations/${body.skey}`, { title: 'Tile quotes', tags: ['reno'] }) + const conversation = await get(`/conversations/${body.skey}`) + expect((conversation.body.info as Json).value).toMatchObject({ + title: 'Tile quotes', + titleSource: 'user', + }) + expect((await get('/conversations')).body.conversations).toMatchObject([ + { title: 'Tile quotes', tags: ['reno'] }, + ]) + }) + + it('deletes a conversation', async () => { + const { get, send } = await setup(mode) + const { body } = await send('POST', '/conversations') + await send('DELETE', `/conversations/${body.skey}`) + expect((await get('/conversations')).body.conversations).toEqual([]) + }) + + it('returns keys and CIDs without record content', async () => { + const { get, send } = await setup(mode) + const { body } = await send('POST', '/conversations', { systemPrompt: 'secret words' }) + const keys = await get(`/conversations/${body.skey}/keys`) + expect(keys.body.keys).toMatchObject([ + { collection: 'network.sharedcomputer.chat.info', rkey: 'self' }, + ]) + expect(JSON.stringify(keys.body)).not.toContain('secret words') + expect((await get('/conversations/keys')).body.keys).toHaveLength(1) + }) + + it('round-trips preferences and rejects invalid ones', async () => { + const { get, send } = await setup(mode) + await send('PUT', '/preferences', { customInstructions: 'Use metric', generateTitles: false }) + expect((await get('/preferences')).body.preferences).toMatchObject({ + customInstructions: 'Use metric', + }) + expect((await send('PUT', '/preferences', { generateTitles: 'yes' })).status).toBe(400) + }) +}) + +describe('account settings', () => { + it('lets the user turn off background sync when the admin allows it', async () => { + const { get, send } = await setup('space') + expect((await get('/account')).body).toEqual({ backgroundSync: true, allowUserOptOut: true }) + await send('PUT', '/account', { backgroundSync: false }) + expect((await get('/account')).body.backgroundSync).toBe(false) + }) + + it('refuses to turn off background sync when the admin does not allow it', async () => { + const { send } = await setup('space', { allowUserOptOut: false }) + expect((await send('PUT', '/account', { backgroundSync: false })).status).toBe(403) + }) +}) + +describe('sync on open and the sync button', () => { + it('syncs a spaces conversation before returning it', async () => { + const { get, send, engine } = await setup('space') + const { body } = await send('POST', '/conversations') + await get(`/conversations/${body.skey}`) + expect(engine.syncConversation).toHaveBeenCalledWith('did:plc:alice', body.skey) + }) + + it('syncs when the sync button is pressed', async () => { + const { send, engine } = await setup('space') + const { body } = await send('POST', '/conversations') + await send('POST', `/conversations/${body.skey}/sync`) + expect(engine.syncConversation).toHaveBeenCalledTimes(1) + }) +}) + +describe('GET /api/events', () => { + it('streams change events for the signed-in user only', async () => { + const { app, cookie, events } = await setup('space') + const controller = new AbortController() + const res = await app.request('/api/events', { headers: { cookie }, signal: controller.signal }) + const reader = res.body?.getReader() + if (!reader) throw new Error('no stream') + events.emit('conversation:changed', { did: 'did:plc:bob', skey: 'nope' }) + events.emit('conversation:changed', { did: 'did:plc:alice', skey: '3aaaaaaaaaaaa' }) + const { value } = await reader.read() + const text = new TextDecoder().decode(value) + expect(text).toContain('event: conversation-changed') + expect(text).toContain('3aaaaaaaaaaaa') + expect(text).not.toContain('nope') + controller.abort() + }) +}) diff --git a/apps/server/tests/storage/space-record-store.test.ts b/apps/server/tests/storage/space-record-store.test.ts new file mode 100644 index 0000000..257ce50 --- /dev/null +++ b/apps/server/tests/storage/space-record-store.test.ts @@ -0,0 +1,96 @@ +import { nsid } from '@scn-chat/lexicons' +import { describe, expect, it } from 'vitest' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { RecordExists, SpaceExists, SpaceNotFound } from '../../src/storage/record-store.ts' +import { SpaceRecordStore } from '../../src/storage/space-record-store.ts' +import { createSqliteDb } from '../helpers/db.ts' +import { FakePds } from '../helpers/fake-pds.ts' + +const did = 'did:plc:alice' +const info = { $type: nsid.info, createdAt: '2026-09-26T00:00:00.000Z' } + +async function setup() { + const db = createSqliteDb() + await migrateToLatest(db) + const pds = await FakePds.create(db) + const store = new SpaceRecordStore(did, async () => pds.client(did) as never) + return { pds, store } +} + +describe('SpaceRecordStore', () => { + it('creates spaces with member-list policies and open app access', async () => { + const { pds, store } = await setup() + await store.createSpace(nsid.conversation, '3aaaaaaaaaaaa') + expect(pds.calls[0]?.input).toMatchObject({ + readPolicy: { $type: 'com.atproto.simplespace.defs#memberListPolicy' }, + writePolicy: { $type: 'com.atproto.simplespace.defs#memberListPolicy' }, + appAccess: { $type: 'com.atproto.simplespace.defs#open' }, + }) + }) + + it('maps SpaceAlreadyExists to SpaceExists', async () => { + const { store } = await setup() + await store.createSpace(nsid.conversation, '3aaaaaaaaaaaa') + await expect(store.createSpace(nsid.conversation, '3aaaaaaaaaaaa')).rejects.toBeInstanceOf( + SpaceExists, + ) + }) + + it('writes records to the user repo without PDS validation', async () => { + const { pds, store } = await setup() + const space = await store.createSpace(nsid.conversation, '3aaaaaaaaaaaa') + await store.putRecord(space, nsid.info, 'self', info) + expect(pds.calls.at(-1)?.input).toMatchObject({ repo: did, validate: false }) + }) + + it('maps RecordAlreadyExists to RecordExists', async () => { + const { store } = await setup() + const space = await store.createSpace(nsid.conversation, '3aaaaaaaaaaaa') + await store.createRecord(space, nsid.info, 'self', info) + await expect(store.createRecord(space, nsid.info, 'self', info)).rejects.toBeInstanceOf( + RecordExists, + ) + }) + + it('returns null for a missing record and throws SpaceNotFound for a missing space', async () => { + const { store } = await setup() + const space = await store.createSpace(nsid.conversation, '3aaaaaaaaaaaa') + expect(await store.getRecord(space, nsid.info, 'self')).toBeNull() + await expect(store.putRecord(`${space}x`, nsid.info, 'self', info)).rejects.toBeInstanceOf( + SpaceNotFound, + ) + }) + + it('round-trips record values as JSON', async () => { + const { store } = await setup() + const space = await store.createSpace(nsid.conversation, '3aaaaaaaaaaaa') + await store.putRecord(space, nsid.info, 'self', { ...info, title: 'hello' }) + expect((await store.getRecord(space, nsid.info, 'self'))?.value).toEqual({ + ...info, + title: 'hello', + }) + expect((await store.listRecords(space, nsid.info))[0]?.value).toEqual({ + ...info, + title: 'hello', + }) + }) + + it('lists ops with the commit and keys without values', async () => { + const { store, pds } = await setup() + const space = await store.createSpace(nsid.conversation, '3aaaaaaaaaaaa') + const { cid } = await store.putRecord(space, nsid.info, 'self', info) + const page = await store.listOps(space) + expect(page.ops).toHaveLength(1) + expect(page.commit).toBeDefined() + expect(await store.listKeys(space)).toEqual([{ collection: nsid.info, rkey: 'self', cid }]) + expect(await store.headRev(space)).toBe(page.rev) + }) + + it('fails when the op log reaches its head without a signed commit', async () => { + const client = { call: async () => ({ ops: [] }) } + const store = new SpaceRecordStore(did, async () => client as never) + await expect( + store.listOps(`at://${did}/space/${nsid.conversation}/3aaaaaaaaaaaa`), + ).rejects.toThrow(/without a commit/) + }) +}) diff --git a/apps/server/tests/sync/coalescer.test.ts b/apps/server/tests/sync/coalescer.test.ts new file mode 100644 index 0000000..77bb4dd --- /dev/null +++ b/apps/server/tests/sync/coalescer.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { Coalescer } from '../../src/sync/coalescer.ts' + +const tick = (ms = 0) => new Promise((resolve) => setTimeout(resolve, ms)) + +describe('Coalescer', () => { + it('runs simultaneous requests once', async () => { + const coalescer = new Coalescer(20) + let runs = 0 + const work = async () => { + runs++ + await tick(5) + } + await Promise.all([ + coalescer.request('a', work), + coalescer.request('a', work), + coalescer.request('a', work), + ]) + expect(runs).toBe(1) + }) + + it('runs once more for requests made while running', async () => { + const coalescer = new Coalescer(10) + let runs = 0 + const work = async () => { + runs++ + await tick(15) + } + const first = coalescer.request('a', work) + await tick(5) + const second = coalescer.request('a', work) + const third = coalescer.request('a', work) + await Promise.all([first, second, third]) + expect(runs).toBe(2) + }) + + it('keeps separate keys independent', async () => { + const coalescer = new Coalescer(10) + const seen: string[] = [] + await Promise.all([ + coalescer.request('a', async () => void seen.push('a')), + coalescer.request('b', async () => void seen.push('b')), + ]) + expect(seen.sort()).toEqual(['a', 'b']) + }) + + it('waits the interval before running again', async () => { + const coalescer = new Coalescer(40) + const starts: number[] = [] + const work = async () => void starts.push(Date.now()) + await coalescer.request('a', work) + await coalescer.request('a', work) + expect((starts[1] ?? 0) - (starts[0] ?? 0)).toBeGreaterThanOrEqual(35) + }) + + it('rejects waiters when the work fails', async () => { + const coalescer = new Coalescer(10) + await expect( + coalescer.request('a', async () => { + throw new Error('boom') + }), + ).rejects.toThrow('boom') + }) +}) diff --git a/apps/server/tests/sync/credentials.test.ts b/apps/server/tests/sync/credentials.test.ts new file mode 100644 index 0000000..b9846e1 --- /dev/null +++ b/apps/server/tests/sync/credentials.test.ts @@ -0,0 +1,82 @@ +import { LexError } from '@atproto/lex-data' +import { describe, expect, it, vi } from 'vitest' +import { + CredentialCache, + CredentialError, + mintSpaceCredential, +} from '../../src/sync/credentials.ts' + +const space = 'at://did:plc:alice/space/network.sharedcomputer.chat.settings/self' + +function deps(response: Response, now = () => 1_000) { + const fetchImpl = vi.fn(async (_request: Request) => response.clone()) + return { + fetchImpl, + deps: { + getPdsClient: async () => + ({ call: vi.fn(async () => ({ token: 'delegation-token' })) }) as never, + resolvePds: vi.fn(async () => 'https://pds.test'), + fetch: fetchImpl as unknown as typeof fetch, + now, + }, + } +} + +describe('mintSpaceCredential', () => { + it('exchanges a delegation token with a DPoP proof at the authority PDS', async () => { + const { deps: d, fetchImpl } = deps(Response.json({ credential: 'space-credential' })) + const credential = await mintSpaceCredential(d, 'did:plc:alice', space) + expect(credential.token).toBe('space-credential') + const request = fetchImpl.mock.calls[0]?.[0] as Request + expect(request.url).toBe('https://pds.test/xrpc/com.atproto.space.getSpaceCredential') + expect(request.headers.get('authorization')).toBe('Bearer delegation-token') + expect(request.headers.get('dpop')).toMatch(/^ey/) + expect(d.resolvePds).toHaveBeenCalledWith('did:plc:alice') + }) + + it('fails with the PDS error code when the exchange is refused', async () => { + const { deps: d } = deps( + Response.json({ error: 'UserNotAuthorized', message: 'no' }, { status: 403 }), + ) + const promise = mintSpaceCredential(d, 'did:plc:alice', space) + await expect(promise).rejects.toBeInstanceOf(CredentialError) + await expect(promise).rejects.toMatchObject({ code: 'UserNotAuthorized' }) + }) + + it('sends a DPoP-bound credential on later requests', async () => { + const { deps: d, fetchImpl } = deps(Response.json({ credential: 'space-credential' })) + const credential = await mintSpaceCredential(d, 'did:plc:alice', space) + fetchImpl.mockResolvedValueOnce(Response.json({})) + await credential.fetch('https://pds.test/xrpc/com.atproto.space.listRepos') + const request = fetchImpl.mock.calls[1]?.[0] as Request + expect(request.headers.get('authorization')).toBe('DPoP space-credential') + expect(request.headers.get('dpop')).toMatch(/^ey/) + }) +}) + +describe('CredentialCache', () => { + it('reuses a credential until close to expiry', async () => { + let now = 0 + const { deps: d, fetchImpl } = deps(Response.json({ credential: 'c' }), () => now) + const cache = new CredentialCache(d) + await cache.get('did:plc:alice', space) + await cache.get('did:plc:alice', space) + expect(fetchImpl).toHaveBeenCalledTimes(1) + now = 2 * 60 * 60_000 + await cache.get('did:plc:alice', space) + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) + + it('retries once with a fresh credential after an auth failure', async () => { + const { deps: d, fetchImpl } = deps(Response.json({ credential: 'c' })) + const cache = new CredentialCache(d) + let attempts = 0 + const result = await cache.withCredential('did:plc:alice', space, async () => { + attempts++ + if (attempts === 1) throw new LexError('ExpiredToken', 'expired') + return 'ok' + }) + expect(result).toBe('ok') + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/server/tests/sync/engine.test.ts b/apps/server/tests/sync/engine.test.ts new file mode 100644 index 0000000..00a9869 --- /dev/null +++ b/apps/server/tests/sync/engine.test.ts @@ -0,0 +1,215 @@ +import { nsid } from '@scn-chat/lexicons' +import { sql } from 'kysely' +import { describe, expect, it } from 'vitest' +import { recordCid } from '../../src/storage/records.ts' +import { ALICE, collect, spacesHarness, userMessage } from '../helpers/spaces.ts' + +const stateOf = async (h: Awaited>, space: string) => + h.db.selectFrom('sync_state').selectAll().where('space_uri', '=', space).executeTakeFirst() + +/** A direct client following the index convention: write the message, then bump the entry. */ +async function writeAsClient( + h: Awaited>, + skey: string, + rkey: string, + record: object, +) { + await h.external.createRecord(h.chats.conversationUri(skey), nsid.message, rkey, record as never) + const ref = await h.external.getRecord(h.chats.settingsUri, nsid.conversationRef, skey) + await h.external.putRecord(h.chats.settingsUri, nsid.conversationRef, skey, { + ...(ref?.value ?? { $type: nsid.conversationRef, conversation: h.chats.conversationUri(skey) }), + updatedAt: new Date().toISOString(), + }) +} + +describe('SyncEngine', () => { + it('turns a direct write noticed through the index into a live message event', async () => { + const h = await spacesHarness() + const { skey } = await h.chats.createConversation() + await h.engine.syncIndex(ALICE) + const changed = collect(h.events, 'message:changed') + await writeAsClient(h, skey, '3mmmmmmmmmmm1', userMessage('hi', { generation: {} })) + await h.engine.syncIndex(ALICE) + expect(changed).toMatchObject([ + { did: ALICE, skey, rkey: '3mmmmmmmmmmm1', kind: 'create', live: true }, + ]) + }) + + it('emits backfill events on the first sync of a conversation', async () => { + const h = await spacesHarness() + const { skey } = await h.chats.createConversation() + await h.external.createRecord( + h.chats.conversationUri(skey), + nsid.message, + '3mmmmmmmmmmm1', + userMessage('old') as never, + ) + const changed = collect(h.events, 'message:changed') + await h.engine.syncConversation(ALICE, skey) + expect(changed).toMatchObject([{ rkey: '3mmmmmmmmmmm1', live: false }]) + }) + + it('tells open tabs when another client deletes a message', async () => { + const h = await spacesHarness() + const { skey, uri } = await h.chats.createConversation() + await h.external.createRecord(uri, nsid.message, '3mmmmmmmmmmm1', userMessage('hi') as never) + await h.engine.syncConversation(ALICE, skey) + const changed = collect(h.events, 'conversation:changed') + await h.external.deleteRecord(uri, nsid.message, '3mmmmmmmmmmm1') + await h.engine.syncConversation(ALICE, skey) + expect(changed).toEqual([{ did: ALICE, skey }]) + }) + + it('does not copy the title of an info record that fails validation into the index', async () => { + const h = await spacesHarness() + const { skey, uri } = await h.chats.createConversation() + await h.engine.syncConversation(ALICE, skey) + const value = { $type: nsid.info, title: 'Bad', createdAt: 5 } + const raw = JSON.stringify(value) + const cid = await recordCid(value) + await sql`update local_record set value_json = ${raw}, cid = ${cid} where space_uri = ${uri}`.execute( + h.db, + ) + await sql`insert into local_op (space_uri, collection, rkey, cid, value_json, created_at) values (${uri}, ${nsid.info}, 'self', ${cid}, ${raw}, 'now')`.execute( + h.db, + ) + await h.engine.syncConversation(ALICE, skey) + const { conversations } = await h.chats.listConversations() + expect(conversations[0]?.title).toBeNull() + }) + + it('emits no message events for the server own writes', async () => { + const h = await spacesHarness() + const { skey } = await h.chats.createConversation() + await h.engine.syncConversation(ALICE, skey) + const changed = collect(h.events, 'message:changed') + await h.chats.createMessage(skey, '3mmmmmmmmmmm1', userMessage('mine') as never) + await h.engine.syncConversation(ALICE, skey) + expect(changed).toEqual([]) + }) + + it('syncs only conversations inside the backfill window on a first index sync, and starts at the head', async () => { + const h = await spacesHarness({ backfillWindowMs: 60 * 60_000 }) + const recent = await h.chats.createConversation() + const old = await h.chats.createConversation() + await h.external.putRecord(h.chats.settingsUri, nsid.conversationRef, old.skey, { + $type: nsid.conversationRef, + conversation: old.uri, + updatedAt: new Date(Date.now() - 2 * 60 * 60_000).toISOString(), + }) + await h.engine.syncIndex(ALICE) + expect(await stateOf(h, recent.uri)).toBeDefined() + expect(await stateOf(h, old.uri)).toBeUndefined() + expect((await stateOf(h, h.chats.settingsUri))?.last_rev).toBe( + await h.external.headRev(h.chats.settingsUri), + ) + }) + + it('reports a synced message that fails validation instead of treating it as a message', async () => { + const h = await spacesHarness() + const { skey, uri } = await h.chats.createConversation() + await h.engine.syncConversation(ALICE, skey) + const invalid = collect(h.events, 'message:invalid') + const changed = collect(h.events, 'message:changed') + const value = { $type: nsid.message, role: 'user', createdAt: new Date().toISOString() } + const raw = JSON.stringify(value) + const cid = await recordCid(value) + await sql`insert into local_record values (${uri}, ${nsid.message}, 'bad', ${raw}, ${cid}, 'now')`.execute( + h.db, + ) + await sql`insert into local_op (space_uri, collection, rkey, cid, value_json, created_at) values (${uri}, ${nsid.message}, 'bad', ${cid}, ${raw}, 'now')`.execute( + h.db, + ) + await h.engine.syncConversation(ALICE, skey) + expect(invalid).toMatchObject([{ rkey: 'bad', error: expect.stringMatching(/content/) }]) + expect(changed).toEqual([]) + }) + + it('drops a conversation whose index entry was deleted', async () => { + const h = await spacesHarness() + const { skey, uri } = await h.chats.createConversation() + await h.engine.syncIndex(ALICE) + const deleted = collect(h.events, 'conversation:deleted') + await h.external.deleteRecord(h.chats.settingsUri, nsid.conversationRef, skey) + await h.engine.syncIndex(ALICE) + expect(deleted).toMatchObject([{ skey }]) + expect(await stateOf(h, uri)).toBeUndefined() + }) + + it('copies an externally changed info title into the index entry', async () => { + const h = await spacesHarness() + const { skey, uri } = await h.chats.createConversation() + await h.engine.syncConversation(ALICE, skey) + await h.external.putRecord(uri, nsid.info, 'self', { + $type: nsid.info, + createdAt: new Date().toISOString(), + title: 'Renamed elsewhere', + }) + await h.engine.syncConversation(ALICE, skey) + expect((await h.chats.listConversations()).conversations[0]?.title).toBe('Renamed elsewhere') + }) + + it('adds a missing index entry for a conversation created by another client, and syncs it', async () => { + const h = await spacesHarness() + const uri = await h.external.createSpace(nsid.conversation, '3cccccccccccc') + await h.external.putRecord(uri, nsid.info, 'self', { + $type: nsid.info, + createdAt: new Date().toISOString(), + title: 'From elsewhere', + }) + await h.external.createRecord(uri, nsid.message, '3mmmmmmmmmmm1', userMessage('hello') as never) + const changed = collect(h.events, 'message:changed') + await h.engine.discover(ALICE) + expect((await h.chats.listConversations()).conversations).toMatchObject([ + { skey: '3cccccccccccc', title: 'From elsewhere' }, + ]) + expect(changed).toMatchObject([{ rkey: '3mmmmmmmmmmm1', live: false }]) + }) + + it('does nothing for a local account', async () => { + const h = await spacesHarness({ storageMode: 'local' }) + const changed = collect(h.events, 'message:changed') + await h.engine.syncIndex(ALICE) + expect(changed).toEqual([]) + expect(await stateOf(h, h.chats.settingsUri)).toBeUndefined() + }) + + it('stores the registration expiry and renews registrations that expire within an hour', async () => { + const h = await spacesHarness() + await h.engine.registerIndex(ALICE) + expect((await stateOf(h, h.chats.settingsUri))?.registered_until).toBeTruthy() + await h.db + .updateTable('sync_state') + .set({ registered_until: new Date(Date.now() + 60_000).toISOString() }) + .execute() + await h.engine.renewRegistrations() + const renewed = Date.parse((await stateOf(h, h.chats.settingsUri))?.registered_until ?? '') + expect(renewed).toBeGreaterThan(Date.now() + 60 * 60_000) + }) +}) + +describe('SyncEngine gap detection', () => { + const signedHarness = spacesHarness + + it('keeps the cursor when the signed commit matches the index', async () => { + const h = await signedHarness() + await h.chats.createConversation() + await h.engine.syncIndex(ALICE) + const before = (await stateOf(h, h.chats.settingsUri))?.last_rev + await h.engine.safetyNet(ALICE) + expect((await stateOf(h, h.chats.settingsUri))?.last_rev).toBe(before) + }) + + it('resets the cursor when the signed commit does not match the index', async () => { + const h = await signedHarness() + const { skey } = await h.chats.createConversation() + await h.engine.syncIndex(ALICE) + await h.db + .deleteFrom('sync_state') + .where('space_uri', '=', h.chats.conversationUri(skey)) + .execute() + h.pds.tamperCommits = true + await h.engine.safetyNet(ALICE) + expect(await stateOf(h, h.chats.conversationUri(skey))).toBeDefined() + }) +}) diff --git a/apps/server/tests/sync/identity.test.ts b/apps/server/tests/sync/identity.test.ts new file mode 100644 index 0000000..05883be --- /dev/null +++ b/apps/server/tests/sync/identity.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest' +import { appDid, didDocument, serviceId } from '../../src/sync/identity.ts' + +describe('app identity', () => { + it('derives a did:web from the public URL, encoding a port', () => { + expect(appDid('https://chat.example.com')).toBe('did:web:chat.example.com') + expect(appDid('http://localhost:3000')).toBe('did:web:localhost%3A3000') + }) + + it('names the service with the #scn_chat fragment', () => { + expect(serviceId('https://chat.example.com')).toBe('did:web:chat.example.com#scn_chat') + }) + + it('publishes one AtprotoSpaceService entry pointing at the public URL', () => { + expect(didDocument('https://chat.example.com')).toEqual({ + '@context': ['https://www.w3.org/ns/did/v1'], + id: 'did:web:chat.example.com', + service: [ + { + id: '#scn_chat', + type: 'AtprotoSpaceService', + serviceEndpoint: 'https://chat.example.com', + }, + ], + }) + }) +}) diff --git a/apps/server/tests/sync/recent-writes.test.ts b/apps/server/tests/sync/recent-writes.test.ts new file mode 100644 index 0000000..a399430 --- /dev/null +++ b/apps/server/tests/sync/recent-writes.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from 'vitest' +import { RecentWrites } from '../../src/sync/recent-writes.ts' + +describe('RecentWrites', () => { + it('remembers a CID until its time to live passes', () => { + const writes = new RecentWrites(1_000) + writes.add('bafy1', 0) + expect(writes.has('bafy1', 500)).toBe(true) + expect(writes.has('bafy1', 1_500)).toBe(false) + expect(writes.has('bafy2', 0)).toBe(false) + }) +}) diff --git a/apps/server/tests/sync/routes.test.ts b/apps/server/tests/sync/routes.test.ts new file mode 100644 index 0000000..3d2512c --- /dev/null +++ b/apps/server/tests/sync/routes.test.ts @@ -0,0 +1,195 @@ +import { Secp256k1Keypair } from '@atproto/crypto' +import { createServiceJwt } from '@atproto/xrpc-server' +import { nsid } from '@scn-chat/lexicons' +import pino from 'pino' +import { describe, expect, it, vi } from 'vitest' +import { createApp } from '../../src/app.ts' +import { recordLogin } from '../../src/auth/accounts.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { SyncEventBus } from '../../src/sync/events.ts' +import { appDid, serviceId } from '../../src/sync/identity.ts' +import { testConfig } from '../helpers/config.ts' +import { createSqliteDb } from '../helpers/db.ts' + +const PUBLIC_URL = 'https://chat.example.com' +const ALICE = 'did:plc:alice' +const settings = `at://${ALICE}/space/${nsid.settings}/self` +const conversation = `at://${ALICE}/space/${nsid.conversation}/3aaaaaaaaaaaa` + +async function setup(options: { keyLookupFails?: boolean } = {}) { + const db = createSqliteDb() + await migrateToLatest(db) + await recordLogin(db, { + did: ALICE, + handle: null, + pdsUrl: 'https://pds.test', + spacesAllowed: true, + }) + await recordLogin(db, { + did: 'did:plc:local', + handle: null, + pdsUrl: 'https://pds.test', + spacesAllowed: false, + }) + const keypair = await Secp256k1Keypair.create() + const engine = { syncIndex: vi.fn(async () => {}), syncConversation: vi.fn(async () => {}) } + const app = createApp({ + config: testConfig(), + db, + logger: pino({ level: 'silent' }), + sync: { + db, + engine: engine as never, + events: new SyncEventBus(), + publicUrl: PUBLIC_URL, + resolveSigningKey: async () => { + if (options.keyLookupFails) throw new Error('plc.directory is down') + return keypair.did() + }, + logger: pino({ level: 'silent' }), + }, + }) + const token = (iss: string, aud: string, lxm: string) => + createServiceJwt({ iss: iss as never, aud, lxm: lxm as never, keypair }) + const post = async (path: string, body: object, jwt: string) => + app.request(path, { + method: 'POST', + headers: { authorization: `Bearer ${jwt}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) + return { app, engine, token, post } +} + +describe('GET /.well-known/did.json', () => { + it('publishes the service entry', async () => { + const { app } = await setup() + const doc = (await (await app.request('/.well-known/did.json')).json()) as { + service: { id: string }[] + } + expect(doc.service[0]?.id).toBe('#scn_chat') + }) +}) + +describe('POST notifyWrite', () => { + const lxm = 'com.atproto.space.notifyWrite' + + it('syncs the index for a valid settings-space notification', async () => { + const { post, token, engine } = await setup() + const res = await post( + `/xrpc/${lxm}`, + { space: settings }, + await token(ALICE, serviceId(PUBLIC_URL), lxm), + ) + expect(res.status).toBe(200) + expect(engine.syncIndex).toHaveBeenCalledWith(ALICE) + }) + + it('returns 400 for a body without a space', async () => { + const { post, token } = await setup() + const res = await post(`/xrpc/${lxm}`, {}, await token(ALICE, serviceId(PUBLIC_URL), lxm)) + expect(res.status).toBe(400) + }) + + it('reports a failed key lookup as a server error', async () => { + const { post, token } = await setup({ keyLookupFails: true }) + const res = await post( + `/xrpc/${lxm}`, + { space: settings }, + await token(ALICE, serviceId(PUBLIC_URL), lxm), + ) + expect(res.status).toBe(500) + }) + + it('returns 401 for the wrong audience', async () => { + const { post, token } = await setup() + expect( + ( + await post( + `/xrpc/${lxm}`, + { space: settings }, + await token(ALICE, 'did:web:other.test#scn_chat', lxm), + ) + ).status, + ).toBe(401) + }) + + it('returns 401 for the wrong method', async () => { + const { post, token } = await setup() + expect( + ( + await post( + `/xrpc/${lxm}`, + { space: settings }, + await token(ALICE, serviceId(PUBLIC_URL), 'com.atproto.space.other'), + ) + ).status, + ).toBe(401) + }) + + it('returns 401 when the issuer is not the space authority', async () => { + const { post, token } = await setup() + expect( + ( + await post( + `/xrpc/${lxm}`, + { space: settings }, + await token('did:plc:mallory', serviceId(PUBLIC_URL), lxm), + ) + ).status, + ).toBe(401) + }) +}) + +describe('POST requestSync', () => { + const lxm = nsid.requestSync + + it('syncs the named conversation for its owner', async () => { + const { post, token, engine } = await setup() + const res = await post( + `/xrpc/${lxm}`, + { conversation }, + await token(ALICE, appDid(PUBLIC_URL), lxm), + ) + expect(res.status).toBe(200) + expect(engine.syncConversation).toHaveBeenCalledWith(ALICE, '3aaaaaaaaaaaa') + }) + + it('syncs the whole index without a conversation', async () => { + const { post, token, engine } = await setup() + await post(`/xrpc/${lxm}`, {}, await token(ALICE, appDid(PUBLIC_URL), lxm)) + expect(engine.syncIndex).toHaveBeenCalledWith(ALICE) + }) + + it('fails with UnknownConversation for someone else conversation', async () => { + const { post, token } = await setup() + const other = `at://did:plc:bob/space/${nsid.conversation}/3aaaaaaaaaaaa` + const res = await post( + `/xrpc/${lxm}`, + { conversation: other }, + await token(ALICE, appDid(PUBLIC_URL), lxm), + ) + expect(res.status).toBe(400) + expect(await res.json()).toMatchObject({ error: 'UnknownConversation' }) + }) + + it('refuses a caller who is not a spaces account', async () => { + const { post, token } = await setup() + expect( + (await post(`/xrpc/${lxm}`, {}, await token('did:plc:local', appDid(PUBLIC_URL), lxm))) + .status, + ).toBe(401) + }) + + it('refuses a token for another method', async () => { + const { post, token } = await setup() + expect( + ( + await post( + `/xrpc/${lxm}`, + {}, + await token(ALICE, appDid(PUBLIC_URL), 'com.atproto.space.notifyWrite'), + ) + ).status, + ).toBe(401) + }) +}) diff --git a/apps/server/tests/sync/scheduler.test.ts b/apps/server/tests/sync/scheduler.test.ts new file mode 100644 index 0000000..630318f --- /dev/null +++ b/apps/server/tests/sync/scheduler.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import { recordLogin, setBackgroundSync } from '../../src/auth/accounts.ts' +import { migrateToLatest } from '../../src/db/migrate.ts' +import { eligibleUsers, resolveSyncConfig } from '../../src/sync/scheduler.ts' +import { createSqliteDb } from '../helpers/db.ts' + +async function accounts() { + const db = createSqliteDb() + await migrateToLatest(db) + const base = { handle: null, pdsUrl: 'https://pds.test' } + await recordLogin(db, { ...base, did: 'did:plc:active', spacesAllowed: true }) + await recordLogin(db, { ...base, did: 'did:plc:optedout', spacesAllowed: true }) + await setBackgroundSync(db, 'did:plc:optedout', false) + await recordLogin(db, { ...base, did: 'did:plc:local', spacesAllowed: false }) + await recordLogin( + db, + { ...base, did: 'did:plc:idle', spacesAllowed: true }, + new Date('2020-01-01T00:00:00Z'), + ) + return db +} + +describe('resolveSyncConfig', () => { + it('fills in the defaults', () => { + expect(resolveSyncConfig()).toEqual({ + safetyNet: { enabled: true, intervalMinutes: 15, activeWithinHours: 24 }, + discovery: { intervalMinutes: 60, activeWithinDays: 30 }, + allowUserOptOut: true, + }) + }) +}) + +describe('eligibleUsers', () => { + const since = new Date(Date.now() - 86_400_000) + + it('includes active spaces users and skips opted-out, local, and idle ones', async () => { + const db = await accounts() + expect(await eligibleUsers(db, resolveSyncConfig(), since)).toEqual(['did:plc:active']) + }) + + it('includes users who opted out when the admin does not allow opting out', async () => { + const db = await accounts() + const users = await eligibleUsers(db, resolveSyncConfig({ allowUserOptOut: false }), since) + expect(users.sort()).toEqual(['did:plc:active', 'did:plc:optedout']) + }) +}) diff --git a/apps/server/tests/turns/accumulator.test.ts b/apps/server/tests/turns/accumulator.test.ts new file mode 100644 index 0000000..ac77db1 --- /dev/null +++ b/apps/server/tests/turns/accumulator.test.ts @@ -0,0 +1,91 @@ +import type { TextStreamPart, ToolSet } from 'ai' +import { describe, expect, it } from 'vitest' +import { PartAccumulator, type StreamEvent } from '../../src/turns/accumulator.ts' + +/** A stream part with only the fields the accumulator reads. */ +const part = (chunk: object) => chunk as TextStreamPart + +describe('PartAccumulator', () => { + it('maps stream parts to lexicon parts in order, JSON-encoding tool input and output', () => { + const events: StreamEvent[] = [] + const acc = new PartAccumulator((event) => events.push(event)) + for (const chunk of [ + { type: 'reasoning-start', id: 'r' }, + { type: 'reasoning-delta', id: 'r', text: 'hmm' }, + { type: 'reasoning-end', id: 'r', providerMetadata: { anthropic: { signature: 'SIG' } } }, + { type: 'tool-call', toolCallId: 'c1', toolName: 'search', input: { q: 'x', lat: 49.28 } }, + { type: 'tool-result', toolCallId: 'c1', toolName: 'search', output: { hits: 2 } }, + { type: 'tool-error', toolCallId: 'c2', toolName: 'fetch', error: new Error('timeout') }, + { type: 'source', sourceType: 'url', id: 's', url: 'https://weather.gc.ca', title: 'EC' }, + { type: 'text-start', id: 't' }, + { type: 'text-delta', id: 't', text: 'Hel' }, + { type: 'text-delta', id: 't', text: 'lo' }, + { type: 'text-end', id: 't' }, + { + type: 'finish', + totalUsage: { + inputTokens: 10, + outputTokens: 5, + outputTokenDetails: { reasoningTokens: 2 }, + }, + }, + ]) { + expect(acc.push(part(chunk))).toBeUndefined() + } + expect(acc.parts).toEqual([ + { + $type: 'network.sharedcomputer.chat.defs#reasoningPart', + text: 'hmm', + providerData: '{"anthropic":{"signature":"SIG"}}', + }, + { + $type: 'network.sharedcomputer.chat.defs#toolCallPart', + callId: 'c1', + tool: 'search', + input: '{"q":"x","lat":49.28}', + }, + { + $type: 'network.sharedcomputer.chat.defs#toolResultPart', + callId: 'c1', + output: '{"hits":2}', + }, + { + $type: 'network.sharedcomputer.chat.defs#toolResultPart', + callId: 'c2', + output: 'timeout', + isError: true, + }, + { + $type: 'network.sharedcomputer.chat.defs#sourcePart', + url: 'https://weather.gc.ca', + title: 'EC', + }, + { $type: 'network.sharedcomputer.chat.defs#textPart', text: 'Hello' }, + ]) + expect(acc.lexiconUsage()).toEqual({ inputTokens: 10, outputTokens: 5, reasoningTokens: 2 }) + expect( + events.filter((e) => e.type === 'delta').map((e) => (e as { text: string }).text), + ).toEqual(['hmm', 'Hel', 'lo']) + }) + + it('stores unparsable tool input JSON-encoded, so it replays as the string the model sent', () => { + const acc = new PartAccumulator() + acc.push(part({ type: 'tool-call', toolCallId: 'c1', toolName: 'search', input: '{"q": ' })) + expect(JSON.parse(acc.parts[0]?.input as string)).toBe('{"q": ') + }) + + it('fails the turn on a generated file, which the lexicon cannot hold yet', () => { + expect(new PartAccumulator().push(part({ type: 'file', file: {} }))).toBeInstanceOf(Error) + }) + + it('returns the error from an error part', () => { + const error = new Error('401 Unauthorized') + expect(new PartAccumulator().push(part({ type: 'error', error }))).toBe(error) + }) + + it('leaves out usage fields that are missing', () => { + const acc = new PartAccumulator() + acc.push(part({ type: 'finish', totalUsage: { inputTokens: 3, outputTokenDetails: {} } })) + expect(acc.lexiconUsage()).toEqual({ inputTokens: 3 }) + }) +}) diff --git a/apps/server/tests/turns/prompt.test.ts b/apps/server/tests/turns/prompt.test.ts new file mode 100644 index 0000000..49f5736 --- /dev/null +++ b/apps/server/tests/turns/prompt.test.ts @@ -0,0 +1,246 @@ +import { nsid } from '@scn-chat/lexicons' +import type { BranchMessage, ModelProvider } from '@scn-chat/plugin-api' +import { describe, expect, it } from 'vitest' +import { + branchTo, + buildInstructions, + lastReplyModel, + TurnInputError, + toModelMessages, +} from '../../src/turns/prompt.ts' + +const d = (name: string) => `${nsid.defs}#${name}` +const msg = (rkey: string, record: Record): BranchMessage => ({ + rkey, + author: 'did:plc:a', + record: record as never, +}) +const content = (...parts: Record[]) => ({ $type: d('plainContent'), parts }) +const text = (value: string, extra: Record = {}) => ({ + $type: d('textPart'), + text: value, + ...extra, +}) +const caps = { vision: true, reasoning: true, tools: true } +const provider = (id: string, replay: 'replay' | 'drop' = 'replay') => + ({ id, replay }) as ModelProvider +const noBlobs = async () => ({ bytes: new Uint8Array(), mimeType: 'text/plain' }) + +describe('branchTo', () => { + it('follows parent keys from the root to the message, leaving out other branches', () => { + const messages = new Map([ + ['a', msg('a', { role: 'user' })], + ['a.r0', msg('a.r0', { role: 'assistant', parent: 'a' })], + ['b', msg('b', { role: 'user', parent: 'a.r0' })], + ['sibling', msg('sibling', { role: 'user', parent: 'a.r0' })], + ]) + expect(branchTo(messages, 'b').map((m) => m.rkey)).toEqual(['a', 'a.r0', 'b']) + }) + + it('stops at a cycle', () => { + const messages = new Map([ + ['a', msg('a', { role: 'user', parent: 'b' })], + ['b', msg('b', { role: 'user', parent: 'a' })], + ]) + expect(branchTo(messages, 'a').map((m) => m.rkey)).toEqual(['b', 'a']) + }) +}) + +describe('lastReplyModel', () => { + it('returns the model of the nearest completed reply', () => { + const branch = [ + msg('a.r0', { role: 'assistant', status: 'complete', model: { provider: 'p', id: 'old' } }), + msg('b.r0', { role: 'assistant', status: 'complete', model: { provider: 'p', id: 'new' } }), + msg('c.r0', { role: 'assistant', status: 'error', model: { provider: 'p', id: 'broken' } }), + ] + expect(lastReplyModel(branch)).toEqual({ provider: 'p', id: 'new' }) + }) +}) + +describe('buildInstructions', () => { + it('joins custom instructions and the system prompt with a blank line', () => { + expect( + buildInstructions({ customInstructions: 'Metric units' }, { systemPrompt: 'Be brief' }), + ).toBe('Metric units\n\nBe brief') + expect(buildInstructions(null, null)).toBe('') + }) +}) + +describe('toModelMessages', () => { + it('leaves out errored and pending replies and keeps cancelled ones', async () => { + const branch = [ + msg('a', { role: 'user', content: content(text('q1')) }), + msg('a.r0', { + role: 'assistant', + status: 'error', + model: { provider: 'p', id: 'm' }, + content: content(text('boom')), + }), + msg('b', { role: 'user', content: content(text('q2')) }), + msg('b.r0', { + role: 'assistant', + status: 'cancelled', + model: { provider: 'p', id: 'm' }, + content: content(text('partial')), + }), + msg('c', { role: 'user', content: content(text('q3')) }), + msg('c.r0', { role: 'assistant', status: 'pending', content: content() }), + ] + const messages = await toModelMessages(branch, { + provider: provider('p'), + capabilities: caps, + readBlob: noBlobs, + }) + expect(JSON.stringify(messages)).not.toContain('boom') + expect(JSON.stringify(messages)).toContain('partial') + expect(messages.map((m) => m.role)).toEqual(['user', 'user', 'assistant', 'user']) + }) + + it('replays reasoning and provider data only for the same provider with a replay policy', async () => { + const reply = (provider: string) => + msg('r', { + role: 'assistant', + status: 'complete', + model: { provider, id: 'm' }, + content: content( + { + $type: d('reasoningPart'), + text: 'thinking', + providerData: JSON.stringify({ anthropic: { signature: 'SIG' } }), + }, + text('answer', { providerData: JSON.stringify({ google: { thoughtSignature: 'T' } }) }), + ), + }) + const same = await toModelMessages([reply('anthropic')], { + provider: provider('anthropic'), + capabilities: caps, + readBlob: noBlobs, + }) + expect(JSON.stringify(same)).toContain('SIG') + expect(JSON.stringify(same)).toContain('"type":"reasoning"') + const other = await toModelMessages([reply('anthropic')], { + provider: provider('openai'), + capabilities: caps, + readBlob: noBlobs, + }) + expect(JSON.stringify(other)).not.toContain('SIG') + expect(JSON.stringify(other)).not.toContain('reasoning') + const dropping = await toModelMessages([reply('local')], { + provider: provider('local', 'drop'), + capabilities: caps, + readBlob: noBlobs, + }) + expect(JSON.stringify(dropping)).not.toContain('thinking') + }) + + it('splits tool calls and results into assistant and tool messages', async () => { + const branch = [ + msg('r', { + role: 'assistant', + status: 'complete', + model: { provider: 'p', id: 'm' }, + content: content( + { + $type: d('toolCallPart'), + callId: 'c1', + tool: 'search', + input: '{"q":"x","lat":49.28}', + }, + { $type: d('toolResultPart'), callId: 'c1', output: 'found it' }, + text('done'), + ), + }), + ] + const messages = await toModelMessages(branch, { + provider: provider('p'), + capabilities: caps, + readBlob: noBlobs, + }) + expect(messages.map((m) => m.role)).toEqual(['assistant', 'tool', 'assistant']) + expect(JSON.stringify(messages[0])).toContain('"lat":49.28') + expect(JSON.stringify(messages[1])).toContain('"toolName":"search"') + }) + + it('sends images as file parts and extracted file text as a headed text part', async () => { + const blobs: Record = { + img: { bytes: new Uint8Array([1, 2]), mimeType: 'image/png' }, + txt: { bytes: new TextEncoder().encode('PDF body'), mimeType: 'text/plain' }, + } + const branch = [ + msg('a', { + role: 'user', + content: content( + { + $type: d('imagePart'), + image: { $type: 'blob', ref: { $link: 'img' }, mimeType: 'image/png', size: 2 }, + }, + { + $type: d('filePart'), + name: 'report.pdf', + file: {}, + extracted: { text: { ref: { $link: 'txt' } }, method: 'text' }, + }, + { $type: d('filePart'), name: 'scan.pdf', file: {} }, + ), + }), + ] + const [user] = await toModelMessages(branch, { + provider: provider('p'), + capabilities: caps, + readBlob: async (cid) => blobs[cid] as never, + }) + const parts = user?.content as { type: string; text?: string; mediaType?: string }[] + expect(parts[0]).toMatchObject({ type: 'file', mediaType: 'image/png' }) + expect(parts[1]?.text).toBe('# report.pdf\n\nPDF body') + expect(parts[2]?.text).toMatch(/scan\.pdf.*could not be read/) + }) + + it('refuses images for a model without vision', async () => { + const branch = [ + msg('a', { + role: 'user', + content: content({ $type: d('imagePart'), image: { ref: { $link: 'x' } } }), + }), + ] + await expect( + toModelMessages(branch, { + provider: provider('p'), + capabilities: { ...caps, vision: false }, + readBlob: noBlobs, + }), + ).rejects.toBeInstanceOf(TurnInputError) + }) + + it('throws on a tool result with no tool call before it', async () => { + const branch = [ + msg('r', { + role: 'assistant', + status: 'complete', + content: content({ $type: d('toolResultPart'), callId: 'c9', output: 'orphan' }), + }), + ] + await expect( + toModelMessages(branch, { provider: provider('p'), capabilities: caps, readBlob: noBlobs }), + ).rejects.toThrow(/c9 has no tool call/) + }) + + it('throws on tool input that is not JSON', async () => { + const branch = [ + msg('r', { + role: 'assistant', + status: 'complete', + content: content({ $type: d('toolCallPart'), callId: 'c1', tool: 't', input: '{"q": ' }), + }), + ] + await expect( + toModelMessages(branch, { provider: provider('p'), capabilities: caps, readBlob: noBlobs }), + ).rejects.toBeInstanceOf(SyntaxError) + }) + + it('fails with a readable error on encrypted content', async () => { + const branch = [msg('a', { role: 'user', content: { $type: d('encryptedContent') } })] + await expect( + toModelMessages(branch, { provider: provider('p'), capabilities: caps, readBlob: noBlobs }), + ).rejects.toThrow(/encrypted/) + }) +}) diff --git a/apps/server/tests/turns/routes.test.ts b/apps/server/tests/turns/routes.test.ts new file mode 100644 index 0000000..7a89689 --- /dev/null +++ b/apps/server/tests/turns/routes.test.ts @@ -0,0 +1,124 @@ +import { nsid } from '@scn-chat/lexicons' +import { describe, expect, it, vi } from 'vitest' +import { createApp } from '../../src/app.ts' +import { buildScope } from '../../src/auth/scope.ts' +import { authDeps, fakeOAuth, fakeSession, ORIGIN, sessionCookie } from '../helpers/auth.ts' +import { testConfig } from '../helpers/config.ts' +import { scriptedModel, textReply, turnsHarness } from '../helpers/turns.ts' + +const text = (value: string) => ({ $type: `${nsid.defs}#textPart`, text: value }) + +async function setup(options: Parameters[0] = {}) { + const h = await turnsHarness(options) + const oauth = fakeOAuth({ + callback: vi.fn(async () => ({ session: fakeSession('did:plc:alice', buildScope('raw')) })), + }) + const app = createApp({ + config: testConfig(), + db: h.db, + logger: h.logger, + auth: authDeps(h.db, { oauth }), + turns: { services: h.services, runner: h.runner, hub: h.hub }, + }) + const cookie = sessionCookie(await app.request('/oauth/callback?code=a&state=b')) + const post = async (path: string, body: object = {}) => { + const res = await app.request(`/api${path}`, { + method: 'POST', + headers: { cookie, origin: ORIGIN, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) + return { status: res.status, body: (await res.json()) as Record } + } + const { skey } = await h.chats.createConversation() + return { ...h, app, cookie, post, skey } +} + +describe('POST /api/conversations/:skey/messages', () => { + it('writes the user message and starts a turn when asked for a reply', async () => { + const { post, skey, runner, messages } = await setup() + const res = await post(`/conversations/${skey}/messages`, { + parts: [text('hi')], + generation: {}, + }) + expect(res.status).toBe(201) + expect(res.body.replyRkey).toBe(`${res.body.rkey}.r0`) + await runner.idle() + expect((await messages(skey)).get(String(res.body.replyRkey))?.status).toBe('complete') + }) + + it('writes only the user message without a generation request', async () => { + const { post, skey, messages } = await setup() + const res = await post(`/conversations/${skey}/messages`, { parts: [text('note to self')] }) + expect(res.body.replyRkey).toBeNull() + expect((await messages(skey)).size).toBe(1) + }) + + it('refuses assistant parts or an empty message', async () => { + const { post, skey } = await setup() + expect((await post(`/conversations/${skey}/messages`, { parts: [] })).status).toBe(400) + const reasoning = { $type: `${nsid.defs}#reasoningPart`, text: 'sneaky' } + expect((await post(`/conversations/${skey}/messages`, { parts: [reasoning] })).status).toBe(400) + }) +}) + +describe('regenerate and cancel', () => { + it('raises the attempt and writes the next reply', async () => { + const { post, skey, runner, messages } = await setup() + const sent = await post(`/conversations/${skey}/messages`, { + parts: [text('hi')], + generation: {}, + }) + await runner.idle() + const res = await post(`/conversations/${skey}/messages/${sent.body.rkey}/regenerate`, { + effort: 'high', + }) + expect(res.body.replyRkey).toBe(`${sent.body.rkey}.r1`) + await runner.idle() + const user = (await messages(skey)).get(String(sent.body.rkey)) + expect(user?.generation).toMatchObject({ attempt: 1, effort: 'high' }) + }) + + it('cancels a running reply', async () => { + const { post, skey, runner, messages } = await setup({ + model: () => scriptedModel(textReply('slow reply here'), { delayMs: 30 }).model, + }) + const sent = await post(`/conversations/${skey}/messages`, { + parts: [text('hi')], + generation: {}, + }) + await new Promise((resolve) => setTimeout(resolve, 80)) + expect( + (await post(`/conversations/${skey}/messages/${sent.body.replyRkey}/cancel`)).body.cancelled, + ).toBe(true) + await runner.idle() + expect((await messages(skey)).get(String(sent.body.replyRkey))?.status).toBe('cancelled') + }) +}) + +describe('GET /api/conversations/:skey/messages/:rkey/stream', () => { + it('reports an unknown status for a reply this server is not running', async () => { + const { app, cookie, skey } = await setup() + const res = await app.request(`/api/conversations/${skey}/messages/nothing.r0/stream`, { + headers: { cookie }, + }) + expect(await res.text()).toContain('"status":"unknown"') + }) + + it('streams a running reply to the end', async () => { + const { app, cookie, post, skey } = await setup({ + model: () => scriptedModel(textReply('stream me'), { delayMs: 10 }).model, + }) + const sent = await post(`/conversations/${skey}/messages`, { + parts: [text('hi')], + generation: {}, + }) + const res = await app.request( + `/api/conversations/${skey}/messages/${sent.body.replyRkey}/stream`, + { headers: { cookie } }, + ) + const body = await res.text() + expect(body).toContain('event: part-start') + expect(body).toContain('event: delta') + expect(body).toContain('"status":"complete"') + }) +}) diff --git a/apps/server/tests/turns/runner.test.ts b/apps/server/tests/turns/runner.test.ts new file mode 100644 index 0000000..a0120cf --- /dev/null +++ b/apps/server/tests/turns/runner.test.ts @@ -0,0 +1,564 @@ +import { nsid } from '@scn-chat/lexicons' +import { definePlugin, type TurnContext } from '@scn-chat/plugin-api' +import pino from 'pino' +import { describe, expect, it, vi } from 'vitest' +import { z } from 'zod' +import { loadPlugins } from '../../src/plugins/host.ts' +import { LocalRecordStore } from '../../src/storage/local-record-store.ts' +import { safeErrorMessage } from '../../src/turns/runner.ts' +import { ALICE, textContent, userMessage } from '../helpers/spaces.ts' +import { scriptedModel, sequenceModel, textReply, turnsHarness } from '../helpers/turns.ts' + +const partsOf = (record: Record | undefined) => + ((record?.content as { parts: { text?: string }[] })?.parts ?? []).map((p) => p.text).join('') + +async function sendUser( + h: Awaited>, + skey: string, + rkey: string, + extra: Record = {}, +) { + await h.chats.createMessage( + skey, + rkey, + userMessage(`question ${rkey}`, { generation: {}, ...extra }) as never, + ) +} + +describe('TurnRunner.start', () => { + it('claims with a pending reply at .r0, then writes the finished reply', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + const started = await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + expect(started).toEqual({ status: 'claimed', replyRkey: '3uuuuuuuuuuu1.r0' }) + await h.runner.idle() + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r0') + expect(reply).toMatchObject({ + role: 'assistant', + parent: '3uuuuuuuuuuu1', + status: 'complete', + model: { provider: 'fake', id: 'default-model' }, + }) + expect(partsOf(reply)).toBe('Hello') + expect(reply?.usage).toEqual({ inputTokens: 10, outputTokens: 5, reasoningTokens: 0 }) + }) + + it('skips a user message without a generation request', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await h.chats.createMessage(skey, '3uuuuuuuuuuu1', userMessage('no reply wanted') as never) + expect(await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1')).toEqual({ status: 'skipped' }) + expect((await h.messages(skey)).size).toBe(1) + }) + + it('runs exactly one generation when two runners notice the same message', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + const results = await Promise.all([ + h.runner.start(ALICE, skey, '3uuuuuuuuuuu1'), + h.runner.start(ALICE, skey, '3uuuuuuuuuuu1'), + ]) + await h.runner.idle() + expect(results.map((r) => r.status).sort()).toEqual(['claimed', 'exists']) + expect(h.created).toHaveLength(1) + }) + + it('writes regenerations at .r and keeps earlier replies', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + const current = (await h.messages(skey)).get('3uuuuuuuuuuu1') as Record + await h.chats.putMessage(skey, '3uuuuuuuuuuu1', { ...current, generation: { attempt: 1 } }) + expect((await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1')).replyRkey).toBe('3uuuuuuuuuuu1.r1') + await h.runner.idle() + const all = await h.messages(skey) + expect(all.has('3uuuuuuuuuuu1.r0')).toBe(true) + expect(all.get('3uuuuuuuuuuu1.r1')?.status).toBe('complete') + }) + + it('gives an edit its own reply as a sibling', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + await sendUser(h, skey, '3uuuuuuuuuuu2', { parent: '3uuuuuuuuuuu1.r0' }) + await sendUser(h, skey, '3uuuuuuuuuuu3', { parent: '3uuuuuuuuuuu1.r0' }) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu2') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu3') + await h.runner.idle() + const all = await h.messages(skey) + expect(all.get('3uuuuuuuuuuu2.r0')?.status).toBe('complete') + expect(all.get('3uuuuuuuuuuu3.r0')?.status).toBe('complete') + }) + + it('resolves the model from the request, then the last reply, then preferences, then the admin default', async () => { + const h = await turnsHarness({ + adminModels: [ + { id: 'admin-default', default: true }, + { id: 'requested' }, + { id: 'preferred' }, + ], + }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + await h.chats.putPreferences({ defaultModel: { provider: 'fake', id: 'preferred' } }) + await sendUser(h, skey, '3uuuuuuuuuuu2', { + parent: '3uuuuuuuuuuu1.r0', + generation: { model: { provider: 'fake', id: 'requested' } }, + }) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu2') + await h.runner.idle() + await sendUser(h, skey, '3uuuuuuuuuuu3', { parent: '3uuuuuuuuuuu2.r0' }) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu3') + await h.runner.idle() + const other = await h.chats.createConversation() + await sendUser(h, other.skey, '3uuuuuuuuuuu4') + await h.runner.start(ALICE, other.skey, '3uuuuuuuuuuu4') + await h.runner.idle() + expect(h.created).toEqual(['admin-default', 'requested', 'requested', 'preferred']) + }) + + it('sends only the branch, with custom instructions and the system prompt as instructions', async () => { + const script = scriptedModel(textReply('ok')) + const h = await turnsHarness({ model: () => script.model }) + const { skey } = await h.chats.createConversation({ systemPrompt: 'Be brief' }) + await h.chats.putPreferences({ customInstructions: 'Metric units' }) + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + await sendUser(h, skey, '3uuuuuuuuuuuX', { parent: '3uuuuuuuuuuu1.r0' }) + await sendUser(h, skey, '3uuuuuuuuuuu2', { parent: '3uuuuuuuuuuu1.r0' }) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu2') + await h.runner.idle() + const prompt = JSON.stringify(script.prompts.at(-1)) + expect(prompt).toContain('Metric units\\n\\nBe brief') + expect(prompt).toContain('question 3uuuuuuuuuuu1') + expect(prompt).toContain('question 3uuuuuuuuuuu2') + expect(prompt).not.toContain('question 3uuuuuuuuuuuX') + }) + + it('ends with status error, a message without secrets, and the partial parts on a provider error', async () => { + const parts = [ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: 't' }, + { type: 'text-delta', id: 't', delta: 'Part' }, + { type: 'error', error: new Error('upstream rejected key sk-live-abcdef123456') }, + ] + const h = await turnsHarness({ model: () => scriptedModel(parts).model }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r0') + expect(reply?.status).toBe('error') + expect(reply?.error).not.toContain('sk-live') + expect(partsOf(reply)).toBe('Part') + }) + + it('ends with status cancelled and keeps partial parts when cancelled', async () => { + const h = await turnsHarness({ + model: () => + scriptedModel(textReply('a long reply that streams slowly'), { delayMs: 20 }).model, + }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await new Promise((resolve) => setTimeout(resolve, 150)) + expect(h.runner.cancel(ALICE, skey, '3uuuuuuuuuuu1.r0')).toBe(true) + await h.runner.idle() + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r0') + expect(reply?.status).toBe('cancelled') + expect(partsOf(reply).length).toBeGreaterThan(0) + expect(partsOf(reply).length).toBeLessThan('a long reply that streams slowly'.length) + }) + + it('cancels a turn that runs past the timeout', async () => { + const h = await turnsHarness({ + timeoutMs: 100, + model: () => scriptedModel(textReply('slow slow slow slow'), { delayMs: 30 }).model, + }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + expect((await h.messages(skey)).get('3uuuuuuuuuuu1.r0')?.status).toBe('cancelled') + }) + + it('moves the largest tool outputs into blobs when the record would pass 900 KB', async () => { + const huge = 'x'.repeat(950_000) + const host = await loadPlugins( + [ + definePlugin({ + id: 'fetcher', + name: 'Fetcher', + apiVersion: 1, + setup: (ctx) => + ctx.tools.register({ + name: 'fetch', + description: 'Fetch a page', + inputSchema: z.object({}) as never, + run: async () => huge, + }), + }), + ], + { + services: {} as never, + logger: pino({ level: 'silent' }), + app: { name: 'T', publicUrl: 'http://x' }, + }, + ) + const finish = { + type: 'finish', + usage: { inputTokens: { total: 1 }, outputTokens: { total: 1 } }, + finishReason: { unified: 'tool-calls', raw: 'tool_use' }, + } + const script = sequenceModel([ + [ + { type: 'stream-start', warnings: [] }, + { type: 'tool-call', toolCallId: 'c1', toolName: 'fetch', input: '{}' }, + finish, + ], + textReply('Summarized.'), + ]) + const h = await turnsHarness({ host, model: () => script.model }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1', { generation: { tools: ['fetch'] } }) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r0') + expect(reply?.status).toBe('complete') + const result = (reply?.content as { parts: Record[] }).parts.find((p) => + String(p.$type).endsWith('toolResultPart'), + ) + expect(result?.outputBlob).toMatchObject({ $type: 'blob', mimeType: 'text/plain' }) + expect(String(result?.output)).toMatch(/stored as a blob/) + expect(h.stored.get(`bafkrei0fake`)?.length).toBeGreaterThan(900_000) + }) + + it('queues past the rate limit without claiming, and starts once the limit allows', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + try { + const h = await turnsHarness({ ratePerMinute: 1 }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await sendUser(h, skey, '3uuuuuuuuuuu2') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + expect(await h.runner.start(ALICE, skey, '3uuuuuuuuuuu2')).toEqual({ + status: 'queued', + replyRkey: '3uuuuuuuuuuu2.r0', + }) + expect((await h.messages(skey)).has('3uuuuuuuuuuu2.r0')).toBe(false) + expect(await h.db.selectFrom('turn_request').selectAll().execute()).toHaveLength(1) + await vi.advanceTimersByTimeAsync(61_000) + await h.runner.idle() + expect((await h.messages(skey)).get('3uuuuuuuuuuu2.r0')?.status).toBe('complete') + expect(h.created).toHaveLength(2) + } finally { + vi.useRealTimers() + } + }) + + it('queues the same unclaimed turn once', async () => { + const h = await turnsHarness({ ratePerMinute: 1 }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await sendUser(h, skey, '3uuuuuuuuuuu2') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu2') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu2') + expect(await h.db.selectFrom('turn_request').selectAll().execute()).toHaveLength(1) + }) + + it('runs the turn:after hook after the final record is written', async () => { + let seen: TurnContext | undefined + const host = await loadPlugins( + [ + definePlugin({ + id: 'watcher', + name: 'Watcher', + apiVersion: 1, + setup: (ctx) => ctx.hooks.on('turn:after', (turn) => void (seen = turn)), + }), + ], + { + services: {} as never, + logger: pino({ level: 'silent' }), + app: { name: 'T', publicUrl: 'http://x' }, + }, + ) + const h = await turnsHarness({ host }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + expect(seen?.reply.rkey).toBe('3uuuuuuuuuuu1.r0') + expect((seen?.reply.record as unknown as { status: string }).status).toBe('complete') + }) + + it('fails the turn with an error naming the plugin when a filter throws', async () => { + const host = await loadPlugins( + [ + definePlugin({ + id: 'strict', + name: 'Strict', + apiVersion: 1, + setup: (ctx) => + ctx.hooks.on('messages:beforeModel', () => { + throw new Error('nope') + }), + }), + ], + { + services: {} as never, + logger: pino({ level: 'silent' }), + app: { name: 'T', publicUrl: 'http://x' }, + }, + ) + const h = await turnsHarness({ host }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r0') + expect(reply).toMatchObject({ status: 'error', error: expect.stringContaining('strict') }) + }) + + it('saves an empty reply when an afterModel filter throws', async () => { + const host = await loadPlugins( + [ + definePlugin({ + id: 'redact', + name: 'Redact', + apiVersion: 1, + setup: (ctx) => + ctx.hooks.on('message:afterModel', () => { + throw new Error('redaction failed') + }), + }), + ], + { + services: {} as never, + logger: pino({ level: 'silent' }), + app: { name: 'T', publicUrl: 'http://x' }, + }, + ) + const h = await turnsHarness({ host }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r0') + expect(reply).toMatchObject({ status: 'error', error: expect.stringContaining('redact') }) + expect(partsOf(reply)).toBe('') + }) + + it('fails the turn on encrypted messages', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await h.chats.createMessage(skey, '3uuuuuuuuuuu1', { + ...userMessage('x', { generation: {} }), + content: { + $type: `${nsid.defs}#encryptedContent`, + scheme: 'test', + keyId: 'k', + nonce: { $bytes: 'AAAAAAAAAAAAAAAA' }, + ciphertext: { $bytes: 'AAAA' }, + }, + } as never) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r0') + expect(reply).toMatchObject({ status: 'error', error: expect.stringContaining('encrypted') }) + }) + + it('fails the turn when it asks for a tool that is not installed', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1', { generation: { tools: ['web_search'] } }) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r0') + expect(reply).toMatchObject({ status: 'error', error: expect.stringContaining('web_search') }) + }) + + it('keeps the claim for recovery when neither the reply nor the error reply can be written', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + const put = vi + .spyOn(LocalRecordStore.prototype, 'putRecord') + .mockRejectedValue(new Error('PDS unavailable')) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + put.mockRestore() + const claims = await h.db.selectFrom('turn_claim').select('reply_rkey').execute() + expect(claims).toEqual([{ reply_rkey: '3uuuuuuuuuuu1.r0' }]) + }) + + it('throws for an account that does not exist', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await expect(h.runner.start('did:plc:nobody', skey, '3uuuuuuuuuuu1')).rejects.toThrow( + /No account/, + ) + }) + + it('publishes stream events that a late subscriber replays before following live', async () => { + const h = await turnsHarness({ + model: () => scriptedModel(textReply('abc'), { delayMs: 15 }).model, + }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await new Promise((resolve) => setTimeout(resolve, 60)) + const seen: string[] = [] + await new Promise((resolve) => + h.hub.subscribe(h.runner.streamKey(ALICE, skey, '3uuuuuuuuuuu1.r0'), (event) => { + seen.push(event.type) + if (event.type === 'status') resolve() + }), + ) + expect(seen[0]).toBe('part-start') + expect(seen.filter((t) => t === 'delta')).toHaveLength(3) + expect(seen.at(-1)).toBe('status') + }) + + it('works the same for a local account', async () => { + const h = await turnsHarness({ storageMode: 'local' }) + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + expect((await h.messages(skey)).get('3uuuuuuuuuuu1.r0')?.status).toBe('complete') + }) +}) + +describe('turns started from sync', () => { + it('starts a turn for a live direct write with a generation request', async () => { + const h = await turnsHarness() + h.runner.attach(h.events) + const { skey } = await h.chats.createConversation() + await h.engine.syncConversation(ALICE, skey) + await h.external.createRecord( + h.chats.conversationUri(skey), + nsid.message, + '3uuuuuuuuuuu1', + userMessage('from a client', { generation: {} }) as never, + ) + await h.engine.syncConversation(ALICE, skey) + await vi.waitFor(async () => + expect((await h.messages(skey)).get('3uuuuuuuuuuu1.r0')?.status).toBe('complete'), + ) + }) + + it('answers recent backfilled messages and ignores old ones', async () => { + const h = await turnsHarness() + h.runner.attach(h.events) + const { skey } = await h.chats.createConversation() + const uri = h.chats.conversationUri(skey) + const old = new Date(Date.now() - 2 * 60 * 60_000).toISOString() + await h.external.createRecord( + uri, + nsid.message, + '3uuuuuuuuuuu1', + userMessage('old', { generation: {}, createdAt: old }) as never, + ) + await h.external.createRecord( + uri, + nsid.message, + '3uuuuuuuuuuu2', + userMessage('recent', { generation: {} }) as never, + ) + await h.engine.syncConversation(ALICE, skey) + await vi.waitFor(async () => + expect((await h.messages(skey)).get('3uuuuuuuuuuu2.r0')?.status).toBe('complete'), + ) + expect((await h.messages(skey)).has('3uuuuuuuuuuu1.r0')).toBe(false) + }) + + it('answers an invalid direct write with an error reply quoting the validation error', async () => { + const h = await turnsHarness() + const { skey } = await h.chats.createConversation() + await h.runner.answerInvalid({ + did: ALICE, + skey, + rkey: '3uuuuuuuuuuu1', + raw: { role: 'user', generation: { attempt: 2 } }, + error: 'Missing required key "content"', + }) + const reply = (await h.messages(skey)).get('3uuuuuuuuuuu1.r2') + expect(reply).toMatchObject({ status: 'error', parent: '3uuuuuuuuuuu1' }) + expect(reply?.error).toContain('Missing required key "content"') + }) +}) + +describe('TurnRunner.recover', () => { + it('marks this server interrupted replies and leaves other pending replies alone', async () => { + const h = await turnsHarness() + const { skey, uri } = await h.chats.createConversation() + const pending = { + $type: nsid.message, + role: 'assistant', + content: textContent(''), + status: 'pending', + createdAt: new Date().toISOString(), + } + await h.chats.createMessage(skey, 'ours.r0', pending as never) + await h.chats.createMessage(skey, 'theirs.r0', pending as never) + await h.db + .insertInto('turn_claim') + .values({ conversation_uri: uri, reply_rkey: 'ours.r0', owner_did: ALICE, claimed_at: 'now' }) + .execute() + await h.runner.recover() + const all = await h.messages(skey) + expect(all.get('ours.r0')).toMatchObject({ status: 'error', error: 'interrupted' }) + expect(all.get('theirs.r0')?.status).toBe('pending') + expect(await h.db.selectFrom('turn_claim').selectAll().execute()).toEqual([]) + }) + + it('retries recent queued turns and drops old ones', async () => { + const h = await turnsHarness() + const { skey, uri } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + await sendUser(h, skey, '3uuuuuuuuuuu2') + await h.db + .insertInto('turn_request') + .values([ + { + conversation_uri: uri, + message_rkey: '3uuuuuuuuuuu1', + attempt: 0, + owner_did: ALICE, + requested_at: new Date().toISOString(), + }, + { + conversation_uri: uri, + message_rkey: '3uuuuuuuuuuu2', + attempt: 0, + owner_did: ALICE, + requested_at: '2020-01-01T00:00:00.000Z', + }, + ]) + .execute() + await h.runner.recover() + await h.runner.idle() + const all = await h.messages(skey) + expect(all.get('3uuuuuuuuuuu1.r0')?.status).toBe('complete') + expect(all.has('3uuuuuuuuuuu2.r0')).toBe(false) + }) +}) + +describe('safeErrorMessage', () => { + it('removes API keys and bearer tokens', () => { + expect(safeErrorMessage(new Error('bad key sk-proj-abc12345678 and Bearer abc.def.ghi'))).toBe( + 'bad key [redacted] and Bearer [redacted]', + ) + }) +}) diff --git a/apps/server/tsconfig.json b/apps/server/tsconfig.json new file mode 100644 index 0000000..71351fd --- /dev/null +++ b/apps/server/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src", "tests"] +} diff --git a/apps/web/index.html b/apps/web/index.html new file mode 100644 index 0000000..a0e85df --- /dev/null +++ b/apps/web/index.html @@ -0,0 +1,13 @@ + + + + + + + __APP_NAME__ + + +
+ + + diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 0000000..d136308 --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,33 @@ +{ + "name": "@scn-chat/web", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "typecheck": "tsc --noEmit -p ." + }, + "dependencies": { + "@scn-chat/lexicons": "workspace:*", + "@sqlite.org/sqlite-wasm": "3.53.4-build1", + "comlink": "4.4.2", + "hono": "4.13.9", + "react": "19.3.0", + "react-dom": "19.3.0", + "wouter": "3.11.0" + }, + "devDependencies": { + "@scn-chat/server": "workspace:*", + "@testing-library/jest-dom": "7.0.1", + "@testing-library/react": "16.3.3", + "@testing-library/user-event": "14.6.7", + "@types/better-sqlite3": "9.6.0", + "@types/react": "19.3.0", + "@types/react-dom": "19.3.0", + "@vitejs/plugin-react": "6.1.1", + "better-sqlite3": "13.0.3", + "jsdom": "30.1.1", + "vite": "8.3.1" + } +} diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx new file mode 100644 index 0000000..6cfb2b1 --- /dev/null +++ b/apps/web/src/App.tsx @@ -0,0 +1,91 @@ +import { useEffect, useState } from 'react' +import { Redirect, Route, Switch, useLocation } from 'wouter' +import { ApiError, api, read } from './api.ts' +import { messageOf } from './components/useAction.ts' +import { ChatPage } from './pages/ChatPage.tsx' +import { LoginPage } from './pages/LoginPage.tsx' +import { SettingsPage } from './pages/SettingsPage.tsx' +import { SharedPage } from './pages/SharedPage.tsx' +import { type Me, MeContext } from './session.tsx' +import { openStore } from './store/client.ts' +import { StoreProvider } from './store/react.tsx' + +type Session = + | { state: 'loading' } + | { state: 'signed-out' } + | { state: 'signed-in'; me: Me } + | { state: 'error'; message: string } + +export function useSession(): Session { + const [session, setSession] = useState({ state: 'loading' }) + useEffect(() => { + read(api.auth.me.$get()) + .then((me) => setSession({ state: 'signed-in', me: me as Me })) + .catch((err: unknown) => + setSession( + err instanceof ApiError && err.status === 401 + ? { state: 'signed-out' } + : { state: 'error', message: messageOf(err) }, + ), + ) + }, []) + return session +} + +function SignedIn({ me }: { me: Me }) { + const store = openStore(me.did) + useEffect( + () => + store.onChange((change) => { + if (change.type !== 'unauthorized') return + // Use a full page load, since the signed-in routes redirect /login to /. + store + .deleteLocalCopy() + .catch((err: unknown) => console.error('Could not delete the local copy', err)) + .finally(() => location.assign('/login')) + }), + [store], + ) + return ( + + + + {(params) => } + + + + + + + + + + + + + ) +} + +export function App() { + const session = useSession() + if (session.state === 'loading') return

Loading...

+ if (session.state === 'error') + return

Could not reach the server: {session.message}

+ const signedIn = session.state === 'signed-in' + return ( + + + {(params) => ( + + )} + + {signedIn ? : } + + ) +} + +function SignedOut() { + const [location] = useLocation() + if (location !== '/login') return + return +} diff --git a/apps/web/src/api.ts b/apps/web/src/api.ts new file mode 100644 index 0000000..f8e4ca6 --- /dev/null +++ b/apps/web/src/api.ts @@ -0,0 +1,61 @@ +import type { + AuthApi, + BlobsApi, + PluginsApi, + ProvidersApi, + SharingApi, + StorageApi, + TurnsApi, +} from '@scn-chat/server/api-types' +import { type ClientResponse, hc } from 'hono/client' +import type { SuccessStatusCode } from 'hono/utils/http-status' + +/** Clients for calling the hono server's /api routes. **/ +export const api = { + auth: hc('/api'), + chats: hc('/api'), + turns: hc('/api'), + providers: hc('/api'), + plugins: hc('/api'), + blobs: hc('/api'), + sharing: hc('/api'), +} + +/** Build the request options for a JSON body with api calls. */ +export const json = (body: unknown) => ({ + init: { body: JSON.stringify(body), headers: { 'content-type': 'application/json' } }, +}) + +export class ApiError extends Error { + readonly status: number + + constructor(status: number, message: string) { + super(message) + this.name = 'ApiError' + this.status = status + } +} + +/** The body of the successful responses among a route's possible responses. */ +type SuccessBody = + R extends ClientResponse + ? S extends SuccessStatusCode + ? T + : never + : never + +/** Read a response's JSON. Throws ApiError on failure. */ +export async function read>( + response: Promise, +): Promise> { + const res = await response + if (!res.ok) { + // Error bodies from a proxy may not be JSON. + const body = (await res.json().catch(() => ({}))) as { message?: string; error?: string } + throw new ApiError( + res.status, + body.message ?? body.error ?? `Request failed with ${res.status}`, + ) + } + return (await res.json()) as SuccessBody +} diff --git a/apps/web/src/app-name.ts b/apps/web/src/app-name.ts new file mode 100644 index 0000000..015c3e0 --- /dev/null +++ b/apps/web/src/app-name.ts @@ -0,0 +1,4 @@ +/** The configured app name, used in index.html. */ +const meta = document.querySelector('meta[name="application-name"]') +if (!meta) throw new Error('index.html has no application-name meta tag') +export const appName = meta.content diff --git a/apps/web/src/components/BusyBanner.tsx b/apps/web/src/components/BusyBanner.tsx new file mode 100644 index 0000000..50f06cc --- /dev/null +++ b/apps/web/src/components/BusyBanner.tsx @@ -0,0 +1,17 @@ +import { appName } from '../app-name.ts' +import { useStore, useStoreState } from '../store/react.tsx' + +/** The fallback message when the current tab fails to take the local store from another tab in time. */ +export function BusyBanner() { + const store = useStore() + const state = useStoreState() + if (state !== 'busy') return null + return ( +
+ {appName} is busy in another tab.{' '} + +
+ ) +} diff --git a/apps/web/src/components/ChatList.tsx b/apps/web/src/components/ChatList.tsx new file mode 100644 index 0000000..e6f4388 --- /dev/null +++ b/apps/web/src/components/ChatList.tsx @@ -0,0 +1,35 @@ +import { Link, useLocation } from 'wouter' +import { api, read } from '../api.ts' +import { useConversations } from '../store/react.tsx' +import { useAction } from './useAction.ts' +import { useSignOut } from './useSignOut.ts' + +export function ChatList() { + const { conversations, error: loadError } = useConversations() + const { error, run } = useAction() + const signOut = useSignOut() + const [, navigate] = useLocation() + const create = async () => { + const created = await read(api.chats.conversations.$post()) + navigate(`/c/${created.skey}`) + } + return ( + + ) +} diff --git a/apps/web/src/components/Composer.tsx b/apps/web/src/components/Composer.tsx new file mode 100644 index 0000000..f0eabda --- /dev/null +++ b/apps/web/src/components/Composer.tsx @@ -0,0 +1,156 @@ +import { nsid } from '@scn-chat/lexicons/nsid' +import { useState } from 'react' +import { api, json, read } from '../api.ts' + +export type ModelOption = { + provider: string + id: string + name: string + capabilities: { vision: boolean; reasoning: boolean; tools: boolean } +} + +export type ComposerProps = { + skey: string + parent?: string + models: ModelOption[] + initialModel?: { provider: string; id: string } | null + initialText?: string + onSent: (sent: { rkey: string; replyRkey: string | null }) => void + onCancel?: () => void +} + +export const modelKey = (m: { provider: string; id: string }) => `${m.provider}/${m.id}` + +/** The message box, with model and effort choice and attachments. */ +export function Composer({ + skey, + parent, + models, + initialModel, + initialText = '', + onSent, + onCancel, +}: ComposerProps) { + const [text, setText] = useState(initialText) + const [modelId, setModelId] = useState(initialModel ? modelKey(initialModel) : '') + const [effort, setEffort] = useState('') + const [attachments, setAttachments] = useState[]>([]) + const [uploading, setUploading] = useState(0) + const [error, setError] = useState(null) + const model = models.find((m) => modelKey(m) === modelId) + + const attach = async (files: FileList | null) => { + for (const file of Array.from(files ?? [])) { + if (file.type.startsWith('image/') && model && !model.capabilities.vision) { + setError('This model cannot read images.') + continue + } + setUploading((n) => n + 1) + try { + const res = await fetch('/api/attachments', { + method: 'POST', + headers: { + 'content-type': file.type || 'application/octet-stream', + 'x-filename': encodeURIComponent(file.name), + }, + body: file, + }) + const body = (await res.json()) as { part?: Record; message?: string } + if (!res.ok || !body.part) throw new Error(body.message ?? 'Upload failed') + setAttachments((current) => [...current, body.part as Record]) + } catch (err) { + setError(err instanceof Error ? err.message : 'Upload failed') + } finally { + setUploading((n) => n - 1) + } + } + } + + const send = async () => { + if (!text.trim() && attachments.length === 0) return + setError(null) + const parts = [ + ...attachments, + ...(text.trim() ? [{ $type: `${nsid.defs}#textPart`, text }] : []), + ] + const generation = { + ...(model ? { model: { provider: model.provider, id: model.id } } : {}), + ...(effort ? { effort } : {}), + } + try { + const sent = await read( + api.turns.conversations[':skey'].messages.$post( + { param: { skey } }, + json({ parent, parts, generation }), + ), + ) + setText('') + setAttachments([]) + onSent({ rkey: sent.rkey, replyRkey: sent.replyRkey }) + } catch (err) { + setError(err instanceof Error ? err.message : 'Sending failed') + } + } + + return ( +
{ + event.preventDefault() + void send() + }} + > +