build: convert to monorepo (migrate tap project)

.gitignore

···
 /.versions
 /.vscode/*
 !/.vscode/extensions.json
-/coverage
-/dist
-/node_modules
-/platforms
-/plugins
-/www
+coverage
+dist
+node_modules
+platforms
+plugins
+www

README.md

···
-# Twisted
+# Twisted Monorepo
 
-A mobile client for [Tangled](https://tangled.org).
+- `apps/twisted`: Ionic/Vue client
+- `packages/api`: Go API copied from `~/Projects/TWISTER`
 
 ## Development
 
-Run the mobile apps with Capacitor:
+Use the top-level `justfile` for common tasks:
 
 ```bash
-pnpm cap run ios
-pnpm cap run android
+just dev
+just build
+just test
 ```
 
-Or to test the web version:
-
-```bash
-pnpm dev
-```
+The existing client package still works directly from `apps/twisted`.

apps/twisted/README.md

···
+# Twisted
+
+A mobile client for [Tangled](https://tangled.org).
+
+## Development
+
+Run the mobile apps with Capacitor:
+
+```bash
+pnpm cap run ios
+pnpm cap run android
+```
+
+Or to test the web version:
+
+```bash
+pnpm dev
+```

apps/twisted/package.json

···
+{
+  "name": "@twisted/app",
+  "private": true,
+  "version": "0.0.1",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vue-tsc && vite build",
+    "preview": "vite preview",
+    "test:e2e": "cypress run",
+    "test:unit": "vitest",
+    "lint": "eslint .",
+    "check": "vue-tsc --noEmit"
+  },
+  "dependencies": {
+    "@atcute/bluesky": "^3.3.0",
+    "@atcute/client": "^4.2.1",
+    "@atcute/tangled": "^1.0.17",
+    "@capacitor/android": "^8.2.0",
+    "@capacitor/app": "8.0.1",
+    "@capacitor/core": "8.2.0",
+    "@capacitor/haptics": "8.0.1",
+    "@capacitor/ios": "^8.2.0",
+    "@capacitor/keyboard": "8.0.1",
+    "@capacitor/status-bar": "8.0.1",
+    "@ionic/vue": "^8.0.0",
+    "@ionic/vue-router": "^8.0.0",
+    "@shikijs/markdown-it": "^4.0.2",
+    "@tanstack/query-persist-client-core": "^5.95.0",
+    "@tanstack/vue-query": "^5.94.5",
+    "dompurify": "^3.3.3",
+    "idb-keyval": "^6.2.2",
+    "ionicons": "^7.0.0",
+    "markdown-it": "^14.1.1",
+    "pinia": "^3.0.4",
+    "shiki": "^4.0.2",
+    "vue": "^3.3.0",
+    "vue-router": "^4.2.0"
+  },
+  "devDependencies": {
+    "@capacitor/cli": "8.2.0",
+    "@eslint/js": "10.0.1",
+    "@types/dompurify": "^3.2.0",
+    "@types/node": "25.5.0",
+    "@vitejs/plugin-legacy": "^5.0.0",
+    "@vitejs/plugin-vue": "^4.0.0",
+    "@vue/test-utils": "^2.3.0",
+    "cypress": "^13.5.0",
+    "eslint": "^10.1.0",
+    "eslint-plugin-vue": "^10.8.0",
+    "globals": "17.4.0",
+    "jsdom": "^22.1.0",
+    "prettier": "^3.8.1",
+    "terser": "^5.4.0",
+    "typescript": "~5.9.0",
+    "typescript-eslint": "^8.57.1",
+    "vite": "^5.0.0",
+    "vitest": "^0.34.6",
+    "vue-eslint-parser": "10.4.0",
+    "vue-tsc": "^2.1.10"
+  },
+  "description": "An Ionic project"
+}

justfile

···
+set shell := ["/bin/zsh", "-cu"]
+
+default:
+    @just --list
+
+dev:
+    pnpm --dir apps/twisted dev
+
+build:
+    pnpm --dir apps/twisted build
+    just --justfile packages/api/justfile build
+
+lint:
+    pnpm --dir apps/twisted lint
+
+check:
+    pnpm --dir apps/twisted check
+
+test:
+    pnpm --dir apps/twisted test:unit
+    just --justfile packages/api/justfile test
+
+app-build:
+    pnpm --dir apps/twisted build
+
+app-preview:
+    pnpm --dir apps/twisted preview
+
+app-test-unit:
+    pnpm --dir apps/twisted test:unit
+
+app-test-e2e:
+    pnpm --dir apps/twisted test:e2e
+
+app-cap-ios:
+    pnpm --dir apps/twisted exec cap run ios
+
+app-cap-android:
+    pnpm --dir apps/twisted exec cap run android
+
+api-build:
+    just --justfile packages/api/justfile build
+
+api-run-api:
+    just --justfile packages/api/justfile run-api
+
+api-run-indexer:
+    just --justfile packages/api/justfile run-indexer
+
+api-test:
+    just --justfile packages/api/justfile test
+
+api-clean:
+    just --justfile packages/api/justfile clean

package.json

···
 {
-  "name": "twisted",
+  "name": "twisted-monorepo",
   "private": true,
-  "version": "0.0.1",
-  "type": "module",
   "scripts": {
-    "dev": "vite",
-    "build": "vue-tsc && vite build",
-    "preview": "vite preview",
-    "test:e2e": "cypress run",
-    "test:unit": "vitest",
-    "lint": "eslint .",
-    "check": "vue-tsc --noEmit"
-  },
-  "dependencies": {
-    "@atcute/bluesky": "^3.3.0",
-    "@atcute/client": "^4.2.1",
-    "@atcute/tangled": "^1.0.17",
-    "@capacitor/android": "^8.2.0",
-    "@capacitor/app": "8.0.1",
-    "@capacitor/core": "8.2.0",
-    "@capacitor/haptics": "8.0.1",
-    "@capacitor/ios": "^8.2.0",
-    "@capacitor/keyboard": "8.0.1",
-    "@capacitor/status-bar": "8.0.1",
-    "@ionic/vue": "^8.0.0",
-    "@ionic/vue-router": "^8.0.0",
-    "@shikijs/markdown-it": "^4.0.2",
-    "@tanstack/query-persist-client-core": "^5.95.0",
-    "@tanstack/vue-query": "^5.94.5",
-    "dompurify": "^3.3.3",
-    "idb-keyval": "^6.2.2",
-    "ionicons": "^7.0.0",
-    "markdown-it": "^14.1.1",
-    "pinia": "^3.0.4",
-    "shiki": "^4.0.2",
-    "vue": "^3.3.0",
-    "vue-router": "^4.2.0"
-  },
-  "devDependencies": {
-    "@capacitor/cli": "8.2.0",
-    "@eslint/js": "10.0.1",
-    "@types/dompurify": "^3.2.0",
-    "@types/node": "25.5.0",
-    "@vitejs/plugin-legacy": "^5.0.0",
-    "@vitejs/plugin-vue": "^4.0.0",
-    "@vue/test-utils": "^2.3.0",
-    "cypress": "^13.5.0",
-    "eslint": "^10.1.0",
-    "eslint-plugin-vue": "^10.8.0",
-    "globals": "17.4.0",
-    "jsdom": "^22.1.0",
-    "prettier": "^3.8.1",
-    "terser": "^5.4.0",
-    "typescript": "~5.9.0",
-    "typescript-eslint": "^8.57.1",
-    "vite": "^5.0.0",
-    "vitest": "^0.34.6",
-    "vue-eslint-parser": "10.4.0",
-    "vue-tsc": "^2.1.10"
-  },
-  "description": "An Ionic project"
+    "dev": "pnpm --dir apps/twisted dev",
+    "build": "pnpm --dir apps/twisted build",
+    "preview": "pnpm --dir apps/twisted preview",
+    "lint": "pnpm --dir apps/twisted lint",
+    "check": "pnpm --dir apps/twisted check",
+    "test:unit": "pnpm --dir apps/twisted test:unit",
+    "test:e2e": "pnpm --dir apps/twisted test:e2e",
+    "app:cap:run:ios": "pnpm --dir apps/twisted exec cap run ios",
+    "app:cap:run:android": "pnpm --dir apps/twisted exec cap run android",
+    "api:build": "just --justfile packages/api/justfile build",
+    "api:test": "just --justfile packages/api/justfile test",
+    "api:run:api": "just --justfile packages/api/justfile run-api",
+    "api:run:indexer": "just --justfile packages/api/justfile run-indexer"
+  }
 }

packages/api/.env.example

···
+TAP_URL=wss://tap-instance.up.railway.app/channel
+TAP_AUTH_PASSWORD=your-tap-admin-password
+
+TURSO_DATABASE_URL=libsql://twister-db.turso.io
+TURSO_AUTH_TOKEN=eyJhbGci...
+
+INDEXED_COLLECTIONS=sh.tangled.*
+
+SEARCH_DEFAULT_LIMIT=20
+SEARCH_MAX_LIMIT=100
+SEARCH_DEFAULT_MODE=keyword
+
+HYBRID_KEYWORD_WEIGHT=0.65
+HYBRID_SEMANTIC_WEIGHT=0.35
+
+# EMBEDDING_PROVIDER=openai
+# EMBEDDING_MODEL=text-embedding-3-small
+# EMBEDDING_API_KEY=sk-...
+# EMBEDDING_API_URL=
+# EMBEDDING_DIM=768
+# EMBEDDING_BATCH_SIZE=32
+
+HTTP_BIND_ADDR=:8080
+LOG_LEVEL=info
+LOG_FORMAT=json
+ENABLE_ADMIN_ENDPOINTS=false
+# ADMIN_AUTH_TOKEN=

packages/api/.gitignore

···
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+twister
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Code coverage profiles and other test artifacts
+*.out
+coverage.*
+*.coverprofile
+profile.cov
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env
+
+# Editor/IDE
+# .idea/
+# .vscode/

packages/api/Dockerfile

···
+FROM golang:1.24-alpine AS builder
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+ARG VERSION=dev
+ARG COMMIT=none
+
+RUN CGO_ENABLED=0 GOOS=linux go build \
+    -ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \
+    -o /app/twister \
+    ./main.go
+
+FROM alpine:3.21
+
+RUN apk add --no-cache ca-certificates tzdata
+
+COPY --from=builder /app/twister /usr/local/bin/twister
+
+EXPOSE 8080 9090 9091
+
+CMD ["twister", "api"]

packages/api/README.md

···
+# Twister
+
+Tap-based search engine for Tangled.

packages/api/docs/specs/01-architecture.md

···
+---
+title: "Spec 01 — Architecture"
+updated: 2026-03-22
+---
+
+## 1. Purpose
+
+Build a Go-based search service for Tangled content on AT Protocol that:
+
+* ingests Tangled records through **Tap** (already deployed on Railway)
+* denormalizes them into internal search documents
+* indexes them in **Turso/libSQL**
+* exposes a search API with **keyword**, **semantic**, and **hybrid** retrieval modes
+
+## 2. Functional Goals
+
+The system shall:
+
+* index Tangled-specific ATProto collections under the `sh.tangled.*` namespace
+* support initial backfill and continuous incremental sync via Tap
+* support lexical retrieval using Turso's Tantivy-backed FTS
+* support semantic retrieval using vector embeddings
+* support hybrid ranking combining lexical and semantic signals
+* expose stable HTTP APIs for search and document lookup
+* support deployment on **Railway**
+
+## 3. Non-Functional Goals
+
+The system shall prioritize:
+
+* **correctness of sync** — cursors never advance ahead of committed data
+* **operational simplicity** — single binary, subcommand-driven
+* **incremental delivery** — keyword search ships before embeddings
+* **small deployable services** — process groups, not microservices
+* **reindexability** — any document or collection can be re-normalized and re-indexed
+* **low coupling** — sync, indexing, and serving are independent concerns
+
+## 4. Out of Scope (v1)
+
+* code-aware symbol search
+* sourcegraph-style structural search
+* personalized ranking
+* access control beyond public/private visibility flags in indexed records
+* full analytics pipeline
+* custom ANN infrastructure outside Turso/libSQL
+
+## 5. Design Principles
+
+1. **Tap owns synchronization correctness.** The application does not consume the raw firehose. Tap handles connection, cryptographic verification, backfill, and filtering.
+
+2. **The indexer owns denormalization.** Raw ATProto records are never queried directly by the public API.
+
+3. **Search serves denormalized documents.** Search ranking depends on the document model, not transport.
+
+4. **Keyword search is the baseline.** Semantic and hybrid search are layered on top.
+
+5. **Embeddings are asynchronous.** Ingestion is never blocked on vector generation unless explicitly configured.
+
+## 6. External Systems
+
+- **AT Protocol network** — source of all Tangled content
+- **Tap** — filtered event delivery from the AT Protocol firehose (deployed on Railway)
+- **Turso/libSQL** — relational storage, Tantivy-backed FTS, and native vector search
+- **Embedding provider** — generates vectors for semantic search
+- **Railway** — deployment platform for Twister services and Tap
+
+## 7. Architecture Summary
+
+```text
+ATProto Firehose / PDS
+        │
+        ▼
+   Tap (Railway)
+        │ WebSocket / webhook JSON events
+        ▼
+  Go Indexer Service
+   ├─ decode Tap events
+   ├─ normalize records → documents
+   ├─ upsert documents
+   ├─ schedule embeddings
+   └─ persist sync cursor
+        │
+        ▼
+    Turso/libSQL
+   ├─ documents table
+   ├─ document_embeddings table
+   ├─ FTS index (Tantivy-backed)
+   ├─ vector index (DiskANN)
+   └─ sync_state table
+        │
+        ▼
+   Go Search API
+   ├─ keyword search (fts_match / fts_score)
+   ├─ semantic search (vector_top_k)
+   ├─ hybrid search (weighted merge)
+   └─ document fetch
+```
+
+## 8. Runtime Units
+
+| Unit           | Role                                | Deployment                      |
+| -------------- | ----------------------------------- | ------------------------------- |
+| `api`          | HTTP search and document API        | Railway service (public)        |
+| `indexer`      | Tap consumer, normalizer, DB writer | Railway service (internal)      |
+| `embed-worker` | Async embedding generation          | Optional Railway service        |
+| `tap`          | ATProto sync                        | Railway (already deployed)      |
+
+## 9. Repository Structure
+
+```text
+main.go
+
+internal/
+  api/          # HTTP handlers, middleware, routes
+  config/       # Config struct, env parsing
+  embed/        # Embedding provider abstraction, worker
+  index/        # FTS and vector index management
+  ingest/       # Tap event consumer, ingestion loop
+  normalize/    # Per-collection record → document adapters
+  observability/# Structured logging, metrics
+  ranking/      # Score normalization, hybrid merge
+  search/       # Search orchestration (keyword, semantic, hybrid)
+  store/        # DB access layer, migrations, domain types
+  tapclient/    # Tap WebSocket/webhook client
+```
+
+## 10. Binary Subcommands
+
+```bash
+twister api           # Start HTTP search API
+twister indexer       # Start Tap consumer / indexer
+twister embed-worker  # Start async embedding worker
+twister reindex       # Re-normalize and upsert documents
+twister reembed       # Re-generate embeddings
+twister backfill      # Bootstrap index from seed users
+twister healthcheck   # One-shot health probe
+```
+
+## 11. Technology Choices
+
+### Language: Go
+
+Go is the implementation language for the API server, indexer, embedding worker, and CLI commands. Rationale: straightforward long-running services, excellent HTTP support, good concurrency model, small container footprint.
+
+### Sync Layer: Tap
+
+Tap is the only supported sync source in v1. It handles firehose connection, cryptographic verification, backfill, and filtering, then delivers simple JSON events via WebSocket or webhook.
+
+**Tap is already deployed on Railway.** Twister connects to it as a WebSocket client.
+
+#### Tap Capabilities
+
+- Validates repo structure, MST integrity, and identity signatures
+- Automatic backfill fetches full repo history from PDS when repos are added
+- Filtered output by DID list, collection, or full network mode
+- Ordering guarantees: historical events (`live: false`) delivered before live events (`live: true`)
+
+#### Tap Delivery Modes
+
+| Mode                       | Config                  | Behavior                                          |
+| -------------------------- | ----------------------- | ------------------------------------------------- |
+| WebSocket + acks (default) | —                       | Client acks each event; no data loss              |
+| Fire-and-forget            | `TAP_DISABLE_ACKS=true` | Events marked acked on receipt; simpler but lossy |
+| Webhook                    | `TAP_WEBHOOK_URL=...`   | Events POSTed as JSON; acked on HTTP 200          |
+
+#### Tap API Endpoints (reference)
+
+| Endpoint              | Method | Purpose                               |
+| --------------------- | ------ | ------------------------------------- |
+| `/health`             | GET    | Health check                          |
+| `/channel`            | WS     | WebSocket event stream                |
+| `/repos/add`          | POST   | Add DIDs to track                     |
+| `/repos/remove`       | POST   | Stop tracking a repo                  |
+| `/info/:did`          | GET    | Repo state, rev, record count, errors |
+| `/stats/repo-count`   | GET    | Total tracked repos                   |
+| `/stats/record-count` | GET    | Total tracked records                 |
+| `/stats/cursors`      | GET    | Firehose and list repos cursors       |
+
+#### Key Tap Configuration
+
+| Variable                 | Default | Purpose                                                                            |
+| ------------------------ | ------- | ---------------------------------------------------------------------------------- |
+| `TAP_SIGNAL_COLLECTION`  | —       | Auto-track repos with records in this collection                                   |
+| `TAP_COLLECTION_FILTERS` | —       | Comma-separated collection filters (e.g., `sh.tangled.repo,sh.tangled.repo.issue`) |
+| `TAP_ADMIN_PASSWORD`     | —       | Basic auth for API access                                                          |
+| `TAP_DISABLE_ACKS`       | `false` | Fire-and-forget mode                                                               |
+| `TAP_WEBHOOK_URL`        | —       | Webhook delivery URL                                                               |
+
+### Storage and Search: Turso/libSQL
+
+Turso/libSQL is used for relational metadata storage, Tantivy-backed full-text search, and native vector search.
+
+#### Go SDK Options
+
+| Package                                            | CGo | Embedded Replicas | Remote |
+| -------------------------------------------------- | --- | ----------------- | ------ |
+| `github.com/tursodatabase/go-libsql`               | Yes | Yes               | Yes    |
+| `github.com/tursodatabase/libsql-client-go/libsql` | No  | No                | Yes    |
+
+Both register as `database/sql` drivers under `"libsql"`. They cannot be imported in the same binary.
+
+**Recommendation:** Use `libsql-client-go` (pure Go, remote-only) unless embedded replicas are needed for local read performance.
+
+#### Connection Patterns
+
+```go
+// Remote only (pure Go, no CGo)
+import _ "github.com/tursodatabase/libsql-client-go/libsql"
+db, err := sql.Open("libsql", "libsql://your-db.turso.io?authToken=TOKEN")
+
+// Embedded replica (CGo required)
+import "github.com/tursodatabase/go-libsql"
+connector, err := libsql.NewEmbeddedReplicaConnector(
+    "local.db", "libsql://your-db.turso.io",
+    libsql.WithAuthToken("TOKEN"),
+    libsql.WithSyncInterval(time.Minute),
+)
+db := sql.OpenDB(connector)
+```
+
+#### Full-Text Search (Tantivy-backed)
+
+Turso FTS is **not** standard SQLite FTS5. It uses Tantivy under the hood.
+
+```sql
+-- Create FTS index with per-column tokenizers and weights
+CREATE INDEX idx_docs_fts ON documents USING fts (
+    title WITH tokenizer=default,
+    body WITH tokenizer=default,
+    summary WITH tokenizer=default,
+    repo_name WITH tokenizer=simple,
+    author_handle WITH tokenizer=raw
+) WITH (weights='title=3.0,repo_name=2.5,author_handle=2.0,summary=1.5,body=1.0');
+
+-- Filter by match
+SELECT id, title FROM documents
+WHERE fts_match(title, body, summary, repo_name, author_handle, 'search query');
+
+-- BM25 scoring
+SELECT id, title, fts_score(title, body, summary, repo_name, author_handle, 'search query') AS score
+FROM documents
+ORDER BY score DESC;
+
+-- Highlighting
+SELECT fts_highlight(title, '<b>', '</b>', 'search query') AS highlighted
+FROM documents;
+```
+
+**Available tokenizers:** `default` (Unicode-aware), `raw` (exact match), `simple` (whitespace+punctuation), `whitespace`, `ngram` (2-3 char n-grams).
+
+**Query syntax (Tantivy):** `database AND search`, `database NOT nosql`, `"exact phrase"`, `data*` (prefix), `title:database` (field-specific), `title:database^2` (boosting).
+
+**Limitations:** No snippet function (use highlighting). No automatic segment merging (manual `OPTIMIZE INDEX` required).
+No read-your-writes within a transaction. No MATCH operator (use `fts_match()` function).
+
+#### Vector Search
+
+```sql
+-- Vector column type
+embedding F32_BLOB(768)
+
+-- Insert
+INSERT INTO document_embeddings (document_id, embedding, ...)
+VALUES (?, vector32(?), ...);  -- ? is JSON array '[0.1, 0.2, ...]'
+
+-- Brute-force similarity search
+SELECT d.id, vector_distance_cos(e.embedding, vector32(?)) AS distance
+FROM documents d
+JOIN document_embeddings e ON d.id = e.document_id
+ORDER BY distance ASC LIMIT 20;
+
+-- Create ANN index (DiskANN)
+CREATE INDEX idx_embeddings ON document_embeddings(
+    libsql_vector_idx(embedding, 'metric=cosine')
+);
+
+-- ANN search via index
+SELECT d.id, d.title
+FROM vector_top_k('idx_embeddings', vector32(?), 20) AS v
+JOIN document_embeddings e ON e.rowid = v.id
+JOIN documents d ON d.id = e.document_id;
+```
+
+**Vector types:** `F32_BLOB` (recommended), `F16_BLOB`, `F64_BLOB`, `F8_BLOB`, `F1BIT_BLOB`.
+
+**Distance functions:** `vector_distance_cos` (cosine), `vector_distance_l2` (Euclidean).
+
+**Max dimensions:** 65,536. Dimension is fixed at table creation.
+
+### Deployment: Railway
+
+Railway is the deployment platform. It supports health checks, autodeploy, per-service scaling, and internal networking. Tap is already deployed here. Twister deploys as separate Railway services (api, indexer, embed-worker) within the same project.

packages/api/docs/specs/02-tangled-lexicons.md

···
+---
+title: "Spec 02 — Tangled Lexicons"
+updated: 2026-03-22
+source: https://github.com/mary-ext/atcute/tree/trunk/packages/definitions/tangled/lexicons/sh/tangled
+---
+
+All Tangled records use the `sh.tangled.*` namespace. Records use TID keys unless noted otherwise.
+
+## 1. Searchable Record Types
+
+These are the primary records Twister indexes for search.
+
+### sh.tangled.repo
+
+Repository metadata. Key: `tid`.
+
+| Field         | Type     | Required | Description                                    |
+| ------------- | -------- | -------- | ---------------------------------------------- |
+| `name`        | string   | yes      | Repository name                                |
+| `knot`        | string   | yes      | Knot (hosting node) where the repo was created |
+| `spindle`     | string   | no       | CI runner for jobs                             |
+| `description` | string   | no       | 1–140 graphemes                                |
+| `website`     | uri      | no       | Related URI                                    |
+| `topics`      | string[] | no       | Up to 50 topic tags, each 1–50 chars           |
+| `source`      | uri      | no       | Upstream source                                |
+| `labels`      | at-uri[] | no       | Label definitions this repo subscribes to      |
+| `createdAt`   | datetime | yes      |                                                |
+
+### sh.tangled.repo.issue
+
+Issue on a repository. Key: `tid`.
+
+| Field        | Type     | Required | Description                      |
+| ------------ | -------- | -------- | -------------------------------- |
+| `repo`       | at-uri   | yes      | AT-URI of the parent repo record |
+| `title`      | string   | yes      | Issue title                      |
+| `body`       | string   | no       | Issue body (markdown)            |
+| `createdAt`  | datetime | yes      |                                  |
+| `mentions`   | did[]    | no       | Mentioned users                  |
+| `references` | at-uri[] | no       | Referenced records               |
+
+### sh.tangled.repo.pull
+
+Pull request. Key: `tid`.
+
+| Field        | Type     | Required | Description                                        |
+| ------------ | -------- | -------- | -------------------------------------------------- |
+| `target`     | object   | yes      | `{repo: at-uri, branch: string}`                   |
+| `title`      | string   | yes      | PR title                                           |
+| `body`       | string   | no       | PR description (markdown)                          |
+| `patchBlob`  | blob     | yes      | Patch content (`text/x-patch`)                     |
+| `source`     | object   | no       | `{branch: string, sha: string(40), repo?: at-uri}` |
+| `createdAt`  | datetime | yes      |                                                    |
+| `mentions`   | did[]    | no       | Mentioned users                                    |
+| `references` | at-uri[] | no       | Referenced records                                 |
+
+### sh.tangled.string
+
+Code snippet / gist. Key: `tid`.
+
+| Field         | Type     | Required | Description         |
+| ------------- | -------- | -------- | ------------------- |
+| `filename`    | string   | yes      | 1–140 graphemes     |
+| `description` | string   | yes      | Up to 280 graphemes |
+| `createdAt`   | datetime | yes      |                     |
+| `contents`    | string   | yes      | Snippet content     |
+
+### sh.tangled.actor.profile
+
+User profile. Key: `literal:self` (singleton per account).
+
+| Field                | Type     | Required | Description                  |
+| -------------------- | -------- | -------- | ---------------------------- |
+| `avatar`             | blob     | no       | PNG/JPEG, max 1MB            |
+| `description`        | string   | no       | Bio, up to 256 graphemes     |
+| `links`              | uri[]    | no       | Up to 5 social/website links |
+| `stats`              | string[] | no       | Up to 2 vanity stat types    |
+| `bluesky`            | boolean  | yes      | Show Bluesky link            |
+| `location`           | string   | no       | Up to 40 graphemes           |
+| `pinnedRepositories` | at-uri[] | no       | Up to 6 pinned repos         |
+| `pronouns`           | string   | no       | Up to 40 chars               |
+
+## 2. Interaction Record Types
+
+These records represent social interactions. They may be indexed for counts/signals but are lower priority for text search.
+
+### sh.tangled.feed.star
+
+Star/favorite on a record. Key: `tid`.
+
+| Field       | Type     | Required |
+| ----------- | -------- | -------- |
+| `subject`   | at-uri   | yes      |
+| `createdAt` | datetime | yes      |
+
+### sh.tangled.feed.reaction
+
+Emoji reaction on a record. Key: `tid`.
+
+| Field       | Type     | Required | Description                     |
+| ----------- | -------- | -------- | ------------------------------- |
+| `subject`   | at-uri   | yes      |                                 |
+| `reaction`  | string   | yes      | One of: 👍 👎 😆 🎉 🫤 ❤️ 🚀 👀 |
+| `createdAt` | datetime | yes      |                                 |
+
+### sh.tangled.graph.follow
+
+Follow a user. Key: `tid`.
+
+| Field       | Type     | Required |
+| ----------- | -------- | -------- |
+| `subject`   | did      | yes      |
+| `createdAt` | datetime | yes      |
+
+## 3. State Record Types
+
+These records track mutable state of issues and PRs.
+
+### sh.tangled.repo.issue.state
+
+| Field   | Type   | Required | Description                                                                |
+| ------- | ------ | -------- | -------------------------------------------------------------------------- |
+| `issue` | at-uri | yes      |                                                                            |
+| `state` | string | yes      | `sh.tangled.repo.issue.state.open` or `sh.tangled.repo.issue.state.closed` |
+
+### sh.tangled.repo.pull.status
+
+| Field    | Type   | Required | Description                                                 |
+| -------- | ------ | -------- | ----------------------------------------------------------- |
+| `pull`   | at-uri | yes      |                                                             |
+| `status` | string | yes      | `sh.tangled.repo.pull.status.open`, `.closed`, or `.merged` |
+
+## 4. Comment Record Types
+
+### sh.tangled.repo.issue.comment
+
+| Field        | Type     | Required | Description                    |
+| ------------ | -------- | -------- | ------------------------------ |
+| `issue`      | at-uri   | yes      | Parent issue                   |
+| `body`       | string   | yes      | Comment body                   |
+| `createdAt`  | datetime | yes      |                                |
+| `replyTo`    | at-uri   | no       | Parent comment (for threading) |
+| `mentions`   | did[]    | no       |                                |
+| `references` | at-uri[] | no       |                                |
+
+### sh.tangled.repo.pull.comment
+
+| Field        | Type     | Required | Description  |
+| ------------ | -------- | -------- | ------------ |
+| `pull`       | at-uri   | yes      | Parent PR    |
+| `body`       | string   | yes      | Comment body |
+| `createdAt`  | datetime | yes      |              |
+| `mentions`   | did[]    | no       |              |
+| `references` | at-uri[] | no       |              |
+
+## 5. Infrastructure Record Types
+
+These are not indexed for search but may be consumed for operational context.
+
+| Collection                    | Description                                          |
+| ----------------------------- | ---------------------------------------------------- |
+| `sh.tangled.label.definition` | Label definitions with name, valueType, scope, color |
+| `sh.tangled.label.op`         | Label application operations                         |
+| `sh.tangled.git.refUpdate`    | Git reference update events                          |
+| `sh.tangled.knot.member`      | Knot membership                                      |
+| `sh.tangled.spindle.member`   | Spindle (CI runner) membership                       |
+| `sh.tangled.pipeline.status`  | CI pipeline status                                   |
+
+## 6. Collection Priority for v1 Indexing
+
+| Priority | Collection                      | Rationale                            |
+| -------- | ------------------------------- | ------------------------------------ |
+| P0       | `sh.tangled.repo`               | Core searchable content              |
+| P0       | `sh.tangled.repo.issue`         | High-signal text content             |
+| P0       | `sh.tangled.repo.pull`          | High-signal text content             |
+| P1       | `sh.tangled.string`             | Searchable code snippets             |
+| P1       | `sh.tangled.actor.profile`      | User/org discovery                   |
+| P2       | `sh.tangled.repo.issue.comment` | Body text, high volume               |
+| P2       | `sh.tangled.repo.pull.comment`  | Body text, high volume               |
+| P2       | `sh.tangled.repo.issue.state`   | State for filtering, not text search |
+| P2       | `sh.tangled.repo.pull.status`   | State for filtering, not text search |
+| P3       | `sh.tangled.feed.star`          | Ranking signal (star count)          |
+| P3       | `sh.tangled.feed.reaction`      | Ranking signal                       |
+| P3       | `sh.tangled.graph.follow`       | Ranking signal                       |
+
+### Tap Collection Filter for v1
+
+```sh
+TAP_COLLECTION_FILTERS=sh.tangled.repo,sh.tangled.repo.issue,sh.tangled.repo.issue.comment,sh.tangled.repo.issue.state,sh.tangled.repo.pull,sh.tangled.repo.pull.comment,sh.tangled.repo.pull.status,sh.tangled.string,sh.tangled.actor.profile,sh.tangled.feed.star
+
+# or sh.tangled.*
+```

packages/api/docs/specs/03-data-model.md

···
+---
+title: "Spec 03 — Data Model"
+updated: 2026-03-22
+---
+
+## 1. Search Document
+
+A **search document** is the internal denormalized representation used for retrieval. It is derived from one or more ATProto records via normalization.
+
+### Stable Identifier
+
+```sh
+id = did + "|" + collection + "|" + rkey
+```
+
+Example: `did:plc:abc123|sh.tangled.repo|3kb3fge5lm32x`
+
+### Required Fields
+
+| Field           | Type    | Description                                                                |
+| --------------- | ------- | -------------------------------------------------------------------------- |
+| `id`            | TEXT PK | Stable composite identifier                                                |
+| `did`           | TEXT    | Author DID                                                                 |
+| `collection`    | TEXT    | ATProto collection NSID                                                    |
+| `rkey`          | TEXT    | Record key (TID)                                                           |
+| `at_uri`        | TEXT    | Full AT-URI                                                                |
+| `cid`           | TEXT    | Content identifier (hash)                                                  |
+| `record_type`   | TEXT    | Normalized type label (e.g., `repo`, `issue`, `pull`, `string`, `profile`) |
+| `title`         | TEXT    | Normalized title                                                           |
+| `body`          | TEXT    | Normalized body text                                                       |
+| `summary`       | TEXT    | Short summary / description                                                |
+| `repo_did`      | TEXT    | DID of the repo owner (resolved from at-uri for issues/PRs)                |
+| `repo_name`     | TEXT    | Repository name (resolved)                                                 |
+| `author_handle` | TEXT    | Author handle (resolved via identity)                                      |
+| `tags_json`     | TEXT    | JSON array of tags/topics                                                  |
+| `language`      | TEXT    | Detected or declared language                                              |
+| `created_at`    | TEXT    | Record creation timestamp (ISO 8601)                                       |
+| `updated_at`    | TEXT    | Last record update timestamp                                               |
+| `indexed_at`    | TEXT    | When this document was last indexed                                        |
+| `deleted_at`    | TEXT    | Soft-delete timestamp (tombstone)                                          |
+
+### Derived Fields (not stored in documents table)
+
+| Field            | Location                               | Description                    |
+| ---------------- | -------------------------------------- | ------------------------------ |
+| Embedding vector | `document_embeddings` table            | F32_BLOB(N)                    |
+| FTS index        | Turso FTS index                        | Tantivy-backed full-text index |
+| Star count       | Aggregated from `sh.tangled.feed.star` | Ranking signal                 |
+
+## 2. Core Documents Table
+
+```sql
+CREATE TABLE documents (
+    id          TEXT PRIMARY KEY,
+    did         TEXT NOT NULL,
+    collection  TEXT NOT NULL,
+    rkey        TEXT NOT NULL,
+    at_uri      TEXT NOT NULL,
+    cid         TEXT NOT NULL,
+    record_type TEXT NOT NULL,
+    title       TEXT,
+    body        TEXT,
+    summary     TEXT,
+    repo_did    TEXT,
+    repo_name   TEXT,
+    author_handle TEXT,
+    tags_json   TEXT,
+    language    TEXT,
+    created_at  TEXT,
+    updated_at  TEXT,
+    indexed_at  TEXT NOT NULL,
+    deleted_at  TEXT
+);
+
+CREATE INDEX idx_documents_did ON documents(did);
+CREATE INDEX idx_documents_collection ON documents(collection);
+CREATE INDEX idx_documents_record_type ON documents(record_type);
+CREATE INDEX idx_documents_repo_did ON documents(repo_did);
+CREATE INDEX idx_documents_created_at ON documents(created_at);
+CREATE INDEX idx_documents_deleted_at ON documents(deleted_at);
+```
+
+## 3. FTS Index
+
+```sql
+CREATE INDEX idx_documents_fts ON documents USING fts (
+    title WITH tokenizer=default,
+    body WITH tokenizer=default,
+    summary WITH tokenizer=default,
+    repo_name WITH tokenizer=simple,
+    author_handle WITH tokenizer=raw,
+    tags_json WITH tokenizer=simple
+) WITH (weights='title=3.0,repo_name=2.5,author_handle=2.0,summary=1.5,tags_json=1.2,body=1.0');
+```
+
+## 4. Embeddings Table
+
+```sql
+CREATE TABLE document_embeddings (
+    document_id     TEXT PRIMARY KEY REFERENCES documents(id),
+    embedding       F32_BLOB(768),
+    embedding_model TEXT NOT NULL,
+    embedded_at     TEXT NOT NULL
+);
+
+CREATE INDEX idx_embeddings_vec ON document_embeddings(
+    libsql_vector_idx(embedding, 'metric=cosine')
+);
+```
+
+The vector dimension (768) is configurable by model. Changing models requires a new column or table migration.
+
+## 5. Sync State Table
+
+```sql
+CREATE TABLE sync_state (
+    consumer_name   TEXT PRIMARY KEY,
+    cursor          TEXT NOT NULL,
+    high_water_mark TEXT,
+    updated_at      TEXT NOT NULL
+);
+```
+
+Stores the Tap event ID that has been successfully committed. On restart, the indexer resumes from this cursor.
+
+## 6. Embedding Jobs Table
+
+```sql
+CREATE TABLE embedding_jobs (
+    document_id  TEXT PRIMARY KEY REFERENCES documents(id),
+    status       TEXT NOT NULL,  -- 'pending', 'processing', 'completed', 'failed'
+    attempts     INTEGER NOT NULL DEFAULT 0,
+    last_error   TEXT,
+    scheduled_at TEXT NOT NULL,
+    updated_at   TEXT NOT NULL
+);
+
+CREATE INDEX idx_embedding_jobs_status ON embedding_jobs(status);
+```
+
+## 7. Issue/PR State Cache (optional)
+
+To support filtering search results by issue state or PR status without joining back to the raw records:
+
+```sql
+CREATE TABLE record_state (
+    subject_uri TEXT PRIMARY KEY,  -- at-uri of the issue or PR
+    state       TEXT NOT NULL,     -- 'open', 'closed', 'merged'
+    updated_at  TEXT NOT NULL
+);
+```
+
+Updated when `sh.tangled.repo.issue.state` or `sh.tangled.repo.pull.status` events are ingested.

packages/api/docs/specs/04-data-pipeline.md

···
+---
+title: "Spec 04 — Data Pipeline"
+updated: 2026-03-22
+---
+
+Covers the full data path: Tap event ingestion, record normalization, and failure handling.
+
+## 1. Tap Event Format
+
+### Record Events
+
+```json
+{
+    "id": 12345,
+    "type": "record",
+    "record": {
+        "live": true,
+        "rev": "3kb3fge5lm32x",
+        "did": "did:plc:abc123",
+        "collection": "sh.tangled.repo",
+        "rkey": "3kb3fge5lm32x",
+        "action": "create",
+        "cid": "bafyreig...",
+        "record": {
+            "$type": "sh.tangled.repo",
+            "name": "my-project",
+            "knot": "knot.tangled.org",
+            "description": "A cool project",
+            "topics": ["go", "search"],
+            "createdAt": "2026-03-22T12:00:00.000Z"
+        }
+    }
+}
+```
+
+Key fields:
+
+- `id` — monotonic event ID, used as cursor
+- `type` — `"record"` or `"identity"`
+- `record.live` — `true` for real-time events, `false` for backfill
+- `record.action` — `"create"`, `"update"`, or `"delete"`
+- `record.did` — author DID
+- `record.collection` — ATProto collection NSID
+- `record.rkey` — record key
+- `record.cid` — content identifier
+- `record.record` — the full ATProto record payload (absent on delete)
+
+### Identity Events
+
+```json
+{
+    "id": 12346,
+    "type": "identity",
+    "identity": {
+        "did": "did:plc:abc123",
+        "handle": "alice.tangled.org",
+        "isActive": true,
+        "status": "active"
+    }
+}
+```
+
+Identity events are always delivered for tracked repos, regardless of collection filters.
+
+## 2. WebSocket Protocol
+
+### Connection
+
+Connect to `wss://<tap-host>/channel` (or `ws://` for local dev).
+
+If `TAP_ADMIN_PASSWORD` is set, authenticate with HTTP Basic auth (`admin:<password>`).
+
+### Acknowledgment Protocol
+
+Default mode requires the client to ack each event by sending the event `id` back over the WebSocket. Events are retried after `TAP_RETRY_TIMEOUT` (default 60s) if unacked.
+
+For simpler development, set `TAP_DISABLE_ACKS=true` on Tap for fire-and-forget delivery.
+
+### Ordering Guarantees
+
+Events are ordered **per-repo** (per-DID), not globally:
+
+- **Historical events** (`live: false`) may be sent concurrently within a repo
+- **Live events** (`live: true`) are synchronization barriers — all prior events for that repo must complete before a live event is sent
+- No ordering guarantee across different repos
+
+Example sequence for one repo: `H1, H2, L1, H3, H4, L2`
+
+- H1 and H2 sent concurrently
+- Wait for completion, send L1 alone
+- Wait for L1, send H3 and H4 concurrently
+- Wait for completion, send L2 alone
+
+### Delivery Guarantee
+
+Events are delivered **at least once**. Duplicates may occur on crashes or ack timeouts. The indexer must handle idempotent upserts.
+
+## 3. Ingestion Contract
+
+For each event, the indexer:
+
+1. Validates `type` is `"record"` (identity events are handled separately)
+2. Checks `record.collection` against the allowlist
+3. Maps `record.action` to an operation:
+    - `create` → upsert document
+    - `update` → upsert document
+    - `delete` → tombstone document (`deleted_at = now`)
+4. Decodes `record.record` into the collection-specific struct
+5. Normalizes to internal `Document`
+6. Upserts into the documents table
+7. Schedules embedding job if eligible
+8. Persists cursor (`event.id`) **only after successful DB commit**
+
+### Cursor Persistence Rules
+
+- If DB commit fails → cursor does not advance → event will be retried
+- If normalization fails → log error, optionally dead-letter, skip → cursor advances
+- If embedding scheduling fails → document remains keyword-searchable → cursor advances
+
+## 4. Backfill Behavior
+
+When a repo is added to Tap (via `/repos/add`, signal collection, or full network mode):
+
+1. Tap fetches full repo history from PDS via `com.atproto.sync.getRepo`
+2. Firehose events for that repo are buffered during backfill
+3. Historical events (`live: false`) are delivered first
+4. After backfill completes, buffered live events drain
+5. New firehose events stream normally (`live: true`)
+
+### Application-Level Backfill Support
+
+The indexer also supports:
+
+- Full reindex from existing corpus (re-normalize all stored documents)
+- Targeted reindex by collection
+- Targeted reindex by DID
+
+These do not involve Tap — they re-process documents already in the database.
+
+## 5. Normalization
+
+Normalization converts heterogeneous `sh.tangled.*` records into the common `Document` shape defined in [03-data-model.md](03-data-model.md).
+
+### Adapter Interface
+
+Each indexed collection provides an adapter:
+
+```go
+type RecordAdapter interface {
+    Collection() string
+    RecordType() string
+    Normalize(event TapRecordEvent) (*Document, error)
+    Searchable(record map[string]any) bool
+}
+```
+
+### Per-Collection Normalization
+
+#### sh.tangled.repo → `repo`
+
+| Document Field | Source                           |
+| -------------- | -------------------------------- |
+| `title`        | `record.name`                    |
+| `body`         | `record.description`             |
+| `summary`      | `record.description` (truncated) |
+| `repo_name`    | `record.name`                    |
+| `repo_did`     | `event.did`                      |
+| `tags_json`    | `json(record.topics)`            |
+| `created_at`   | `record.createdAt`               |
+
+**Searchable:** Always (unless empty name).
+
+#### sh.tangled.repo.issue → `issue`
+
+| Document Field | Source                                      |
+| -------------- | ------------------------------------------- |
+| `title`        | `record.title`                              |
+| `body`         | `record.body`                               |
+| `summary`      | First ~200 chars of `record.body`           |
+| `repo_did`     | Extracted from `record.repo` AT-URI         |
+| `repo_name`    | Resolved from repo AT-URI                   |
+| `tags_json`    | `[]` (labels resolved separately if needed) |
+| `created_at`   | `record.createdAt`                          |
+
+**Searchable:** Always.
+
+#### sh.tangled.repo.pull → `pull`
+
+| Document Field | Source                                     |
+| -------------- | ------------------------------------------ |
+| `title`        | `record.title`                             |
+| `body`         | `record.body`                              |
+| `summary`      | First ~200 chars of `record.body`          |
+| `repo_did`     | Extracted from `record.target.repo` AT-URI |
+| `repo_name`    | Resolved from target repo AT-URI           |
+| `tags_json`    | `[]`                                       |
+| `created_at`   | `record.createdAt`                         |
+
+**Searchable:** Always.
+
+#### sh.tangled.string → `string`
+
+| Document Field | Source               |
+| -------------- | -------------------- |
+| `title`        | `record.filename`    |
+| `body`         | `record.contents`    |
+| `summary`      | `record.description` |
+| `repo_name`    | —                    |
+| `repo_did`     | —                    |
+| `tags_json`    | `[]`                 |
+| `created_at`   | `record.createdAt`   |
+
+**Searchable:** Always (content is required).
+
+#### sh.tangled.actor.profile → `profile`
+
+| Document Field | Source                                               |
+| -------------- | ---------------------------------------------------- |
+| `title`        | Author handle (resolved from DID)                    |
+| `body`         | `record.description`                                 |
+| `summary`      | `record.description` (truncated) + `record.location` |
+| `repo_name`    | —                                                    |
+| `repo_did`     | —                                                    |
+| `tags_json`    | `[]`                                                 |
+| `created_at`   | — (profiles don't have createdAt)                    |
+
+**Searchable:** If `description` is non-empty.
+
+#### sh.tangled.repo.issue.comment → `issue_comment`
+
+| Document Field | Source                                    |
+| -------------- | ----------------------------------------- |
+| `title`        | — (derived: "Comment on {issue title}")   |
+| `body`         | `record.body`                             |
+| `summary`      | First ~200 chars of `record.body`         |
+| `repo_did`     | Resolved from `record.issue` AT-URI chain |
+| `repo_name`    | Resolved                                  |
+| `created_at`   | `record.createdAt`                        |
+
+**Searchable:** If body is non-empty.
+
+#### sh.tangled.repo.pull.comment → `pull_comment`
+
+Same pattern as issue comments, using `record.pull` instead of `record.issue`.
+
+### State Event Handling
+
+State and status records (`sh.tangled.repo.issue.state`, `sh.tangled.repo.pull.status`) do **not** produce new search documents. Instead, they update the `record_state` cache table (see [03-data-model.md](03-data-model.md)).
+
+### Interaction Event Handling
+
+Stars (`sh.tangled.feed.star`) and reactions (`sh.tangled.feed.reaction`) do not produce search documents. They may be aggregated for ranking signals in later phases.
+
+### Embedding Input Text
+
+For documents eligible for embedding, compose the input as:
+
+```sh
+{title}\n{repo_name}\n{author_handle}\n{tags}\n{summary}\n{body}
+```
+
+Fields are joined with newlines. Empty fields are omitted.
+
+### Repo Name Resolution
+
+Issues, PRs, and comments reference their parent repo via AT-URI (e.g., `at://did:plc:abc/sh.tangled.repo/tid`). Resolving the repo name requires either:
+
+1. Looking up the repo document in the local `documents` table
+2. Caching repo metadata in a lightweight lookup table
+
+Option 1 is preferred for v1. If the repo document hasn't been indexed yet, `repo_name` is left empty and backfilled on the next reindex pass.
+
+## 6. Identity Event Handling
+
+Identity events should be used to maintain an author handle cache:
+
+```sh
+did → handle mapping
+```
+
+When an identity event arrives with a new handle, update `author_handle` on all documents with that DID. This ensures search by handle returns current results.
+
+## 7. Repo Management
+
+To add repos for tracking, POST to Tap's `/repos/add` endpoint:
+
+```bash
+curl -u admin:PASSWORD -X POST https://tap-host/repos/add \
+  -H "Content-Type: application/json" \
+  -d '{"dids": ["did:plc:abc123", "did:plc:def456"]}'
+```
+
+Alternatively, use `TAP_SIGNAL_COLLECTION=sh.tangled.repo` to auto-track any repo that has Tangled repo records.
+
+## 8. Failure Handling
+
+### Ingestion Failures
+
+If Tap event processing fails before DB commit:
+
+- Log the failure with event ID, DID, collection, rkey, and error class
+- Retry with exponential backoff (for transient errors like DB timeouts)
+- Do **not** advance cursor — the event will be re-delivered by Tap
+- After max retries for a persistent error, log and skip (cursor advances)
+
+### Normalization Failures
+
+If a record cannot be normalized:
+
+- Log collection, DID, rkey, CID, and error class
+- Do not crash the process
+- Skip the event and advance cursor
+- Optionally insert into a `dead_letter` table for manual inspection
+
+### Embedding Failures
+
+If embedding generation fails:
+
+- The document remains keyword-searchable
+- The embedding job is marked `failed` with `last_error` and incremented `attempts`
+- Jobs are retried with exponential backoff up to a max attempt count
+- After max attempts, the job enters `dead` state
+- The embed-worker exposes failed job count as a metric
+
+### DB Failures
+
+If Turso/libSQL is unreachable:
+
+- **API** returns `503` for search endpoints; `/healthz` still returns 200 (liveness), `/readyz` returns 503
+- **Indexer** pauses event processing and retries DB connection with backoff; cursor does not advance
+- **Embed-worker** pauses job processing and retries
+
+### Tap Connection Failures
+
+If the WebSocket connection to Tap drops:
+
+- Reconnect with exponential backoff
+- Resume from the last persisted cursor
+- Log reconnection attempts and success
+
+Tap itself handles firehose reconnection independently — a Tap restart does not require indexer intervention beyond reconnecting the WebSocket.
+
+### Duplicate Event Handling
+
+Tap delivers events **at least once**. Duplicates are handled by:
+
+- Using `id = did|collection|rkey` as the primary key
+- All writes are upserts (`INSERT OR REPLACE` / `ON CONFLICT ... DO UPDATE`)
+- CID comparison can detect true no-ops (same content) vs. actual updates
+
+### Startup Recovery
+
+On indexer startup:
+
+1. Read `cursor` from `sync_state` table
+2. Connect to Tap WebSocket
+3. Tap replays events from the stored cursor position
+4. Processing resumes normally
+
+If no cursor exists (first run), Tap delivers all historical events from backfill.

packages/api/docs/specs/05-search.md

···
+---
+title: "Spec 05 — Search"
+updated: 2026-03-22
+---
+
+Covers all search modes, the API contract, scoring, and filtering.
+
+## 1. Search Modes
+
+| Mode | Backing | Available |
+|------|---------|-----------|
+| `keyword` | Turso Tantivy-backed FTS | MVP |
+| `semantic` | Vector similarity (DiskANN index) | Phase 2 |
+| `hybrid` | Weighted merge of keyword + semantic | Phase 3 |
+
+## 2. Keyword Search
+
+### Implementation
+
+Uses Turso's `fts_score()` function for BM25 ranking:
+
+```sql
+SELECT
+    d.id, d.title, d.summary, d.repo_name, d.author_handle,
+    d.collection, d.record_type, d.updated_at,
+    fts_score(d.title, d.body, d.summary, d.repo_name, d.author_handle, d.tags_json, ?) AS score
+FROM documents d
+WHERE fts_match(d.title, d.body, d.summary, d.repo_name, d.author_handle, d.tags_json, ?)
+  AND d.deleted_at IS NULL
+ORDER BY score DESC
+LIMIT ? OFFSET ?;
+```
+
+### Field Weights
+
+Configured in the FTS index definition:
+
+| Field | Weight | Rationale |
+|-------|--------|-----------|
+| `title` | 3.0 | Highest signal for relevance |
+| `repo_name` | 2.5 | Exact repo lookups should rank first |
+| `author_handle` | 2.0 | Author search is common |
+| `summary` | 1.5 | More focused than body |
+| `tags_json` | 1.2 | Topic matching |
+| `body` | 1.0 | Baseline |
+
+### Query Features
+
+Tantivy query syntax is exposed to users:
+
+- Boolean: `go AND search`, `rust NOT unsafe`
+- Phrase: `"pull request"`
+- Prefix: `tang*`
+- Field-specific: `title:parser`
+
+### Snippets
+
+Use `fts_highlight()` to generate highlighted snippets:
+
+```sql
+fts_highlight(d.body, '<mark>', '</mark>', ?) AS body_snippet
+```
+
+## 3. Semantic Search
+
+### Query Flow
+
+1. Convert user query text to embedding via the configured provider
+2. Query `vector_top_k` for nearest neighbors
+3. Join back to `documents` to get metadata
+4. Filter out deleted/hidden documents
+5. Return results with distance as score
+
+```sql
+SELECT d.id, d.title, d.summary, d.repo_name, d.author_handle,
+       d.collection, d.record_type, d.updated_at
+FROM vector_top_k('idx_embeddings_vec', vector32(?), ?) AS v
+JOIN document_embeddings e ON e.rowid = v.id
+JOIN documents d ON d.id = e.document_id
+WHERE d.deleted_at IS NULL;
+```
+
+### Score Normalization
+
+Cosine distance ranges from 0 (identical) to 2 (opposite). Normalize to a 0–1 relevance score:
+
+```
+semantic_score = 1.0 - (distance / 2.0)
+```
+
+## 4. Hybrid Search
+
+### v1: Weighted Score Blending
+
+```
+hybrid_score = 0.65 * keyword_score_normalized + 0.35 * semantic_score_normalized
+```
+
+### Score Normalization for Blending
+
+Keyword (BM25) scores are unbounded. Normalize using min-max within the result set:
+
+```
+keyword_normalized = (score - min_score) / (max_score - min_score)
+```
+
+Semantic scores are already bounded after the distance-to-relevance conversion.
+
+### Merge Strategy
+
+1. Fetch top N keyword results (e.g., N=50)
+2. Fetch top N semantic results
+3. Merge on `document_id`
+4. For documents appearing in both sets, combine scores
+5. For documents in only one set, use that score (with 0 for the missing signal)
+6. Sort by `hybrid_score` descending
+7. Deduplicate
+8. Apply limit/offset
+
+### v2: Reciprocal Rank Fusion (future)
+
+If keyword and semantic score scales prove unstable under weighted blending, replace with RRF:
+
+```
+rrf_score = Σ 1 / (k + rank_i)
+```
+
+where `k` is a constant (typically 60) and `rank_i` is the document's rank in each result list.
+
+## 5. Filtering
+
+All search modes support these filters, applied as SQL WHERE clauses:
+
+| Filter | Parameter | SQL |
+|--------|-----------|-----|
+| Collection | `collection` | `d.collection = ?` |
+| Author | `author` | `d.author_handle = ?` or `d.did = ?` |
+| Repo | `repo` | `d.repo_name = ?` or `d.repo_did = ?` |
+| Record type | `type` | `d.record_type = ?` |
+| Language | `language` | `d.language = ?` |
+| Date range | `from`, `to` | `d.created_at >= ?` and `d.created_at <= ?` |
+| State | `state` | Join to `record_state` table |
+
+## 6. Embedding Eligibility
+
+A document is eligible for embedding if:
+
+- `deleted_at IS NULL`
+- `record_type` is one of: `repo`, `issue`, `pull`, `string`, `profile`
+- At least one of `title`, `body`, or `summary` is non-empty
+- Total text length exceeds a minimum threshold (e.g., 20 characters)
+
+## 7. API Endpoints
+
+### Health
+
+| Method | Path       | Description                      |
+| ------ | ---------- | -------------------------------- |
+| GET    | `/healthz` | Liveness — process is responsive |
+| GET    | `/readyz`  | Readiness — DB is reachable      |
+
+### Search
+
+| Method | Path               | Description                                      |
+| ------ | ------------------ | ------------------------------------------------ |
+| GET    | `/search`          | Search with configurable mode (default: keyword) |
+| GET    | `/search/keyword`  | Keyword-only search                              |
+| GET    | `/search/semantic` | Semantic-only search                             |
+| GET    | `/search/hybrid`   | Hybrid search                                    |
+
+### Documents
+
+| Method | Path              | Description                   |
+| ------ | ----------------- | ----------------------------- |
+| GET    | `/documents/{id}` | Fetch a single document by ID |
+
+### Admin
+
+| Method | Path             | Description          |
+| ------ | ---------------- | -------------------- |
+| POST   | `/admin/reindex` | Trigger reindex      |
+| POST   | `/admin/reembed` | Trigger re-embedding |
+
+Admin endpoints are disabled by default. Enable with `ENABLE_ADMIN_ENDPOINTS=true`.
+
+## 8. Query Parameters
+
+| Parameter    | Type   | Default   | Description                                                          |
+| ------------ | ------ | --------- | -------------------------------------------------------------------- |
+| `q`          | string | required  | Search query                                                         |
+| `mode`       | string | `keyword` | `keyword`, `semantic`, or `hybrid`                                   |
+| `limit`      | int    | 20        | Results per page (max: `SEARCH_MAX_LIMIT`)                           |
+| `offset`     | int    | 0         | Pagination offset                                                    |
+| `collection` | string | —         | Filter by `sh.tangled.*` collection                                  |
+| `type`       | string | —         | Filter by record type (`repo`, `issue`, `pull`, `string`, `profile`) |
+| `author`     | string | —         | Filter by author handle or DID                                       |
+| `repo`       | string | —         | Filter by repo name or repo DID                                      |
+| `language`   | string | —         | Filter by language                                                   |
+| `from`       | string | —         | Created after (ISO 8601)                                             |
+| `to`         | string | —         | Created before (ISO 8601)                                            |
+| `state`      | string | —         | Filter by state (`open`, `closed`, `merged`)                         |
+
+## 9. Search Response
+
+```json
+{
+    "query": "rust markdown tui",
+    "mode": "hybrid",
+    "total": 142,
+    "limit": 20,
+    "offset": 0,
+    "results": [
+        {
+            "id": "did:plc:abc|sh.tangled.repo|3kb3fge5lm32x",
+            "collection": "sh.tangled.repo",
+            "record_type": "repo",
+            "title": "glow-rs",
+            "body_snippet": "A TUI markdown viewer inspired by <mark>Glow</mark>...",
+            "summary": "Rust TUI markdown viewer",
+            "repo_name": "glow-rs",
+            "author_handle": "desertthunder.dev",
+            "score": 0.842,
+            "matched_by": ["keyword", "semantic"],
+            "created_at": "2026-03-20T10:00:00Z",
+            "updated_at": "2026-03-22T15:03:11Z"
+        }
+    ]
+}
+```
+
+### Result Fields
+
+| Field           | Type     | Description                             |
+| --------------- | -------- | --------------------------------------- |
+| `id`            | string   | Document stable ID                      |
+| `collection`    | string   | ATProto collection NSID                 |
+| `record_type`   | string   | Normalized type label                   |
+| `title`         | string   | Document title                          |
+| `body_snippet`  | string   | Highlighted body excerpt                |
+| `summary`       | string   | Short description                       |
+| `repo_name`     | string   | Repository name (if applicable)         |
+| `author_handle` | string   | Author handle                           |
+| `score`         | float    | Relevance score (0–1)                   |
+| `matched_by`    | string[] | Which search modes produced this result |
+| `created_at`    | string   | ISO 8601 creation timestamp             |
+| `updated_at`    | string   | ISO 8601 last update timestamp          |
+
+## 10. Document Response
+
+`GET /documents/{id}` returns the full document:
+
+```json
+{
+    "id": "did:plc:abc|sh.tangled.repo|3kb3fge5lm32x",
+    "did": "did:plc:abc",
+    "collection": "sh.tangled.repo",
+    "rkey": "3kb3fge5lm32x",
+    "at_uri": "at://did:plc:abc/sh.tangled.repo/3kb3fge5lm32x",
+    "cid": "bafyreig...",
+    "record_type": "repo",
+    "title": "glow-rs",
+    "body": "A TUI markdown viewer inspired by Glow, written in Rust.",
+    "summary": "Rust TUI markdown viewer",
+    "repo_name": "glow-rs",
+    "author_handle": "desertthunder.dev",
+    "tags_json": "[\"rust\", \"tui\", \"markdown\"]",
+    "language": "en",
+    "created_at": "2026-03-20T10:00:00Z",
+    "updated_at": "2026-03-22T15:03:11Z",
+    "indexed_at": "2026-03-22T15:05:00Z",
+    "has_embedding": true
+}
+```
+
+## 11. Error Responses
+
+| Status | Condition                                                          |
+| ------ | ------------------------------------------------------------------ |
+| 400    | Missing `q` parameter, invalid `limit`/`offset`, malformed filters |
+| 404    | Document not found                                                 |
+| 503    | DB unreachable (readiness failure)                                 |
+
+```json
+{
+    "error": "invalid_parameter",
+    "message": "limit must be between 1 and 100"
+}
+```
+
+## 12. API Behavior
+
+- `keyword` returns only lexical matches via `fts_match`/`fts_score`
+- `semantic` returns only embedding-backed matches via `vector_top_k`
+- `hybrid` merges both result sets and reranks
+- All modes exclude documents with `deleted_at IS NOT NULL` by default
+- Pagination uses `limit`/`offset` (cursor-based pagination deferred)

packages/api/docs/specs/06-operations.md

···
+---
+title: "Spec 06 — Operations"
+updated: 2026-03-22
+---
+
+Covers configuration, observability, security, and deployment.
+
+## 1. Configuration
+
+All configuration is via environment variables.
+
+### Required
+
+| Variable              | Description                                                 |
+| --------------------- | ----------------------------------------------------------- |
+| `TAP_URL`             | Tap WebSocket URL (e.g., `wss://tap.example.com/channel`)   |
+| `TAP_AUTH_PASSWORD`   | Tap admin password for Basic auth (if set on Tap)           |
+| `TURSO_DATABASE_URL`  | Turso connection URL (e.g., `libsql://db-name.turso.io`)    |
+| `TURSO_AUTH_TOKEN`    | Turso JWT auth token                                        |
+| `INDEXED_COLLECTIONS` | Comma-separated list of `sh.tangled.*` collections to index |
+
+### Search
+
+| Variable               | Default   | Description              |
+| ---------------------- | --------- | ------------------------ |
+| `SEARCH_DEFAULT_LIMIT` | `20`      | Default results per page |
+| `SEARCH_MAX_LIMIT`     | `100`     | Maximum results per page |
+| `SEARCH_DEFAULT_MODE`  | `keyword` | Default search mode      |
+
+### Embedding
+
+| Variable               | Default | Description                                          |
+| ---------------------- | ------- | ---------------------------------------------------- |
+| `EMBEDDING_PROVIDER`   | —       | Provider name (e.g., `openai`, `ollama`, `voyageai`) |
+| `EMBEDDING_MODEL`      | —       | Model name (e.g., `text-embedding-3-small`)          |
+| `EMBEDDING_API_KEY`    | —       | Provider API key                                     |
+| `EMBEDDING_API_URL`    | —       | Provider base URL (for self-hosted)                  |
+| `EMBEDDING_DIM`        | `768`   | Vector dimensionality                                |
+| `EMBEDDING_BATCH_SIZE` | `32`    | Batch size for embed-worker                          |
+
+### Hybrid Search
+
+| Variable                 | Default | Description                             |
+| ------------------------ | ------- | --------------------------------------- |
+| `HYBRID_KEYWORD_WEIGHT`  | `0.65`  | Keyword score weight in hybrid ranking  |
+| `HYBRID_SEMANTIC_WEIGHT` | `0.35`  | Semantic score weight in hybrid ranking |
+
+### Server
+
+| Variable                 | Default | Description                                 |
+| ------------------------ | ------- | ------------------------------------------- |
+| `HTTP_BIND_ADDR`         | `:8080` | API server bind address                     |
+| `LOG_LEVEL`              | `info`  | Log level: `debug`, `info`, `warn`, `error` |
+| `LOG_FORMAT`             | `json`  | Log format: `json` or `text`                |
+| `ENABLE_ADMIN_ENDPOINTS` | `false` | Enable `/admin/*` endpoints                 |
+| `ADMIN_AUTH_TOKEN`       | —       | Bearer token for admin endpoints            |
+
+### Example `.env`
+
+```bash
+# Tap (deployed on Railway)
+TAP_URL=wss://tap-instance.up.railway.app/channel
+TAP_AUTH_PASSWORD=your-tap-admin-password
+
+# Turso
+TURSO_DATABASE_URL=libsql://twister-db.turso.io
+TURSO_AUTH_TOKEN=eyJhbGci...
+
+# Collections
+INDEXED_COLLECTIONS=sh.tangled.repo,sh.tangled.repo.issue,sh.tangled.repo.pull,sh.tangled.string,sh.tangled.actor.profile,sh.tangled.repo.issue.comment,sh.tangled.repo.pull.comment,sh.tangled.repo.issue.state,sh.tangled.repo.pull.status,sh.tangled.feed.star
+
+# Search
+SEARCH_DEFAULT_LIMIT=20
+SEARCH_MAX_LIMIT=100
+
+# Embedding (Phase 2)
+# EMBEDDING_PROVIDER=openai
+# EMBEDDING_MODEL=text-embedding-3-small
+# EMBEDDING_API_KEY=sk-...
+# EMBEDDING_DIM=768
+
+# Server
+HTTP_BIND_ADDR=:8080
+LOG_LEVEL=info
+ENABLE_ADMIN_ENDPOINTS=false
+```
+
+## 2. Observability
+
+### Structured Logging
+
+Use Go's `slog` with JSON output. Every log entry includes:
+
+| Field     | Description                         |
+| --------- | ----------------------------------- |
+| `ts`      | Timestamp (RFC 3339)                |
+| `level`   | Log level                           |
+| `service` | `api`, `indexer`, or `embed-worker` |
+| `msg`     | Human-readable message              |
+
+#### Context Fields (where applicable)
+
+| Field         | When                     |
+| ------------- | ------------------------ |
+| `event_name`  | Tap event processing     |
+| `event_id`    | Tap event ID             |
+| `document_id` | Document operations      |
+| `did`         | Any DID-scoped operation |
+| `collection`  | Record processing        |
+| `rkey`        | Record processing        |
+| `cursor`      | Cursor persistence       |
+| `error_class` | Error handling           |
+| `duration_ms` | Timed operations         |
+
+### Metrics
+
+Recommended counters and gauges (via logs, Prometheus, or platform metrics):
+
+#### Ingestion
+
+| Metric                         | Type      | Description                        |
+| ------------------------------ | --------- | ---------------------------------- |
+| `events_processed_total`       | counter   | Total Tap events processed         |
+| `events_failed_total`          | counter   | Events that failed processing      |
+| `normalization_failures_total` | counter   | Normalization errors by collection |
+| `upsert_duration_ms`           | histogram | DB upsert latency                  |
+| `cursor_position`              | gauge     | Current Tap cursor position        |
+
+#### Embedding
+
+| Metric                     | Type      | Description                    |
+| -------------------------- | --------- | ------------------------------ |
+| `embedding_queue_depth`    | gauge     | Pending embedding jobs         |
+| `embedding_failures_total` | counter   | Failed embedding attempts      |
+| `embedding_duration_ms`    | histogram | Per-document embedding latency |
+
+#### Search
+
+| Metric                  | Type      | Description                |
+| ----------------------- | --------- | -------------------------- |
+| `search_requests_total` | counter   | Requests by mode           |
+| `search_duration_ms`    | histogram | Query latency by mode      |
+| `search_results_count`  | histogram | Results returned per query |
+
+### Health Checks
+
+#### API Process
+
+| Endpoint       | Check                 | Healthy             |
+| -------------- | --------------------- | ------------------- |
+| `GET /healthz` | Process is responsive | Always (liveness)   |
+| `GET /readyz`  | DB connection works   | `SELECT 1` succeeds |
+
+#### Indexer Process
+
+The indexer exposes a top-level health probe (not HTTP-routed):
+
+- Tap WebSocket connected or reconnecting
+- Cursor advancing or intentionally idle
+- DB reachable
+
+On Railway, this is a health check endpoint on a separate port (9090).
+
+#### Embed Worker
+
+- DB reachable
+- Embedding provider reachable (periodic test call)
+- Job queue not stalled (jobs processing within expected timeframe)
+
+## 3. Security
+
+### Secrets Management
+
+Secrets are injected through platform secret management:
+
+- **Railway:** Environment variables in the dashboard or `railway variables`
+
+Secrets are never stored in code, config files, or Docker images.
+
+Required secrets:
+
+| Secret              | Purpose                           |
+| ------------------- | --------------------------------- |
+| `TURSO_AUTH_TOKEN`  | Turso database authentication     |
+| `TAP_AUTH_PASSWORD` | Tap admin API authentication      |
+| `EMBEDDING_API_KEY` | Embedding provider authentication |
+| `ADMIN_AUTH_TOKEN`  | Admin endpoint authentication     |
+
+### Admin Endpoints
+
+Admin endpoints (`/admin/reindex`, `/admin/reembed`) are:
+
+- Disabled by default (`ENABLE_ADMIN_ENDPOINTS=false`)
+- When enabled, protected by bearer token (`ADMIN_AUTH_TOKEN`)
+- Alternatively, exposed only on internal networking (Railway private networking)
+
+### Input Validation
+
+The search API shall:
+
+- Validate `limit` is between 1 and `SEARCH_MAX_LIMIT`
+- Validate `offset` is non-negative
+- Reject unknown or malformed filter parameters with 400
+- Sanitize query strings before passing to FTS (Tantivy query parser handles this, but validate basic structure)
+- Bound hybrid requests (limit concurrent vector searches)
+
+### Tap Authentication
+
+The indexer authenticates to Tap using HTTP Basic auth (`admin:<TAP_AUTH_PASSWORD>`). The WebSocket upgrade request includes the auth header.
+
+### Data Privacy
+
+- All indexed content is public ATProto data
+- No private or authenticated content is ingested
+- Deleted records are tombstoned (`deleted_at` set) and excluded from search results
+- Tombstoned documents are periodically purged (configurable retention)
+
+## 4. Deployment
+
+### Railway (Primary)
+
+All Twister services deploy as separate Railway services within the same project. Tap is already deployed here.
+
+#### Service Layout
+
+| Service      | Start Command          | Health Check       | Public |
+| ------------ | ---------------------- | ------------------ | ------ |
+| tap          | (already deployed)     | `GET /health`      | no     |
+| api          | `twister api`          | `GET /healthz`     | yes    |
+| indexer      | `twister indexer`      | `GET :9090/health` | no     |
+| embed-worker | `twister embed-worker` | `GET :9091/health` | no     |
+
+All services share the same Docker image. Railway uses the start command to select the subcommand.
+
+#### Environment Variables
+
+Set per-service in the Railway dashboard or via `railway variables`:
+
+```bash
+# Shared across services
+TURSO_DATABASE_URL=libsql://twister-db.turso.io
+TURSO_AUTH_TOKEN=eyJ...
+LOG_LEVEL=info
+LOG_FORMAT=json
+
+# API service
+HTTP_BIND_ADDR=:8080
+SEARCH_DEFAULT_LIMIT=20
+SEARCH_MAX_LIMIT=100
+ENABLE_ADMIN_ENDPOINTS=false
+
+# Indexer service
+TAP_URL=wss://${{tap.RAILWAY_PUBLIC_DOMAIN}}/channel  # Railway service reference
+TAP_AUTH_PASSWORD=...
+INDEXED_COLLECTIONS=sh.tangled.repo,sh.tangled.repo.issue,sh.tangled.repo.pull,sh.tangled.string,sh.tangled.actor.profile
+
+# Embed-worker (Phase 2)
+# EMBEDDING_PROVIDER=openai
+# EMBEDDING_MODEL=text-embedding-3-small
+# EMBEDDING_API_KEY=sk-...
+```
+
+Railway supports referencing other services' variables with `${{service.VAR}}` syntax, which is useful for linking the indexer to Tap's domain.
+
+#### Health Checks
+
+Railway activates deployments based on health check responses. Configure per-service:
+
+- **api:** HTTP health check on `/healthz` port 8080
+- **indexer:** HTTP health check on `/health` port 9090
+- **embed-worker:** HTTP health check on `/health` port 9091
+
+#### Autodeploy
+
+Connect the GitHub repository for automatic deployments on push. Railway builds from the Dockerfile and uses the start command configured per service.
+
+#### Internal Networking
+
+Railway services within the same project can communicate over private networking using `service.railway.internal` hostnames. The indexer connects to Tap via this internal network when both are in the same project.
+
+### Dockerfile
+
+```dockerfile
+FROM golang:1.24-alpine AS builder
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN CGO_ENABLED=0 GOOS=linux go build \
+    -ldflags="-s -w" \
+    -o /app/twister \
+    ./main.go
+
+FROM alpine:3.21
+
+RUN apk add --no-cache ca-certificates tzdata
+
+COPY --from=builder /app/twister /usr/local/bin/twister
+
+EXPOSE 8080 9090 9091
+
+CMD ["twister", "api"]
+```
+
+Notes:
+
+- `CGO_ENABLED=0` for static binary (required if using `libsql-client-go`; not compatible with `go-libsql` which needs CGo)
+- Railway overrides `CMD` with the start command configured per service
+- Multiple ports exposed: 8080 (API), 9090 (indexer health), 9091 (embed-worker health)
+
+### Graceful Shutdown
+
+All processes handle `SIGTERM` and `SIGINT`:
+
+1. Stop accepting new requests/events
+2. Drain in-flight work (with timeout)
+3. Persist current cursor (indexer)
+4. Close DB connections
+5. Exit 0
+
+Railway sends `SIGTERM` during deployments and restarts.

packages/api/docs/specs/07-graph-backfill.md

···
+---
+title: "Spec 07 — Graph Backfill"
+updated: 2026-03-22
+---
+
+## 1. Purpose
+
+Bootstrap the search index with existing Tangled content by discovering users from a seed set and triggering Tap backfill for their repositories. Without this, the index only captures new events after deployment.
+
+## 2. Seed Set
+
+A manually curated list of known Tangled users (DIDs or handles), stored in a plain text file:
+
+```text
+# Known active Tangled users
+did:plc:abc123
+did:plc:def456
+alice.tangled.sh
+bob.tangled.sh
+# Add more as discovered
+```
+
+Format:
+- One entry per line
+- Lines starting with `#` are comments
+- Blank lines are ignored
+- Entries can be DIDs (`did:plc:...`) or handles (`alice.tangled.sh`)
+- Handles are resolved to DIDs before processing
+
+## 3. Fan-Out Strategy
+
+From each seed user, discover connected users to expand the crawl set:
+
+### Discovery Sources
+
+1. **Follows**: Fetch `sh.tangled.graph.follow` records for the user → extract `subject` DIDs
+2. **Collaborators**: For repos owned by the user, identify other users who have created issues, PRs, or comments → extract their DIDs
+
+### Depth Limit
+
+Fan-out is configurable with a max hops parameter (default: 2):
+
+- **Hop 0**: Seed users themselves
+- **Hop 1**: Direct follows and collaborators of seed users
+- **Hop 2**: Follows and collaborators of hop-1 users
+
+Higher hop counts discover more users but increase time and may pull in loosely related accounts. Start with 2 hops and adjust based on the size of the Tangled network.
+
+### Crawl Queue
+
+Discovered DIDs are added to a queue, deduplicated by DID. Each entry tracks:
+- DID
+- Discovery hop (distance from seed)
+- Source (which seed/user led to discovery)
+
+## 4. Backfill Mechanism
+
+For each discovered user:
+
+1. **Check if already tracked**: Query Tap's `/info/:did` endpoint — if the repo is already tracked and backfilled, skip
+2. **Register with Tap**: POST to `/repos/add` with the DID — Tap handles the actual repo export and event delivery
+3. **Tap backfill flow**: Tap fetches full repo history from PDS via `com.atproto.sync.getRepo`, then delivers historical events (`live: false`) through the normal WebSocket channel
+4. **Indexer processes normally**: The indexer's existing ingestion loop handles backfill events the same as live events — no special backfill code path needed
+
+### Rate Limiting
+
+- Batch `/repos/add` calls (e.g., 10 DIDs per request)
+- Add configurable delay between batches to avoid overwhelming Tap
+- Respect Tap's processing capacity — monitor `/stats/repo-count` to track progress
+
+## 5. Deduplication
+
+- **User-level**: Maintain a visited set of DIDs during fan-out; skip already-seen DIDs
+- **Tap-level**: Tap's `/repos/add` is idempotent — adding an already-tracked DID is a no-op
+- **Record-level**: The indexer's upsert logic (keyed on `did|collection|rkey`) handles duplicate events naturally
+
+## 6. CLI Interface
+
+```bash
+# Basic backfill from seed file
+twister backfill --seeds seeds.txt
+
+# Limit fan-out depth
+twister backfill --seeds seeds.txt --max-hops 1
+
+# Preview discovered users without triggering backfill
+twister backfill --seeds seeds.txt --dry-run
+
+# Control parallelism
+twister backfill --seeds seeds.txt --concurrency 5
+```
+
+### Flags
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--seeds` | required | Path to seed file |
+| `--max-hops` | `2` | Max fan-out depth from seed users |
+| `--dry-run` | `false` | List discovered users without submitting to Tap |
+| `--concurrency` | `5` | Parallel discovery workers |
+| `--batch-size` | `10` | DIDs per `/repos/add` call |
+| `--batch-delay` | `1s` | Delay between batches |
+
+### Output
+
+Progress is logged to stdout:
+
+```text
+[hop 0] Processing 5 seed users...
+[hop 0] did:plc:abc123 → 12 follows, 3 collaborators
+[hop 0] did:plc:def456 → 8 follows, 1 collaborator
+[hop 1] Processing 24 discovered users (18 new)...
+...
+[done] Discovered 142 unique users across 2 hops
+[done] Submitted 98 new DIDs to Tap (44 already tracked)
+```
+
+## 7. Idempotency
+
+The entire backfill process is safe to re-run:
+
+- Seed file parsing is stateless
+- Fan-out discovery is deterministic for a given network state
+- Tap's `/repos/add` is idempotent
+- The indexer's upsert logic handles re-delivered events
+- No local state is persisted between runs (the crawl queue is in-memory)
+
+## 8. Configuration
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `TAP_URL` | (existing) | Tap base URL for API calls |
+| `TAP_AUTH_PASSWORD` | (existing) | Tap admin auth |
+| `TURSO_DATABASE_URL` | (existing) | For checking existing records |
+| `TURSO_AUTH_TOKEN` | (existing) | DB auth |
+
+No new environment variables are needed — backfill reuses existing Tap and DB configuration.

packages/api/docs/specs/README.md

···
+---
+title: "Twister — Technical Specification Index"
+updated: 2026-03-22
+---
+
+# Twister Technical Specifications
+
+Twister is a Go-based search service for [Tangled](https://tangled.org) content on AT Protocol.
+It ingests records through [Tap](https://github.com/bluesky-social/indigo/tree/main/cmd/tap), denormalizes them into search documents, indexes them in [Turso/libSQL](https://docs.turso.tech), and exposes keyword, semantic, and hybrid search APIs.
+
+## Specifications
+
+| # | Document | Description |
+|---|----------|-------------|
+| 1 | [Architecture](01-architecture.md) | Purpose, goals, design principles, system context, tech choices |
+| 2 | [Tangled Lexicons](02-tangled-lexicons.md) | `sh.tangled.*` record schemas and fields |
+| 3 | [Data Model](03-data-model.md) | Database schema, search documents, sync state |
+| 4 | [Data Pipeline](04-data-pipeline.md) | Tap integration, normalization, failure handling |
+| 5 | [Search](05-search.md) | Search modes, API contract, scoring, filtering |
+| 6 | [Operations](06-operations.md) | Configuration, observability, security, deployment |
+| 7 | [Graph Backfill](07-graph-backfill.md) | Seed-based user discovery and content backfill |

packages/api/docs/tasks/README.md

···
+---
+title: "Twister — Task Index"
+updated: 2026-03-22
+---
+
+# Twister Tasks
+
+Assumes Go, Tap (deployed on Railway), Turso/libSQL, and Railway for deployment.
+
+## Delivery Strategy
+
+Build in four phases:
+
+1. **MVP** — ingestion, keyword search, deployment, operational tooling, graph backfill
+2. **Semantic Search** — embeddings, vector retrieval
+3. **Hybrid Search** — weighted merge of keyword + semantic
+4. **Quality Polish** — ranking refinement, advanced filters, analytics
+
+Ship keyword search before embeddings. That gives a testable, inspectable baseline before introducing model behavior.
+
+## Phases
+
+| Phase | Title | Document | Status |
+| ----- | ----- | -------- | ------ |
+| 1 | MVP | [phase-1-mvp.md](phase-1-mvp.md) | In progress (M0–M2 complete) |
+| 2 | Semantic Search | [phase-2-semantic.md](phase-2-semantic.md) | Not started |
+| 3 | Hybrid Search | [phase-3-hybrid.md](phase-3-hybrid.md) | Not started |
+| 4 | Quality Polish | [phase-4-quality.md](phase-4-quality.md) | Not started |
+
+## MVP Complete When
+
+- Tap ingests tracked `sh.tangled.*` records
+- Documents normalize into a stable store
+- Keyword search works publicly
+- API and indexer are deployed on Railway
+- Restart does not lose sync position
+- Reindex exists for repair
+- Graph backfill populates initial content from seed users

packages/api/docs/tasks/phase-1-mvp.md

···
+---
+title: "Phase 1 — MVP"
+updated: 2026-03-22
+---
+
+# Phase 1 — MVP
+
+Get a searchable product online: ingestion, keyword search, deployment, and operational tooling.
+
+## MVP Complete When
+
+- Tap ingests tracked `sh.tangled.*` records
+- Documents normalize into a stable store
+- Keyword search works publicly
+- API and indexer are deployed on Railway
+- Restart does not lose sync position
+- Reindex exists for repair
+- Graph backfill populates initial content from seed users
+
+---
+
+## M0 — Repository Bootstrap ✅
+
+Executable layout, local tooling, and development conventions (completed 2026-03-22).
+
+---
+
+## M1 — Database Schema and Store Layer ✅
+
+refs: [specs/03-data-model.md](../specs/03-data-model.md)
+
+Implemented the Turso/libSQL schema and Go store package for document persistence.
+
+---
+
+## M2 — Normalization Layer ✅
+
+refs: [specs/02-tangled-lexicons.md](../specs/02-tangled-lexicons.md), [specs/04-data-pipeline.md](../specs/04-data-pipeline.md)
+
+Translate `sh.tangled.*` records into internal search documents.
+
+---
+
+## M3 — Tap Client and Ingestion Loop
+
+refs: [specs/04-data-pipeline.md](../specs/04-data-pipeline.md), [specs/01-architecture.md](../specs/01-architecture.md)
+
+### Goal
+
+Connect the indexer to Tap (on Railway) and process live events into the store.
+
+### Why Now
+
+Tap is the point of truth for synchronized ATProto ingestion. It is already deployed on Railway.
+
+### Deliverables
+
+- Tap WebSocket client package (`internal/tapclient/`)
+- Event decode layer (record events + identity events)
+- Ingestion loop with retry/backoff
+- Cursor persistence coupled to successful DB commits
+- Identity event handler (DID → handle cache)
+
+### Tasks
+
+- [ ] Define Tap event DTOs matching the documented event shape:
+
+  ```go
+  type TapEvent struct {
+      ID       int64           `json:"id"`
+      Type     string          `json:"type"`     // "record" or "identity"
+      Record   *TapRecord      `json:"record"`
+      Identity *TapIdentity    `json:"identity"`
+  }
+  type TapRecord struct {
+      Live       bool            `json:"live"`
+      Rev        string          `json:"rev"`
+      DID        string          `json:"did"`
+      Collection string          `json:"collection"`
+      RKey       string          `json:"rkey"`
+      Action     string          `json:"action"`  // "create", "update", "delete"
+      CID        string          `json:"cid"`
+      Record     json.RawMessage `json:"record"`
+  }
+  type TapIdentity struct {
+      DID      string `json:"did"`
+      Handle   string `json:"handle"`
+      IsActive bool   `json:"isActive"`
+      Status   string `json:"status"`
+  }
+  ```
+
+- [ ] Implement WebSocket client:
+  - Connect to `TAP_URL` (e.g., `wss://tap.railway.internal/channel`)
+  - HTTP Basic auth with `admin:TAP_AUTH_PASSWORD`
+  - Auto-reconnect with exponential backoff
+  - Ack protocol: send event `id` back after successful processing
+- [ ] Implement ingestion loop:
+  1. Receive event from WebSocket
+  2. If `type == "identity"` → update handle cache, ack, continue
+  3. If `type == "record"` → check collection allowlist
+  4. Map `action` to operation (create/update → upsert, delete → tombstone)
+  5. Decode `record.record` via adapter registry
+  6. Normalize to `Document`
+  7. Upsert to store
+  8. Schedule embedding job if eligible (Phase 2)
+  9. Persist cursor (event ID) after successful DB commit
+  10. Ack the event
+- [ ] Implement collection allowlist from `INDEXED_COLLECTIONS` config
+- [ ] Handle state events (`sh.tangled.repo.issue.state`, `sh.tangled.repo.pull.status`) → update `record_state`
+- [ ] Handle normalization failures: log, skip, advance cursor
+- [ ] Handle DB failures: retry with backoff, do not advance cursor
+
+### Verification
+
+- [ ] Indexer connects to Tap via WebSocket in development
+- [ ] A newly created tracked record appears in `documents` table
+- [ ] An updated record changes the existing row (CID changes)
+- [ ] A delete event tombstones the row (`deleted_at` set)
+- [ ] Killing and restarting the indexer resumes from persisted cursor without duplication
+- [ ] Identity events update handle cache
+- [ ] Unsupported collections are silently skipped
+- [ ] Connection drops trigger automatic reconnection
+
+### Exit Criteria
+
+The system continuously ingests and persists `sh.tangled.*` records from Tap.
+
+---
+
+## M4 — Keyword Search API
+
+refs: [specs/05-search.md](../specs/05-search.md)
+
+### Goal
+
+Expose a usable public search API backed by Turso's Tantivy-backed FTS.
+
+### Why Now
+
+First real product milestone. Searchable Tangled content without waiting for embeddings.
+
+### Deliverables
+
+- HTTP server (chi or net/http)
+- `GET /healthz` — liveness
+- `GET /readyz` — readiness (DB connectivity)
+- `GET /search` — keyword search with configurable mode
+- `GET /search/keyword` — keyword-only search
+- `GET /documents/{id}` — document lookup
+- Search repository layer (FTS queries isolated from handlers)
+- Pagination, filtering, snippets
+
+### Tasks
+
+- [ ] Set up HTTP server with chi router
+- [ ] Implement `/healthz` (always 200) and `/readyz` (SELECT 1 against DB)
+- [ ] Implement search repository with FTS queries:
+
+  ```sql
+  SELECT id, title, summary, repo_name, author_handle, collection, record_type,
+         created_at, updated_at,
+         fts_score(title, body, summary, repo_name, author_handle, tags_json, ?) AS score,
+         fts_highlight(body, '<mark>', '</mark>', ?) AS body_snippet
+  FROM documents
+  WHERE fts_match(title, body, summary, repo_name, author_handle, tags_json, ?)
+    AND deleted_at IS NULL
+  ORDER BY score DESC
+  LIMIT ? OFFSET ?;
+  ```
+
+- [ ] Implement request validation:
+  - `q` required, non-empty
+  - `limit` 1–100, default 20
+  - `offset` >= 0, default 0
+  - Reject unknown parameters with 400
+- [ ] Implement filters (as WHERE clauses):
+  - `collection` → `d.collection = ?`
+  - `type` → `d.record_type = ?`
+  - `author` → `d.author_handle = ?` or `d.did = ?`
+  - `repo` → `d.repo_name = ?`
+- [ ] Implement `/documents/{id}` — full document response
+- [ ] Implement stable JSON response contract (see spec 05-search.md)
+- [ ] Exclude tombstoned documents (`deleted_at IS NOT NULL`) by default
+- [ ] Add request logging middleware (method, path, status, duration)
+- [ ] Add CORS headers if needed
+
+### Verification
+
+- [ ] Searching by exact repo name returns the expected repo first
+- [ ] Searching by title term returns expected documents
+- [ ] Searching by author handle returns relevant docs
+- [ ] Tombstoned documents do not appear
+- [ ] Malformed query parameters return 400 with error JSON
+- [ ] DB outage causes `/readyz` to fail (503)
+- [ ] Pagination works: `offset=0&limit=5` then `offset=5&limit=5` returns different results
+- [ ] Filter by collection returns only matching docs
+
+### Exit Criteria
+
+A user can search Tangled content reliably with keyword search.
+
+---
+
+## M5 — Railway Deployment
+
+refs: [specs/06-operations.md](../specs/06-operations.md)
+
+### Goal
+
+Deploy the API and indexer as Railway services alongside Tap.
+
+### Why Now
+
+At this point, the product is useful enough to run continuously.
+
+### Deliverables
+
+- Finalized Dockerfile
+- Railway project with services: `api`, `indexer`
+- Health checks configured per service
+- Secrets/env vars set
+- Production startup commands documented
+
+### Tasks
+
+- [ ] Finalize Dockerfile (multi-stage, CGO_ENABLED=0, Alpine runtime)
+- [ ] Create Railway services:
+  - `api` — start command: `twister api`
+  - `indexer` — start command: `twister indexer`
+- [ ] Configure environment variables per service:
+  - Shared: `TURSO_DATABASE_URL`, `TURSO_AUTH_TOKEN`, `LOG_LEVEL`, `LOG_FORMAT`
+  - API: `HTTP_BIND_ADDR`, `SEARCH_DEFAULT_LIMIT`, `SEARCH_MAX_LIMIT`
+  - Indexer: `TAP_URL` (reference Tap service domain), `TAP_AUTH_PASSWORD`, `INDEXED_COLLECTIONS`
+- [ ] Configure health checks:
+  - API: HTTP check on `/healthz` port 8080
+  - Indexer: HTTP check on `/health` port 9090
+- [ ] Use Railway internal networking for indexer → Tap connection
+- [ ] Connect GitHub repo for autodeploy
+- [ ] Test graceful shutdown on redeploy (SIGTERM handling)
+- [ ] Document deploy steps
+
+### Verification
+
+- [ ] API service becomes healthy and routable (public URL)
+- [ ] Indexer service starts and stays healthy
+- [ ] A new Tangled record ingested post-deploy becomes searchable
+- [ ] A redeploy preserves API availability
+- [ ] A restart does not lose sync position (cursor persisted)
+- [ ] Health checks correctly report status
+
+### Exit Criteria
+
+The system runs as a deployed service with health-checked processes on Railway.
+
+---
+
+## M6 — Reindex and Repair
+
+refs: [specs/05-search.md](../specs/05-search.md)
+
+### Goal
+
+Make the system recoverable and operable with repair tools.
+
+### Why Now
+
+Search systems are never perfect on first ingestion. Repair tools are needed before production.
+
+### Deliverables
+
+- `twister reindex` command with scoping options
+- Dry-run mode
+- Admin reindex endpoint (optional)
+- Progress logging and error summary
+
+### Tasks
+
+- [ ] Implement `reindex` subcommand with flags:
+  - `--collection` — reindex one collection
+  - `--did` — reindex one DID's documents
+  - `--document` — reindex one document by ID
+  - `--dry-run` — show intended work without writes
+  - No flags → reindex all
+- [ ] Implement reindex logic:
+  1. Select documents matching scope
+  2. For each document, re-run normalization from stored fields (or re-fetch if source available)
+  3. Update FTS-relevant fields
+  4. Upsert back to store
+  5. Log progress (N/total, errors)
+- [ ] Implement `POST /admin/reindex` endpoint (behind `ENABLE_ADMIN_ENDPOINTS` + `ADMIN_AUTH_TOKEN`)
+- [ ] Add error summary output on completion
+- [ ] Exit non-zero on unrecoverable failures
+
+### Verification
+
+- [ ] Reindexing one document updates its stored normalized text
+- [ ] Reindexing one collection repairs intentionally corrupted rows
+- [ ] Dry-run shows intended work without writes
+- [ ] Reindex command exits non-zero on failures
+- [ ] Admin endpoint triggers reindex when enabled
+
+### Exit Criteria
+
+Operators can repair bad indexes without rebuilding everything manually.
+
+---
+
+## M7 — Observability
+
+refs: [specs/06-operations.md](../specs/06-operations.md)
+
+### Goal
+
+Make the system diagnosable in production.
+
+### Deliverables
+
+- Structured slog fields across all services
+- Error classification
+- Ingestion lag visibility
+- Periodic state logs
+- Operator documentation
+
+### Tasks
+
+- [ ] Standardize slog fields across all packages:
+  - `service`, `event_name`, `event_id`, `did`, `collection`, `rkey`, `document_id`, `cursor`, `error_class`, `duration_ms`
+- [ ] Add error classification (normalize_error, db_error, tap_error, embed_error)
+- [ ] Add periodic state logs in indexer:
+  - Current cursor position
+  - Events processed since last log
+  - Documents in store (count)
+- [ ] Add request logging in API (method, path, status, duration, query)
+- [ ] Add search latency logging per query mode
+- [ ] Write operator documentation:
+  - Restart procedure
+  - Reindex procedure
+  - Backfill notes
+  - Failure triage guide
+
+### Verification
+
+- [ ] A failed Tap decode surfaces enough context to debug (collection, DID, rkey, error class)
+- [ ] DB connectivity failures are visible in logs and readiness
+- [ ] Operator can follow the runbook to diagnose a broken indexer
+- [ ] Search latency is logged per request
+
+### Exit Criteria
+
+The system is maintainable without guesswork.
+
+---
+
+## M-New — Graph Backfill from Seed Users
+
+refs: [specs/07-graph-backfill.md](../specs/07-graph-backfill.md)
+
+### Goal
+
+Bootstrap the search index with existing Tangled content by discovering and backfilling users from a seed set.
+
+### Why Now
+
+Before MVP launch, the index needs existing content. Live ingestion only captures new events — backfill populates historical data.
+
+### Deliverables
+
+- `twister backfill` CLI command
+- Seed file parser
+- Graph fan-out discovery (follows/collaborators)
+- Tap `/repos/add` integration for discovered users
+- Deduplication against already-indexed users
+- Progress logging
+
+### Tasks
+
+- [ ] Implement `backfill` subcommand with flags:
+  - `--seeds <file>` — path to seed file (one DID or handle per line)
+  - `--max-hops <n>` — depth limit for fan-out (default: 2)
+  - `--dry-run` — show discovered users without triggering backfill
+  - `--concurrency <n>` — parallel discovery workers (default: 5)
+- [ ] Implement seed file parser (supports DIDs and handles, comments with `#`)
+- [ ] Implement graph fan-out:
+  1. For each seed user, resolve DID if handle provided
+  2. Fetch `sh.tangled.graph.follow` records for the user
+  3. Fetch collaborators from repos owned by the user
+  4. Add discovered DIDs to the crawl queue
+  5. Repeat up to `max-hops` depth
+- [ ] Integrate with Tap `/repos/add` to register discovered DIDs for tracking
+- [ ] Deduplicate: skip DIDs already tracked by Tap (check via `/info/:did`)
+- [ ] Log progress: seeds processed, users discovered per hop, DIDs submitted to Tap
+- [ ] Handle rate limiting and errors gracefully (retry with backoff)
+- [ ] Make idempotent: safe to re-run; Tap handles duplicate `/repos/add` calls
+
+### Verification
+
+- [ ] Running with a seed file of 3 known users discovers their followers
+- [ ] `--max-hops 1` limits discovery to direct connections only
+- [ ] `--dry-run` lists discovered DIDs without calling Tap
+- [ ] Already-tracked users are skipped
+- [ ] Re-running the same seed file produces no duplicate work
+- [ ] Tap begins backfilling records for newly added DIDs
+
+### Exit Criteria
+
+The index contains historical content from the seed user graph, not just new events.

packages/api/docs/tasks/phase-2-semantic.md

···
+---
+title: "Phase 2 — Semantic Search"
+updated: 2026-03-22
+---
+
+# Phase 2 — Semantic Search
+
+Add embedding generation and vector-based retrieval on top of the keyword baseline.
+
+---
+
+## M8 — Embedding Pipeline
+
+refs: [specs/03-data-model.md](../specs/03-data-model.md), [specs/05-search.md](../specs/05-search.md)
+
+### Goal
+
+Add asynchronous embedding generation without blocking ingestion.
+
+### Why Now
+
+Only after keyword search is stable should semantic complexity be added.
+
+### Deliverables
+
+- `embedding_jobs` table operational (schema from M1)
+- `embed-worker` subcommand
+- Embedding provider abstraction (OpenAI, Voyage, Ollama)
+- Retry and dead-letter behavior
+- `twister reembed` command
+
+### Tasks
+
+- [ ] Define embedding provider interface:
+
+  ```go
+  type EmbeddingProvider interface {
+      Embed(ctx context.Context, texts []string) ([][]float32, error)
+      Model() string
+      Dimension() int
+  }
+  ```
+
+- [ ] Implement OpenAI provider (or preferred provider)
+- [ ] Implement embedding input text composition (see spec 04-data-pipeline.md, section 5):
+  `title\nrepo_name\nauthor_handle\ntags\nsummary\nbody`
+- [ ] Add job enqueueing: on document upsert, insert `embedding_jobs` row with `status=pending`
+- [ ] Implement `embed-worker` loop:
+  1. Poll for `pending` jobs (batch by `EMBEDDING_BATCH_SIZE`)
+  2. Compose input text per document
+  3. Call embedding provider
+  4. Store vectors in `document_embeddings` with `vector32(?)`
+  5. Mark job `completed`
+  6. On failure: increment `attempts`, set `last_error`, backoff
+  7. After max attempts: mark `dead`
+- [ ] Create DiskANN vector index: `CREATE INDEX idx_embeddings_vec ON document_embeddings(libsql_vector_idx(embedding, 'metric=cosine'))`
+- [ ] Implement `reembed` command (re-generate all embeddings, useful for model migration)
+- [ ] Skip deleted documents in embedding pipeline
+- [ ] Add health check endpoint for embed-worker (port 9091)
+
+### Verification
+
+- [ ] Creating a new searchable document enqueues an embedding job
+- [ ] Worker processes the job and stores a vector in `document_embeddings`
+- [ ] Failed embedding calls retry with bounded attempts
+- [ ] Keyword search still works when embed-worker is down
+- [ ] `reembed` regenerates embeddings for all eligible documents
+
+### Exit Criteria
+
+Embeddings are produced asynchronously and stored durably.
+
+---
+
+## M9 — Semantic Search
+
+refs: [specs/05-search.md](../specs/05-search.md)
+
+### Goal
+
+Expose vector-based semantic retrieval.
+
+### Why Now
+
+Natural next step once embeddings exist. Turso/libSQL has native vector search with `vector_top_k`.
+
+### Deliverables
+
+- `GET /search/semantic` endpoint
+- Query-time embedding (convert query text → vector)
+- Vector similarity search via `vector_top_k`
+- Response parity with keyword search
+
+### Tasks
+
+- [ ] Implement query embedding: call embedding provider with user's query text
+- [ ] Implement semantic search repository:
+
+  ```sql
+  SELECT d.id, d.title, d.summary, d.repo_name, d.author_handle,
+         d.collection, d.record_type, d.created_at, d.updated_at
+  FROM vector_top_k('idx_embeddings_vec', vector32(?), ?) AS v
+  JOIN document_embeddings e ON e.rowid = v.id
+  JOIN documents d ON d.id = e.document_id
+  WHERE d.deleted_at IS NULL;
+  ```
+
+- [ ] Normalize distance to relevance score: `score = 1.0 - (distance / 2.0)`
+- [ ] Apply same filters as keyword search (collection, author, repo, type)
+- [ ] Add timeout and cost controls (limit vector search to reasonable K)
+- [ ] Wire `/search/semantic` handler
+- [ ] Return `matched_by: ["semantic"]` in results
+
+### Verification
+
+- [ ] Semantically similar queries retrieve expected documents even with little lexical overlap
+- [ ] Documents without embeddings are omitted from semantic results
+- [ ] Semantic search returns the same JSON schema as keyword search
+- [ ] Latency is acceptable under small test load
+- [ ] Filters work correctly with semantic results
+
+### Exit Criteria
+
+The API supports true semantic search over Tangled documents.

packages/api/docs/tasks/phase-3-hybrid.md

···
+---
+title: "Phase 3 — Hybrid Search"
+updated: 2026-03-22
+---
+
+# Phase 3 — Hybrid Search
+
+Merge lexical and semantic search into the default high-quality retrieval mode.
+
+---
+
+## M10 — Hybrid Search
+
+refs: [specs/05-search.md](../specs/05-search.md)
+
+### Deliverables
+
+- `GET /search/hybrid` endpoint
+- Weighted score blending (keyword 0.65 + semantic 0.35)
+- Score normalization
+- Result deduplication
+- `matched_by` metadata showing which modes contributed
+
+### Tasks
+
+- [ ] Implement hybrid search orchestrator:
+  1. Fetch top N keyword results (N=50 or configurable)
+  2. Fetch top N semantic results
+  3. Normalize keyword scores (min-max within result set)
+  4. Semantic scores already normalized (0–1)
+  5. Merge on `document_id`
+  6. For documents in both sets: `hybrid_score = 0.65 * keyword + 0.35 * semantic`
+  7. For documents in one set: use available score (other = 0)
+  8. Sort by hybrid_score descending
+  9. Deduplicate
+  10. Apply limit/offset
+- [ ] Populate `matched_by` field: `["keyword"]`, `["semantic"]`, or `["keyword", "semantic"]`
+- [ ] Make weights configurable via `HYBRID_KEYWORD_WEIGHT` / `HYBRID_SEMANTIC_WEIGHT`
+- [ ] Wire `/search/hybrid` handler
+- [ ] Make `/search?mode=hybrid` work
+
+### Verification
+
+- [ ] Hybrid returns documents found by either source
+- [ ] Duplicates are merged correctly (no duplicate IDs in results)
+- [ ] Exact-match queries still favor lexical relevance
+- [ ] Exploratory natural-language queries improve over keyword-only results
+- [ ] Score ordering is stable across repeated runs on the same corpus
+- [ ] `matched_by` accurately reflects which modes produced each result
+
+### Exit Criteria
+
+Hybrid search becomes the preferred default search mode.

packages/api/docs/tasks/phase-4-quality.md

···
+---
+title: "Phase 4 — Ranking and Quality Polish"
+updated: 2026-03-22
+---
+
+# Phase 4 — Ranking and Quality Polish
+
+Improve search quality without changing the core architecture.
+
+---
+
+## M11 — Ranking and Quality Polish
+
+refs: [specs/05-search.md](../specs/05-search.md)
+
+### Deliverables
+
+- Boosted field weighting refinement
+- Recency boost
+- Collection-aware ranking
+- Better snippets/highlights
+- Issue/PR state filtering
+- Star count as ranking signal
+- Optional query analytics
+
+### Tasks
+
+- [ ] Tune FTS index weights based on real query results
+- [ ] Add small recency boost to ranking (e.g., decay function on `created_at`)
+- [ ] Add collection-aware ranking adjustments (repos ranked differently from comments)
+- [ ] Index `sh.tangled.repo.issue.comment` and `sh.tangled.repo.pull.comment` (P2 collections)
+- [ ] Aggregate `sh.tangled.feed.star` counts per repo and use as ranking signal
+- [ ] Implement `state` filter (open/closed/merged) using `record_state` table
+- [ ] Improve snippets: better truncation, multi-field highlights
+- [ ] Add curated relevance test fixtures (expected queries → expected top results)
+- [ ] Run `OPTIMIZE INDEX idx_documents_fts` as maintenance task
+- [ ] Optional: log queries for analytics (anonymized)
+
+### Verification
+
+- [ ] Exact repo lookups reliably rank the repo first
+- [ ] Recent active content gets a reasonable small boost without overwhelming exact relevance
+- [ ] Snippets show useful matched context
+- [ ] Ranking regression tests catch obvious degradations
+- [ ] State filter correctly excludes closed/merged items when requested
+
+### Exit Criteria
+
+Search quality is noticeably improved and more predictable.

packages/api/go.mod

···
+module tangled.org/desertthunder.dev/twister
+
+go 1.25.0
+
+require (
+	github.com/spf13/cobra v1.10.2
+	github.com/tursodatabase/libsql-client-go v0.0.0-20251219100830-236aa1ff8acc
+	modernc.org/sqlite v1.47.0
+)
+
+require (
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/coder/websocket v1.8.12 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/spf13/pflag v1.0.9 // indirect
+	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	modernc.org/libc v1.70.0 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+)

packages/api/go.sum

···
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
+github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo=
+github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/tursodatabase/libsql-client-go v0.0.0-20251219100830-236aa1ff8acc h1:lzi/5fg2EfinRlh3v//YyIhnc4tY7BTqazQGwb1ar+0=
+github.com/tursodatabase/libsql-client-go v0.0.0-20251219100830-236aa1ff8acc/go.mod h1:08inkKyguB6CGGssc/JzhmQWwBgFQBgjlYFjxjRh7nU=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
+golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
+modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw=
+modernc.org/ccgo/v4 v4.32.0/go.mod h1:6F08EBCx5uQc38kMGl+0Nm0oWczoo1c7cgpzEry7Uc0=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
+modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw=
+modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.47.0 h1:R1XyaNpoW4Et9yly+I2EeX7pBza/w+pmYee/0HJDyKk=
+modernc.org/sqlite v1.47.0/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

packages/api/internal/api/api.go

···
+package api

packages/api/internal/config/config.go

···
+package config
+
+import (
+	"errors"
+	"os"
+	"strconv"
+	"strings"
+)
+
+type Config struct {
+	TursoURL             string
+	TursoToken           string
+	TapURL               string
+	TapAuthPassword      string
+	IndexedCollections   string
+	SearchDefaultLimit   int
+	SearchMaxLimit       int
+	SearchDefaultMode    string
+	EmbeddingProvider    string
+	EmbeddingModel       string
+	EmbeddingAPIKey      string
+	EmbeddingAPIURL      string
+	EmbeddingDim         int
+	EmbeddingBatchSize   int
+	HybridKeywordWeight  float64
+	HybridSemanticWeight float64
+	HTTPBindAddr         string
+	LogLevel             string
+	LogFormat            string
+	EnableAdminEndpoints bool
+	AdminAuthToken       string
+}
+
+func Load() (*Config, error) {
+	cfg := &Config{
+		TursoURL:             os.Getenv("TURSO_DATABASE_URL"),
+		TursoToken:           os.Getenv("TURSO_AUTH_TOKEN"),
+		TapURL:               os.Getenv("TAP_URL"),
+		TapAuthPassword:      os.Getenv("TAP_AUTH_PASSWORD"),
+		IndexedCollections:   os.Getenv("INDEXED_COLLECTIONS"),
+		SearchDefaultMode:    envOrDefault("SEARCH_DEFAULT_MODE", "keyword"),
+		EmbeddingProvider:    os.Getenv("EMBEDDING_PROVIDER"),
+		EmbeddingModel:       os.Getenv("EMBEDDING_MODEL"),
+		EmbeddingAPIKey:      os.Getenv("EMBEDDING_API_KEY"),
+		EmbeddingAPIURL:      os.Getenv("EMBEDDING_API_URL"),
+		HTTPBindAddr:         envOrDefault("HTTP_BIND_ADDR", ":8080"),
+		LogLevel:             envOrDefault("LOG_LEVEL", "info"),
+		LogFormat:            envOrDefault("LOG_FORMAT", "json"),
+		AdminAuthToken:       os.Getenv("ADMIN_AUTH_TOKEN"),
+		SearchDefaultLimit:   envInt("SEARCH_DEFAULT_LIMIT", 20),
+		SearchMaxLimit:       envInt("SEARCH_MAX_LIMIT", 100),
+		EmbeddingDim:         envInt("EMBEDDING_DIM", 768),
+		EmbeddingBatchSize:   envInt("EMBEDDING_BATCH_SIZE", 32),
+		HybridKeywordWeight:  envFloat("HYBRID_KEYWORD_WEIGHT", 0.65),
+		HybridSemanticWeight: envFloat("HYBRID_SEMANTIC_WEIGHT", 0.35),
+		EnableAdminEndpoints: envBool("ENABLE_ADMIN_ENDPOINTS", false),
+	}
+
+	var errs []error
+	if cfg.TursoURL == "" {
+		errs = append(errs, errors.New("TURSO_DATABASE_URL is required"))
+	}
+	if cfg.TursoToken == "" && !strings.HasPrefix(cfg.TursoURL, "file:") {
+		errs = append(errs, errors.New("TURSO_AUTH_TOKEN is required for non-file URLs"))
+	}
+	if len(errs) > 0 {
+		return nil, errors.Join(errs...)
+	}
+	return cfg, nil
+}
+
+func envOrDefault(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+func envInt(key string, def int) int {
+	v := os.Getenv(key)
+	if v == "" {
+		return def
+	}
+	n, err := strconv.Atoi(v)
+	if err != nil {
+		return def
+	}
+	return n
+}
+
+func envFloat(key string, def float64) float64 {
+	v := os.Getenv(key)
+	if v == "" {
+		return def
+	}
+	f, err := strconv.ParseFloat(v, 64)
+	if err != nil {
+		return def
+	}
+	return f
+}
+
+func envBool(key string, def bool) bool {
+	v := os.Getenv(key)
+	if v == "" {
+		return def
+	}
+	b, err := strconv.ParseBool(v)
+	if err != nil {
+		return def
+	}
+	return b
+}

packages/api/internal/embed/embed.go

···
+package embed

packages/api/internal/index/index.go

···
+package index

packages/api/internal/ingest/ingest.go

···
+package ingest

packages/api/internal/normalize/issue.go

···
+package normalize
+
+import (
+	"fmt"
+
+	"tangled.org/desertthunder.dev/twister/internal/store"
+)
+
+const collectionIssue = "sh.tangled.repo.issue"
+
+// IssueAdapter normalizes sh.tangled.repo.issue records.
+type IssueAdapter struct{}
+
+func (a *IssueAdapter) Collection() string { return collectionIssue }
+func (a *IssueAdapter) RecordType() string { return "issue" }
+
+func (a *IssueAdapter) Searchable(_ map[string]any) bool { return true }
+
+func (a *IssueAdapter) Normalize(event TapRecordEvent) (*store.Document, error) {
+	r := event.Record
+	rec := r.Record
+
+	title := str(rec, "title")
+	body := str(rec, "body")
+	repoURI := str(rec, "repo")
+
+	repoDID := ""
+	if repoURI != "" {
+		did, _, _, err := ParseATURI(repoURI)
+		if err != nil {
+			return nil, fmt.Errorf("issue repo AT-URI: %w", err)
+		}
+		repoDID = did
+	}
+
+	return &store.Document{
+		ID:         StableID(r.DID, r.Collection, r.RKey),
+		DID:        r.DID,
+		Collection: r.Collection,
+		RKey:       r.RKey,
+		ATURI:      BuildATURI(r.DID, r.Collection, r.RKey),
+		CID:        r.CID,
+		RecordType: a.RecordType(),
+		Title:      title,
+		Body:       body,
+		Summary:    truncate(body, 200),
+		RepoDID:    repoDID,
+		TagsJSON:   "[]",
+		CreatedAt:  str(rec, "createdAt"),
+	}, nil
+}

packages/api/internal/normalize/normalize.go

···
+package normalize
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"tangled.org/desertthunder.dev/twister/internal/store"
+)
+
+// TapRecord is the inner record object from a Tap event.
+type TapRecord struct {
+	Live       bool           `json:"live"`
+	Rev        string         `json:"rev"`
+	DID        string         `json:"did"`
+	Collection string         `json:"collection"`
+	RKey       string         `json:"rkey"`
+	Action     string         `json:"action"`
+	CID        string         `json:"cid"`
+	Record     map[string]any `json:"record"`
+}
+
+// TapIdentity is the identity payload from a Tap identity event.
+type TapIdentity struct {
+	DID      string `json:"did"`
+	Handle   string `json:"handle"`
+	IsActive bool   `json:"isActive"`
+	Status   string `json:"status"`
+}
+
+// TapRecordEvent is the top-level event received from the Tap WebSocket.
+type TapRecordEvent struct {
+	ID       int64        `json:"id"`
+	Type     string       `json:"type"`
+	Record   *TapRecord   `json:"record,omitempty"`
+	Identity *TapIdentity `json:"identity,omitempty"`
+}
+
+// RecordAdapter normalizes a Tap record event into a search Document.
+type RecordAdapter interface {
+	Collection() string
+	RecordType() string
+	Normalize(event TapRecordEvent) (*store.Document, error)
+	Searchable(record map[string]any) bool
+}
+
+// StateUpdate represents a record_state change derived from a state/status record.
+type StateUpdate struct {
+	SubjectURI string
+	State      string
+}
+
+// StateHandler processes state/status records that update record_state,
+// not the documents table.
+type StateHandler interface {
+	Collection() string
+	HandleState(event TapRecordEvent) (*StateUpdate, error)
+}
+
+// StableID returns the canonical document ID: did|collection|rkey.
+func StableID(did, collection, rkey string) string {
+	return fmt.Sprintf("%s|%s|%s", did, collection, rkey)
+}
+
+// BuildATURI constructs the AT-URI for a record.
+func BuildATURI(did, collection, rkey string) string {
+	return fmt.Sprintf("at://%s/%s/%s", did, collection, rkey)
+}
+
+// ParseATURI extracts the DID, collection, and rkey from an AT-URI of the
+// form at://did:plc:abc123/sh.tangled.repo/3kb3fge5lm32x.
+func ParseATURI(uri string) (did, collection, rkey string, err error) {
+	trimmed := strings.TrimPrefix(uri, "at://")
+	parts := strings.SplitN(trimmed, "/", 3)
+	if len(parts) != 3 || parts[0] == "" {
+		return "", "", "", fmt.Errorf("invalid AT-URI: %q", uri)
+	}
+	return parts[0], parts[1], parts[2], nil
+}
+
+// truncate returns s truncated to at most n bytes.
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n]
+}
+
+// marshalTags serializes a value as JSON, returning "[]" on nil or error.
+func marshalTags(v any) string {
+	if v == nil {
+		return "[]"
+	}
+	b, err := json.Marshal(v)
+	if err != nil {
+		return "[]"
+	}
+	return string(b)
+}
+
+// str safely extracts a string field from a map[string]any.
+func str(m map[string]any, key string) string {
+	if v, ok := m[key]; ok {
+		if s, ok := v.(string); ok {
+			return s
+		}
+	}
+	return ""
+}
+
+// nestedMap safely extracts a nested map[string]any from a map.
+func nestedMap(m map[string]any, key string) map[string]any {
+	if v, ok := m[key]; ok {
+		if nested, ok := v.(map[string]any); ok {
+			return nested
+		}
+	}
+	return nil
+}

packages/api/internal/normalize/normalize_test.go

···
+package normalize_test
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"tangled.org/desertthunder.dev/twister/internal/normalize"
+)
+
+func loadFixture(t *testing.T, name string) normalize.TapRecordEvent {
+	t.Helper()
+	path := filepath.Join("testdata", name)
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read fixture %s: %v", name, err)
+	}
+	var event normalize.TapRecordEvent
+	if err := json.Unmarshal(data, &event); err != nil {
+		t.Fatalf("decode fixture %s: %v", name, err)
+	}
+	return event
+}
+
+// TestStableID verifies deterministic ID generation.
+func TestStableID(t *testing.T) {
+	id := normalize.StableID("did:plc:abc", "sh.tangled.repo", "3kb3fge5lm32x")
+	want := "did:plc:abc|sh.tangled.repo|3kb3fge5lm32x"
+	if id != want {
+		t.Errorf("StableID = %q, want %q", id, want)
+	}
+}
+
+// TestParseATURI covers the happy path and malformed input.
+func TestParseATURI(t *testing.T) {
+	t.Run("valid", func(t *testing.T) {
+		did, col, rkey, err := normalize.ParseATURI("at://did:plc:abc123/sh.tangled.repo/3kb3fge5lm32x")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if did != "did:plc:abc123" || col != "sh.tangled.repo" || rkey != "3kb3fge5lm32x" {
+			t.Errorf("got (%s, %s, %s)", did, col, rkey)
+		}
+	})
+
+	t.Run("invalid", func(t *testing.T) {
+		_, _, _, err := normalize.ParseATURI("not-an-at-uri")
+		if err == nil {
+			t.Error("expected error for malformed AT-URI")
+		}
+	})
+
+	t.Run("missing rkey", func(t *testing.T) {
+		_, _, _, err := normalize.ParseATURI("at://did:plc:abc/sh.tangled.repo")
+		if err == nil {
+			t.Error("expected error for AT-URI missing rkey segment")
+		}
+	})
+}
+
+// TestRepoAdapter verifies field mapping for sh.tangled.repo.
+func TestRepoAdapter(t *testing.T) {
+	event := loadFixture(t, "repo.json")
+	adapter := &normalize.RepoAdapter{}
+
+	if adapter.Collection() != "sh.tangled.repo" {
+		t.Errorf("Collection = %q", adapter.Collection())
+	}
+	if adapter.RecordType() != "repo" {
+		t.Errorf("RecordType = %q", adapter.RecordType())
+	}
+
+	doc, err := adapter.Normalize(event)
+	if err != nil {
+		t.Fatalf("Normalize: %v", err)
+	}
+
+	if doc.ID != "did:plc:abc123|sh.tangled.repo|3kb3fge5lm32x" {
+		t.Errorf("ID = %q", doc.ID)
+	}
+	if doc.Title != "my-project" {
+		t.Errorf("Title = %q", doc.Title)
+	}
+	if doc.Body != "A cool project for searching things" {
+		t.Errorf("Body = %q", doc.Body)
+	}
+	if doc.RepoName != "my-project" {
+		t.Errorf("RepoName = %q", doc.RepoName)
+	}
+	if doc.RepoDID != "did:plc:abc123" {
+		t.Errorf("RepoDID = %q", doc.RepoDID)
+	}
+	if doc.TagsJSON != `["go","search","atproto"]` {
+		t.Errorf("TagsJSON = %q", doc.TagsJSON)
+	}
+	if doc.RecordType != "repo" {
+		t.Errorf("RecordType = %q", doc.RecordType)
+	}
+	if doc.ATURI != "at://did:plc:abc123/sh.tangled.repo/3kb3fge5lm32x" {
+		t.Errorf("ATURI = %q", doc.ATURI)
+	}
+
+	// Searchable
+	if !adapter.Searchable(event.Record.Record) {
+		t.Error("Searchable returned false for a named repo")
+	}
+	if adapter.Searchable(map[string]any{"name": ""}) {
+		t.Error("Searchable returned true for empty name")
+	}
+}
+
+// TestIssueAdapter verifies field mapping for sh.tangled.repo.issue.
+func TestIssueAdapter(t *testing.T) {
+	event := loadFixture(t, "issue.json")
+	adapter := &normalize.IssueAdapter{}
+
+	doc, err := adapter.Normalize(event)
+	if err != nil {
+		t.Fatalf("Normalize: %v", err)
+	}
+
+	if doc.Title != "Fix search ranking for repos" {
+		t.Errorf("Title = %q", doc.Title)
+	}
+	if doc.RepoDID != "did:plc:repoowner" {
+		t.Errorf("RepoDID = %q, want did:plc:repoowner", doc.RepoDID)
+	}
+	if doc.RecordType != "issue" {
+		t.Errorf("RecordType = %q", doc.RecordType)
+	}
+	if len(doc.Summary) > 200 {
+		t.Errorf("Summary too long: %d chars", len(doc.Summary))
+	}
+	if doc.TagsJSON != "[]" {
+		t.Errorf("TagsJSON = %q, want []", doc.TagsJSON)
+	}
+
+	// Deterministic output
+	doc2, _ := adapter.Normalize(event)
+	if doc.ID != doc2.ID {
+		t.Error("Normalize is not deterministic")
+	}
+}
+
+// TestIssueAdapter_BadATURI ensures malformed repo AT-URI returns an error.
+func TestIssueAdapter_BadATURI(t *testing.T) {
+	event := normalize.TapRecordEvent{
+		ID:   9999,
+		Type: "record",
+		Record: &normalize.TapRecord{
+			DID:        "did:plc:abc",
+			Collection: "sh.tangled.repo.issue",
+			RKey:       "rkey1",
+			CID:        "cid1",
+			Record: map[string]any{
+				"title": "Test",
+				"body":  "Body",
+				"repo":  "not-an-at-uri",
+			},
+		},
+	}
+	adapter := &normalize.IssueAdapter{}
+	_, err := adapter.Normalize(event)
+	if err == nil {
+		t.Error("expected error for invalid repo AT-URI")
+	}
+}
+
+// TestPullAdapter verifies field mapping for sh.tangled.repo.pull.
+func TestPullAdapter(t *testing.T) {
+	event := loadFixture(t, "pull.json")
+	adapter := &normalize.PullAdapter{}
+
+	doc, err := adapter.Normalize(event)
+	if err != nil {
+		t.Fatalf("Normalize: %v", err)
+	}
+
+	if doc.Title != "Add star-based ranking signal" {
+		t.Errorf("Title = %q", doc.Title)
+	}
+	if doc.RepoDID != "did:plc:repoowner" {
+		t.Errorf("RepoDID = %q, want did:plc:repoowner", doc.RepoDID)
+	}
+	if doc.RecordType != "pull" {
+		t.Errorf("RecordType = %q", doc.RecordType)
+	}
+}
+
+// TestPullAdapter_NoTarget verifies that a missing target is handled gracefully.
+func TestPullAdapter_NoTarget(t *testing.T) {
+	event := normalize.TapRecordEvent{
+		ID:   9998,
+		Type: "record",
+		Record: &normalize.TapRecord{
+			DID:        "did:plc:abc",
+			Collection: "sh.tangled.repo.pull",
+			RKey:       "rkey2",
+			CID:        "cid2",
+			Record: map[string]any{
+				"title": "PR without target",
+				"body":  "Body",
+			},
+		},
+	}
+	adapter := &normalize.PullAdapter{}
+	doc, err := adapter.Normalize(event)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if doc.RepoDID != "" {
+		t.Errorf("RepoDID = %q, want empty", doc.RepoDID)
+	}
+}
+
+// TestStringAdapter verifies field mapping for sh.tangled.string.
+func TestStringAdapter(t *testing.T) {
+	event := loadFixture(t, "string.json")
+	adapter := &normalize.StringAdapter{}
+
+	doc, err := adapter.Normalize(event)
+	if err != nil {
+		t.Fatalf("Normalize: %v", err)
+	}
+
+	if doc.Title != "search.go" {
+		t.Errorf("Title = %q", doc.Title)
+	}
+	if doc.Summary != "BM25 scoring function for full-text search" {
+		t.Errorf("Summary = %q", doc.Summary)
+	}
+	if doc.RecordType != "string" {
+		t.Errorf("RecordType = %q", doc.RecordType)
+	}
+
+	// Searchable only when contents is non-empty
+	if !adapter.Searchable(event.Record.Record) {
+		t.Error("Searchable = false for non-empty contents")
+	}
+	if adapter.Searchable(map[string]any{"contents": ""}) {
+		t.Error("Searchable = true for empty contents")
+	}
+}
+
+// TestProfileAdapter verifies field mapping for sh.tangled.actor.profile.
+func TestProfileAdapter(t *testing.T) {
+	event := loadFixture(t, "profile.json")
+	adapter := &normalize.ProfileAdapter{}
+
+	doc, err := adapter.Normalize(event)
+	if err != nil {
+		t.Fatalf("Normalize: %v", err)
+	}
+
+	if doc.Body != "Building search infrastructure for the open social web. Go enthusiast." {
+		t.Errorf("Body = %q", doc.Body)
+	}
+	if doc.Summary == "" {
+		t.Error("Summary is empty; expected description + location")
+	}
+	if doc.RecordType != "profile" {
+		t.Errorf("RecordType = %q", doc.RecordType)
+	}
+	// Title is intentionally empty (handle resolved separately via identity events)
+	if doc.Title != "" {
+		t.Errorf("Title = %q, want empty (handle resolved externally)", doc.Title)
+	}
+
+	// Searchable only when description is non-empty
+	if !adapter.Searchable(event.Record.Record) {
+		t.Error("Searchable = false for non-empty description")
+	}
+	if adapter.Searchable(map[string]any{"description": ""}) {
+		t.Error("Searchable = true for empty description")
+	}
+}
+
+// TestIssueStateHandler verifies record_state extraction.
+func TestIssueStateHandler(t *testing.T) {
+	event := loadFixture(t, "issue_state.json")
+	handler := &normalize.IssueStateHandler{}
+
+	if handler.Collection() != "sh.tangled.repo.issue.state" {
+		t.Errorf("Collection = %q", handler.Collection())
+	}
+
+	update, err := handler.HandleState(event)
+	if err != nil {
+		t.Fatalf("HandleState: %v", err)
+	}
+	if update.SubjectURI != "at://did:plc:abc123/sh.tangled.repo.issue/3kb3fge5lm32y" {
+		t.Errorf("SubjectURI = %q", update.SubjectURI)
+	}
+	if update.State != "closed" {
+		t.Errorf("State = %q, want closed", update.State)
+	}
+}
+
+// TestPullStatusHandler verifies record_state extraction for PRs.
+func TestPullStatusHandler(t *testing.T) {
+	event := loadFixture(t, "pull_status.json")
+	handler := &normalize.PullStatusHandler{}
+
+	if handler.Collection() != "sh.tangled.repo.pull.status" {
+		t.Errorf("Collection = %q", handler.Collection())
+	}
+
+	update, err := handler.HandleState(event)
+	if err != nil {
+		t.Fatalf("HandleState: %v", err)
+	}
+	if update.State != "merged" {
+		t.Errorf("State = %q, want merged", update.State)
+	}
+}
+
+// TestIssueStateHandler_MissingFields ensures errors on missing required fields.
+func TestIssueStateHandler_MissingFields(t *testing.T) {
+	handler := &normalize.IssueStateHandler{}
+
+	t.Run("missing subject", func(t *testing.T) {
+		event := normalize.TapRecordEvent{
+			Record: &normalize.TapRecord{
+				Record: map[string]any{"status": "closed"},
+			},
+		}
+		_, err := handler.HandleState(event)
+		if err == nil {
+			t.Error("expected error for missing subject")
+		}
+	})
+
+	t.Run("missing status", func(t *testing.T) {
+		event := normalize.TapRecordEvent{
+			Record: &normalize.TapRecord{
+				Record: map[string]any{"subject": "at://did:plc:x/col/rkey"},
+			},
+		}
+		_, err := handler.HandleState(event)
+		if err == nil {
+			t.Error("expected error for missing status")
+		}
+	})
+}
+
+// TestRegistry verifies adapter and state handler lookup.
+func TestRegistry(t *testing.T) {
+	reg := normalize.NewRegistry()
+
+	collections := []string{
+		"sh.tangled.repo",
+		"sh.tangled.repo.issue",
+		"sh.tangled.repo.pull",
+		"sh.tangled.string",
+		"sh.tangled.actor.profile",
+	}
+	for _, col := range collections {
+		if _, ok := reg.Adapter(col); !ok {
+			t.Errorf("no adapter registered for %q", col)
+		}
+	}
+
+	stateCollections := []string{
+		"sh.tangled.repo.issue.state",
+		"sh.tangled.repo.pull.status",
+	}
+	for _, col := range stateCollections {
+		if _, ok := reg.StateHandler(col); !ok {
+			t.Errorf("no state handler registered for %q", col)
+		}
+	}
+
+	// Unsupported collections return false
+	if _, ok := reg.Adapter("sh.tangled.unknown"); ok {
+		t.Error("expected no adapter for unknown collection")
+	}
+}

packages/api/internal/normalize/profile.go

···
+package normalize
+
+import "tangled.org/desertthunder.dev/twister/internal/store"
+
+const collectionProfile = "sh.tangled.actor.profile"
+
+// ProfileAdapter normalizes sh.tangled.actor.profile records.
+// Title (author handle) is not available in the record payload — it is set by
+// the indexer after identity-event handle resolution.
+type ProfileAdapter struct{}
+
+func (a *ProfileAdapter) Collection() string { return collectionProfile }
+func (a *ProfileAdapter) RecordType() string { return "profile" }
+
+func (a *ProfileAdapter) Searchable(record map[string]any) bool {
+	return str(record, "description") != ""
+}
+
+func (a *ProfileAdapter) Normalize(event TapRecordEvent) (*store.Document, error) {
+	r := event.Record
+	rec := r.Record
+
+	description := str(rec, "description")
+	location := str(rec, "location")
+
+	summary := description
+	if location != "" {
+		if summary != "" {
+			summary = summary + " · " + location
+		} else {
+			summary = location
+		}
+	}
+
+	return &store.Document{
+		ID:         StableID(r.DID, r.Collection, r.RKey),
+		DID:        r.DID,
+		Collection: r.Collection,
+		RKey:       r.RKey,
+		ATURI:      BuildATURI(r.DID, r.Collection, r.RKey),
+		CID:        r.CID,
+		RecordType: a.RecordType(),
+		// Title (handle) is resolved from DID by the indexer via identity events.
+		Title:    "",
+		Body:     description,
+		Summary:  truncate(summary, 200),
+		TagsJSON: "[]",
+	}, nil
+}

packages/api/internal/normalize/pull.go

···
+package normalize
+
+import (
+	"fmt"
+
+	"tangled.org/desertthunder.dev/twister/internal/store"
+)
+
+const collectionPull = "sh.tangled.repo.pull"
+
+// PullAdapter normalizes sh.tangled.repo.pull records.
+type PullAdapter struct{}
+
+func (a *PullAdapter) Collection() string { return collectionPull }
+func (a *PullAdapter) RecordType() string { return "pull" }
+
+func (a *PullAdapter) Searchable(_ map[string]any) bool { return true }
+
+func (a *PullAdapter) Normalize(event TapRecordEvent) (*store.Document, error) {
+	r := event.Record
+	rec := r.Record
+
+	title := str(rec, "title")
+	body := str(rec, "body")
+
+	// repo DID is extracted from record.target.repo AT-URI
+	repoDID := ""
+	target := nestedMap(rec, "target")
+	if target != nil {
+		repoURI := str(target, "repo")
+		if repoURI != "" {
+			did, _, _, err := ParseATURI(repoURI)
+			if err != nil {
+				return nil, fmt.Errorf("pull target repo AT-URI: %w", err)
+			}
+			repoDID = did
+		}
+	}
+
+	return &store.Document{
+		ID:         StableID(r.DID, r.Collection, r.RKey),
+		DID:        r.DID,
+		Collection: r.Collection,
+		RKey:       r.RKey,
+		ATURI:      BuildATURI(r.DID, r.Collection, r.RKey),
+		CID:        r.CID,
+		RecordType: a.RecordType(),
+		Title:      title,
+		Body:       body,
+		Summary:    truncate(body, 200),
+		RepoDID:    repoDID,
+		TagsJSON:   "[]",
+		CreatedAt:  str(rec, "createdAt"),
+	}, nil
+}

packages/api/internal/normalize/registry.go

···
+package normalize
+
+// Registry maps collection NSIDs to their adapters or state handlers.
+type Registry struct {
+	adapters      map[string]RecordAdapter
+	stateHandlers map[string]StateHandler
+}
+
+// NewRegistry returns a Registry pre-loaded with all supported adapters.
+func NewRegistry() *Registry {
+	r := &Registry{
+		adapters:      make(map[string]RecordAdapter),
+		stateHandlers: make(map[string]StateHandler),
+	}
+	for _, a := range []RecordAdapter{
+		&RepoAdapter{},
+		&IssueAdapter{},
+		&PullAdapter{},
+		&StringAdapter{},
+		&ProfileAdapter{},
+	} {
+		r.adapters[a.Collection()] = a
+	}
+	for _, h := range []StateHandler{
+		&IssueStateHandler{},
+		&PullStatusHandler{},
+	} {
+		r.stateHandlers[h.Collection()] = h
+	}
+	return r
+}
+
+// Adapter returns the RecordAdapter for a collection, if supported.
+func (r *Registry) Adapter(collection string) (RecordAdapter, bool) {
+	a, ok := r.adapters[collection]
+	return a, ok
+}
+
+// StateHandler returns the StateHandler for a collection, if supported.
+func (r *Registry) StateHandler(collection string) (StateHandler, bool) {
+	h, ok := r.stateHandlers[collection]
+	return h, ok
+}

packages/api/internal/normalize/repo.go

···
+package normalize
+
+import "tangled.org/desertthunder.dev/twister/internal/store"
+
+const collectionRepo = "sh.tangled.repo"
+
+// RepoAdapter normalizes sh.tangled.repo records.
+type RepoAdapter struct{}
+
+func (a *RepoAdapter) Collection() string { return collectionRepo }
+func (a *RepoAdapter) RecordType() string { return "repo" }
+
+func (a *RepoAdapter) Searchable(record map[string]any) bool {
+	return str(record, "name") != ""
+}
+
+func (a *RepoAdapter) Normalize(event TapRecordEvent) (*store.Document, error) {
+	r := event.Record
+	rec := r.Record
+
+	name := str(rec, "name")
+	description := str(rec, "description")
+
+	return &store.Document{
+		ID:         StableID(r.DID, r.Collection, r.RKey),
+		DID:        r.DID,
+		Collection: r.Collection,
+		RKey:       r.RKey,
+		ATURI:      BuildATURI(r.DID, r.Collection, r.RKey),
+		CID:        r.CID,
+		RecordType: a.RecordType(),
+		Title:      name,
+		Body:       description,
+		Summary:    truncate(description, 200),
+		RepoDID:    r.DID,
+		RepoName:   name,
+		TagsJSON:   marshalTags(rec["topics"]),
+		CreatedAt:  str(rec, "createdAt"),
+	}, nil
+}

packages/api/internal/normalize/state.go

···
+package normalize
+
+import "fmt"
+
+const (
+	collectionIssueState string = "sh.tangled.repo.issue.state"
+	collectionPullStatus string = "sh.tangled.repo.pull.status"
+)
+
+// IssueStateHandler processes sh.tangled.repo.issue.state records.
+type IssueStateHandler struct{}
+
+func (h *IssueStateHandler) Collection() string { return collectionIssueState }
+
+func (h *IssueStateHandler) HandleState(event TapRecordEvent) (*StateUpdate, error) {
+	r := event.Record
+	rec := r.Record
+
+	subject := str(rec, "subject")
+	if subject == "" {
+		return nil, fmt.Errorf("issue state record missing subject field")
+	}
+	state := str(rec, "status")
+	if state == "" {
+		return nil, fmt.Errorf("issue state record missing status field")
+	}
+
+	return &StateUpdate{
+		SubjectURI: subject,
+		State:      state,
+	}, nil
+}
+
+// PullStatusHandler processes sh.tangled.repo.pull.status records.
+type PullStatusHandler struct{}
+
+func (h *PullStatusHandler) Collection() string { return collectionPullStatus }
+
+func (h *PullStatusHandler) HandleState(event TapRecordEvent) (*StateUpdate, error) {
+	r := event.Record
+	rec := r.Record
+
+	subject := str(rec, "subject")
+	if subject == "" {
+		return nil, fmt.Errorf("pull status record missing subject field")
+	}
+	status := str(rec, "status")
+	if status == "" {
+		return nil, fmt.Errorf("pull status record missing status field")
+	}
+
+	return &StateUpdate{
+		SubjectURI: subject,
+		State:      status,
+	}, nil
+}

packages/api/internal/normalize/string.go

···
+package normalize
+
+import "tangled.org/desertthunder.dev/twister/internal/store"
+
+const collectionString = "sh.tangled.string"
+
+// StringAdapter normalizes sh.tangled.string records (code snippets/gists).
+type StringAdapter struct{}
+
+func (a *StringAdapter) Collection() string { return collectionString }
+func (a *StringAdapter) RecordType() string { return "string" }
+
+func (a *StringAdapter) Searchable(record map[string]any) bool {
+	return str(record, "contents") != ""
+}
+
+func (a *StringAdapter) Normalize(event TapRecordEvent) (*store.Document, error) {
+	r := event.Record
+	rec := r.Record
+
+	return &store.Document{
+		ID:         StableID(r.DID, r.Collection, r.RKey),
+		DID:        r.DID,
+		Collection: r.Collection,
+		RKey:       r.RKey,
+		ATURI:      BuildATURI(r.DID, r.Collection, r.RKey),
+		CID:        r.CID,
+		RecordType: a.RecordType(),
+		Title:      str(rec, "filename"),
+		Body:       str(rec, "contents"),
+		Summary:    str(rec, "description"),
+		TagsJSON:   "[]",
+		CreatedAt:  str(rec, "createdAt"),
+	}, nil
+}

packages/api/internal/normalize/testdata/issue.json

···
+{
+    "id": 1002,
+    "type": "record",
+    "record": {
+        "live": true,
+        "rev": "3kb3fge5lm32y",
+        "did": "did:plc:abc123",
+        "collection": "sh.tangled.repo.issue",
+        "rkey": "3kb3fge5lm32y",
+        "action": "create",
+        "cid": "bafyreigabc124",
+        "record": {
+            "$type": "sh.tangled.repo.issue",
+            "title": "Fix search ranking for repos",
+            "body": "The current ranking algorithm does not account for star count. We should add a star-based signal to improve relevance for popular repositories.",
+            "repo": "at://did:plc:repoowner/sh.tangled.repo/3kb3fge5lm30a",
+            "createdAt": "2026-03-22T13:00:00.000Z"
+        }
+    }
+}

packages/api/internal/normalize/testdata/issue_state.json

···
+{
+  "id": 1006,
+  "type": "record",
+  "record": {
+    "live": true,
+    "rev": "3kb3fge5lm33c",
+    "did": "did:plc:abc123",
+    "collection": "sh.tangled.repo.issue.state",
+    "rkey": "3kb3fge5lm33c",
+    "action": "create",
+    "cid": "bafyreigabc128",
+    "record": {
+      "$type": "sh.tangled.repo.issue.state",
+      "subject": "at://did:plc:abc123/sh.tangled.repo.issue/3kb3fge5lm32y",
+      "status": "closed"
+    }
+  }
+}

packages/api/internal/normalize/testdata/profile.json

···
+{
+  "id": 1005,
+  "type": "record",
+  "record": {
+    "live": true,
+    "rev": "3kb3fge5lm33b",
+    "did": "did:plc:abc123",
+    "collection": "sh.tangled.actor.profile",
+    "rkey": "self",
+    "action": "create",
+    "cid": "bafyreigabc127",
+    "record": {
+      "$type": "sh.tangled.actor.profile",
+      "description": "Building search infrastructure for the open social web. Go enthusiast.",
+      "location": "San Francisco, CA",
+      "website": "https://example.tangled.org"
+    }
+  }
+}

packages/api/internal/normalize/testdata/pull.json

···
+{
+  "id": 1003,
+  "type": "record",
+  "record": {
+    "live": true,
+    "rev": "3kb3fge5lm32z",
+    "did": "did:plc:abc123",
+    "collection": "sh.tangled.repo.pull",
+    "rkey": "3kb3fge5lm32z",
+    "action": "create",
+    "cid": "bafyreigabc125",
+    "record": {
+      "$type": "sh.tangled.repo.pull",
+      "title": "Add star-based ranking signal",
+      "body": "Implements the star count signal discussed in issue #42. Adds a weight multiplier based on repository star count, capped at 1000 stars.",
+      "target": {
+        "repo": "at://did:plc:repoowner/sh.tangled.repo/3kb3fge5lm30a",
+        "branch": "main"
+      },
+      "source": {
+        "repo": "at://did:plc:abc123/sh.tangled.repo/3kb3fge5lm30b",
+        "branch": "feature/star-ranking"
+      },
+      "createdAt": "2026-03-22T14:00:00.000Z"
+    }
+  }
+}

packages/api/internal/normalize/testdata/pull_status.json

···
+{
+  "id": 1007,
+  "type": "record",
+  "record": {
+    "live": true,
+    "rev": "3kb3fge5lm33d",
+    "did": "did:plc:abc123",
+    "collection": "sh.tangled.repo.pull.status",
+    "rkey": "3kb3fge5lm33d",
+    "action": "create",
+    "cid": "bafyreigabc129",
+    "record": {
+      "$type": "sh.tangled.repo.pull.status",
+      "subject": "at://did:plc:abc123/sh.tangled.repo.pull/3kb3fge5lm32z",
+      "status": "merged"
+    }
+  }
+}

packages/api/internal/normalize/testdata/repo.json

···
+{
+  "id": 1001,
+  "type": "record",
+  "record": {
+    "live": true,
+    "rev": "3kb3fge5lm32x",
+    "did": "did:plc:abc123",
+    "collection": "sh.tangled.repo",
+    "rkey": "3kb3fge5lm32x",
+    "action": "create",
+    "cid": "bafyreigabc123",
+    "record": {
+      "$type": "sh.tangled.repo",
+      "name": "my-project",
+      "knot": "knot.tangled.org",
+      "description": "A cool project for searching things",
+      "topics": ["go", "search", "atproto"],
+      "createdAt": "2026-03-22T12:00:00.000Z"
+    }
+  }
+}

packages/api/internal/normalize/testdata/string.json

···
+{
+  "id": 1004,
+  "type": "record",
+  "record": {
+    "live": true,
+    "rev": "3kb3fge5lm33a",
+    "did": "did:plc:abc123",
+    "collection": "sh.tangled.string",
+    "rkey": "3kb3fge5lm33a",
+    "action": "create",
+    "cid": "bafyreigabc126",
+    "record": {
+      "$type": "sh.tangled.string",
+      "filename": "search.go",
+      "contents": "package search\n\nfunc BM25Score(tf, df, n, avgdl float64) float64 {\n\tk1 := 1.2\n\tb := 0.75\n\treturn tf * (k1 + 1) / (tf + k1*(1-b+b*n/avgdl))\n}",
+      "description": "BM25 scoring function for full-text search",
+      "createdAt": "2026-03-22T15:00:00.000Z"
+    }
+  }
+}

packages/api/internal/observability/log.go

···
+package observability
+
+import (
+	"log/slog"
+	"os"
+
+	"tangled.org/desertthunder.dev/twister/internal/config"
+)
+
+func NewLogger(cfg *config.Config) *slog.Logger {
+	level := slog.LevelInfo
+	switch cfg.LogLevel {
+	case "debug":
+		level = slog.LevelDebug
+	case "warn":
+		level = slog.LevelWarn
+	case "error":
+		level = slog.LevelError
+	}
+
+	opts := &slog.HandlerOptions{Level: level}
+
+	var handler slog.Handler
+	if cfg.LogFormat == "text" {
+		handler = slog.NewTextHandler(os.Stdout, opts)
+	} else {
+		handler = slog.NewJSONHandler(os.Stdout, opts)
+	}
+
+	return slog.New(handler)
+}

packages/api/internal/ranking/ranking.go

···
+package ranking

packages/api/internal/search/search.go

···
+package search

packages/api/internal/store/db.go

···
+package store
+
+import (
+	"database/sql"
+	"embed"
+	"fmt"
+	"log/slog"
+	"sort"
+	"strings"
+
+	_ "github.com/tursodatabase/libsql-client-go/libsql"
+	_ "modernc.org/sqlite"
+)
+
+//go:embed migrations/*.sql
+var migrationsFS embed.FS
+
+// Open establishes a connection to the database.
+// For remote Turso URLs (libsql:// or https://) it uses the libsql-client-go driver.
+// For local file: URLs it uses the pure-Go SQLite driver (no CGo required).
+func Open(url, token string) (*sql.DB, error) {
+	driver, dsn := driverAndDSN(url, token)
+	db, err := sql.Open(driver, dsn)
+	if err != nil {
+		return nil, fmt.Errorf("open db: %w", err)
+	}
+	if err := db.Ping(); err != nil {
+		db.Close()
+		return nil, fmt.Errorf("ping db: %w", err)
+	}
+	return db, nil
+}
+
+// driverAndDSN returns the sql driver name and DSN for the given URL.
+// file: URLs use the pure-Go "sqlite" driver; all others use "libsql".
+func driverAndDSN(url, token string) (driver, dsn string) {
+	if strings.HasPrefix(url, "file:") {
+		return "sqlite", strings.TrimPrefix(url, "file:")
+	}
+	if token == "" || strings.Contains(url, "?") {
+		return "libsql", url
+	}
+	return "libsql", url + "?authToken=" + token
+}
+
+// Migrate runs all embedded SQL migration files in order.
+func Migrate(db *sql.DB) error {
+	entries, err := migrationsFS.ReadDir("migrations")
+	if err != nil {
+		return fmt.Errorf("read migrations dir: %w", err)
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].Name() < entries[j].Name()
+	})
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".sql") {
+			continue
+		}
+		data, err := migrationsFS.ReadFile("migrations/" + entry.Name())
+		if err != nil {
+			return fmt.Errorf("read migration %s: %w", entry.Name(), err)
+		}
+		if err := execMigration(db, entry.Name(), string(data)); err != nil {
+			return err
+		}
+		slog.Info("migration applied", "file", entry.Name())
+	}
+	return nil
+}
+
+func execMigration(db *sql.DB, name, content string) error {
+	for _, stmt := range splitStatements(content) {
+		if _, err := db.Exec(stmt); err != nil {
+			upper := strings.ToUpper(stmt)
+			if strings.Contains(upper, "USING FTS") || strings.Contains(upper, "LIBSQL_VECTOR_IDX") {
+				slog.Warn("migration: skipping extension index (not supported in this environment)",
+					"migration", name, "err", err)
+				continue
+			}
+			return fmt.Errorf("migration %s: exec failed: %w\nstatement: %s", name, err, stmt)
+		}
+	}
+	return nil
+}
+
+func splitStatements(content string) []string {
+	var stmts []string
+	for _, s := range strings.Split(content, ";") {
+		s = strings.TrimSpace(s)
+		if s != "" {
+			stmts = append(stmts, s)
+		}
+	}
+	return stmts
+}

packages/api/internal/store/migrations/001_initial.sql

···
+CREATE TABLE IF NOT EXISTS documents (
+    id            TEXT PRIMARY KEY,
+    did           TEXT NOT NULL,
+    collection    TEXT NOT NULL,
+    rkey          TEXT NOT NULL,
+    at_uri        TEXT NOT NULL,
+    cid           TEXT NOT NULL,
+    record_type   TEXT NOT NULL,
+    title         TEXT,
+    body          TEXT,
+    summary       TEXT,
+    repo_did      TEXT,
+    repo_name     TEXT,
+    author_handle TEXT,
+    tags_json     TEXT,
+    language      TEXT,
+    created_at    TEXT,
+    updated_at    TEXT,
+    indexed_at    TEXT NOT NULL,
+    deleted_at    TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_documents_did ON documents(did);
+
+CREATE INDEX IF NOT EXISTS idx_documents_collection ON documents(collection);
+
+CREATE INDEX IF NOT EXISTS idx_documents_record_type ON documents(record_type);
+
+CREATE INDEX IF NOT EXISTS idx_documents_repo_did ON documents(repo_did);
+
+CREATE INDEX IF NOT EXISTS idx_documents_created_at ON documents(created_at);
+
+CREATE INDEX IF NOT EXISTS idx_documents_deleted_at ON documents(deleted_at);
+
+CREATE INDEX IF NOT EXISTS idx_documents_fts ON documents USING fts (
+    title WITH tokenizer=default,
+    body WITH tokenizer=default,
+    summary WITH tokenizer=default,
+    repo_name WITH tokenizer=simple,
+    author_handle WITH tokenizer=raw,
+    tags_json WITH tokenizer=simple
+) WITH (weights='title=3.0,repo_name=2.5,author_handle=2.0,summary=1.5,tags_json=1.2,body=1.0');
+
+CREATE TABLE IF NOT EXISTS sync_state (
+    consumer_name   TEXT PRIMARY KEY,
+    cursor          TEXT NOT NULL,
+    high_water_mark TEXT,
+    updated_at      TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS document_embeddings (
+    document_id     TEXT PRIMARY KEY REFERENCES documents(id),
+    embedding       F32_BLOB(768),
+    embedding_model TEXT NOT NULL,
+    embedded_at     TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_embeddings_vec ON document_embeddings(
+    libsql_vector_idx(embedding, 'metric=cosine')
+);
+
+CREATE TABLE IF NOT EXISTS embedding_jobs (
+    document_id  TEXT PRIMARY KEY REFERENCES documents(id),
+    status       TEXT NOT NULL,
+    attempts     INTEGER NOT NULL DEFAULT 0,
+    last_error   TEXT,
+    scheduled_at TEXT NOT NULL,
+    updated_at   TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_embedding_jobs_status ON embedding_jobs(status);
+
+CREATE TABLE IF NOT EXISTS record_state (
+    subject_uri TEXT PRIMARY KEY,
+    state       TEXT NOT NULL,
+    updated_at  TEXT NOT NULL
+);

packages/api/internal/store/sql_store.go

···
+package store
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+)
+
+// SQLStore implements Store against a libSQL database.
+type SQLStore struct {
+	db *sql.DB
+}
+
+// New wraps an open *sql.DB in a Store implementation.
+func New(db *sql.DB) Store {
+	return &SQLStore{db: db}
+}
+
+func (s *SQLStore) UpsertDocument(ctx context.Context, doc *Document) error {
+	doc.IndexedAt = time.Now().UTC().Format(time.RFC3339)
+	_, err := s.db.ExecContext(ctx, `
+		INSERT INTO documents (
+			id, did, collection, rkey, at_uri, cid, record_type,
+			title, body, summary, repo_did, repo_name, author_handle,
+			tags_json, language, created_at, updated_at, indexed_at, deleted_at
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		ON CONFLICT(id) DO UPDATE SET
+			did           = excluded.did,
+			collection    = excluded.collection,
+			rkey          = excluded.rkey,
+			at_uri        = excluded.at_uri,
+			cid           = excluded.cid,
+			record_type   = excluded.record_type,
+			title         = excluded.title,
+			body          = excluded.body,
+			summary       = excluded.summary,
+			repo_did      = excluded.repo_did,
+			repo_name     = excluded.repo_name,
+			author_handle = excluded.author_handle,
+			tags_json     = excluded.tags_json,
+			language      = excluded.language,
+			created_at    = excluded.created_at,
+			updated_at    = excluded.updated_at,
+			indexed_at    = excluded.indexed_at,
+			deleted_at    = excluded.deleted_at`,
+		doc.ID, doc.DID, doc.Collection, doc.RKey, doc.ATURI, doc.CID, doc.RecordType,
+		doc.Title, doc.Body, doc.Summary, doc.RepoDID, doc.RepoName, doc.AuthorHandle,
+		doc.TagsJSON, doc.Language, doc.CreatedAt, doc.UpdatedAt, doc.IndexedAt, nullableStr(doc.DeletedAt),
+	)
+	if err != nil {
+		return fmt.Errorf("upsert document: %w", err)
+	}
+	return nil
+}
+
+func (s *SQLStore) GetDocument(ctx context.Context, id string) (*Document, error) {
+	row := s.db.QueryRowContext(ctx, `
+		SELECT id, did, collection, rkey, at_uri, cid, record_type,
+		       title, body, summary, repo_did, repo_name, author_handle,
+		       tags_json, language, created_at, updated_at, indexed_at, deleted_at
+		FROM documents WHERE id = ?`, id)
+
+	doc, err := scanDocument(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get document: %w", err)
+	}
+	return doc, nil
+}
+
+func (s *SQLStore) MarkDeleted(ctx context.Context, id string) error {
+	now := time.Now().UTC().Format(time.RFC3339)
+	_, err := s.db.ExecContext(ctx,
+		`UPDATE documents SET deleted_at = ? WHERE id = ?`, now, id)
+	if err != nil {
+		return fmt.Errorf("mark deleted: %w", err)
+	}
+	return nil
+}
+
+func (s *SQLStore) GetSyncState(ctx context.Context, consumer string) (*SyncState, error) {
+	row := s.db.QueryRowContext(ctx, `
+		SELECT consumer_name, cursor, high_water_mark, updated_at
+		FROM sync_state WHERE consumer_name = ?`, consumer)
+
+	ss := &SyncState{}
+	var hwm sql.NullString
+	err := row.Scan(&ss.ConsumerName, &ss.Cursor, &hwm, &ss.UpdatedAt)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get sync state: %w", err)
+	}
+	ss.HighWaterMark = hwm.String
+	return ss, nil
+}
+
+func (s *SQLStore) SetSyncState(ctx context.Context, consumer string, cursor string) error {
+	now := time.Now().UTC().Format(time.RFC3339)
+	_, err := s.db.ExecContext(ctx, `
+		INSERT INTO sync_state (consumer_name, cursor, updated_at) VALUES (?, ?, ?)
+		ON CONFLICT(consumer_name) DO UPDATE SET
+			cursor     = excluded.cursor,
+			updated_at = excluded.updated_at`,
+		consumer, cursor, now,
+	)
+	if err != nil {
+		return fmt.Errorf("set sync state: %w", err)
+	}
+	return nil
+}
+
+func (s *SQLStore) UpdateRecordState(ctx context.Context, subjectURI string, state string) error {
+	now := time.Now().UTC().Format(time.RFC3339)
+	_, err := s.db.ExecContext(ctx, `
+		INSERT INTO record_state (subject_uri, state, updated_at) VALUES (?, ?, ?)
+		ON CONFLICT(subject_uri) DO UPDATE SET
+			state      = excluded.state,
+			updated_at = excluded.updated_at`,
+		subjectURI, state, now,
+	)
+	if err != nil {
+		return fmt.Errorf("update record state: %w", err)
+	}
+	return nil
+}
+
+func scanDocument(row *sql.Row) (*Document, error) {
+	doc := &Document{}
+	var (
+		title, body, summary, repoDID, repoName, authorHandle sql.NullString
+		tagsJSON, language, createdAt, updatedAt, deletedAt   sql.NullString
+	)
+	err := row.Scan(
+		&doc.ID, &doc.DID, &doc.Collection, &doc.RKey, &doc.ATURI, &doc.CID, &doc.RecordType,
+		&title, &body, &summary, &repoDID, &repoName, &authorHandle,
+		&tagsJSON, &language, &createdAt, &updatedAt, &doc.IndexedAt, &deletedAt,
+	)
+	if err != nil {
+		return nil, err
+	}
+	doc.Title = title.String
+	doc.Body = body.String
+	doc.Summary = summary.String
+	doc.RepoDID = repoDID.String
+	doc.RepoName = repoName.String
+	doc.AuthorHandle = authorHandle.String
+	doc.TagsJSON = tagsJSON.String
+	doc.Language = language.String
+	doc.CreatedAt = createdAt.String
+	doc.UpdatedAt = updatedAt.String
+	doc.DeletedAt = deletedAt.String
+	return doc, nil
+}
+
+func nullableStr(s string) any {
+	if s == "" {
+		return nil
+	}
+	return s
+}

packages/api/internal/store/store.go

···
+package store
+
+import "context"
+
+// Document is the denormalized search document stored in the database.
+type Document struct {
+	ID           string
+	DID          string
+	Collection   string
+	RKey         string
+	ATURI        string
+	CID          string
+	RecordType   string
+	Title        string
+	Body         string
+	Summary      string
+	RepoDID      string
+	RepoName     string
+	AuthorHandle string
+	TagsJSON     string
+	Language     string
+	CreatedAt    string
+	UpdatedAt    string
+	IndexedAt    string
+	DeletedAt    string
+}
+
+// SyncState tracks the Tap consumer cursor for resuming on restart.
+type SyncState struct {
+	ConsumerName  string
+	Cursor        string
+	HighWaterMark string
+	UpdatedAt     string
+}
+
+// RecordState caches issue/PR open-closed-merged state.
+type RecordState struct {
+	SubjectURI string
+	State      string
+	UpdatedAt  string
+}
+
+// Store is the persistence interface for Twister.
+type Store interface {
+	UpsertDocument(ctx context.Context, doc *Document) error
+	GetDocument(ctx context.Context, id string) (*Document, error)
+	MarkDeleted(ctx context.Context, id string) error
+	GetSyncState(ctx context.Context, consumer string) (*SyncState, error)
+	SetSyncState(ctx context.Context, consumer string, cursor string) error
+	UpdateRecordState(ctx context.Context, subjectURI string, state string) error
+}

packages/api/internal/store/store_test.go

···
+package store_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"tangled.org/desertthunder.dev/twister/internal/store"
+)
+
+func TestIntegration(t *testing.T) {
+	dir := t.TempDir()
+	dbPath := filepath.Join(dir, "test.db")
+	url := "file:" + dbPath
+
+	db, err := store.Open(url, "")
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() {
+		db.Close()
+		os.Remove(dbPath)
+	})
+
+	if err := store.Migrate(db); err != nil {
+		t.Fatalf("migrate: %v", err)
+	}
+
+	st := store.New(db)
+	ctx := context.Background()
+
+	t.Run("upsert and get document", func(t *testing.T) {
+		doc := &store.Document{
+			ID:         "did:plc:abc|sh.tangled.repo|abc123",
+			DID:        "did:plc:abc",
+			Collection: "sh.tangled.repo",
+			RKey:       "abc123",
+			ATURI:      "at://did:plc:abc/sh.tangled.repo/abc123",
+			CID:        "bafyreiabc",
+			RecordType: "repo",
+			Title:      "my-repo",
+			Body:       "A test repository",
+			RepoName:   "my-repo",
+		}
+		if err := st.UpsertDocument(ctx, doc); err != nil {
+			t.Fatalf("upsert: %v", err)
+		}
+
+		got, err := st.GetDocument(ctx, doc.ID)
+		if err != nil {
+			t.Fatalf("get: %v", err)
+		}
+		if got == nil {
+			t.Fatal("expected document, got nil")
+		}
+		if got.Title != "my-repo" {
+			t.Errorf("title: got %q, want %q", got.Title, "my-repo")
+		}
+		if got.IndexedAt == "" {
+			t.Error("indexed_at should be set by upsert")
+		}
+	})
+
+	t.Run("upsert is idempotent", func(t *testing.T) {
+		doc := &store.Document{
+			ID:         "did:plc:abc|sh.tangled.repo|abc123",
+			DID:        "did:plc:abc",
+			Collection: "sh.tangled.repo",
+			RKey:       "abc123",
+			ATURI:      "at://did:plc:abc/sh.tangled.repo/abc123",
+			CID:        "bafyreiabc2",
+			RecordType: "repo",
+			Title:      "my-repo-v2",
+		}
+		if err := st.UpsertDocument(ctx, doc); err != nil {
+			t.Fatalf("upsert: %v", err)
+		}
+		got, err := st.GetDocument(ctx, doc.ID)
+		if err != nil {
+			t.Fatalf("get: %v", err)
+		}
+		if got.Title != "my-repo-v2" {
+			t.Errorf("title: got %q, want %q", got.Title, "my-repo-v2")
+		}
+		if got.CID != "bafyreiabc2" {
+			t.Errorf("cid: got %q, want updated CID", got.CID)
+		}
+	})
+
+	t.Run("tombstone sets deleted_at", func(t *testing.T) {
+		if err := st.MarkDeleted(ctx, "did:plc:abc|sh.tangled.repo|abc123"); err != nil {
+			t.Fatalf("mark deleted: %v", err)
+		}
+		got, err := st.GetDocument(ctx, "did:plc:abc|sh.tangled.repo|abc123")
+		if err != nil {
+			t.Fatalf("get: %v", err)
+		}
+		if got.DeletedAt == "" {
+			t.Error("deleted_at should be set after tombstone")
+		}
+	})
+
+	t.Run("get missing document returns nil", func(t *testing.T) {
+		got, err := st.GetDocument(ctx, "nonexistent")
+		if err != nil {
+			t.Fatalf("get: %v", err)
+		}
+		if got != nil {
+			t.Error("expected nil for missing document")
+		}
+	})
+
+	t.Run("sync state CRUD", func(t *testing.T) {
+		got, err := st.GetSyncState(ctx, "tap-consumer")
+		if err != nil {
+			t.Fatalf("get sync state: %v", err)
+		}
+		if got != nil {
+			t.Error("expected nil for missing sync state")
+		}
+
+		if err := st.SetSyncState(ctx, "tap-consumer", "cursor-001"); err != nil {
+			t.Fatalf("set sync state: %v", err)
+		}
+
+		got, err = st.GetSyncState(ctx, "tap-consumer")
+		if err != nil {
+			t.Fatalf("get sync state: %v", err)
+		}
+		if got == nil {
+			t.Fatal("expected sync state, got nil")
+		}
+		if got.Cursor != "cursor-001" {
+			t.Errorf("cursor: got %q, want %q", got.Cursor, "cursor-001")
+		}
+
+		if err := st.SetSyncState(ctx, "tap-consumer", "cursor-002"); err != nil {
+			t.Fatalf("update sync state: %v", err)
+		}
+		got, err = st.GetSyncState(ctx, "tap-consumer")
+		if err != nil {
+			t.Fatalf("get: %v", err)
+		}
+		if got.Cursor != "cursor-002" {
+			t.Errorf("cursor after update: got %q, want %q", got.Cursor, "cursor-002")
+		}
+	})
+
+	t.Run("record state upsert", func(t *testing.T) {
+		uri := "at://did:plc:abc/sh.tangled.repo.issue/1"
+		if err := st.UpdateRecordState(ctx, uri, "open"); err != nil {
+			t.Fatalf("update record state: %v", err)
+		}
+		if err := st.UpdateRecordState(ctx, uri, "closed"); err != nil {
+			t.Fatalf("update record state to closed: %v", err)
+		}
+	})
+}

packages/api/internal/tapclient/tapclient.go

···
+package tapclient

packages/api/justfile

···
+version := `git describe --tags --always --dirty 2>/dev/null || echo dev`
+commit  := `git rev-parse --short HEAD 2>/dev/null || echo none`
+ldflags := "-s -w -X main.version=" + version + " -X main.commit=" + commit
+
+build:
+    CGO_ENABLED=0 go build -ldflags "{{ldflags}}" -o twister ./main.go
+
+run-api:
+    go run -ldflags "{{ldflags}}" ./main.go api
+
+run-indexer:
+    go run -ldflags "{{ldflags}}" ./main.go indexer
+
+test:
+    go test ./...
+
+clean:
+    rm -f twister

packages/api/main.go

···
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/spf13/cobra"
+	"tangled.org/desertthunder.dev/twister/internal/config"
+	"tangled.org/desertthunder.dev/twister/internal/observability"
+)
+
+var (
+	version = "dev"
+	commit  = "none"
+)
+
+func main() {
+	root := &cobra.Command{
+		Use:     "twister",
+		Short:   "Tangled search service",
+		Version: fmt.Sprintf("%s (%s)", version, commit),
+	}
+
+	root.AddCommand(
+		newAPICmd(),
+		newIndexerCmd(),
+		newEmbedWorkerCmd(),
+		newReindexCmd(),
+		newReembedCmd(),
+		newHealthcheckCmd(),
+	)
+
+	if err := root.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func baseContext() (context.Context, context.CancelFunc) {
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		ch := make(chan os.Signal, 1)
+		signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
+		<-ch
+		cancel()
+	}()
+	return ctx, cancel
+}
+
+func newAPICmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "api",
+		Short: "Start the HTTP search API",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load()
+			if err != nil {
+				return fmt.Errorf("config: %w", err)
+			}
+			log := observability.NewLogger(cfg)
+			log.Info("starting api", slog.String("service", "api"), slog.String("version", version), slog.String("addr", cfg.HTTPBindAddr))
+			ctx, cancel := baseContext()
+			defer cancel()
+			<-ctx.Done()
+			log.Info("shutting down api")
+			return nil
+		},
+	}
+}
+
+func newIndexerCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "indexer",
+		Short: "Start the Tap consumer and indexer",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load()
+			if err != nil {
+				return fmt.Errorf("config: %w", err)
+			}
+			log := observability.NewLogger(cfg)
+			log.Info("starting indexer", slog.String("service", "indexer"), slog.String("version", version))
+			ctx, cancel := baseContext()
+			defer cancel()
+			<-ctx.Done()
+			log.Info("shutting down indexer")
+			return nil
+		},
+	}
+}
+
+func newEmbedWorkerCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "embed-worker",
+		Short: "Start the async embedding worker",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load()
+			if err != nil {
+				return fmt.Errorf("config: %w", err)
+			}
+			log := observability.NewLogger(cfg)
+			log.Info("starting embed-worker", slog.String("service", "embed-worker"), slog.String("version", version))
+			ctx, cancel := baseContext()
+			defer cancel()
+			<-ctx.Done()
+			log.Info("shutting down embed-worker")
+			return nil
+		},
+	}
+}
+
+func newReindexCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "reindex",
+		Short: "Re-normalize and upsert all documents",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load()
+			if err != nil {
+				return fmt.Errorf("config: %w", err)
+			}
+			log := observability.NewLogger(cfg)
+			log.Info("reindex: not yet implemented")
+			return nil
+		},
+	}
+}
+
+func newReembedCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "reembed",
+		Short: "Re-generate all embeddings",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load()
+			if err != nil {
+				return fmt.Errorf("config: %w", err)
+			}
+			log := observability.NewLogger(cfg)
+			log.Info("reembed: not yet implemented")
+			return nil
+		},
+	}
+}
+
+func newHealthcheckCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "healthcheck",
+		Short: "One-shot health probe",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := config.Load()
+			if err != nil {
+				return fmt.Errorf("config: %w", err)
+			}
+			log := observability.NewLogger(cfg)
+			log.Info("healthcheck: ok")
+			return nil
+		},
+	}
+}

pnpm-lock.yaml

···
 
 importers:
 
-  .:
+  .: {}
+
+  apps/twisted:
     dependencies:
       '@atcute/bluesky':
         specifier: ^3.3.0

pnpm-workspace.yaml

···
+packages:
+  - "apps/*"
+  - "packages/*"
+
 onlyBuiltDependencies:
   - core-js
   - cypress