medsky radical trim of bigsky (relay) sync 1.1 induction firehose add getRepo 302 redirect note subscribeRepos message deprecation time-seq rename everything from 'bigsky' to 'relay' don't bounce auth key off db handleFedEvent clean-er deprecated {handle,migrate,tombstone} BGS.createExternalUser() is now syncPDSAccount() metric events_warn_counter{pds,warn} a bunch of rename 'user' to 'account' tiny ram blockstore even more minimal than ipld/ipfs do deprecate migrate/handle/tombstone (but don't delete yet so we have the option of measuring stale data received)

.github/workflows/container-relay-aws.yaml

···
+name: container-relay-aws
+on: [push]
+env:
+  REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
+  USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}
+  PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }}
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: relay
+
+jobs:
+  container-relay-aws:
+    if: github.repository == 'bluesky-social/indigo'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME }}
+          password: ${{ env.PASSWORD }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,enable=true,priority=100,prefix=,suffix=,format=long
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./cmd/relay/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

atproto/identity/cache_directory.go

···
 	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
+// CacheDirectory is an implementation of identity.Directory with local cache of Handle and DID
 type CacheDirectory struct {
 	Inner             Directory
 	ErrTTL            time.Duration

atproto/repo/car.go

···
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 
 	"github.com/bluesky-social/indigo/atproto/repo/mst"
 	"github.com/bluesky-social/indigo/atproto/syntax"
 
-	"github.com/ipfs/go-datastore"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipld/go-car"
 )
 
 func LoadFromCAR(ctx context.Context, r io.Reader) (*Commit, *Repo, error) {
 
-	bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
+	//bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
+	bs := NewTinyBlockstore()
 
 	cr, err := car.NewCarReader(r)
 	if err != nil {
···
 	}
 	return &commit, &repo, nil
 }
+
+var ErrNoRoot = errors.New("CAR file missing root CID")
+var ErrNoCommit = errors.New("no commit")
+
+// LoadCARCommit is like LoadFromCAR() but filters to only return the commit object.
+// useful for subscribeRepos/firehose `#sync` message
+func LoadCARCommit(ctx context.Context, r io.Reader) (*Commit, error) {
+	cr, err := car.NewCarReader(r)
+	if err != nil {
+		return nil, err
+	}
+	if cr.Header.Version != 1 {
+		return nil, fmt.Errorf("unsupported CAR file version: %d", cr.Header.Version)
+	}
+	if len(cr.Header.Roots) < 1 {
+		return nil, ErrNoRoot
+	}
+	commitCID := cr.Header.Roots[0]
+	var commitBlock blocks.Block
+	for {
+		blk, err := cr.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+
+		if blk.Cid().Equals(commitCID) {
+			commitBlock = blk
+			break
+		}
+	}
+	if commitBlock == nil {
+		return nil, ErrNoCommit
+	}
+	var commit Commit
+	if err := commit.UnmarshalCBOR(bytes.NewReader(commitBlock.RawData())); err != nil {
+		return nil, fmt.Errorf("parsing commit block from CAR file: %w", err)
+	}
+	if err := commit.VerifyStructure(); err != nil {
+		return nil, fmt.Errorf("parsing commit block from CAR file: %w", err)
+	}
+	return &commit, nil
+}

atproto/repo/mst/encoding.go

···
 	return c, nil
 }
 
-func loadNodeFromStore(ctx context.Context, bs blockstore.Blockstore, ref cid.Cid) (*Node, error) {
+func loadNodeFromStore(ctx context.Context, bs MSTBlockSource, ref cid.Cid) (*Node, error) {
 	block, err := bs.Get(ctx, ref)
 	if err != nil {
 		return nil, err

atproto/repo/mst/tree.go

···
 	"errors"
 	"fmt"
 
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 )
···
 	}
 }
 
-func LoadTreeFromStore(ctx context.Context, bs blockstore.Blockstore, root cid.Cid) (*Tree, error) {
+func LoadTreeFromStore(ctx context.Context, bs MSTBlockSource, root cid.Cid) (*Tree, error) {
 	n, err := loadNodeFromStore(ctx, bs, root)
 	if err != nil {
 		return nil, err
···
 	return &Tree{
 		Root: n,
 	}, nil
+}
+
+// subset of Blockstore that we actually need
+type MSTBlockSource interface {
+	Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 }
 
 // Walks the tree, encodes any "dirty" nodes as CBOR data, and writes that data as blocks to the provided blockstore. Returns root CID.

atproto/repo/repo.go

···
 	"github.com/bluesky-social/indigo/atproto/repo/mst"
 	"github.com/bluesky-social/indigo/atproto/syntax"
 
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
 )
 
 // Version of the repo data format implemented in this package
···
 	DID   syntax.DID
 	Clock *syntax.TIDClock
 
-	RecordStore blockstore.Blockstore
+	RecordStore RepoBlockSource // formerly blockstore.Blockstore
 	MST         mst.Tree
 }
 
+// subset of Blockstore that we actually need
+type RepoBlockSource interface {
+	Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
+}
+
 var ErrNotFound = errors.New("record not found in repository")
 
-func NewEmptyRepo(did syntax.DID) Repo {
-	clk := syntax.NewTIDClock(0)
-	return Repo{
-		DID:         did,
-		Clock:       &clk,
-		RecordStore: blockstore.NewBlockstore(datastore.NewMapDatastore()),
-		MST:         mst.NewEmptyTree(),
-	}
-}
+//func NewEmptyRepo(did syntax.DID) Repo {
+//	clk := syntax.NewTIDClock(0)
+//	return Repo{
+//		DID:         did,
+//		Clock:       &clk,
+//		RecordStore: blockstore.NewBlockstore(datastore.NewMapDatastore()),
+//		MST:         mst.NewEmptyTree(),
+//	}
+//}
 
 func (repo *Repo) GetRecordCID(ctx context.Context, collection syntax.NSID, rkey syntax.RecordKey) (*cid.Cid, error) {
 	path := collection.String() + "/" + rkey.String()

atproto/repo/tiny_blockstore.go

···
+package repo
+
+import (
+	"context"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+)
+
+type TinyBlockstore struct {
+	blocks map[string]blocks.Block
+}
+
+func NewTinyBlockstore() *TinyBlockstore {
+	return &TinyBlockstore{blocks: make(map[string]blocks.Block, 20)}
+}
+
+func (tb *TinyBlockstore) Put(_ context.Context, block blocks.Block) error {
+	ncid := block.Cid()
+	key := ncid.KeyString()
+	tb.blocks[key] = block
+	return nil
+}
+
+func (tb *TinyBlockstore) Get(_ context.Context, ncid cid.Cid) (blocks.Block, error) {
+	key := ncid.KeyString()
+	block, found := tb.blocks[key]
+	if found {
+		return block, nil
+	}
+	return nil, &ipld.ErrNotFound{Cid: ncid}
+}

cmd/bigsky/main.go

···
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
 
-	_, err := cliutil.SetupSlog(cliutil.LogOptions{})
+	_, _, err := cliutil.SetupSlog(cliutil.LogOptions{})
 	if err != nil {
 		return err
 	}

cmd/gosky/main.go

···
 		},
 	}
 
-	_, err := cliutil.SetupSlog(cliutil.LogOptions{})
+	_, _, err := cliutil.SetupSlog(cliutil.LogOptions{})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "logging setup error: %s\n", err.Error())
 		os.Exit(1)

cmd/relay/Dockerfile

···
+# Run this dockerfile from the top level of the indigo git repository like:
+#
+#   podman build -f ./cmd/relay/Dockerfile -t relay .
+
+### Compile stage
+FROM golang:1.23-alpine3.20 AS build-env
+RUN apk add --no-cache build-base make git
+
+ADD . /dockerbuild
+WORKDIR /dockerbuild
+
+# timezone data for alpine builds
+ENV GOEXPERIMENT=loopvar
+RUN GIT_VERSION=$(git describe --tags --long --always) && \
+    go build -tags timetzdata -o /relay ./cmd/relay
+
+### Build Frontend stage
+FROM node:18-alpine as web-builder
+
+WORKDIR /app
+
+COPY ts/bgs-dash /app/
+
+RUN yarn install --frozen-lockfile
+
+RUN yarn build
+
+### Run stage
+FROM alpine:3.20
+
+RUN apk add --no-cache --update dumb-init ca-certificates runit
+ENTRYPOINT ["dumb-init", "--"]
+
+WORKDIR /
+RUN mkdir -p data/relay
+COPY --from=build-env /relay /
+COPY --from=web-builder /app/dist/ public/
+
+# small things to make golang binaries work well under alpine
+ENV GODEBUG=netdns=go
+ENV TZ=Etc/UTC
+
+EXPOSE 2470
+
+CMD ["/relay"]
+
+LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo
+LABEL org.opencontainers.image.description="atproto Relay"
+LABEL org.opencontainers.image.licenses=MIT

cmd/relay/README.md

···
+
+atproto Relay Service
+===============================
+
+*NOTE: "Relays" used to be called "Big Graph Servers", or "BGS", or "bigsky". Many variables and packages still reference "bgs"*
+
+This is the implementation of an atproto Relay which is running in the production network, written and operated by Bluesky.
+
+In atproto, a Relay subscribes to multiple PDS hosts and outputs a combined "firehose" event stream. Downstream services can subscribe to this single firehose a get all relevant events for the entire network, or a specific sub-graph of the network. The Relay maintains a mirror of repo data from all accounts on the upstream PDS instances, and verifies repo data structure integrity and identity signatures. It is agnostic to applications, and does not validate data against atproto Lexicon schemas.
+
+This Relay implementation is designed to subscribe to the entire global network. The current state of the codebase is informally expected to scale to around 50 million accounts in the network, and thousands of repo events per second (peak).
+
+Features and design decisions:
+
+- runs on a single server
+- crawling and account state: stored in SQL database
+- SQL driver: gorm, with PostgreSQL in production and sqlite for testing
+- highly concurrent: not particularly CPU intensive
+- single golang binary for easy deployment
+- observability: logging, prometheus metrics, OTEL traces
+- admin web interface: configure limits, add upstream PDS instances, etc
+
+This software is not as packaged, documented, and supported for self-hosting as our PDS distribution or Ozone service. But it is relatively simple and inexpensive to get running.
+
+A note and reminder about Relays in general are that they are more of a convenience in the protocol than a hard requirement. The "firehose" API is the exact same on the PDS and on a Relay. Any service which subscribes to the Relay could instead connect to one or more PDS instances directly.
+
+
+## Development Tips
+
+The README and Makefile at the top level of this git repo have some generic helpers for testing, linting, formatting code, etc.
+
+To re-build and run the Relay locally:
+
+    make run-dev-relay
+
+You can re-build and run the command directly to get a list of configuration flags and env vars; env vars will be loaded from `.env` if that file exists:
+
+    RELAY_ADMIN_KEY=localdev go run ./cmd/relay/ --help
+
+By default, the daemon will use sqlite for databases (in the directory `./data/bigsky/`), CAR data will be stored as individual shard files in `./data/bigsky/carstore/`), and the HTTP API will be bound to localhost port 2470.
+
+When the daemon isn't running, sqlite database files can be inspected with:
+
+    sqlite3 data/bigsky/bgs.sqlite
+    [...]
+    sqlite> .schema
+
+Wipe all local data:
+
+    # careful! double-check this destructive command
+    rm -rf ./data/bigsky/*
+
+There is a basic web dashboard, though it will not be included unless built and copied to a local directory `./public/`. Run `make build-relay-ui`, and then when running the daemon the dashboard will be available at: <http://localhost:2470/dash/>. Paste in the admin key, eg `localdev`.
+
+The local admin routes can also be accessed by passing the admin key as a bearer token, for example:
+
+    http get :2470/admin/pds/list Authorization:"Bearer localdev"
+
+Request crawl of an individual PDS instance like:
+
+    http post :2470/admin/pds/requestCrawl Authorization:"Bearer localdev" hostname=pds.example.com
+
+
+## Docker Containers
+
+One way to deploy is running a docker image. You can pull and/or run a specific version of bigsky, referenced by git commit, from the Bluesky Github container registry. For example:
+
+    docker pull ghcr.io/bluesky-social/indigo:relay-fd66f93ce1412a3678a1dd3e6d53320b725978a6
+    docker run ghcr.io/bluesky-social/indigo:relay-fd66f93ce1412a3678a1dd3e6d53320b725978a6
+
+There is a Dockerfile in this directory, which can be used to build customized/patched versions of the Relay as a container, republish them, run locally, deploy to servers, deploy to an orchestrated cluster, etc. See docs and guides for docker and cluster management systems for details.
+
+
+## Database Setup
+
+PostgreSQL and Sqlite are both supported. When using Sqlite, separate files are used for Relay metadata and CarStore metadata. With PostgreSQL a single database server, user, and logical database can all be reused: table names will not conflict.
+
+Database configuration is passed via the `DATABASE_URL` and `CARSTORE_DATABASE_URL` environment variables, or the corresponding CLI args.
+
+For PostgreSQL, the user and database must already be configured. Some example SQL commands are:
+
+    CREATE DATABASE bgs;
+    CREATE DATABASE carstore;
+
+    CREATE USER ${username} WITH PASSWORD '${password}';
+    GRANT ALL PRIVILEGES ON DATABASE bgs TO ${username};
+    GRANT ALL PRIVILEGES ON DATABASE carstore TO ${username};
+
+This service currently uses `gorm` to automatically run database migrations as the regular user. There is no concept of running a separate set of migrations under more privileged database user.
+
+
+## Deployment
+
+*NOTE: this is not a complete guide to operating a Relay. There are decisions to be made and communicated about policies, bandwidth use, PDS crawling and rate-limits, financial sustainability, etc, which are not covered here. This is just a quick overview of how to technically get a relay up and running.*
+
+In a real-world system, you will probably want to use PostgreSQL.
+
+Some notable configuration env vars to set:
+
+- `ENVIRONMENT`: eg, `production`
+- `DATABASE_URL`: see section below
+- `DATA_DIR`: misc data will go in a subdirectory
+- `GOLOG_LOG_LEVEL`: log verbosity
+- `RESOLVE_ADDRESS`: DNS server to use
+- `FORCE_DNS_UDP`: recommend "true"
+
+There is a health check endpoint at `/xrpc/_health`. Prometheus metrics are exposed by default on port 2471, path `/metrics`. The service logs fairly verbosely to stderr; use `GOLOG_LOG_LEVEL` to control log volume.
+
+As a rough guideline for the compute resources needed to run a full-network Relay, in June 2024 an example Relay for over 5 million repositories used:
+
+- roughly 1 TByte of disk for PostgreSQL
+- roughly 1 TByte of disk for event playback buffer
+- roughly 5k disk I/O operations per second (all combined)
+- roughly 100% of one CPU core (quite low CPU utilization)
+- roughly 5GB of RAM for `relay`, and as much RAM as available for PostgreSQL and page cache
+- on the order of 1 megabit inbound bandwidth (crawling PDS instances) and 1 megabit outbound per connected client. 1 mbit continuous is approximately 350 GByte/month
+
+Be sure to double-check bandwidth usage and pricing if running a public relay! Bandwidth prices can vary widely between providers, and popular cloud services (AWS, Google Cloud, Azure) are very expensive compared to alternatives like OVH or Hetzner.
+
+
+## Bootstrapping the Network
+
+To bootstrap the entire network, you'll want to start with a list of large PDS instances to backfill from. You could pull from a public dashboard of instances (like [mackuba's](https://blue.mackuba.eu/directory/pdses)), or scrape the full DID PLC directory, parse out all PDS service declarations, and sort by count.
+
+Once you have a set of PDS hosts, you can put the bare hostnames (not URLs: no `https://` prefix, port, or path suffix) in a `hosts.txt` file, and then use the `crawl_pds.sh` script to backfill and configure limits for all of them:
+
+    export RELAY_HOST=your.pds.hostname.tld
+    export RELAY_ADMIN_KEY=your-secret-key
+
+    # both request crawl, and set generous crawl limits for each
+	cat hosts.txt | parallel -j1 ./crawl_pds.sh {}
+
+Just consuming from the firehose for a few hours will only backfill accounts with activity during that period. This is fine to get the backfill process started, but eventually you'll want to do full "resync" of all the repositories on the PDS host to the most recent repo rev version. To enqueue that for all the PDS instances:
+
+    # start sync/backfill of all accounts
+	cat hosts.txt | parallel -j1 ./sync_pds.sh {}
+
+Lastly, can monitor progress of any ongoing re-syncs:
+
+    # check sync progress for all hosts
+	cat hosts.txt | parallel -j1 ./sync_pds.sh {}
+
+
+## Admin API
+
+The relay has a number of admin HTTP API endpoints. Given a relay setup listening on port 2470 and with a reasonably secure admin secret:
+
+```
+RELAY_ADMIN_PASSWORD=$(openssl rand --hex 16)
+relay  --api-listen :2470 --admin-key ${RELAY_ADMIN_PASSWORD} ...
+```
+
+One can, for example, begin compaction of all repos
+
+```
+curl -H 'Authorization: Bearer '${RELAY_ADMIN_PASSWORD} -H 'Content-Type: application/x-www-form-urlencoded' --data '' http://127.0.0.1:2470/admin/repo/compactAll
+```
+
+### /admin/subs/getUpstreamConns
+
+Return list of PDS host names in json array of strings: ["host", ...]
+
+### /admin/subs/perDayLimit
+
+Return `{"limit": int}` for the number of new PDS subscriptions that the relay may start in a rolling 24 hour window.
+
+### /admin/subs/setPerDayLimit
+
+POST with `?limit={int}` to set the number of new PDS subscriptions that the relay may start in a rolling 24 hour window.
+
+### /admin/subs/setEnabled
+
+POST with param `?enabled=true` or `?enabled=false` to enable or disable PDS-requested new-PDS crawling.
+
+### /admin/subs/getEnabled
+
+Return `{"enabled": bool}` if non-admin new PDS crawl requests are enabled
+
+### /admin/subs/killUpstream
+
+POST with `?host={pds host name}` to disconnect from their firehose.
+
+Optionally add `&block=true` to prevent connecting to them in the future.
+
+### /admin/subs/listDomainBans
+
+Return `{"banned_domains": ["host name", ...]}`
+
+### /admin/subs/banDomain
+
+POST `{"Domain": "host name"}` to ban a domain
+
+### /admin/subs/unbanDomain
+
+POST `{"Domain": "host name"}` to un-ban a domain
+
+### /admin/repo/takeDown
+
+POST `{"did": "did:..."}` to take-down a bad repo; deletes all local data for the repo
+
+### /admin/repo/reverseTakedown
+
+POST `?did={did:...}` to reverse a repo take-down
+
+### /admin/repo/compact
+
+POST `?did={did:...}` to compact a repo. Optionally `&fast=true`. HTTP blocks until the compaction finishes.
+
+### /admin/repo/compactAll
+
+POST to begin compaction of all repos. Optional query params:
+
+ * `fast=true`
+ * `limit={int}` maximum number of repos to compact (biggest first) (default 50)
+ * `threhsold={int}` minimum number of shard files a repo must have on disk to merit compaction (default 20)
+
+### /admin/repo/reset
+
+POST `?did={did:...}` deletes all local data for the repo
+
+### /admin/repo/verify
+
+POST  `?did={did:...}` checks that all repo data is accessible. HTTP blocks until done.
+
+### /admin/pds/requestCrawl
+
+POST `{"hostname":"pds host"}` to start crawling a PDS
+
+### /admin/pds/list
+
+GET returns JSON list of records
+```json
+[{
+  "Host": string,
+  "Did": string,
+  "SSL": bool,
+  "Cursor": int,
+  "Registered": bool,
+  "Blocked": bool,
+  "RateLimit": float,
+  "CrawlRateLimit": float,
+  "RepoCount": int,
+  "RepoLimit": int,
+  "HourlyEventLimit": int,
+  "DailyEventLimit": int,
+
+  "HasActiveConnection": bool,
+  "EventsSeenSinceStartup": int,
+  "PerSecondEventRate": {"Max": float, "Window": float seconds},
+  "PerHourEventRate": {"Max": float, "Window": float seconds},
+  "PerDayEventRate": {"Max": float, "Window": float seconds},
+  "CrawlRate": {"Max": float, "Window": float seconds},
+  "UserCount": int,
+}, ...]
+```
+
+### /admin/pds/resync
+
+POST `?host={host}` to start a resync of a PDS
+
+GET `?host={host}` to get status of a PDS resync, return
+
+```json
+{"resync": {
+  "pds": {
+    "Host": string,
+    "Did": string,
+    "SSL": bool,
+    "Cursor": int,
+    "Registered": bool,
+    "Blocked": bool,
+    "RateLimit": float,
+    "CrawlRateLimit": float,
+    "RepoCount": int,
+    "RepoLimit": int,
+    "HourlyEventLimit": int,
+    "DailyEventLimit": int,
+  },
+  "numRepoPages": int,
+  "numRepos": int,
+  "numReposChecked": int,
+  "numReposToResync": int,
+  "status": string,
+  "statusChangedAt": time,
+}}
+```
+
+### /admin/pds/changeLimits
+
+POST to set the limits for a PDS. body:
+
+```json
+{
+  "host": string,
+  "per_second": int,
+  "per_hour": int,
+  "per_day": int,
+  "crawl_rate": int,
+  "repo_limit": int,
+}
+```
+
+### /admin/pds/block
+
+POST `?host={host}` to block a PDS
+
+### /admin/pds/unblock
+
+POST `?host={host}` to un-block a PDS
+
+
+### /admin/pds/addTrustedDomain
+
+POST `?domain={}` to make a domain trusted
+
+### /admin/consumers/list
+
+GET returns list json of clients currently reading from the relay firehose
+
+```json
+[{
+  "id": int,
+  "remote_addr": string,
+  "user_agent": string,
+  "events_consumed": int,
+  "connected_at": time,
+}, ...]
+```

cmd/relay/bgs/admin.go

···
+package bgs
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/bluesky-social/indigo/cmd/relay/models"
+	"github.com/labstack/echo/v4"
+	dto "github.com/prometheus/client_model/go"
+	"gorm.io/gorm"
+)
+
+func (bgs *BGS) handleAdminSetSubsEnabled(e echo.Context) error {
+	enabled, err := strconv.ParseBool(e.QueryParam("enabled"))
+	if err != nil {
+		return &echo.HTTPError{
+			Code:    400,
+			Message: err.Error(),
+		}
+	}
+
+	return bgs.slurper.SetNewSubsDisabled(!enabled)
+}
+
+func (bgs *BGS) handleAdminGetSubsEnabled(e echo.Context) error {
+	return e.JSON(200, map[string]bool{
+		"enabled": !bgs.slurper.GetNewSubsDisabledState(),
+	})
+}
+
+func (bgs *BGS) handleAdminGetNewPDSPerDayRateLimit(e echo.Context) error {
+	limit := bgs.slurper.GetNewPDSPerDayLimit()
+	return e.JSON(200, map[string]int64{
+		"limit": limit,
+	})
+}
+
+func (bgs *BGS) handleAdminSetNewPDSPerDayRateLimit(e echo.Context) error {
+	limit, err := strconv.ParseInt(e.QueryParam("limit"), 10, 64)
+	if err != nil {
+		return &echo.HTTPError{
+			Code:    400,
+			Message: fmt.Errorf("failed to parse limit: %w", err).Error(),
+		}
+	}
+
+	err = bgs.slurper.SetNewPDSPerDayLimit(limit)
+	if err != nil {
+		return &echo.HTTPError{
+			Code:    500,
+			Message: fmt.Errorf("failed to set new PDS per day rate limit: %w", err).Error(),
+		}
+	}
+
+	return nil
+}
+
+func (bgs *BGS) handleAdminTakeDownRepo(e echo.Context) error {
+	ctx := e.Request().Context()
+
+	var body map[string]string
+	if err := e.Bind(&body); err != nil {
+		return err
+	}
+	did, ok := body["did"]
+	if !ok {
+		return &echo.HTTPError{
+			Code:    400,
+			Message: "must specify did parameter in body",
+		}
+	}
+
+	err := bgs.TakeDownRepo(ctx, did)
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return &echo.HTTPError{
+				Code:    http.StatusNotFound,
+				Message: "repo not found",
+			}
+		}
+		return &echo.HTTPError{
+			Code:    http.StatusInternalServerError,
+			Message: err.Error(),
+		}
+	}
+	return nil
+}
+
+func (bgs *BGS) handleAdminReverseTakedown(e echo.Context) error {
+	did := e.QueryParam("did")
+	ctx := e.Request().Context()
+	err := bgs.ReverseTakedown(ctx, did)
+
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return &echo.HTTPError{
+				Code:    http.StatusNotFound,
+				Message: "repo not found",
+			}
+		}
+		return &echo.HTTPError{
+			Code:    http.StatusInternalServerError,
+			Message: err.Error(),
+		}
+	}
+
+	return nil
+}
+
+type ListTakedownsResponse struct {
+	Dids   []string `json:"dids"`
+	Cursor int64    `json:"cursor,omitempty"`
+}
+
+func (bgs *BGS) handleAdminListRepoTakeDowns(e echo.Context) error {
+	ctx := e.Request().Context()
+	haveMinId := false
+	minId := int64(-1)
+	qmin := e.QueryParam("cursor")
+	if qmin != "" {
+		tmin, err := strconv.ParseInt(qmin, 10, 64)
+		if err != nil {
+			return &echo.HTTPError{Code: 400, Message: "bad cursor"}
+		}
+		minId = tmin
+		haveMinId = true
+	}
+	limit := 1000
+	wat := bgs.db.Model(Account{}).WithContext(ctx).Select("id", "did").Where("taken_down = TRUE")
+	if haveMinId {
+		wat = wat.Where("id > ?", minId)
+	}
+	//var users []Account
+	rows, err := wat.Order("id").Limit(limit).Rows()
+	if err != nil {
+		return echo.NewHTTPError(http.StatusInternalServerError, "oops").WithInternal(err)
+	}
+	var out ListTakedownsResponse
+	for rows.Next() {
+		var id int64
+		var did string
+		err := rows.Scan(&id, &did)
+		if err != nil {
+			return echo.NewHTTPError(http.StatusInternalServerError, "oops").WithInternal(err)
+		}
+		out.Dids = append(out.Dids, did)
+		out.Cursor = id
+	}
+	if len(out.Dids) < limit {
+		out.Cursor = 0
+	}
+	return e.JSON(200, out)
+}
+
+func (bgs *BGS) handleAdminGetUpstreamConns(e echo.Context) error {
+	return e.JSON(200, bgs.slurper.GetActiveList())
+}
+
+type rateLimit struct {
+	Max           float64 `json:"Max"`
+	WindowSeconds float64 `json:"Window"`
+}
+
+type enrichedPDS struct {
+	models.PDS
+	HasActiveConnection    bool      `json:"HasActiveConnection"`
+	EventsSeenSinceStartup uint64    `json:"EventsSeenSinceStartup"`
+	PerSecondEventRate     rateLimit `json:"PerSecondEventRate"`
+	PerHourEventRate       rateLimit `json:"PerHourEventRate"`
+	PerDayEventRate        rateLimit `json:"PerDayEventRate"`
+	UserCount              int64     `json:"UserCount"`
+}
+
+type UserCount struct {
+	PDSID     uint  `gorm:"column:pds"`
+	UserCount int64 `gorm:"column:user_count"`
+}
+
+func (bgs *BGS) handleListPDSs(e echo.Context) error {
+	var pds []models.PDS
+	if err := bgs.db.Find(&pds).Error; err != nil {
+		return err
+	}
+
+	enrichedPDSs := make([]enrichedPDS, len(pds))
+
+	activePDSHosts := bgs.slurper.GetActiveList()
+
+	for i, p := range pds {
+		enrichedPDSs[i].PDS = p
+		enrichedPDSs[i].HasActiveConnection = false
+		for _, host := range activePDSHosts {
+			if strings.ToLower(host) == strings.ToLower(p.Host) {
+				enrichedPDSs[i].HasActiveConnection = true
+				break
+			}
+		}
+		var m = &dto.Metric{}
+		if err := eventsReceivedCounter.WithLabelValues(p.Host).Write(m); err != nil {
+			enrichedPDSs[i].EventsSeenSinceStartup = 0
+			continue
+		}
+		enrichedPDSs[i].EventsSeenSinceStartup = uint64(m.Counter.GetValue())
+
+		enrichedPDSs[i].PerSecondEventRate = rateLimit{
+			Max:           p.RateLimit,
+			WindowSeconds: 1,
+		}
+
+		enrichedPDSs[i].PerHourEventRate = rateLimit{
+			Max:           float64(p.HourlyEventLimit),
+			WindowSeconds: 3600,
+		}
+
+		enrichedPDSs[i].PerDayEventRate = rateLimit{
+			Max:           float64(p.DailyEventLimit),
+			WindowSeconds: 86400,
+		}
+	}
+
+	return e.JSON(200, enrichedPDSs)
+}
+
+type consumer struct {
+	ID             uint64    `json:"id"`
+	RemoteAddr     string    `json:"remote_addr"`
+	UserAgent      string    `json:"user_agent"`
+	EventsConsumed uint64    `json:"events_consumed"`
+	ConnectedAt    time.Time `json:"connected_at"`
+}
+
+func (bgs *BGS) handleAdminListConsumers(e echo.Context) error {
+	bgs.consumersLk.RLock()
+	defer bgs.consumersLk.RUnlock()
+
+	consumers := make([]consumer, 0, len(bgs.consumers))
+	for id, c := range bgs.consumers {
+		var m = &dto.Metric{}
+		if err := c.EventsSent.Write(m); err != nil {
+			continue
+		}
+		consumers = append(consumers, consumer{
+			ID:             id,
+			RemoteAddr:     c.RemoteAddr,
+			UserAgent:      c.UserAgent,
+			EventsConsumed: uint64(m.Counter.GetValue()),
+			ConnectedAt:    c.ConnectedAt,
+		})
+	}
+
+	return e.JSON(200, consumers)
+}
+
+func (bgs *BGS) handleAdminKillUpstreamConn(e echo.Context) error {
+	host := strings.TrimSpace(e.QueryParam("host"))
+	if host == "" {
+		return &echo.HTTPError{
+			Code:    400,
+			Message: "must pass a valid host",
+		}
+	}
+
+	block := strings.ToLower(e.QueryParam("block")) == "true"
+
+	if err := bgs.slurper.KillUpstreamConnection(host, block); err != nil {
+		if errors.Is(err, ErrNoActiveConnection) {
+			return &echo.HTTPError{
+				Code:    400,
+				Message: "no active connection to given host",
+			}
+		}
+		return err
+	}
+
+	return e.JSON(200, map[string]any{
+		"success": "true",
+	})
+}
+
+func (bgs *BGS) handleBlockPDS(e echo.Context) error {
+	host := strings.TrimSpace(e.QueryParam("host"))
+	if host == "" {
+		return &echo.HTTPError{
+			Code:    400,
+			Message: "must pass a valid host",
+		}
+	}
+
+	// Set the block flag to true in the DB
+	if err := bgs.db.Model(&models.PDS{}).Where("host = ?", host).Update("blocked", true).Error; err != nil {
+		return err
+	}
+
+	// don't care if this errors, but we should try to disconnect something we just blocked
+	_ = bgs.slurper.KillUpstreamConnection(host, false)
+
+	return e.JSON(200, map[string]any{
+		"success": "true",
+	})
+}
+
+func (bgs *BGS) handleUnblockPDS(e echo.Context) error {
+	host := strings.TrimSpace(e.QueryParam("host"))
+	if host == "" {
+		return &echo.HTTPError{
+			Code:    400,
+			Message: "must pass a valid host",
+		}
+	}
+
+	// Set the block flag to false in the DB
+	if err := bgs.db.Model(&models.PDS{}).Where("host = ?", host).Update("blocked", false).Error; err != nil {
+		return err
+	}
+
+	return e.JSON(200, map[string]any{
+		"success": "true",
+	})
+}
+
+type bannedDomains struct {
+	BannedDomains []string `json:"banned_domains"`
+}
+
+func (bgs *BGS) handleAdminListDomainBans(c echo.Context) error {
+	var all []DomainBan
+	if err := bgs.db.Find(&all).Error; err != nil {
+		return err
+	}
+
+	resp := bannedDomains{
+		BannedDomains: []string{},
+	}
+	for _, b := range all {
+		resp.BannedDomains = append(resp.BannedDomains, b.Domain)
+	}
+
+	return c.JSON(200, resp)
+}
+
+type banDomainBody struct {
+	Domain string
+}
+
+func (bgs *BGS) handleAdminBanDomain(c echo.Context) error {
+	var body banDomainBody
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+
+	// Check if the domain is already banned
+	var existing DomainBan
+	if err := bgs.db.Where("domain = ?", body.Domain).First(&existing).Error; err == nil {
+		return &echo.HTTPError{
+			Code:    400,
+			Message: "domain is already banned",
+		}
+	}
+
+	if err := bgs.db.Create(&DomainBan{
+		Domain: body.Domain,
+	}).Error; err != nil {
+		return err
+	}
+
+	return c.JSON(200, map[string]any{
+		"success": "true",
+	})
+}
+
+func (bgs *BGS) handleAdminUnbanDomain(c echo.Context) error {
+	var body banDomainBody
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+
+	if err := bgs.db.Where("domain = ?", body.Domain).Delete(&DomainBan{}).Error; err != nil {
+		return err
+	}
+
+	return c.JSON(200, map[string]any{
+		"success": "true",
+	})
+}
+
+type PDSRates struct {
+	// core event rate, counts firehose events
+	PerSecond int64 `json:"per_second,omitempty"`
+	PerHour   int64 `json:"per_hour,omitempty"`
+	PerDay    int64 `json:"per_day,omitempty"`
+
+	RepoLimit int64 `json:"repo_limit,omitempty"`
+}
+
+func (pr *PDSRates) FromSlurper(s *Slurper) {
+	if pr.PerSecond == 0 {
+		pr.PerHour = s.DefaultPerSecondLimit
+	}
+	if pr.PerHour == 0 {
+		pr.PerHour = s.DefaultPerHourLimit
+	}
+	if pr.PerDay == 0 {
+		pr.PerDay = s.DefaultPerDayLimit
+	}
+	if pr.RepoLimit == 0 {
+		pr.RepoLimit = s.DefaultRepoLimit
+	}
+}
+
+type RateLimitChangeRequest struct {
+	Host string `json:"host"`
+	PDSRates
+}
+
+func (bgs *BGS) handleAdminChangePDSRateLimits(e echo.Context) error {
+	var body RateLimitChangeRequest
+	if err := e.Bind(&body); err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid body: %s", err))
+	}
+
+	// Get the PDS from the DB
+	var pds models.PDS
+	if err := bgs.db.Where("host = ?", body.Host).First(&pds).Error; err != nil {
+		return err
+	}
+
+	// Update the rate limits in the DB
+	pds.RateLimit = float64(body.PerSecond)
+	pds.HourlyEventLimit = body.PerHour
+	pds.DailyEventLimit = body.PerDay
+	pds.RepoLimit = body.RepoLimit
+
+	if err := bgs.db.Save(&pds).Error; err != nil {
+		return echo.NewHTTPError(http.StatusInternalServerError, fmt.Errorf("failed to save rate limit changes: %w", err))
+	}
+
+	// Update the rate limit in the limiter
+	limits := bgs.slurper.GetOrCreateLimiters(pds.ID, body.PerSecond, body.PerHour, body.PerDay)
+	limits.PerSecond.SetLimit(body.PerSecond)
+	limits.PerHour.SetLimit(body.PerHour)
+	limits.PerDay.SetLimit(body.PerDay)
+
+	return e.JSON(200, map[string]any{
+		"success": "true",
+	})
+}
+
+func (bgs *BGS) handleAdminAddTrustedDomain(e echo.Context) error {
+	domain := e.QueryParam("domain")
+	if domain == "" {
+		return fmt.Errorf("must specify domain in query parameter")
+	}
+
+	// Check if the domain is already trusted
+	trustedDomains := bgs.slurper.GetTrustedDomains()
+	if slices.Contains(trustedDomains, domain) {
+		return &echo.HTTPError{
+			Code:    400,
+			Message: "domain is already trusted",
+		}
+	}
+
+	if err := bgs.slurper.AddTrustedDomain(domain); err != nil {
+		return err
+	}
+
+	return e.JSON(200, map[string]any{
+		"success": true,
+	})
+}
+
+type AdminRequestCrawlRequest struct {
+	Hostname string `json:"hostname"`
+
+	// optional:
+	PDSRates
+}
+
+func (bgs *BGS) handleAdminRequestCrawl(e echo.Context) error {
+	ctx := e.Request().Context()
+
+	var body AdminRequestCrawlRequest
+	if err := e.Bind(&body); err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid body: %s", err))
+	}
+
+	host := body.Hostname
+	if host == "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname")
+	}
+
+	if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
+		if bgs.ssl {
+			host = "https://" + host
+		} else {
+			host = "http://" + host
+		}
+	}
+
+	u, err := url.Parse(host)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, "failed to parse hostname")
+	}
+
+	if u.Scheme == "http" && bgs.ssl {
+		return echo.NewHTTPError(http.StatusBadRequest, "this server requires https")
+	}
+
+	if u.Scheme == "https" && !bgs.ssl {
+		return echo.NewHTTPError(http.StatusBadRequest, "this server does not support https")
+	}
+
+	if u.Path != "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without path")
+	}
+
+	if u.Query().Encode() != "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without query")
+	}
+
+	host = u.Host // potentially hostname:port
+
+	banned, err := bgs.domainIsBanned(ctx, host)
+	if banned {
+		return echo.NewHTTPError(http.StatusUnauthorized, "domain is banned")
+	}
+
+	// Skip checking if the server is online for now
+	rateOverrides := body.PDSRates
+	rateOverrides.FromSlurper(bgs.slurper)
+
+	return bgs.slurper.SubscribeToPds(ctx, host, true, true, &rateOverrides) // Override Trusted Domain Check
+}

cmd/relay/bgs/bgs.go

···
+package bgs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/bluesky-social/indigo/atproto/identity"
+	"github.com/bluesky-social/indigo/atproto/syntax"
+	"github.com/ipfs/go-cid"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	_ "net/http/pprof"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	comatproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/cmd/relay/events"
+	"github.com/bluesky-social/indigo/cmd/relay/models"
+	lexutil "github.com/bluesky-social/indigo/lex/util"
+	"github.com/bluesky-social/indigo/xrpc"
+
+	"github.com/gorilla/websocket"
+	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+	promclient "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	dto "github.com/prometheus/client_model/go"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"gorm.io/gorm"
+)
+
+var tracer = otel.Tracer("bgs")
+
+// serverListenerBootTimeout is how long to wait for the requested server socket
+// to become available for use. This is an arbitrary timeout that should be safe
+// on any platform, but there's no great way to weave this timeout without
+// adding another parameter to the (at time of writing) long signature of
+// NewServer.
+const serverListenerBootTimeout = 5 * time.Second
+
+type BGS struct {
+	db      *gorm.DB
+	slurper *Slurper
+	events  *events.EventManager
+	didd    identity.Directory
+
+	// TODO: work on doing away with this flag in favor of more pluggable
+	// pieces that abstract the need for explicit ssl checks
+	ssl bool
+
+	// extUserLk serializes a section of syncPDSAccount()
+	// TODO: at some point we will want to lock specific DIDs, this lock as is
+	// is overly broad, but i dont expect it to be a bottleneck for now
+	extUserLk sync.Mutex
+
+	validator *Validator
+
+	// Management of Socket Consumers
+	consumersLk    sync.RWMutex
+	nextConsumerID uint64
+	consumers      map[uint64]*SocketConsumer
+
+	// Account cache
+	userCache *lru.Cache[string, *Account]
+
+	// nextCrawlers gets forwarded POST /xrpc/com.atproto.sync.requestCrawl
+	nextCrawlers []*url.URL
+	httpClient   http.Client
+
+	log               *slog.Logger
+	inductionTraceLog *slog.Logger
+
+	config BGSConfig
+}
+
+type SocketConsumer struct {
+	UserAgent   string
+	RemoteAddr  string
+	ConnectedAt time.Time
+	EventsSent  promclient.Counter
+}
+
+type BGSConfig struct {
+	SSL               bool
+	DefaultRepoLimit  int64
+	ConcurrencyPerPDS int64
+	MaxQueuePerPDS    int64
+
+	// NextCrawlers gets forwarded POST /xrpc/com.atproto.sync.requestCrawl
+	NextCrawlers []*url.URL
+
+	ApplyPDSClientSettings func(c *xrpc.Client)
+	InductionTraceLog      *slog.Logger
+
+	// AdminToken checked against "Authorization: Bearer {}" header
+	AdminToken string
+}
+
+func DefaultBGSConfig() *BGSConfig {
+	return &BGSConfig{
+		SSL:               true,
+		DefaultRepoLimit:  100,
+		ConcurrencyPerPDS: 100,
+		MaxQueuePerPDS:    1_000,
+	}
+}
+
+func NewBGS(db *gorm.DB, validator *Validator, evtman *events.EventManager, didd identity.Directory, config *BGSConfig) (*BGS, error) {
+
+	if config == nil {
+		config = DefaultBGSConfig()
+	}
+	if err := db.AutoMigrate(DomainBan{}); err != nil {
+		panic(err)
+	}
+	if err := db.AutoMigrate(models.PDS{}); err != nil {
+		panic(err)
+	}
+	if err := db.AutoMigrate(Account{}); err != nil {
+		panic(err)
+	}
+	if err := db.AutoMigrate(AccountPreviousState{}); err != nil {
+		panic(err)
+	}
+
+	uc, _ := lru.New[string, *Account](1_000_000)
+
+	bgs := &BGS{
+		db: db,
+
+		validator: validator,
+		events:    evtman,
+		didd:      didd,
+		ssl:       config.SSL,
+
+		consumersLk: sync.RWMutex{},
+		consumers:   make(map[uint64]*SocketConsumer),
+
+		userCache: uc,
+
+		log: slog.Default().With("system", "bgs"),
+
+		config: *config,
+
+		inductionTraceLog: config.InductionTraceLog,
+	}
+
+	slOpts := DefaultSlurperOptions()
+	slOpts.SSL = config.SSL
+	slOpts.DefaultRepoLimit = config.DefaultRepoLimit
+	slOpts.ConcurrencyPerPDS = config.ConcurrencyPerPDS
+	slOpts.MaxQueuePerPDS = config.MaxQueuePerPDS
+	slOpts.Logger = bgs.log
+	s, err := NewSlurper(db, bgs.handleFedEvent, slOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	bgs.slurper = s
+
+	if err := bgs.slurper.RestartAll(); err != nil {
+		return nil, err
+	}
+
+	bgs.nextCrawlers = config.NextCrawlers
+	bgs.httpClient.Timeout = time.Second * 5
+
+	return bgs, nil
+}
+
+func (bgs *BGS) StartMetrics(listen string) error {
+	http.Handle("/metrics", promhttp.Handler())
+	return http.ListenAndServe(listen, nil)
+}
+
+func (bgs *BGS) Start(addr string, logWriter io.Writer) error {
+	var lc net.ListenConfig
+	ctx, cancel := context.WithTimeout(context.Background(), serverListenerBootTimeout)
+	defer cancel()
+
+	li, err := lc.Listen(ctx, "tcp", addr)
+	if err != nil {
+		return err
+	}
+	return bgs.StartWithListener(li, logWriter)
+}
+
+func (bgs *BGS) StartWithListener(listen net.Listener, logWriter io.Writer) error {
+	e := echo.New()
+	e.Logger.SetOutput(logWriter)
+	e.HideBanner = true
+
+	e.Use(middleware.CORSWithConfig(middleware.CORSConfig{
+		AllowOrigins: []string{"*"},
+		AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderContentType, echo.HeaderAccept, echo.HeaderAuthorization},
+	}))
+
+	if !bgs.ssl {
+		e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
+			Format: "method=${method}, uri=${uri}, status=${status} latency=${latency_human}\n",
+		}))
+	} else {
+		e.Use(middleware.LoggerWithConfig(middleware.DefaultLoggerConfig))
+	}
+
+	// React uses a virtual router, so we need to serve the index.html for all
+	// routes that aren't otherwise handled or in the /assets directory.
+	e.File("/dash", "public/index.html")
+	e.File("/dash/*", "public/index.html")
+	e.Static("/assets", "public/assets")
+
+	e.Use(MetricsMiddleware)
+
+	e.HTTPErrorHandler = func(err error, ctx echo.Context) {
+		switch err := err.(type) {
+		case *echo.HTTPError:
+			if err2 := ctx.JSON(err.Code, map[string]any{
+				"error": err.Message,
+			}); err2 != nil {
+				bgs.log.Error("Failed to write http error", "err", err2)
+			}
+		default:
+			sendHeader := true
+			if ctx.Path() == "/xrpc/com.atproto.sync.subscribeRepos" {
+				sendHeader = false
+			}
+
+			bgs.log.Warn("HANDLER ERROR: (%s) %s", ctx.Path(), err)
+
+			if strings.HasPrefix(ctx.Path(), "/admin/") {
+				ctx.JSON(500, map[string]any{
+					"error": err.Error(),
+				})
+				return
+			}
+
+			if sendHeader {
+				ctx.Response().WriteHeader(500)
+			}
+		}
+	}
+
+	// TODO: this API is temporary until we formalize what we want here
+
+	e.GET("/xrpc/com.atproto.sync.subscribeRepos", bgs.EventsHandler)
+	e.POST("/xrpc/com.atproto.sync.requestCrawl", bgs.HandleComAtprotoSyncRequestCrawl)
+	e.GET("/xrpc/com.atproto.sync.listRepos", bgs.HandleComAtprotoSyncListRepos)
+	e.GET("/xrpc/com.atproto.sync.getRepo", bgs.HandleComAtprotoSyncGetRepo) // just returns 3xx redirect to source PDS
+	e.GET("/xrpc/com.atproto.sync.getLatestCommit", bgs.HandleComAtprotoSyncGetLatestCommit)
+	e.GET("/xrpc/_health", bgs.HandleHealthCheck)
+	e.GET("/_health", bgs.HandleHealthCheck)
+	e.GET("/", bgs.HandleHomeMessage)
+
+	admin := e.Group("/admin", bgs.checkAdminAuth)
+
+	// Slurper-related Admin API
+	admin.GET("/subs/getUpstreamConns", bgs.handleAdminGetUpstreamConns)
+	admin.GET("/subs/getEnabled", bgs.handleAdminGetSubsEnabled)
+	admin.GET("/subs/perDayLimit", bgs.handleAdminGetNewPDSPerDayRateLimit)
+	admin.POST("/subs/setEnabled", bgs.handleAdminSetSubsEnabled)
+	admin.POST("/subs/killUpstream", bgs.handleAdminKillUpstreamConn)
+	admin.POST("/subs/setPerDayLimit", bgs.handleAdminSetNewPDSPerDayRateLimit)
+
+	// Domain-related Admin API
+	admin.GET("/subs/listDomainBans", bgs.handleAdminListDomainBans)
+	admin.POST("/subs/banDomain", bgs.handleAdminBanDomain)
+	admin.POST("/subs/unbanDomain", bgs.handleAdminUnbanDomain)
+
+	// Repo-related Admin API
+	admin.POST("/repo/takeDown", bgs.handleAdminTakeDownRepo)
+	admin.POST("/repo/reverseTakedown", bgs.handleAdminReverseTakedown)
+	admin.GET("/repo/takedowns", bgs.handleAdminListRepoTakeDowns)
+
+	// PDS-related Admin API
+	admin.POST("/pds/requestCrawl", bgs.handleAdminRequestCrawl)
+	admin.GET("/pds/list", bgs.handleListPDSs)
+	admin.POST("/pds/changeLimits", bgs.handleAdminChangePDSRateLimits)
+	admin.POST("/pds/block", bgs.handleBlockPDS)
+	admin.POST("/pds/unblock", bgs.handleUnblockPDS)
+	admin.POST("/pds/addTrustedDomain", bgs.handleAdminAddTrustedDomain)
+
+	// Consumer-related Admin API
+	admin.GET("/consumers/list", bgs.handleAdminListConsumers)
+
+	// In order to support booting on random ports in tests, we need to tell the
+	// Echo instance it's already got a port, and then use its StartServer
+	// method to re-use that listener.
+	e.Listener = listen
+	srv := &http.Server{}
+	return e.StartServer(srv)
+}
+
+func (bgs *BGS) Shutdown() []error {
+	errs := bgs.slurper.Shutdown()
+
+	if err := bgs.events.Shutdown(context.TODO()); err != nil {
+		errs = append(errs, err)
+	}
+
+	return errs
+}
+
+type HealthStatus struct {
+	Status  string `json:"status"`
+	Message string `json:"msg,omitempty"`
+}
+
+func (bgs *BGS) HandleHealthCheck(c echo.Context) error {
+	if err := bgs.db.Exec("SELECT 1").Error; err != nil {
+		bgs.log.Error("healthcheck can't connect to database", "err", err)
+		return c.JSON(500, HealthStatus{Status: "error", Message: "can't connect to database"})
+	} else {
+		return c.JSON(200, HealthStatus{Status: "ok"})
+	}
+}
+
+var homeMessage string = `
+.########..########.##..........###....##....##
+.##.....##.##.......##.........##.##....##..##.
+.##.....##.##.......##........##...##....####..
+.########..######...##.......##.....##....##...
+.##...##...##.......##.......#########....##...
+.##....##..##.......##.......##.....##....##...
+.##.....##.########.########.##.....##....##...
+
+This is an atproto [https://atproto.com] relay instance, running the 'bigsky' codebase [https://github.com/bluesky-social/indigo]
+
+The firehose WebSocket path is at:  /xrpc/com.atproto.sync.subscribeRepos
+`
+
+func (bgs *BGS) HandleHomeMessage(c echo.Context) error {
+	return c.String(http.StatusOK, homeMessage)
+}
+
+const authorizationBearerPrefix = "Bearer "
+
+func (bgs *BGS) checkAdminAuth(next echo.HandlerFunc) echo.HandlerFunc {
+	return func(e echo.Context) error {
+		authheader := e.Request().Header.Get("Authorization")
+		if !strings.HasPrefix(authheader, authorizationBearerPrefix) {
+			return echo.ErrForbidden
+		}
+
+		token := authheader[len(authorizationBearerPrefix):]
+
+		if bgs.config.AdminToken != token {
+			return echo.ErrForbidden
+		}
+
+		return next(e)
+	}
+}
+
+type Account struct {
+	ID        models.Uid `gorm:"primarykey"`
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt gorm.DeletedAt `gorm:"index"`
+	Did       string         `gorm:"uniqueIndex"`
+	PDS       uint           // foreign key on models.PDS.ID
+
+	// TakenDown is set to true if the user in question has been taken down by an admin action at this relay.
+	// A user in this state will have all future events related to it dropped
+	// and no data about this user will be served.
+	TakenDown bool
+
+	// UpstreamStatus is the state of the user as reported by the upstream PDS through #account messages.
+	// Additionally, the non-standard string "active" is set to represent an upstream #account message with the active bool true.
+	UpstreamStatus string `gorm:"index"`
+
+	lk sync.Mutex
+}
+
+func (account *Account) GetDid() string {
+	return account.Did
+}
+
+func (account *Account) GetUid() models.Uid {
+	return account.ID
+}
+
+func (account *Account) SetTakenDown(v bool) {
+	account.lk.Lock()
+	defer account.lk.Unlock()
+	account.TakenDown = v
+}
+
+func (account *Account) GetTakenDown() bool {
+	account.lk.Lock()
+	defer account.lk.Unlock()
+	return account.TakenDown
+}
+
+func (account *Account) SetPDS(pdsId uint) {
+	account.lk.Lock()
+	defer account.lk.Unlock()
+	account.PDS = pdsId
+}
+
+func (account *Account) GetPDS() uint {
+	account.lk.Lock()
+	defer account.lk.Unlock()
+	return account.PDS
+}
+
+func (account *Account) SetUpstreamStatus(v string) {
+	account.lk.Lock()
+	defer account.lk.Unlock()
+	account.UpstreamStatus = v
+}
+
+func (account *Account) GetUpstreamStatus() string {
+	account.lk.Lock()
+	defer account.lk.Unlock()
+	return account.UpstreamStatus
+}
+
+type AccountPreviousState struct {
+	Uid models.Uid   `gorm:"column:uid;primaryKey"`
+	Cid models.DbCID `gorm:"column:cid"`
+	Rev string       `gorm:"column:rev"`
+	Seq int64        `gorm:"column:seq"`
+}
+
+func (ups *AccountPreviousState) GetCid() cid.Cid {
+	return ups.Cid.CID
+}
+func (ups *AccountPreviousState) GetRev() syntax.TID {
+	xt, _ := syntax.ParseTID(ups.Rev)
+	return xt
+}
+
+type addTargetBody struct {
+	Host string `json:"host"`
+}
+
+func (bgs *BGS) registerConsumer(c *SocketConsumer) uint64 {
+	bgs.consumersLk.Lock()
+	defer bgs.consumersLk.Unlock()
+
+	id := bgs.nextConsumerID
+	bgs.nextConsumerID++
+
+	bgs.consumers[id] = c
+
+	return id
+}
+
+func (bgs *BGS) cleanupConsumer(id uint64) {
+	bgs.consumersLk.Lock()
+	defer bgs.consumersLk.Unlock()
+
+	c := bgs.consumers[id]
+
+	var m = &dto.Metric{}
+	if err := c.EventsSent.Write(m); err != nil {
+		bgs.log.Error("failed to get sent counter", "err", err)
+	}
+
+	bgs.log.Info("consumer disconnected",
+		"consumer_id", id,
+		"remote_addr", c.RemoteAddr,
+		"user_agent", c.UserAgent,
+		"events_sent", m.Counter.GetValue())
+
+	delete(bgs.consumers, id)
+}
+
+// GET+websocket /xrpc/com.atproto.sync.subscribeRepos
+func (bgs *BGS) EventsHandler(c echo.Context) error {
+	var since *int64
+	if sinceVal := c.QueryParam("cursor"); sinceVal != "" {
+		sval, err := strconv.ParseInt(sinceVal, 10, 64)
+		if err != nil {
+			return err
+		}
+		since = &sval
+	}
+
+	ctx, cancel := context.WithCancel(c.Request().Context())
+	defer cancel()
+
+	conn, err := websocket.Upgrade(c.Response(), c.Request(), c.Response().Header(), 10<<10, 10<<10)
+	if err != nil {
+		return fmt.Errorf("upgrading websocket: %w", err)
+	}
+
+	defer conn.Close()
+
+	lastWriteLk := sync.Mutex{}
+	lastWrite := time.Now()
+
+	// Start a goroutine to ping the client every 30 seconds to check if it's
+	// still alive. If the client doesn't respond to a ping within 5 seconds,
+	// we'll close the connection and teardown the consumer.
+	go func() {
+		ticker := time.NewTicker(30 * time.Second)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				lastWriteLk.Lock()
+				lw := lastWrite
+				lastWriteLk.Unlock()
+
+				if time.Since(lw) < 30*time.Second {
+					continue
+				}
+
+				if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(5*time.Second)); err != nil {
+					bgs.log.Warn("failed to ping client", "err", err)
+					cancel()
+					return
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	conn.SetPingHandler(func(message string) error {
+		err := conn.WriteControl(websocket.PongMessage, []byte(message), time.Now().Add(time.Second*60))
+		if err == websocket.ErrCloseSent {
+			return nil
+		} else if e, ok := err.(net.Error); ok && e.Temporary() {
+			return nil
+		}
+		return err
+	})
+
+	// Start a goroutine to read messages from the client and discard them.
+	go func() {
+		for {
+			_, _, err := conn.ReadMessage()
+			if err != nil {
+				bgs.log.Warn("failed to read message from client", "err", err)
+				cancel()
+				return
+			}
+		}
+	}()
+
+	ident := c.RealIP() + "-" + c.Request().UserAgent()
+
+	evts, cleanup, err := bgs.events.Subscribe(ctx, ident, func(evt *events.XRPCStreamEvent) bool { return true }, since)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	// Keep track of the consumer for metrics and admin endpoints
+	consumer := SocketConsumer{
+		RemoteAddr:  c.RealIP(),
+		UserAgent:   c.Request().UserAgent(),
+		ConnectedAt: time.Now(),
+	}
+	sentCounter := eventsSentCounter.WithLabelValues(consumer.RemoteAddr, consumer.UserAgent)
+	consumer.EventsSent = sentCounter
+
+	consumerID := bgs.registerConsumer(&consumer)
+	defer bgs.cleanupConsumer(consumerID)
+
+	logger := bgs.log.With(
+		"consumer_id", consumerID,
+		"remote_addr", consumer.RemoteAddr,
+		"user_agent", consumer.UserAgent,
+	)
+
+	logger.Info("new consumer", "cursor", since)
+
+	for {
+		select {
+		case evt, ok := <-evts:
+			if !ok {
+				logger.Error("event stream closed unexpectedly")
+				return nil
+			}
+
+			wc, err := conn.NextWriter(websocket.BinaryMessage)
+			if err != nil {
+				logger.Error("failed to get next writer", "err", err)
+				return err
+			}
+
+			if evt.Preserialized != nil {
+				_, err = wc.Write(evt.Preserialized)
+			} else {
+				err = evt.Serialize(wc)
+			}
+			if err != nil {
+				return fmt.Errorf("failed to write event: %w", err)
+			}
+
+			if err := wc.Close(); err != nil {
+				logger.Warn("failed to flush-close our event write", "err", err)
+				return nil
+			}
+
+			lastWriteLk.Lock()
+			lastWrite = time.Now()
+			lastWriteLk.Unlock()
+			sentCounter.Inc()
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
+// domainIsBanned checks if the given host is banned, starting with the host
+// itself, then checking every parent domain up to the tld
+func (s *BGS) domainIsBanned(ctx context.Context, host string) (bool, error) {
+	// ignore ports when checking for ban status
+	hostport := strings.Split(host, ":")
+
+	segments := strings.Split(hostport[0], ".")
+
+	// TODO: use normalize method once that merges
+	var cleaned []string
+	for _, s := range segments {
+		if s == "" {
+			continue
+		}
+		s = strings.ToLower(s)
+
+		cleaned = append(cleaned, s)
+	}
+	segments = cleaned
+
+	for i := 0; i < len(segments)-1; i++ {
+		dchk := strings.Join(segments[i:], ".")
+		found, err := s.findDomainBan(ctx, dchk)
+		if err != nil {
+			return false, err
+		}
+
+		if found {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (s *BGS) findDomainBan(ctx context.Context, host string) (bool, error) {
+	var db DomainBan
+	if err := s.db.Find(&db, "domain = ?", host).Error; err != nil {
+		return false, err
+	}
+
+	if db.ID == 0 {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+var ErrNotFound = errors.New("not found")
+
+func (bgs *BGS) DidToUid(ctx context.Context, did string) (models.Uid, error) {
+	xu, err := bgs.lookupUserByDid(ctx, did)
+	if err != nil {
+		return 0, err
+	}
+	if xu == nil {
+		return 0, ErrNotFound
+	}
+	return xu.ID, nil
+}
+
+func (bgs *BGS) lookupUserByDid(ctx context.Context, did string) (*Account, error) {
+	ctx, span := tracer.Start(ctx, "lookupUserByDid")
+	defer span.End()
+
+	cu, ok := bgs.userCache.Get(did)
+	if ok {
+		return cu, nil
+	}
+
+	var u Account
+	if err := bgs.db.Find(&u, "did = ?", did).Error; err != nil {
+		return nil, err
+	}
+
+	if u.ID == 0 {
+		return nil, gorm.ErrRecordNotFound
+	}
+
+	bgs.userCache.Add(did, &u)
+
+	return &u, nil
+}
+
+func (bgs *BGS) lookupUserByUID(ctx context.Context, uid models.Uid) (*Account, error) {
+	ctx, span := tracer.Start(ctx, "lookupUserByUID")
+	defer span.End()
+
+	var u Account
+	if err := bgs.db.Find(&u, "id = ?", uid).Error; err != nil {
+		return nil, err
+	}
+
+	if u.ID == 0 {
+		return nil, gorm.ErrRecordNotFound
+	}
+
+	return &u, nil
+}
+
+func stringLink(lnk *lexutil.LexLink) string {
+	if lnk == nil {
+		return "<nil>"
+	}
+
+	return lnk.String()
+}
+
+// handleFedEvent() is the callback passed to Slurper called from Slurper.handleConnection()
+func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *events.XRPCStreamEvent) error {
+	ctx, span := tracer.Start(ctx, "handleFedEvent")
+	defer span.End()
+
+	start := time.Now()
+	defer func() {
+		eventsHandleDuration.WithLabelValues(host.Host).Observe(time.Since(start).Seconds())
+	}()
+
+	eventsReceivedCounter.WithLabelValues(host.Host).Add(1)
+
+	switch {
+	case env.RepoCommit != nil:
+		repoCommitsReceivedCounter.WithLabelValues(host.Host).Add(1)
+		return bgs.handleCommit(ctx, host, env.RepoCommit)
+	case env.RepoSync != nil:
+		repoSyncReceivedCounter.WithLabelValues(host.Host).Add(1)
+		return bgs.handleSync(ctx, host, env.RepoSync)
+	case env.RepoHandle != nil:
+		eventsWarningsCounter.WithLabelValues(host.Host, "handle").Add(1)
+		// TODO: rate limit warnings per PDS before we (temporarily?) block them
+		return nil
+	case env.RepoIdentity != nil:
+		bgs.log.Info("bgs got identity event", "did", env.RepoIdentity.Did)
+		// Flush any cached DID documents for this user
+		bgs.purgeDidCache(ctx, env.RepoIdentity.Did)
+
+		// Refetch the DID doc and update our cached keys and handle etc.
+		account, err := bgs.syncPDSAccount(ctx, env.RepoIdentity.Did, host, nil)
+		if err != nil {
+			return err
+		}
+
+		// Broadcast the identity event to all consumers
+		err = bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{
+			RepoIdentity: &comatproto.SyncSubscribeRepos_Identity{
+				Did:    env.RepoIdentity.Did,
+				Seq:    env.RepoIdentity.Seq,
+				Time:   env.RepoIdentity.Time,
+				Handle: env.RepoIdentity.Handle,
+			},
+			PrivUid: account.ID,
+		})
+		if err != nil {
+			bgs.log.Error("failed to broadcast Identity event", "error", err, "did", env.RepoIdentity.Did)
+			return fmt.Errorf("failed to broadcast Identity event: %w", err)
+		}
+
+		return nil
+	case env.RepoAccount != nil:
+		span.SetAttributes(
+			attribute.String("did", env.RepoAccount.Did),
+			attribute.Int64("seq", env.RepoAccount.Seq),
+			attribute.Bool("active", env.RepoAccount.Active),
+		)
+
+		if env.RepoAccount.Status != nil {
+			span.SetAttributes(attribute.String("repo_status", *env.RepoAccount.Status))
+		}
+		bgs.log.Info("bgs got account event", "did", env.RepoAccount.Did)
+
+		if !env.RepoAccount.Active && env.RepoAccount.Status == nil {
+			accountVerifyWarnings.WithLabelValues(host.Host, "nostat").Inc()
+			return nil
+		}
+
+		// Flush any cached DID documents for this user
+		bgs.purgeDidCache(ctx, env.RepoAccount.Did)
+
+		// Refetch the DID doc to make sure the PDS is still authoritative
+		account, err := bgs.syncPDSAccount(ctx, env.RepoAccount.Did, host, nil)
+		if err != nil {
+			span.RecordError(err)
+			return err
+		}
+
+		// Check if the PDS is still authoritative
+		// if not we don't want to be propagating this account event
+		if account.GetPDS() != host.ID {
+			bgs.log.Error("account event from non-authoritative pds",
+				"seq", env.RepoAccount.Seq,
+				"did", env.RepoAccount.Did,
+				"event_from", host.Host,
+				"did_doc_declared_pds", account.GetPDS(),
+				"account_evt", env.RepoAccount,
+			)
+			return fmt.Errorf("event from non-authoritative pds")
+		}
+
+		// Process the account status change
+		repoStatus := events.AccountStatusActive
+		if !env.RepoAccount.Active && env.RepoAccount.Status != nil {
+			repoStatus = *env.RepoAccount.Status
+		}
+
+		account.SetUpstreamStatus(repoStatus)
+		err = bgs.db.Save(account).Error
+		if err != nil {
+			span.RecordError(err)
+			return fmt.Errorf("failed to update account status: %w", err)
+		}
+
+		shouldBeActive := env.RepoAccount.Active
+		status := env.RepoAccount.Status
+
+		// override with local status
+		if account.GetTakenDown() {
+			shouldBeActive = false
+			status = &events.AccountStatusTakendown
+		}
+
+		// Broadcast the account event to all consumers
+		err = bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{
+			RepoAccount: &comatproto.SyncSubscribeRepos_Account{
+				Active: shouldBeActive,
+				Did:    env.RepoAccount.Did,
+				Seq:    env.RepoAccount.Seq,
+				Status: status,
+				Time:   env.RepoAccount.Time,
+			},
+			PrivUid: account.ID,
+		})
+		if err != nil {
+			bgs.log.Error("failed to broadcast Account event", "error", err, "did", env.RepoAccount.Did)
+			return fmt.Errorf("failed to broadcast Account event: %w", err)
+		}
+
+		return nil
+	case env.RepoMigrate != nil:
+		eventsWarningsCounter.WithLabelValues(host.Host, "migrate").Add(1)
+		// TODO: rate limit warnings per PDS before we (temporarily?) block them
+		return nil
+	case env.RepoTombstone != nil:
+		eventsWarningsCounter.WithLabelValues(host.Host, "tombstone").Add(1)
+		// TODO: rate limit warnings per PDS before we (temporarily?) block them
+		return nil
+	default:
+		return fmt.Errorf("invalid fed event")
+	}
+}
+
+func (bgs *BGS) newUser(ctx context.Context, host *models.PDS, did string) (*Account, error) {
+	newUsersDiscovered.Inc()
+	start := time.Now()
+	account, err := bgs.syncPDSAccount(ctx, did, host, nil)
+	newUserDiscoveryDuration.Observe(time.Since(start).Seconds())
+	if err != nil {
+		repoCommitsResultCounter.WithLabelValues(host.Host, "uerr").Inc()
+		return nil, fmt.Errorf("fed event create external user: %w", err)
+	}
+	return account, nil
+}
+
+var ErrCommitNoUser = errors.New("commit no user")
+
+func (bgs *BGS) handleCommit(ctx context.Context, host *models.PDS, evt *comatproto.SyncSubscribeRepos_Commit) error {
+	bgs.log.Debug("bgs got repo append event", "seq", evt.Seq, "pdsHost", host.Host, "repo", evt.Repo)
+
+	account, err := bgs.lookupUserByDid(ctx, evt.Repo)
+	if err != nil {
+		if !errors.Is(err, gorm.ErrRecordNotFound) {
+			repoCommitsResultCounter.WithLabelValues(host.Host, "nou").Inc()
+			return fmt.Errorf("looking up event user: %w", err)
+		}
+
+		account, err = bgs.newUser(ctx, host, evt.Repo)
+		if err != nil {
+			repoCommitsResultCounter.WithLabelValues(host.Host, "nuerr").Inc()
+			return err
+		}
+	}
+	if account == nil {
+		repoCommitsResultCounter.WithLabelValues(host.Host, "nou2").Inc()
+		return ErrCommitNoUser
+	}
+
+	ustatus := account.GetUpstreamStatus()
+
+	if account.GetTakenDown() || ustatus == events.AccountStatusTakendown {
+		bgs.log.Debug("dropping commit event from taken down user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+		repoCommitsResultCounter.WithLabelValues(host.Host, "tdu").Inc()
+		return nil
+	}
+
+	if ustatus == events.AccountStatusSuspended {
+		bgs.log.Debug("dropping commit event from suspended user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+		repoCommitsResultCounter.WithLabelValues(host.Host, "susu").Inc()
+		return nil
+	}
+
+	if ustatus == events.AccountStatusDeactivated {
+		bgs.log.Debug("dropping commit event from deactivated user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+		repoCommitsResultCounter.WithLabelValues(host.Host, "du").Inc()
+		return nil
+	}
+
+	if evt.Rebase {
+		repoCommitsResultCounter.WithLabelValues(host.Host, "rebase").Inc()
+		return fmt.Errorf("rebase was true in event seq:%d,host:%s", evt.Seq, host.Host)
+	}
+
+	accountPDSId := account.GetPDS()
+	if host.ID != accountPDSId && accountPDSId != 0 {
+		bgs.log.Warn("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", accountPDSId, "gotPds", host.Host)
+		// Flush any cached DID documents for this user
+		bgs.purgeDidCache(ctx, evt.Repo)
+
+		account, err = bgs.syncPDSAccount(ctx, evt.Repo, host, account)
+		if err != nil {
+			repoCommitsResultCounter.WithLabelValues(host.Host, "uerr2").Inc()
+			return err
+		}
+
+		if account.GetPDS() != host.ID {
+			repoCommitsResultCounter.WithLabelValues(host.Host, "noauth").Inc()
+			return fmt.Errorf("event from non-authoritative pds")
+		}
+	}
+
+	var prevState AccountPreviousState
+	err = bgs.db.First(&prevState, account.ID).Error
+	prevP := &prevState
+	if errors.Is(err, gorm.ErrRecordNotFound) {
+		prevP = nil
+	} else if err != nil {
+		bgs.log.Error("failed to get previous root", "err", err)
+		prevP = nil
+	}
+	dbPrevRootStr := ""
+	dbPrevSeqStr := ""
+	if prevP != nil {
+		if prevState.Seq >= evt.Seq && ((prevState.Seq - evt.Seq) < 2000) {
+			// ignore catchup overlap of 200 on some subscribeRepos restarts
+			repoCommitsResultCounter.WithLabelValues(host.Host, "dup").Inc()
+			return nil
+		}
+		dbPrevRootStr = prevState.Cid.CID.String()
+		dbPrevSeqStr = strconv.FormatInt(prevState.Seq, 10)
+	}
+	evtPrevDataStr := ""
+	if evt.PrevData != nil {
+		evtPrevDataStr = ((*cid.Cid)(evt.PrevData)).String()
+	}
+	newRootCid, err := bgs.validator.HandleCommit(ctx, host, account, evt, prevP)
+	if err != nil {
+		bgs.inductionTraceLog.Error("commit bad", "seq", evt.Seq, "pseq", dbPrevSeqStr, "pdsHost", host.Host, "repo", evt.Repo, "prev", evtPrevDataStr, "dbprev", dbPrevRootStr, "err", err)
+		bgs.log.Warn("failed handling event", "err", err, "pdsHost", host.Host, "seq", evt.Seq, "repo", account.Did, "commit", evt.Commit.String())
+		repoCommitsResultCounter.WithLabelValues(host.Host, "err").Inc()
+		return fmt.Errorf("handle user event failed: %w", err)
+	} else {
+		// store now verified new repo state
+		err = bgs.upsertPrevState(account.ID, newRootCid, evt.Rev, evt.Seq)
+		if err != nil {
+			return fmt.Errorf("failed to set previous root uid=%d: %w", account.ID, err)
+		}
+	}
+
+	repoCommitsResultCounter.WithLabelValues(host.Host, "ok").Inc()
+
+	// Broadcast the identity event to all consumers
+	commitCopy := *evt
+	err = bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{
+		RepoCommit: &commitCopy,
+		PrivUid:    account.GetUid(),
+	})
+	if err != nil {
+		bgs.log.Error("failed to broadcast commit event", "error", err, "did", evt.Repo)
+		return fmt.Errorf("failed to broadcast commit event: %w", err)
+	}
+
+	return nil
+}
+
+// handleSync processes #sync messages
+func (bgs *BGS) handleSync(ctx context.Context, host *models.PDS, evt *comatproto.SyncSubscribeRepos_Sync) error {
+	account, err := bgs.lookupUserByDid(ctx, evt.Did)
+	if err != nil {
+		if !errors.Is(err, gorm.ErrRecordNotFound) {
+			repoCommitsResultCounter.WithLabelValues(host.Host, "nou").Inc()
+			return fmt.Errorf("looking up event user: %w", err)
+		}
+
+		account, err = bgs.newUser(ctx, host, evt.Did)
+	}
+	if err != nil {
+		return fmt.Errorf("could not get user for did %#v: %w", evt.Did, err)
+	}
+
+	newRootCid, err := bgs.validator.HandleSync(ctx, host, evt)
+	if err != nil {
+		return err
+	}
+	err = bgs.upsertPrevState(account.ID, newRootCid, evt.Rev, evt.Seq)
+	if err != nil {
+		return fmt.Errorf("could not sync set previous state uid=%d: %w", account.ID, err)
+	}
+
+	// Broadcast the sync event to all consumers
+	evtCopy := *evt
+	err = bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{
+		RepoSync: &evtCopy,
+	})
+	if err != nil {
+		bgs.log.Error("failed to broadcast sync event", "error", err, "did", evt.Did)
+		return fmt.Errorf("failed to broadcast sync event: %w", err)
+	}
+
+	return nil
+}
+
+func (bgs *BGS) upsertPrevState(accountID models.Uid, newRootCid *cid.Cid, rev string, seq int64) error {
+	cidBytes := newRootCid.Bytes()
+	return bgs.db.Exec(
+		"INSERT INTO account_previous_states (uid, cid, rev, seq) VALUES (?, ?, ?, ?) ON CONFLICT (uid) DO UPDATE SET cid = EXCLUDED.cid, rev = EXCLUDED.rev, seq = EXCLUDED.seq",
+		accountID, cidBytes, rev, seq,
+	).Error
+}
+
+func (bgs *BGS) purgeDidCache(ctx context.Context, did string) {
+	ati, err := syntax.ParseAtIdentifier(did)
+	if err != nil {
+		return
+	}
+	_ = bgs.didd.Purge(ctx, *ati)
+}
+
+// syncPDSAccount ensures that a DID has an account record in the database attached to a PDS record in the database
+// Some fields may be updated if needed.
+// did is the user
+// host is the PDS we received this from, not necessarily the canonical PDS in the DID document
+// cachedAccount is (optionally) the account that we have already looked up from cache or database
+func (bgs *BGS) syncPDSAccount(ctx context.Context, did string, host *models.PDS, cachedAccount *Account) (*Account, error) {
+	ctx, span := tracer.Start(ctx, "syncPDSAccount")
+	defer span.End()
+
+	externalUserCreationAttempts.Inc()
+
+	bgs.log.Debug("create external user", "did", did)
+
+	// lookup identity so that we know a DID's canonical source PDS
+	pdid, err := syntax.ParseDID(did)
+	if err != nil {
+		return nil, fmt.Errorf("bad did %#v, %w", did, err)
+	}
+	ident, err := bgs.didd.LookupDID(ctx, pdid)
+	if err != nil {
+		return nil, fmt.Errorf("no ident for did %s, %w", did, err)
+	}
+	if len(ident.Services) == 0 {
+		return nil, fmt.Errorf("no services for did %s", did)
+	}
+	pdsService, ok := ident.Services["atproto_pds"]
+	if !ok {
+		return nil, fmt.Errorf("no atproto_pds service for did %s", did)
+	}
+	durl, err := url.Parse(pdsService.URL)
+	if err != nil {
+		return nil, fmt.Errorf("pds bad url %#v, %w", pdsService.URL, err)
+	}
+
+	// is the canonical PDS banned?
+	ban, err := bgs.domainIsBanned(ctx, durl.Host)
+	if err != nil {
+		return nil, fmt.Errorf("failed to check pds ban status: %w", err)
+	}
+	if ban {
+		return nil, fmt.Errorf("cannot create user on pds with banned domain")
+	}
+
+	if strings.HasPrefix(durl.Host, "localhost:") {
+		durl.Scheme = "http"
+	}
+
+	var canonicalHost *models.PDS
+	if host.Host == durl.Host {
+		// we got the message from the canonical PDS, convenient!
+		canonicalHost = host
+	} else {
+		// we got the message from an intermediate relay
+		// check our db for info on canonical PDS
+		var peering models.PDS
+		if err := bgs.db.Find(&peering, "host = ?", durl.Host).Error; err != nil {
+			bgs.log.Error("failed to find pds", "host", durl.Host)
+			return nil, err
+		}
+		canonicalHost = &peering
+	}
+
+	if canonicalHost.Blocked {
+		return nil, fmt.Errorf("refusing to create user with blocked PDS")
+	}
+
+	if canonicalHost.ID == 0 {
+		// we got an event from a non-canonical PDS (an intermediate relay)
+		// a non-canonical PDS we haven't seen before; ping it to make sure it's real
+		// TODO: what do we actually want to track about the source we immediately got this message from vs the canonical PDS?
+		bgs.log.Warn("pds discovered in new user flow", "pds", durl.String(), "did", did)
+
+		// Do a trivial API request against the PDS to verify that it exists
+		pclient := &xrpc.Client{Host: durl.String()}
+		bgs.config.ApplyPDSClientSettings(pclient)
+		cfg, err := comatproto.ServerDescribeServer(ctx, pclient)
+		if err != nil {
+			// TODO: failing this shouldn't halt our indexing
+			return nil, fmt.Errorf("failed to check unrecognized pds: %w", err)
+		}
+
+		// since handles can be anything, checking against this list doesn't matter...
+		_ = cfg
+
+		// could check other things, a valid response is good enough for now
+		canonicalHost.Host = durl.Host
+		canonicalHost.SSL = (durl.Scheme == "https")
+		canonicalHost.RateLimit = float64(bgs.slurper.DefaultPerSecondLimit)
+		canonicalHost.HourlyEventLimit = bgs.slurper.DefaultPerHourLimit
+		canonicalHost.DailyEventLimit = bgs.slurper.DefaultPerDayLimit
+		canonicalHost.RepoLimit = bgs.slurper.DefaultRepoLimit
+
+		if bgs.ssl && !canonicalHost.SSL {
+			return nil, fmt.Errorf("did references non-ssl PDS, this is disallowed in prod: %q %q", did, pdsService.URL)
+		}
+
+		if err := bgs.db.Create(&canonicalHost).Error; err != nil {
+			return nil, err
+		}
+	}
+
+	if canonicalHost.ID == 0 {
+		panic("somehow failed to create a pds entry?")
+	}
+
+	if canonicalHost.RepoCount >= canonicalHost.RepoLimit {
+		// TODO: soft-limit / hard-limit ? create account in 'throttled' state, unless there are _really_ too many accounts
+		return nil, fmt.Errorf("refusing to create user on PDS at max repo limit for pds %q", canonicalHost.Host)
+	}
+
+	// this lock just governs the lower half of this function
+	bgs.extUserLk.Lock()
+	defer bgs.extUserLk.Unlock()
+
+	if cachedAccount == nil {
+		cachedAccount, err = bgs.lookupUserByDid(ctx, did)
+	}
+	if errors.Is(err, ErrNotFound) || errors.Is(err, gorm.ErrRecordNotFound) {
+		err = nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if cachedAccount != nil {
+		caPDS := cachedAccount.GetPDS()
+		if caPDS != canonicalHost.ID {
+			// Account is now on a different PDS, update
+			err = bgs.db.Transaction(func(tx *gorm.DB) error {
+				if caPDS != 0 {
+					// decrement prior PDS's account count
+					tx.Model(&models.PDS{}).Where("id = ?", caPDS).Update("repo_count", gorm.Expr("repo_count - 1"))
+				}
+				// update user's PDS ID
+				res := tx.Model(Account{}).Where("id = ?", cachedAccount.ID).Update("pds", canonicalHost.ID)
+				if res.Error != nil {
+					return fmt.Errorf("failed to update users pds: %w", res.Error)
+				}
+				// increment new PDS's account count
+				res = tx.Model(&models.PDS{}).Where("id = ? AND repo_count < repo_limit", canonicalHost.ID).Update("repo_count", gorm.Expr("repo_count + 1"))
+				return nil
+			})
+
+			cachedAccount.SetPDS(canonicalHost.ID)
+		}
+		return cachedAccount, nil
+	}
+
+	newAccount := Account{
+		Did: did,
+		PDS: canonicalHost.ID,
+	}
+
+	err = bgs.db.Transaction(func(tx *gorm.DB) error {
+		res := tx.Model(&models.PDS{}).Where("id = ? AND repo_count < repo_limit", canonicalHost.ID).Update("repo_count", gorm.Expr("repo_count + 1"))
+		if res.Error != nil {
+			return fmt.Errorf("failed to increment repo count for pds %q: %w", canonicalHost.Host, res.Error)
+		}
+		if terr := bgs.db.Create(&newAccount).Error; terr != nil {
+			bgs.log.Error("failed to create user", "did", newAccount.Did, "err", terr)
+			return fmt.Errorf("failed to create other pds user: %w", terr)
+		}
+		return nil
+	})
+	if err != nil {
+		bgs.log.Error("user create and pds inc err", "err", err)
+		return nil, err
+	}
+
+	bgs.userCache.Add(did, &newAccount)
+
+	return &newAccount, nil
+}
+
+func (bgs *BGS) TakeDownRepo(ctx context.Context, did string) error {
+	u, err := bgs.lookupUserByDid(ctx, did)
+	if err != nil {
+		return err
+	}
+
+	if err := bgs.db.Model(Account{}).Where("id = ?", u.ID).Update("taken_down", true).Error; err != nil {
+		return err
+	}
+	u.SetTakenDown(true)
+
+	if err := bgs.events.TakeDownRepo(ctx, u.ID); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (bgs *BGS) ReverseTakedown(ctx context.Context, did string) error {
+	u, err := bgs.lookupUserByDid(ctx, did)
+	if err != nil {
+		return err
+	}
+
+	if err := bgs.db.Model(Account{}).Where("id = ?", u.ID).Update("taken_down", false).Error; err != nil {
+		return err
+	}
+	u.SetTakenDown(false)
+
+	return nil
+}
+
+func (bgs *BGS) GetRepoRoot(ctx context.Context, user models.Uid) (cid.Cid, error) {
+	var prevState AccountPreviousState
+	err := bgs.db.First(&prevState, user).Error
+	if err == nil {
+		return prevState.Cid.CID, nil
+	} else if errors.Is(err, gorm.ErrRecordNotFound) {
+		return cid.Cid{}, ErrUserStatusUnavailable
+	} else {
+		bgs.log.Error("user db err", "err", err)
+		return cid.Cid{}, fmt.Errorf("user prev db err, %w", err)
+	}
+}

cmd/relay/bgs/fedmgr.go

···
+package bgs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math/rand"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/RussellLuo/slidingwindow"
+	comatproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/cmd/relay/events"
+	"github.com/bluesky-social/indigo/cmd/relay/events/schedulers/parallel"
+	"github.com/bluesky-social/indigo/cmd/relay/models"
+
+	"github.com/gorilla/websocket"
+	pq "github.com/lib/pq"
+	"gorm.io/gorm"
+)
+
+type IndexCallback func(context.Context, *models.PDS, *events.XRPCStreamEvent) error
+
+type Slurper struct {
+	cb IndexCallback
+
+	db *gorm.DB
+
+	lk     sync.Mutex
+	active map[string]*activeSub
+
+	LimitMux              sync.RWMutex
+	Limiters              map[uint]*Limiters
+	DefaultPerSecondLimit int64
+	DefaultPerHourLimit   int64
+	DefaultPerDayLimit    int64
+
+	DefaultRepoLimit  int64
+	ConcurrencyPerPDS int64
+	MaxQueuePerPDS    int64
+
+	NewPDSPerDayLimiter *slidingwindow.Limiter
+
+	newSubsDisabled bool
+	trustedDomains  []string
+
+	shutdownChan   chan bool
+	shutdownResult chan []error
+
+	ssl bool
+
+	log *slog.Logger
+}
+
+type Limiters struct {
+	PerSecond *slidingwindow.Limiter
+	PerHour   *slidingwindow.Limiter
+	PerDay    *slidingwindow.Limiter
+}
+
+type SlurperOptions struct {
+	SSL                   bool
+	DefaultPerSecondLimit int64
+	DefaultPerHourLimit   int64
+	DefaultPerDayLimit    int64
+	DefaultRepoLimit      int64
+	ConcurrencyPerPDS     int64
+	MaxQueuePerPDS        int64
+
+	Logger *slog.Logger
+}
+
+func DefaultSlurperOptions() *SlurperOptions {
+	return &SlurperOptions{
+		SSL:                   false,
+		DefaultPerSecondLimit: 50,
+		DefaultPerHourLimit:   2500,
+		DefaultPerDayLimit:    20_000,
+		DefaultRepoLimit:      100,
+		ConcurrencyPerPDS:     100,
+		MaxQueuePerPDS:        1_000,
+
+		Logger: slog.Default(),
+	}
+}
+
+type activeSub struct {
+	pds    *models.PDS
+	lk     sync.RWMutex
+	ctx    context.Context
+	cancel func()
+}
+
+func (sub *activeSub) updateCursor(curs int64) {
+	sub.lk.Lock()
+	defer sub.lk.Unlock()
+	sub.pds.Cursor = curs
+}
+
+func NewSlurper(db *gorm.DB, cb IndexCallback, opts *SlurperOptions) (*Slurper, error) {
+	if opts == nil {
+		opts = DefaultSlurperOptions()
+	}
+	err := db.AutoMigrate(&SlurpConfig{})
+	if err != nil {
+		return nil, err
+	}
+	s := &Slurper{
+		cb:                    cb,
+		db:                    db,
+		active:                make(map[string]*activeSub),
+		Limiters:              make(map[uint]*Limiters),
+		DefaultPerSecondLimit: opts.DefaultPerSecondLimit,
+		DefaultPerHourLimit:   opts.DefaultPerHourLimit,
+		DefaultPerDayLimit:    opts.DefaultPerDayLimit,
+		DefaultRepoLimit:      opts.DefaultRepoLimit,
+		ConcurrencyPerPDS:     opts.ConcurrencyPerPDS,
+		MaxQueuePerPDS:        opts.MaxQueuePerPDS,
+		ssl:                   opts.SSL,
+		shutdownChan:          make(chan bool),
+		shutdownResult:        make(chan []error),
+		log:                   opts.Logger,
+	}
+	if err := s.loadConfig(); err != nil {
+		return nil, err
+	}
+
+	// Start a goroutine to flush cursors to the DB every 30s
+	go func() {
+		for {
+			select {
+			case <-s.shutdownChan:
+				s.log.Info("flushing PDS cursors on shutdown")
+				ctx := context.Background()
+				var errs []error
+				if errs = s.flushCursors(ctx); len(errs) > 0 {
+					for _, err := range errs {
+						s.log.Error("failed to flush cursors on shutdown", "err", err)
+					}
+				}
+				s.log.Info("done flushing PDS cursors on shutdown")
+				s.shutdownResult <- errs
+				return
+			case <-time.After(time.Second * 10):
+				s.log.Debug("flushing PDS cursors")
+				ctx := context.Background()
+				if errs := s.flushCursors(ctx); len(errs) > 0 {
+					for _, err := range errs {
+						s.log.Error("failed to flush cursors", "err", err)
+					}
+				}
+				s.log.Debug("done flushing PDS cursors")
+			}
+		}
+	}()
+
+	return s, nil
+}
+
+func windowFunc() (slidingwindow.Window, slidingwindow.StopFunc) {
+	return slidingwindow.NewLocalWindow()
+}
+
+func (s *Slurper) GetLimiters(pdsID uint) *Limiters {
+	s.LimitMux.RLock()
+	defer s.LimitMux.RUnlock()
+	return s.Limiters[pdsID]
+}
+
+func (s *Slurper) GetOrCreateLimiters(pdsID uint, perSecLimit int64, perHourLimit int64, perDayLimit int64) *Limiters {
+	s.LimitMux.RLock()
+	defer s.LimitMux.RUnlock()
+	lim, ok := s.Limiters[pdsID]
+	if !ok {
+		perSec, _ := slidingwindow.NewLimiter(time.Second, perSecLimit, windowFunc)
+		perHour, _ := slidingwindow.NewLimiter(time.Hour, perHourLimit, windowFunc)
+		perDay, _ := slidingwindow.NewLimiter(time.Hour*24, perDayLimit, windowFunc)
+		lim = &Limiters{
+			PerSecond: perSec,
+			PerHour:   perHour,
+			PerDay:    perDay,
+		}
+		s.Limiters[pdsID] = lim
+	}
+
+	return lim
+}
+
+func (s *Slurper) SetLimits(pdsID uint, perSecLimit int64, perHourLimit int64, perDayLimit int64) {
+	s.LimitMux.Lock()
+	defer s.LimitMux.Unlock()
+	lim, ok := s.Limiters[pdsID]
+	if !ok {
+		perSec, _ := slidingwindow.NewLimiter(time.Second, perSecLimit, windowFunc)
+		perHour, _ := slidingwindow.NewLimiter(time.Hour, perHourLimit, windowFunc)
+		perDay, _ := slidingwindow.NewLimiter(time.Hour*24, perDayLimit, windowFunc)
+		lim = &Limiters{
+			PerSecond: perSec,
+			PerHour:   perHour,
+			PerDay:    perDay,
+		}
+		s.Limiters[pdsID] = lim
+	}
+
+	lim.PerSecond.SetLimit(perSecLimit)
+	lim.PerHour.SetLimit(perHourLimit)
+	lim.PerDay.SetLimit(perDayLimit)
+}
+
+// Shutdown shuts down the slurper
+func (s *Slurper) Shutdown() []error {
+	s.shutdownChan <- true
+	s.log.Info("waiting for slurper shutdown")
+	errs := <-s.shutdownResult
+	if len(errs) > 0 {
+		for _, err := range errs {
+			s.log.Error("shutdown error", "err", err)
+		}
+	}
+	s.log.Info("slurper shutdown complete")
+	return errs
+}
+
+func (s *Slurper) loadConfig() error {
+	var sc SlurpConfig
+	if err := s.db.Find(&sc).Error; err != nil {
+		return err
+	}
+
+	if sc.ID == 0 {
+		if err := s.db.Create(&SlurpConfig{}).Error; err != nil {
+			return err
+		}
+	}
+
+	s.newSubsDisabled = sc.NewSubsDisabled
+	s.trustedDomains = sc.TrustedDomains
+
+	s.NewPDSPerDayLimiter, _ = slidingwindow.NewLimiter(time.Hour*24, sc.NewPDSPerDayLimit, windowFunc)
+
+	return nil
+}
+
+type SlurpConfig struct {
+	gorm.Model
+
+	NewSubsDisabled   bool
+	TrustedDomains    pq.StringArray `gorm:"type:text[]"`
+	NewPDSPerDayLimit int64
+}
+
+func (s *Slurper) SetNewSubsDisabled(dis bool) error {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("new_subs_disabled", dis).Error; err != nil {
+		return err
+	}
+
+	s.newSubsDisabled = dis
+	return nil
+}
+
+func (s *Slurper) GetNewSubsDisabledState() bool {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+	return s.newSubsDisabled
+}
+
+func (s *Slurper) SetNewPDSPerDayLimit(limit int64) error {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("new_pds_per_day_limit", limit).Error; err != nil {
+		return err
+	}
+
+	s.NewPDSPerDayLimiter.SetLimit(limit)
+	return nil
+}
+
+func (s *Slurper) GetNewPDSPerDayLimit() int64 {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+	return s.NewPDSPerDayLimiter.Limit()
+}
+
+func (s *Slurper) AddTrustedDomain(domain string) error {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("trusted_domains", gorm.Expr("array_append(trusted_domains, ?)", domain)).Error; err != nil {
+		return err
+	}
+
+	s.trustedDomains = append(s.trustedDomains, domain)
+	return nil
+}
+
+func (s *Slurper) RemoveTrustedDomain(domain string) error {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("trusted_domains", gorm.Expr("array_remove(trusted_domains, ?)", domain)).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil
+		}
+		return err
+	}
+
+	for i, d := range s.trustedDomains {
+		if d == domain {
+			s.trustedDomains = append(s.trustedDomains[:i], s.trustedDomains[i+1:]...)
+			break
+		}
+	}
+
+	return nil
+}
+
+func (s *Slurper) SetTrustedDomains(domains []string) error {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("trusted_domains", domains).Error; err != nil {
+		return err
+	}
+
+	s.trustedDomains = domains
+	return nil
+}
+
+func (s *Slurper) GetTrustedDomains() []string {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+	return s.trustedDomains
+}
+
+var ErrNewSubsDisabled = fmt.Errorf("new subscriptions temporarily disabled")
+
+// Checks whether a host is allowed to be subscribed to
+// must be called with the slurper lock held
+func (s *Slurper) canSlurpHost(host string) bool {
+	// Check if we're over the limit for new PDSs today
+	if !s.NewPDSPerDayLimiter.Allow() {
+		return false
+	}
+
+	// Check if the host is a trusted domain
+	for _, d := range s.trustedDomains {
+		// If the domain starts with a *., it's a wildcard
+		if strings.HasPrefix(d, "*.") {
+			// Cut off the * so we have .domain.com
+			if strings.HasSuffix(host, strings.TrimPrefix(d, "*")) {
+				return true
+			}
+		} else {
+			if host == d {
+				return true
+			}
+		}
+	}
+
+	return !s.newSubsDisabled
+}
+
+func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, adminOverride bool, rateOverrides *PDSRates) error {
+	// TODO: for performance, lock on the hostname instead of global
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	_, ok := s.active[host]
+	if ok {
+		return nil
+	}
+
+	var peering models.PDS
+	if err := s.db.Find(&peering, "host = ?", host).Error; err != nil {
+		return err
+	}
+
+	if peering.Blocked {
+		return fmt.Errorf("cannot subscribe to blocked pds")
+	}
+
+	newHost := false
+
+	if peering.ID == 0 {
+		if !adminOverride && !s.canSlurpHost(host) {
+			return ErrNewSubsDisabled
+		}
+		// New PDS!
+		npds := models.PDS{
+			Host:             host,
+			SSL:              s.ssl,
+			Registered:       reg,
+			RateLimit:        float64(s.DefaultPerSecondLimit),
+			HourlyEventLimit: s.DefaultPerHourLimit,
+			DailyEventLimit:  s.DefaultPerDayLimit,
+			RepoLimit:        s.DefaultRepoLimit,
+		}
+		if rateOverrides != nil {
+			npds.RateLimit = float64(rateOverrides.PerSecond)
+			npds.HourlyEventLimit = rateOverrides.PerHour
+			npds.DailyEventLimit = rateOverrides.PerDay
+			npds.RepoLimit = rateOverrides.RepoLimit
+		}
+		if err := s.db.Create(&npds).Error; err != nil {
+			return err
+		}
+
+		newHost = true
+		peering = npds
+	}
+
+	if !peering.Registered && reg {
+		peering.Registered = true
+		if err := s.db.Model(models.PDS{}).Where("id = ?", peering.ID).Update("registered", true).Error; err != nil {
+			return err
+		}
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	sub := activeSub{
+		pds:    &peering,
+		ctx:    ctx,
+		cancel: cancel,
+	}
+	s.active[host] = &sub
+
+	s.GetOrCreateLimiters(peering.ID, int64(peering.RateLimit), peering.HourlyEventLimit, peering.DailyEventLimit)
+
+	go s.subscribeWithRedialer(ctx, &peering, &sub, newHost)
+
+	return nil
+}
+
+func (s *Slurper) RestartAll() error {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	var all []models.PDS
+	if err := s.db.Find(&all, "registered = true AND blocked = false").Error; err != nil {
+		return err
+	}
+
+	for _, pds := range all {
+		pds := pds
+
+		ctx, cancel := context.WithCancel(context.Background())
+		sub := activeSub{
+			pds:    &pds,
+			ctx:    ctx,
+			cancel: cancel,
+		}
+		s.active[pds.Host] = &sub
+
+		// Check if we've already got a limiter for this PDS
+		s.GetOrCreateLimiters(pds.ID, int64(pds.RateLimit), pds.HourlyEventLimit, pds.DailyEventLimit)
+		go s.subscribeWithRedialer(ctx, &pds, &sub, false)
+	}
+
+	return nil
+}
+
+func (s *Slurper) subscribeWithRedialer(ctx context.Context, host *models.PDS, sub *activeSub, newHost bool) {
+	defer func() {
+		s.lk.Lock()
+		defer s.lk.Unlock()
+
+		delete(s.active, host.Host)
+	}()
+
+	d := websocket.Dialer{
+		HandshakeTimeout: time.Second * 5,
+	}
+
+	protocol := "ws"
+	if s.ssl {
+		protocol = "wss"
+	}
+
+	// Special case `.host.bsky.network` PDSs to rewind cursor by 200 events to smooth over unclean shutdowns
+	if strings.HasSuffix(host.Host, ".host.bsky.network") && host.Cursor > 200 {
+		host.Cursor -= 200
+	}
+
+	cursor := host.Cursor
+
+	connectedInbound.Inc()
+	defer connectedInbound.Dec()
+	// TODO:? maybe keep a gauge of 'in retry backoff' sources?
+
+	var backoff int
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		var url string
+		if newHost {
+			url = fmt.Sprintf("%s://%s/xrpc/com.atproto.sync.subscribeRepos", protocol, host.Host)
+		} else {
+			url = fmt.Sprintf("%s://%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", protocol, host.Host, cursor)
+		}
+		con, res, err := d.DialContext(ctx, url, nil)
+		if err != nil {
+			s.log.Warn("dialing failed", "pdsHost", host.Host, "err", err, "backoff", backoff)
+			time.Sleep(sleepForBackoff(backoff))
+			backoff++
+
+			if backoff > 15 {
+				s.log.Warn("pds does not appear to be online, disabling for now", "pdsHost", host.Host)
+				if err := s.db.Model(&models.PDS{}).Where("id = ?", host.ID).Update("registered", false).Error; err != nil {
+					s.log.Error("failed to unregister failing pds", "err", err)
+				}
+
+				return
+			}
+
+			continue
+		}
+
+		s.log.Info("event subscription response", "code", res.StatusCode, "url", url)
+
+		curCursor := cursor
+		if err := s.handleConnection(ctx, host, con, &cursor, sub); err != nil {
+			if errors.Is(err, ErrTimeoutShutdown) {
+				s.log.Info("shutting down pds subscription after timeout", "host", host.Host, "time", EventsTimeout)
+				return
+			}
+			s.log.Warn("connection to failed", "host", host.Host, "err", err)
+			// TODO: measure the last N connection error times and if they're coming too fast reconnect slower or don't reconnect and wait for requestCrawl
+		}
+
+		if cursor > curCursor {
+			backoff = 0
+		}
+	}
+}
+
+func sleepForBackoff(b int) time.Duration {
+	if b == 0 {
+		return 0
+	}
+
+	if b < 10 {
+		return (time.Duration(b) * 2) + (time.Millisecond * time.Duration(rand.Intn(1000)))
+	}
+
+	return time.Second * 30
+}
+
+var ErrTimeoutShutdown = fmt.Errorf("timed out waiting for new events")
+
+var EventsTimeout = time.Minute
+
+func (s *Slurper) handleConnection(ctx context.Context, host *models.PDS, con *websocket.Conn, lastCursor *int64, sub *activeSub) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	rsc := &events.RepoStreamCallbacks{
+		RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error {
+			s.log.Debug("got remote repo event", "pdsHost", host.Host, "repo", evt.Repo, "seq", evt.Seq)
+			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
+				RepoCommit: evt,
+			}); err != nil {
+				s.log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
+			}
+			*lastCursor = evt.Seq
+
+			sub.updateCursor(*lastCursor)
+
+			return nil
+		},
+		RepoSync: func(evt *comatproto.SyncSubscribeRepos_Sync) error {
+			s.log.Debug("got remote repo event", "pdsHost", host.Host, "repo", evt.Did, "seq", evt.Seq)
+			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
+				RepoSync: evt,
+			}); err != nil {
+				s.log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
+			}
+			*lastCursor = evt.Seq
+
+			sub.updateCursor(*lastCursor)
+
+			return nil
+		},
+		RepoHandle: func(evt *comatproto.SyncSubscribeRepos_Handle) error {
+			s.log.Debug("got remote handle update event", "pdsHost", host.Host, "did", evt.Did, "handle", evt.Handle)
+			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
+				RepoHandle: evt,
+			}); err != nil {
+				s.log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
+			}
+			*lastCursor = evt.Seq
+
+			sub.updateCursor(*lastCursor)
+
+			return nil
+		},
+		RepoMigrate: func(evt *comatproto.SyncSubscribeRepos_Migrate) error {
+			s.log.Debug("got remote repo migrate event", "pdsHost", host.Host, "did", evt.Did, "migrateTo", evt.MigrateTo)
+			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
+				RepoMigrate: evt,
+			}); err != nil {
+				s.log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
+			}
+			*lastCursor = evt.Seq
+
+			sub.updateCursor(*lastCursor)
+
+			return nil
+		},
+		RepoTombstone: func(evt *comatproto.SyncSubscribeRepos_Tombstone) error {
+			s.log.Debug("got remote repo tombstone event", "pdsHost", host.Host, "did", evt.Did)
+			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
+				RepoTombstone: evt,
+			}); err != nil {
+				s.log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
+			}
+			*lastCursor = evt.Seq
+
+			sub.updateCursor(*lastCursor)
+
+			return nil
+		},
+		RepoInfo: func(info *comatproto.SyncSubscribeRepos_Info) error {
+			s.log.Debug("info event", "name", info.Name, "message", info.Message, "pdsHost", host.Host)
+			return nil
+		},
+		RepoIdentity: func(ident *comatproto.SyncSubscribeRepos_Identity) error {
+			s.log.Debug("identity event", "did", ident.Did)
+			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
+				RepoIdentity: ident,
+			}); err != nil {
+				s.log.Error("failed handling event", "host", host.Host, "seq", ident.Seq, "err", err)
+			}
+			*lastCursor = ident.Seq
+
+			sub.updateCursor(*lastCursor)
+
+			return nil
+		},
+		RepoAccount: func(acct *comatproto.SyncSubscribeRepos_Account) error {
+			s.log.Debug("account event", "did", acct.Did, "status", acct.Status)
+			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
+				RepoAccount: acct,
+			}); err != nil {
+				s.log.Error("failed handling event", "host", host.Host, "seq", acct.Seq, "err", err)
+			}
+			*lastCursor = acct.Seq
+
+			sub.updateCursor(*lastCursor)
+
+			return nil
+		},
+		// TODO: all the other event types (handle change, migration, etc)
+		Error: func(errf *events.ErrorFrame) error {
+			switch errf.Error {
+			case "FutureCursor":
+				// if we get a FutureCursor frame, reset our sequence number for this host
+				if err := s.db.Table("pds").Where("id = ?", host.ID).Update("cursor", 0).Error; err != nil {
+					return err
+				}
+
+				*lastCursor = 0
+				return fmt.Errorf("got FutureCursor frame, reset cursor tracking for host")
+			default:
+				return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message)
+			}
+		},
+	}
+
+	lims := s.GetOrCreateLimiters(host.ID, int64(host.RateLimit), host.HourlyEventLimit, host.DailyEventLimit)
+
+	limiters := []*slidingwindow.Limiter{
+		lims.PerSecond,
+		lims.PerHour,
+		lims.PerDay,
+	}
+
+	instrumentedRSC := events.NewInstrumentedRepoStreamCallbacks(limiters, rsc.EventHandler)
+
+	pool := parallel.NewScheduler(
+		100,
+		1_000,
+		con.RemoteAddr().String(),
+		instrumentedRSC.EventHandler,
+	)
+	return events.HandleRepoStream(ctx, con, pool, nil)
+}
+
+type cursorSnapshot struct {
+	id     uint
+	cursor int64
+}
+
+// flushCursors updates the PDS cursors in the DB for all active subscriptions
+func (s *Slurper) flushCursors(ctx context.Context) []error {
+	start := time.Now()
+	//ctx, span := otel.Tracer("feedmgr").Start(ctx, "flushCursors")
+	//defer span.End()
+
+	var cursors []cursorSnapshot
+
+	s.lk.Lock()
+	// Iterate over active subs and copy the current cursor
+	for _, sub := range s.active {
+		sub.lk.RLock()
+		cursors = append(cursors, cursorSnapshot{
+			id:     sub.pds.ID,
+			cursor: sub.pds.Cursor,
+		})
+		sub.lk.RUnlock()
+	}
+	s.lk.Unlock()
+
+	errs := []error{}
+	okcount := 0
+
+	tx := s.db.WithContext(ctx).Begin()
+	for _, cursor := range cursors {
+		if err := tx.WithContext(ctx).Model(models.PDS{}).Where("id = ?", cursor.id).UpdateColumn("cursor", cursor.cursor).Error; err != nil {
+			errs = append(errs, err)
+		} else {
+			okcount++
+		}
+	}
+	if err := tx.WithContext(ctx).Commit().Error; err != nil {
+		errs = append(errs, err)
+	}
+	dt := time.Since(start)
+	s.log.Info("flushCursors", "dt", dt, "ok", okcount, "errs", len(errs))
+
+	return errs
+}
+
+func (s *Slurper) GetActiveList() []string {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+	var out []string
+	for k := range s.active {
+		out = append(out, k)
+	}
+
+	return out
+}
+
+var ErrNoActiveConnection = fmt.Errorf("no active connection to host")
+
+func (s *Slurper) KillUpstreamConnection(host string, block bool) error {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	ac, ok := s.active[host]
+	if !ok {
+		return fmt.Errorf("killing connection %q: %w", host, ErrNoActiveConnection)
+	}
+	ac.cancel()
+	// cleanup in the run thread subscribeWithRedialer() will delete(s.active, host)
+
+	if block {
+		if err := s.db.Model(models.PDS{}).Where("id = ?", ac.pds.ID).UpdateColumn("blocked", true).Error; err != nil {
+			return fmt.Errorf("failed to set host as blocked: %w", err)
+		}
+	}
+
+	return nil
+}

cmd/relay/bgs/handlers.go

···
+package bgs
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	atproto "github.com/bluesky-social/indigo/api/atproto"
+	comatprototypes "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/cmd/relay/events"
+	"gorm.io/gorm"
+
+	"github.com/bluesky-social/indigo/xrpc"
+	"github.com/labstack/echo/v4"
+)
+
+func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context, body *comatprototypes.SyncRequestCrawl_Input) error {
+	host := body.Hostname
+	if host == "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname")
+	}
+
+	if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
+		if s.ssl {
+			host = "https://" + host
+		} else {
+			host = "http://" + host
+		}
+	}
+
+	u, err := url.Parse(host)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, "failed to parse hostname")
+	}
+
+	if u.Scheme == "http" && s.ssl {
+		return echo.NewHTTPError(http.StatusBadRequest, "this server requires https")
+	}
+
+	if u.Scheme == "https" && !s.ssl {
+		return echo.NewHTTPError(http.StatusBadRequest, "this server does not support https")
+	}
+
+	if u.Path != "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without path")
+	}
+
+	if u.Query().Encode() != "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without query")
+	}
+
+	host = u.Host // potentially hostname:port
+
+	banned, err := s.domainIsBanned(ctx, host)
+	if banned {
+		return echo.NewHTTPError(http.StatusUnauthorized, "domain is banned")
+	}
+
+	s.log.Warn("TODO: better host validation for crawl requests")
+
+	clientHost := fmt.Sprintf("%s://%s", u.Scheme, host)
+
+	c := &xrpc.Client{
+		Host:   clientHost,
+		Client: http.DefaultClient, // not using the client that auto-retries
+	}
+
+	desc, err := atproto.ServerDescribeServer(ctx, c)
+	if err != nil {
+		errMsg := fmt.Sprintf("requested host (%s) failed to respond to describe request", clientHost)
+		return echo.NewHTTPError(http.StatusBadRequest, errMsg)
+	}
+
+	// Maybe we could do something with this response later
+	_ = desc
+
+	if len(s.nextCrawlers) != 0 {
+		blob, err := json.Marshal(body)
+		if err != nil {
+			s.log.Warn("could not forward requestCrawl, json err", "err", err)
+		} else {
+			go func(bodyBlob []byte) {
+				for _, rpu := range s.nextCrawlers {
+					pu := rpu.JoinPath("/xrpc/com.atproto.sync.requestCrawl")
+					response, err := s.httpClient.Post(pu.String(), "application/json", bytes.NewReader(bodyBlob))
+					if response != nil && response.Body != nil {
+						response.Body.Close()
+					}
+					if err != nil || response == nil {
+						s.log.Warn("requestCrawl forward failed", "host", rpu, "err", err)
+					} else if response.StatusCode != http.StatusOK {
+						s.log.Warn("requestCrawl forward failed", "host", rpu, "status", response.Status)
+					} else {
+						s.log.Info("requestCrawl forward successful", "host", rpu)
+					}
+				}
+			}(blob)
+		}
+	}
+
+	return s.slurper.SubscribeToPds(ctx, host, true, false, nil)
+}
+
+func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor int64, limit int) (*comatprototypes.SyncListRepos_Output, error) {
+	// Load the accounts
+	accounts := []*Account{}
+	if err := s.db.Model(&Account{}).Where("id > ? AND NOT taken_down AND (upstream_status IS NULL OR upstream_status = 'active')", cursor).Order("id").Limit(limit).Find(&accounts).Error; err != nil {
+		if err == gorm.ErrRecordNotFound {
+			return &comatprototypes.SyncListRepos_Output{}, nil
+		}
+		s.log.Error("failed to query accounts", "err", err)
+		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to query accounts")
+	}
+
+	if len(accounts) == 0 {
+		// resp.Repos is an explicit empty array, not just 'nil'
+		return &comatprototypes.SyncListRepos_Output{
+			Repos: []*comatprototypes.SyncListRepos_Repo{},
+		}, nil
+	}
+
+	resp := &comatprototypes.SyncListRepos_Output{
+		Repos: make([]*comatprototypes.SyncListRepos_Repo, len(accounts)),
+	}
+
+	// Fetch the repo roots for each user
+	for i := range accounts {
+		user := accounts[i]
+
+		root, err := s.GetRepoRoot(ctx, user.ID)
+		if err != nil {
+			s.log.Error("failed to get repo root", "err", err, "did", user.Did)
+			return nil, echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("failed to get repo root for (%s): %v", user.Did, err.Error()))
+		}
+
+		resp.Repos[i] = &comatprototypes.SyncListRepos_Repo{
+			Did:  user.Did,
+			Head: root.String(),
+		}
+	}
+
+	// If this is not the last page, set the cursor
+	if len(accounts) >= limit && len(accounts) > 1 {
+		nextCursor := fmt.Sprintf("%d", accounts[len(accounts)-1].ID)
+		resp.Cursor = &nextCursor
+	}
+
+	return resp, nil
+}
+
+var ErrUserStatusUnavailable = errors.New("user status unavailable")
+
+func (s *BGS) handleComAtprotoSyncGetLatestCommit(ctx context.Context, did string) (*comatprototypes.SyncGetLatestCommit_Output, error) {
+	u, err := s.lookupUserByDid(ctx, did)
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil, echo.NewHTTPError(http.StatusNotFound, "user not found")
+		}
+		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user")
+	}
+
+	if u.GetTakenDown() {
+		return nil, fmt.Errorf("account was taken down by the Relay")
+	}
+
+	ustatus := u.GetUpstreamStatus()
+	if ustatus == events.AccountStatusTakendown {
+		return nil, fmt.Errorf("account was taken down by its PDS")
+	}
+
+	if ustatus == events.AccountStatusDeactivated {
+		return nil, fmt.Errorf("account is temporarily deactivated")
+	}
+
+	if ustatus == events.AccountStatusSuspended {
+		return nil, fmt.Errorf("account is suspended by its PDS")
+	}
+
+	var prevState AccountPreviousState
+	err = s.db.First(&prevState, u.ID).Error
+	if err == nil {
+		// okay!
+	} else if errors.Is(err, gorm.ErrRecordNotFound) {
+		return nil, ErrUserStatusUnavailable
+	} else {
+		s.log.Error("user db err", "err", err)
+		return nil, fmt.Errorf("user prev db err, %w", err)
+	}
+
+	return &comatprototypes.SyncGetLatestCommit_Output{
+		Cid: prevState.Cid.CID.String(),
+		Rev: prevState.Rev,
+	}, nil
+}

cmd/relay/bgs/metrics.go

···
+package bgs
+
+import (
+	"errors"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/labstack/echo/v4"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var eventsReceivedCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "events_received_counter",
+	Help: "The total number of events received",
+}, []string{"pds"})
+
+var eventsWarningsCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "events_warn_counter",
+	Help: "Events received with warnings",
+}, []string{"pds", "warn"})
+
+var eventsHandleDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "events_handle_duration",
+	Help:    "A histogram of handleFedEvent latencies",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+}, []string{"pds"})
+
+var repoCommitsReceivedCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "repo_commits_received_counter",
+	Help: "The total number of commit events received",
+}, []string{"pds"})
+var repoSyncReceivedCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "repo_sync_received_counter",
+	Help: "The total number of sync events received",
+}, []string{"pds"})
+
+var repoCommitsResultCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "repo_commits_result_counter",
+	Help: "The results of commit events received",
+}, []string{"pds", "status"})
+
+var eventsSentCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "events_sent_counter",
+	Help: "The total number of events sent to consumers",
+}, []string{"remote_addr", "user_agent"})
+
+var externalUserCreationAttempts = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "bgs_external_user_creation_attempts",
+	Help: "The total number of external users created",
+})
+
+var connectedInbound = promauto.NewGauge(prometheus.GaugeOpts{
+	Name: "bgs_connected_inbound",
+	Help: "Number of inbound firehoses we are consuming",
+})
+
+var newUsersDiscovered = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "bgs_new_users_discovered",
+	Help: "The total number of new users discovered directly from the firehose (not from refs)",
+})
+
+var reqSz = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "http_request_size_bytes",
+	Help:    "A histogram of request sizes for requests.",
+	Buckets: prometheus.ExponentialBuckets(100, 10, 8),
+}, []string{"code", "method", "path"})
+
+var reqDur = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "http_request_duration_seconds",
+	Help:    "A histogram of latencies for requests.",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+}, []string{"code", "method", "path"})
+
+var reqCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "http_requests_total",
+	Help: "A counter for requests to the wrapped handler.",
+}, []string{"code", "method", "path"})
+
+var resSz = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "http_response_size_bytes",
+	Help:    "A histogram of response sizes for requests.",
+	Buckets: prometheus.ExponentialBuckets(100, 10, 8),
+}, []string{"code", "method", "path"})
+
+var newUserDiscoveryDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "relay_new_user_discovery_duration",
+	Help:    "A histogram of new user discovery latencies",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+})
+
+var commitVerifyStarts = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "validator_commit_verify_starts",
+})
+
+var commitVerifyWarnings = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "validator_commit_verify_warnings",
+}, []string{"host", "warn"})
+
+// verify error and short code for why
+var commitVerifyErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "validator_commit_verify_errors",
+}, []string{"host", "err"})
+
+// ok and *fully verified*
+var commitVerifyOk = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "validator_commit_verify_ok",
+}, []string{"host"})
+
+// it's ok, but... {old protocol, no previous root cid, ...}
+var commitVerifyOkish = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "validator_commit_verify_okish",
+}, []string{"host", "but"})
+
+// verify error and short code for why
+var syncVerifyErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "validator_sync_verify_errors",
+}, []string{"host", "err"})
+
+var accountVerifyWarnings = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "validator_account_verify_warnings",
+	Help: "things that have been a little bit wrong with account messages",
+}, []string{"host", "warn"})
+
+// MetricsMiddleware defines handler function for metrics middleware
+func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		path := c.Path()
+		if path == "/metrics" || path == "/_health" {
+			return next(c)
+		}
+
+		start := time.Now()
+		requestSize := computeApproximateRequestSize(c.Request())
+
+		err := next(c)
+
+		status := c.Response().Status
+		if err != nil {
+			var httpError *echo.HTTPError
+			if errors.As(err, &httpError) {
+				status = httpError.Code
+			}
+			if status == 0 || status == http.StatusOK {
+				status = http.StatusInternalServerError
+			}
+		}
+
+		elapsed := float64(time.Since(start)) / float64(time.Second)
+
+		statusStr := strconv.Itoa(status)
+		method := c.Request().Method
+
+		responseSize := float64(c.Response().Size)
+
+		reqDur.WithLabelValues(statusStr, method, path).Observe(elapsed)
+		reqCnt.WithLabelValues(statusStr, method, path).Inc()
+		reqSz.WithLabelValues(statusStr, method, path).Observe(float64(requestSize))
+		resSz.WithLabelValues(statusStr, method, path).Observe(responseSize)
+
+		return err
+	}
+}
+
+func computeApproximateRequestSize(r *http.Request) int {
+	s := 0
+	if r.URL != nil {
+		s = len(r.URL.Path)
+	}
+
+	s += len(r.Method)
+	s += len(r.Proto)
+	for name, values := range r.Header {
+		s += len(name)
+		for _, value := range values {
+			s += len(value)
+		}
+	}
+	s += len(r.Host)
+
+	// N.B. r.Form and r.MultipartForm are assumed to be included in r.URL.
+
+	if r.ContentLength != -1 {
+		s += int(r.ContentLength)
+	}
+	return s
+}

cmd/relay/bgs/models.go

···
+package bgs
+
+import "gorm.io/gorm"
+
+type DomainBan struct {
+	gorm.Model
+	Domain string `gorm:"unique"`
+}

cmd/relay/bgs/stubs.go

···
+package bgs
+
+import (
+	"errors"
+	"fmt"
+	"gorm.io/gorm"
+	"net/http"
+	"strconv"
+
+	comatprototypes "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/atproto/syntax"
+	"github.com/labstack/echo/v4"
+	"go.opentelemetry.io/otel"
+)
+
+type XRPCError struct {
+	Message string `json:"message"`
+}
+
+func (s *BGS) RegisterHandlersAppBsky(e *echo.Echo) error {
+	return nil
+}
+
+func (s *BGS) RegisterHandlersComAtproto(e *echo.Echo) error {
+	e.GET("/xrpc/com.atproto.sync.getLatestCommit", s.HandleComAtprotoSyncGetLatestCommit)
+	e.GET("/xrpc/com.atproto.sync.listRepos", s.HandleComAtprotoSyncListRepos)
+	e.POST("/xrpc/com.atproto.sync.requestCrawl", s.HandleComAtprotoSyncRequestCrawl)
+	return nil
+}
+
+func (s *BGS) HandleComAtprotoSyncGetLatestCommit(c echo.Context) error {
+	ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncGetLatestCommit")
+	defer span.End()
+	did := c.QueryParam("did")
+
+	_, err := syntax.ParseDID(did)
+	if err != nil {
+		return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid did: %s", did)})
+	}
+
+	var out *comatprototypes.SyncGetLatestCommit_Output
+	var handleErr error
+	// func (s *BGS) handleComAtprotoSyncGetLatestCommit(ctx context.Context,did string) (*comatprototypes.SyncGetLatestCommit_Output, error)
+	out, handleErr = s.handleComAtprotoSyncGetLatestCommit(ctx, did)
+	if handleErr != nil {
+		return handleErr
+	}
+	return c.JSON(200, out)
+}
+
+func (s *BGS) HandleComAtprotoSyncListRepos(c echo.Context) error {
+	ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncListRepos")
+	defer span.End()
+
+	cursorQuery := c.QueryParam("cursor")
+	limitQuery := c.QueryParam("limit")
+
+	var err error
+
+	limit := 500
+	if limitQuery != "" {
+		limit, err = strconv.Atoi(limitQuery)
+		if err != nil || limit < 1 || limit > 1000 {
+			return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid limit: %s", limitQuery)})
+		}
+	}
+
+	cursor := int64(0)
+	if cursorQuery != "" {
+		cursor, err = strconv.ParseInt(cursorQuery, 10, 64)
+		if err != nil || cursor < 0 {
+			return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid cursor: %s", cursorQuery)})
+		}
+	}
+
+	out, handleErr := s.handleComAtprotoSyncListRepos(ctx, cursor, limit)
+	if handleErr != nil {
+		return handleErr
+	}
+	return c.JSON(200, out)
+}
+
+// HandleComAtprotoSyncGetRepo handles /xrpc/com.atproto.sync.getRepo
+// returns 3xx to same URL at source PDS
+func (s *BGS) HandleComAtprotoSyncGetRepo(c echo.Context) error {
+	// no request object, only params
+	params := c.QueryParams()
+	var did string
+	hasDid := false
+	for paramName, pvl := range params {
+		switch paramName {
+		case "did":
+			if len(pvl) == 1 {
+				did = pvl[0]
+				hasDid = true
+			} else if len(pvl) > 1 {
+				return c.JSON(http.StatusBadRequest, XRPCError{Message: "only allow one did param"})
+			}
+		case "since":
+			// ok
+		default:
+			return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid param: %s", paramName)})
+		}
+	}
+	if !hasDid {
+		return c.JSON(http.StatusBadRequest, XRPCError{Message: "need did param"})
+	}
+
+	var pdsHostname string
+	err := s.db.Raw("SELECT pds.host FROM users JOIN pds ON users.pds = pds.id WHERE users.did = ?", did).Scan(&pdsHostname).Error
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return c.JSON(http.StatusNotFound, XRPCError{Message: "NULL"})
+		}
+		s.log.Error("user.pds.host lookup", "err", err)
+		return c.JSON(http.StatusInternalServerError, XRPCError{Message: "sorry"})
+	}
+
+	nextUrl := *(c.Request().URL)
+	nextUrl.Host = pdsHostname
+	if nextUrl.Scheme == "" {
+		nextUrl.Scheme = "https"
+	}
+	return c.Redirect(http.StatusFound, nextUrl.String())
+}
+
+func (s *BGS) HandleComAtprotoSyncRequestCrawl(c echo.Context) error {
+	ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncRequestCrawl")
+	defer span.End()
+
+	var body comatprototypes.SyncRequestCrawl_Input
+	if err := c.Bind(&body); err != nil {
+		return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid body: %s", err)})
+	}
+	var handleErr error
+	// func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context,body *comatprototypes.SyncRequestCrawl_Input) error
+	handleErr = s.handleComAtprotoSyncRequestCrawl(ctx, &body)
+	if handleErr != nil {
+		return handleErr
+	}
+	return nil
+}

cmd/relay/bgs/validator.go

···
+package bgs
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log/slog"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	atproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/atproto/identity"
+	atrepo "github.com/bluesky-social/indigo/atproto/repo"
+	"github.com/bluesky-social/indigo/atproto/syntax"
+	"github.com/bluesky-social/indigo/cmd/relay/models"
+	"github.com/ipfs/go-cid"
+	"go.opentelemetry.io/otel"
+)
+
+const defaultMaxRevFuture = time.Hour
+
+func NewValidator(directory identity.Directory, inductionTraceLog *slog.Logger) *Validator {
+	maxRevFuture := defaultMaxRevFuture // TODO: configurable
+	ErrRevTooFarFuture := fmt.Errorf("new rev is > %s in the future", maxRevFuture)
+
+	return &Validator{
+		userLocks:         make(map[models.Uid]*userLock),
+		log:               slog.Default().With("system", "validator"),
+		inductionTraceLog: inductionTraceLog,
+		directory:         directory,
+
+		maxRevFuture:           maxRevFuture,
+		ErrRevTooFarFuture:     ErrRevTooFarFuture,
+		AllowSignatureNotFound: true, // TODO: configurable
+	}
+}
+
+// Validator contains the context and code necessary to validate #commit and #sync messages
+type Validator struct {
+	lklk      sync.Mutex
+	userLocks map[models.Uid]*userLock
+
+	log               *slog.Logger
+	inductionTraceLog *slog.Logger
+
+	directory identity.Directory
+
+	// maxRevFuture is added to time.Now() for a limit of clock skew we'll accept a `rev` in the future for
+	maxRevFuture time.Duration
+
+	// ErrRevTooFarFuture is the error we return
+	// held here because we fmt.Errorf() once with our configured maxRevFuture into the message
+	ErrRevTooFarFuture error
+
+	// AllowSignatureNotFound enables counting messages without findable public key to pass through with a warning counter
+	// TODO: refine this for what kind of 'not found' we accept.
+	AllowSignatureNotFound bool
+}
+
+type NextCommitHandler interface {
+	HandleCommit(ctx context.Context, host *models.PDS, uid models.Uid, did string, commit *atproto.SyncSubscribeRepos_Commit) error
+}
+
+type userLock struct {
+	lk      sync.Mutex
+	waiters atomic.Int32
+}
+
+// lockUser re-serializes access per-user after events may have been fanned out to many worker threads by events/schedulers/parallel
+func (val *Validator) lockUser(ctx context.Context, user models.Uid) func() {
+	ctx, span := otel.Tracer("validator").Start(ctx, "userLock")
+	defer span.End()
+
+	val.lklk.Lock()
+
+	ulk, ok := val.userLocks[user]
+	if !ok {
+		ulk = &userLock{}
+		val.userLocks[user] = ulk
+	}
+
+	ulk.waiters.Add(1)
+
+	val.lklk.Unlock()
+
+	ulk.lk.Lock()
+
+	return func() {
+		val.lklk.Lock()
+		defer val.lklk.Unlock()
+
+		ulk.lk.Unlock()
+
+		nv := ulk.waiters.Add(-1)
+
+		if nv == 0 {
+			delete(val.userLocks, user)
+		}
+	}
+}
+
+func (val *Validator) HandleCommit(ctx context.Context, host *models.PDS, account *Account, commit *atproto.SyncSubscribeRepos_Commit, prevRoot *AccountPreviousState) (newRoot *cid.Cid, err error) {
+	uid := account.GetUid()
+	unlock := val.lockUser(ctx, uid)
+	defer unlock()
+	repoFragment, err := val.VerifyCommitMessage(ctx, host, commit, prevRoot)
+	if err != nil {
+		return nil, err
+	}
+	newRootCid, err := repoFragment.MST.RootCID()
+	if err != nil {
+		return nil, err
+	}
+	return newRootCid, nil
+}
+
+type revOutOfOrderError struct {
+	dt time.Duration
+}
+
+func (roooe *revOutOfOrderError) Error() string {
+	return fmt.Sprintf("new rev is before previous rev by %s", roooe.dt.String())
+}
+
+var ErrNewRevBeforePrevRev = &revOutOfOrderError{}
+
+func (val *Validator) VerifyCommitMessage(ctx context.Context, host *models.PDS, msg *atproto.SyncSubscribeRepos_Commit, prevRoot *AccountPreviousState) (*atrepo.Repo, error) {
+	hostname := host.Host
+	hasWarning := false
+	commitVerifyStarts.Inc()
+	logger := slog.Default().With("did", msg.Repo, "rev", msg.Rev, "seq", msg.Seq, "time", msg.Time)
+
+	did, err := syntax.ParseDID(msg.Repo)
+	if err != nil {
+		commitVerifyErrors.WithLabelValues(hostname, "did").Inc()
+		return nil, err
+	}
+	rev, err := syntax.ParseTID(msg.Rev)
+	if err != nil {
+		commitVerifyErrors.WithLabelValues(hostname, "tid").Inc()
+		return nil, err
+	}
+	if prevRoot != nil {
+		prevRev := prevRoot.GetRev()
+		curTime := rev.Time()
+		prevTime := prevRev.Time()
+		if curTime.Before(prevTime) {
+			commitVerifyErrors.WithLabelValues(hostname, "revb").Inc()
+			dt := prevTime.Sub(curTime)
+			return nil, &revOutOfOrderError{dt}
+		}
+	}
+	if rev.Time().After(time.Now().Add(val.maxRevFuture)) {
+		commitVerifyErrors.WithLabelValues(hostname, "revf").Inc()
+		return nil, val.ErrRevTooFarFuture
+	}
+	_, err = syntax.ParseDatetime(msg.Time)
+	if err != nil {
+		commitVerifyErrors.WithLabelValues(hostname, "time").Inc()
+		return nil, err
+	}
+
+	if msg.TooBig {
+		//logger.Warn("event with tooBig flag set")
+		commitVerifyWarnings.WithLabelValues(hostname, "big").Inc()
+		val.inductionTraceLog.Warn("commit tooBig", "seq", msg.Seq, "pdsHost", host.Host, "repo", msg.Repo)
+		hasWarning = true
+	}
+	if msg.Rebase {
+		//logger.Warn("event with rebase flag set")
+		commitVerifyWarnings.WithLabelValues(hostname, "reb").Inc()
+		val.inductionTraceLog.Warn("commit rebase", "seq", msg.Seq, "pdsHost", host.Host, "repo", msg.Repo)
+		hasWarning = true
+	}
+
+	commit, repoFragment, err := atrepo.LoadFromCAR(ctx, bytes.NewReader([]byte(msg.Blocks)))
+	if err != nil {
+		commitVerifyErrors.WithLabelValues(hostname, "car").Inc()
+		return nil, err
+	}
+
+	if commit.Rev != rev.String() {
+		commitVerifyErrors.WithLabelValues(hostname, "rev").Inc()
+		return nil, fmt.Errorf("rev did not match commit")
+	}
+	if commit.DID != did.String() {
+		commitVerifyErrors.WithLabelValues(hostname, "did2").Inc()
+		return nil, fmt.Errorf("rev did not match commit")
+	}
+
+	err = val.VerifyCommitSignature(ctx, commit, hostname, &hasWarning)
+	if err != nil {
+		// signature errors are metrics counted inside VerifyCommitSignature()
+		return nil, err
+	}
+
+	// load out all the records
+	for _, op := range msg.Ops {
+		if (op.Action == "create" || op.Action == "update") && op.Cid != nil {
+			c := (*cid.Cid)(op.Cid)
+			nsid, rkey, err := syntax.ParseRepoPath(op.Path)
+			if err != nil {
+				commitVerifyErrors.WithLabelValues(hostname, "opp").Inc()
+				return nil, fmt.Errorf("invalid repo path in ops list: %w", err)
+			}
+			val, err := repoFragment.GetRecordCID(ctx, nsid, rkey)
+			if err != nil {
+				commitVerifyErrors.WithLabelValues(hostname, "rcid").Inc()
+				return nil, err
+			}
+			if *c != *val {
+				commitVerifyErrors.WithLabelValues(hostname, "opc").Inc()
+				return nil, fmt.Errorf("record op doesn't match MST tree value")
+			}
+			_, _, err = repoFragment.GetRecordBytes(ctx, nsid, rkey)
+			if err != nil {
+				commitVerifyErrors.WithLabelValues(hostname, "rec").Inc()
+				return nil, err
+			}
+		}
+	}
+
+	// TODO: once firehose format is fully shipped, remove this
+	for _, o := range msg.Ops {
+		switch o.Action {
+		case "delete":
+			if o.Prev == nil {
+				logger.Debug("can't invert legacy op", "action", o.Action)
+				val.inductionTraceLog.Warn("commit delete op", "seq", msg.Seq, "pdsHost", host.Host, "repo", msg.Repo)
+				commitVerifyOkish.WithLabelValues(hostname, "del").Inc()
+				return repoFragment, nil
+			}
+		case "update":
+			if o.Prev == nil {
+				logger.Debug("can't invert legacy op", "action", o.Action)
+				val.inductionTraceLog.Warn("commit update op", "seq", msg.Seq, "pdsHost", host.Host, "repo", msg.Repo)
+				commitVerifyOkish.WithLabelValues(hostname, "up").Inc()
+				return repoFragment, nil
+			}
+		}
+	}
+
+	if msg.PrevData != nil {
+		c := (*cid.Cid)(msg.PrevData)
+		if prevRoot != nil {
+			if *c != prevRoot.GetCid() {
+				commitVerifyWarnings.WithLabelValues(hostname, "pr").Inc()
+				val.inductionTraceLog.Warn("commit prevData mismatch", "seq", msg.Seq, "pdsHost", host.Host, "repo", msg.Repo)
+				hasWarning = true
+			}
+		} else {
+			// see counter below for okish "new"
+		}
+
+		// check internal consistency that claimed previous root matches the rest of this message
+		ops, err := ParseCommitOps(msg.Ops)
+		if err != nil {
+			commitVerifyErrors.WithLabelValues(hostname, "pop").Inc()
+			return nil, err
+		}
+		ops, err = atrepo.NormalizeOps(ops)
+		if err != nil {
+			commitVerifyErrors.WithLabelValues(hostname, "nop").Inc()
+			return nil, err
+		}
+
+		invTree := repoFragment.MST.Copy()
+		for _, op := range ops {
+			if err := atrepo.InvertOp(&invTree, &op); err != nil {
+				commitVerifyErrors.WithLabelValues(hostname, "inv").Inc()
+				return nil, err
+			}
+		}
+		computed, err := invTree.RootCID()
+		if err != nil {
+			commitVerifyErrors.WithLabelValues(hostname, "it").Inc()
+			return nil, err
+		}
+		if *computed != *c {
+			// this is self-inconsistent malformed data
+			commitVerifyErrors.WithLabelValues(hostname, "pd").Inc()
+			return nil, fmt.Errorf("inverted tree root didn't match prevData")
+		}
+		//logger.Debug("prevData matched", "prevData", c.String(), "computed", computed.String())
+
+		if prevRoot == nil {
+			commitVerifyOkish.WithLabelValues(hostname, "new").Inc()
+		} else if hasWarning {
+			commitVerifyOkish.WithLabelValues(hostname, "warn").Inc()
+		} else {
+			// TODO: would it be better to make everything "okish"?
+			// commitVerifyOkish.WithLabelValues(hostname, "ok").Inc()
+			commitVerifyOk.WithLabelValues(hostname).Inc()
+		}
+	} else {
+		// this source is still on old protocol without new prevData field
+		commitVerifyOkish.WithLabelValues(hostname, "old").Inc()
+	}
+
+	return repoFragment, nil
+}
+
+// HandleSync checks signed commit from a #sync message
+func (val *Validator) HandleSync(ctx context.Context, host *models.PDS, msg *atproto.SyncSubscribeRepos_Sync) (newRoot *cid.Cid, err error) {
+	hostname := host.Host
+	hasWarning := false
+
+	did, err := syntax.ParseDID(msg.Did)
+	if err != nil {
+		syncVerifyErrors.WithLabelValues(hostname, "did").Inc()
+		return nil, err
+	}
+	rev, err := syntax.ParseTID(msg.Rev)
+	if err != nil {
+		syncVerifyErrors.WithLabelValues(hostname, "tid").Inc()
+		return nil, err
+	}
+	if rev.Time().After(time.Now().Add(val.maxRevFuture)) {
+		syncVerifyErrors.WithLabelValues(hostname, "revf").Inc()
+		return nil, val.ErrRevTooFarFuture
+	}
+	_, err = syntax.ParseDatetime(msg.Time)
+	if err != nil {
+		syncVerifyErrors.WithLabelValues(hostname, "time").Inc()
+		return nil, err
+	}
+
+	commit, err := atrepo.LoadCARCommit(ctx, bytes.NewReader([]byte(msg.Blocks)))
+	if err != nil {
+		commitVerifyErrors.WithLabelValues(hostname, "car").Inc()
+		return nil, err
+	}
+
+	if commit.Rev != rev.String() {
+		commitVerifyErrors.WithLabelValues(hostname, "rev").Inc()
+		return nil, fmt.Errorf("rev did not match commit")
+	}
+	if commit.DID != did.String() {
+		commitVerifyErrors.WithLabelValues(hostname, "did2").Inc()
+		return nil, fmt.Errorf("rev did not match commit")
+	}
+
+	err = val.VerifyCommitSignature(ctx, commit, hostname, &hasWarning)
+	if err != nil {
+		// signature errors are metrics counted inside VerifyCommitSignature()
+		return nil, err
+	}
+
+	return &commit.Data, nil
+}
+
+// TODO: lift back to indigo/atproto/repo util code?
+func ParseCommitOps(ops []*atproto.SyncSubscribeRepos_RepoOp) ([]atrepo.Operation, error) {
+	out := []atrepo.Operation{}
+	for _, rop := range ops {
+		switch rop.Action {
+		case "create":
+			if rop.Cid == nil || rop.Prev != nil {
+				return nil, fmt.Errorf("invalid repoOp: create")
+			}
+			op := atrepo.Operation{
+				Path:  rop.Path,
+				Prev:  nil,
+				Value: (*cid.Cid)(rop.Cid),
+			}
+			out = append(out, op)
+		case "delete":
+			if rop.Cid != nil || rop.Prev == nil {
+				return nil, fmt.Errorf("invalid repoOp: delete")
+			}
+			op := atrepo.Operation{
+				Path:  rop.Path,
+				Prev:  (*cid.Cid)(rop.Prev),
+				Value: nil,
+			}
+			out = append(out, op)
+		case "update":
+			if rop.Cid == nil || rop.Prev == nil {
+				return nil, fmt.Errorf("invalid repoOp: update")
+			}
+			op := atrepo.Operation{
+				Path:  rop.Path,
+				Prev:  (*cid.Cid)(rop.Prev),
+				Value: (*cid.Cid)(rop.Cid),
+			}
+			out = append(out, op)
+		default:
+			return nil, fmt.Errorf("invalid repoOp action: %s", rop.Action)
+		}
+	}
+	return out, nil
+}
+
+// VerifyCommitSignature get's repo's registered public key from Identity Directory, verifies Commit
+// hostname is just for metrics in case of error
+func (val *Validator) VerifyCommitSignature(ctx context.Context, commit *atrepo.Commit, hostname string, hasWarning *bool) error {
+	if val.directory == nil {
+		return nil
+	}
+	xdid, err := syntax.ParseDID(commit.DID)
+	if err != nil {
+		commitVerifyErrors.WithLabelValues(hostname, "sig1").Inc()
+		return fmt.Errorf("bad car DID, %w", err)
+	}
+	ident, err := val.directory.LookupDID(ctx, xdid)
+	if err != nil {
+		if val.AllowSignatureNotFound {
+			// allow not-found conditions to pass without signature check
+			commitVerifyWarnings.WithLabelValues(hostname, "nok").Inc()
+			if hasWarning != nil {
+				*hasWarning = true
+			}
+			return nil
+		}
+		commitVerifyErrors.WithLabelValues(hostname, "sig2").Inc()
+		return fmt.Errorf("DID lookup failed, %w", err)
+	}
+	pk, err := ident.GetPublicKey("atproto")
+	if err != nil {
+		commitVerifyErrors.WithLabelValues(hostname, "sig3").Inc()
+		return fmt.Errorf("no atproto pubkey, %w", err)
+	}
+	err = commit.VerifySignature(pk)
+	if err != nil {
+		// TODO: if the DID document was stale, force re-fetch from source and re-try if pubkey has changed
+		commitVerifyErrors.WithLabelValues(hostname, "sig4").Inc()
+		return fmt.Errorf("invalid signature, %w", err)
+	}
+	return nil
+}

cmd/relay/events/cbor_gen.go

···
+// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
+
+package events
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"sort"
+
+	cid "github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	xerrors "golang.org/x/xerrors"
+)
+
+var _ = xerrors.Errorf
+var _ = cid.Undef
+var _ = math.E
+var _ = sort.Sort
+
+func (t *EventHeader) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
+		return err
+	}
+
+	// t.MsgType (string) (string)
+	if len("t") > 1000000 {
+		return xerrors.Errorf("Value in field \"t\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("t"))); err != nil {
+		return err
+	}
+	if _, err := cw.WriteString(string("t")); err != nil {
+		return err
+	}
+
+	if len(t.MsgType) > 1000000 {
+		return xerrors.Errorf("Value in field t.MsgType was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.MsgType))); err != nil {
+		return err
+	}
+	if _, err := cw.WriteString(string(t.MsgType)); err != nil {
+		return err
+	}
+
+	// t.Op (int64) (int64)
+	if len("op") > 1000000 {
+		return xerrors.Errorf("Value in field \"op\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("op"))); err != nil {
+		return err
+	}
+	if _, err := cw.WriteString(string("op")); err != nil {
+		return err
+	}
+
+	if t.Op >= 0 {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Op)); err != nil {
+			return err
+		}
+	} else {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Op-1)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (t *EventHeader) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = EventHeader{}
+
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
+	if maj != cbg.MajMap {
+		return fmt.Errorf("cbor input should be of type map")
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("EventHeader: map struct too large (%d)", extra)
+	}
+
+	n := extra
+
+	nameBuf := make([]byte, 2)
+	for i := uint64(0); i < n; i++ {
+		nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000)
+		if err != nil {
+			return err
+		}
+
+		if !ok {
+			// Field doesn't exist on this type, so ignore it
+			if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil {
+				return err
+			}
+			continue
+		}
+
+		switch string(nameBuf[:nameLen]) {
+		// t.MsgType (string) (string)
+		case "t":
+
+			{
+				sval, err := cbg.ReadStringWithMax(cr, 1000000)
+				if err != nil {
+					return err
+				}
+
+				t.MsgType = string(sval)
+			}
+			// t.Op (int64) (int64)
+		case "op":
+			{
+				maj, extra, err := cr.ReadHeader()
+				if err != nil {
+					return err
+				}
+				var extraI int64
+				switch maj {
+				case cbg.MajUnsignedInt:
+					extraI = int64(extra)
+					if extraI < 0 {
+						return fmt.Errorf("int64 positive overflow")
+					}
+				case cbg.MajNegativeInt:
+					extraI = int64(extra)
+					if extraI < 0 {
+						return fmt.Errorf("int64 negative overflow")
+					}
+					extraI = -1 - extraI
+				default:
+					return fmt.Errorf("wrong type for int64 field: %d", maj)
+				}
+
+				t.Op = int64(extraI)
+			}
+
+		default:
+			// Field doesn't exist on this type, so ignore it
+			if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+func (t *ErrorFrame) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
+		return err
+	}
+
+	// t.Error (string) (string)
+	if len("error") > 1000000 {
+		return xerrors.Errorf("Value in field \"error\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("error"))); err != nil {
+		return err
+	}
+	if _, err := cw.WriteString(string("error")); err != nil {
+		return err
+	}
+
+	if len(t.Error) > 1000000 {
+		return xerrors.Errorf("Value in field t.Error was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Error))); err != nil {
+		return err
+	}
+	if _, err := cw.WriteString(string(t.Error)); err != nil {
+		return err
+	}
+
+	// t.Message (string) (string)
+	if len("message") > 1000000 {
+		return xerrors.Errorf("Value in field \"message\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil {
+		return err
+	}
+	if _, err := cw.WriteString(string("message")); err != nil {
+		return err
+	}
+
+	if len(t.Message) > 1000000 {
+		return xerrors.Errorf("Value in field t.Message was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Message))); err != nil {
+		return err
+	}
+	if _, err := cw.WriteString(string(t.Message)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *ErrorFrame) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = ErrorFrame{}
+
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
+	if maj != cbg.MajMap {
+		return fmt.Errorf("cbor input should be of type map")
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("ErrorFrame: map struct too large (%d)", extra)
+	}
+
+	n := extra
+
+	nameBuf := make([]byte, 7)
+	for i := uint64(0); i < n; i++ {
+		nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000)
+		if err != nil {
+			return err
+		}
+
+		if !ok {
+			// Field doesn't exist on this type, so ignore it
+			if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil {
+				return err
+			}
+			continue
+		}
+
+		switch string(nameBuf[:nameLen]) {
+		// t.Error (string) (string)
+		case "error":
+
+			{
+				sval, err := cbg.ReadStringWithMax(cr, 1000000)
+				if err != nil {
+					return err
+				}
+
+				t.Error = string(sval)
+			}
+			// t.Message (string) (string)
+		case "message":
+
+			{
+				sval, err := cbg.ReadStringWithMax(cr, 1000000)
+				if err != nil {
+					return err
+				}
+
+				t.Message = string(sval)
+			}
+
+		default:
+			// Field doesn't exist on this type, so ignore it
+			if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

cmd/relay/events/consumer.go

···
+package events
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"time"
+
+	"github.com/RussellLuo/slidingwindow"
+	comatproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/gorilla/websocket"
+)
+
+type RepoStreamCallbacks struct {
+	RepoCommit    func(evt *comatproto.SyncSubscribeRepos_Commit) error
+	RepoSync      func(evt *comatproto.SyncSubscribeRepos_Sync) error
+	RepoHandle    func(evt *comatproto.SyncSubscribeRepos_Handle) error
+	RepoIdentity  func(evt *comatproto.SyncSubscribeRepos_Identity) error
+	RepoAccount   func(evt *comatproto.SyncSubscribeRepos_Account) error
+	RepoInfo      func(evt *comatproto.SyncSubscribeRepos_Info) error
+	RepoMigrate   func(evt *comatproto.SyncSubscribeRepos_Migrate) error
+	RepoTombstone func(evt *comatproto.SyncSubscribeRepos_Tombstone) error
+	LabelLabels   func(evt *comatproto.LabelSubscribeLabels_Labels) error
+	LabelInfo     func(evt *comatproto.LabelSubscribeLabels_Info) error
+	Error         func(evt *ErrorFrame) error
+}
+
+func (rsc *RepoStreamCallbacks) EventHandler(ctx context.Context, xev *XRPCStreamEvent) error {
+	switch {
+	case xev.RepoCommit != nil && rsc.RepoCommit != nil:
+		return rsc.RepoCommit(xev.RepoCommit)
+	case xev.RepoSync != nil && rsc.RepoSync != nil:
+		return rsc.RepoSync(xev.RepoSync)
+	case xev.RepoHandle != nil && rsc.RepoHandle != nil:
+		return rsc.RepoHandle(xev.RepoHandle)
+	case xev.RepoInfo != nil && rsc.RepoInfo != nil:
+		return rsc.RepoInfo(xev.RepoInfo)
+	case xev.RepoMigrate != nil && rsc.RepoMigrate != nil:
+		return rsc.RepoMigrate(xev.RepoMigrate)
+	case xev.RepoIdentity != nil && rsc.RepoIdentity != nil:
+		return rsc.RepoIdentity(xev.RepoIdentity)
+	case xev.RepoAccount != nil && rsc.RepoAccount != nil:
+		return rsc.RepoAccount(xev.RepoAccount)
+	case xev.RepoTombstone != nil && rsc.RepoTombstone != nil:
+		return rsc.RepoTombstone(xev.RepoTombstone)
+	case xev.LabelLabels != nil && rsc.LabelLabels != nil:
+		return rsc.LabelLabels(xev.LabelLabels)
+	case xev.LabelInfo != nil && rsc.LabelInfo != nil:
+		return rsc.LabelInfo(xev.LabelInfo)
+	case xev.Error != nil && rsc.Error != nil:
+		return rsc.Error(xev.Error)
+	default:
+		return nil
+	}
+}
+
+type InstrumentedRepoStreamCallbacks struct {
+	limiters []*slidingwindow.Limiter
+	Next     func(ctx context.Context, xev *XRPCStreamEvent) error
+}
+
+func NewInstrumentedRepoStreamCallbacks(limiters []*slidingwindow.Limiter, next func(ctx context.Context, xev *XRPCStreamEvent) error) *InstrumentedRepoStreamCallbacks {
+	return &InstrumentedRepoStreamCallbacks{
+		limiters: limiters,
+		Next:     next,
+	}
+}
+
+func waitForLimiter(ctx context.Context, lim *slidingwindow.Limiter) error {
+	if lim.Allow() {
+		return nil
+	}
+
+	// wait until the limiter is ready (check every 100ms)
+	t := time.NewTicker(100 * time.Millisecond)
+	defer t.Stop()
+
+	for !lim.Allow() {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+
+	return nil
+}
+
+func (rsc *InstrumentedRepoStreamCallbacks) EventHandler(ctx context.Context, xev *XRPCStreamEvent) error {
+	// Wait on all limiters before calling the next handler
+	for _, lim := range rsc.limiters {
+		if err := waitForLimiter(ctx, lim); err != nil {
+			return err
+		}
+	}
+	return rsc.Next(ctx, xev)
+}
+
+type instrumentedReader struct {
+	r            io.Reader
+	addr         string
+	bytesCounter prometheus.Counter
+}
+
+func (sr *instrumentedReader) Read(p []byte) (int, error) {
+	n, err := sr.r.Read(p)
+	sr.bytesCounter.Add(float64(n))
+	return n, err
+}
+
+// HandleRepoStream
+// con is source of events
+// sched gets AddWork for each event
+// log may be nil for default logger
+func HandleRepoStream(ctx context.Context, con *websocket.Conn, sched Scheduler, log *slog.Logger) error {
+	if log == nil {
+		log = slog.Default().With("system", "events")
+	}
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	defer sched.Shutdown()
+
+	remoteAddr := con.RemoteAddr().String()
+
+	go func() {
+		t := time.NewTicker(time.Second * 30)
+		defer t.Stop()
+		failcount := 0
+
+		for {
+
+			select {
+			case <-t.C:
+				if err := con.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second*10)); err != nil {
+					log.Warn("failed to ping", "err", err)
+					failcount++
+					if failcount >= 4 {
+						log.Error("too many ping fails", "count", failcount)
+						con.Close()
+						return
+					}
+				} else {
+					failcount = 0 // ok ping
+				}
+			case <-ctx.Done():
+				con.Close()
+				return
+			}
+		}
+	}()
+
+	con.SetPingHandler(func(message string) error {
+		err := con.WriteControl(websocket.PongMessage, []byte(message), time.Now().Add(time.Second*60))
+		if err == websocket.ErrCloseSent {
+			return nil
+		} else if e, ok := err.(net.Error); ok && e.Temporary() {
+			return nil
+		}
+		return err
+	})
+
+	con.SetPongHandler(func(_ string) error {
+		if err := con.SetReadDeadline(time.Now().Add(time.Minute)); err != nil {
+			log.Error("failed to set read deadline", "err", err)
+		}
+
+		return nil
+	})
+
+	lastSeq := int64(-1)
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		mt, rawReader, err := con.NextReader()
+		if err != nil {
+			return fmt.Errorf("con err at read: %w", err)
+		}
+
+		switch mt {
+		default:
+			return fmt.Errorf("expected binary message from subscription endpoint")
+		case websocket.BinaryMessage:
+			// ok
+		}
+
+		r := &instrumentedReader{
+			r:            rawReader,
+			addr:         remoteAddr,
+			bytesCounter: bytesFromStreamCounter.WithLabelValues(remoteAddr),
+		}
+
+		var header EventHeader
+		if err := header.UnmarshalCBOR(r); err != nil {
+			return fmt.Errorf("reading header: %w", err)
+		}
+
+		eventsFromStreamCounter.WithLabelValues(remoteAddr).Inc()
+
+		switch header.Op {
+		case EvtKindMessage:
+			switch header.MsgType {
+			case "#commit":
+				var evt comatproto.SyncSubscribeRepos_Commit
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return fmt.Errorf("reading repoCommit event: %w", err)
+				}
+
+				if evt.Seq < lastSeq {
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
+				}
+
+				lastSeq = evt.Seq
+
+				if err := sched.AddWork(ctx, evt.Repo, &XRPCStreamEvent{
+					RepoCommit: &evt,
+				}); err != nil {
+					return err
+				}
+			case "#sync":
+				var evt comatproto.SyncSubscribeRepos_Sync
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return fmt.Errorf("reading repoSync event: %w", err)
+				}
+
+				if evt.Seq < lastSeq {
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
+				}
+
+				lastSeq = evt.Seq
+
+				if err := sched.AddWork(ctx, evt.Did, &XRPCStreamEvent{
+					RepoSync: &evt,
+				}); err != nil {
+					return err
+				}
+			case "#handle":
+				// TODO: DEPRECATED message; warning/counter; drop message
+				var evt comatproto.SyncSubscribeRepos_Handle
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return err
+				}
+
+				if evt.Seq < lastSeq {
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
+				}
+				lastSeq = evt.Seq
+
+				if err := sched.AddWork(ctx, evt.Did, &XRPCStreamEvent{
+					RepoHandle: &evt,
+				}); err != nil {
+					return err
+				}
+			case "#identity":
+				var evt comatproto.SyncSubscribeRepos_Identity
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return err
+				}
+
+				if evt.Seq < lastSeq {
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
+				}
+				lastSeq = evt.Seq
+
+				if err := sched.AddWork(ctx, evt.Did, &XRPCStreamEvent{
+					RepoIdentity: &evt,
+				}); err != nil {
+					return err
+				}
+			case "#account":
+				var evt comatproto.SyncSubscribeRepos_Account
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return err
+				}
+
+				if evt.Seq < lastSeq {
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
+				}
+				lastSeq = evt.Seq
+
+				if err := sched.AddWork(ctx, evt.Did, &XRPCStreamEvent{
+					RepoAccount: &evt,
+				}); err != nil {
+					return err
+				}
+			case "#info":
+				// TODO: this might also be a LabelInfo (as opposed to RepoInfo)
+				var evt comatproto.SyncSubscribeRepos_Info
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return err
+				}
+
+				if err := sched.AddWork(ctx, "", &XRPCStreamEvent{
+					RepoInfo: &evt,
+				}); err != nil {
+					return err
+				}
+			case "#migrate":
+				// TODO: DEPRECATED message; warning/counter; drop message
+				var evt comatproto.SyncSubscribeRepos_Migrate
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return err
+				}
+
+				if evt.Seq < lastSeq {
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
+				}
+				lastSeq = evt.Seq
+
+				if err := sched.AddWork(ctx, evt.Did, &XRPCStreamEvent{
+					RepoMigrate: &evt,
+				}); err != nil {
+					return err
+				}
+			case "#tombstone":
+				// TODO: DEPRECATED message; warning/counter; drop message
+				var evt comatproto.SyncSubscribeRepos_Tombstone
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return err
+				}
+
+				if evt.Seq < lastSeq {
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
+				}
+				lastSeq = evt.Seq
+
+				if err := sched.AddWork(ctx, evt.Did, &XRPCStreamEvent{
+					RepoTombstone: &evt,
+				}); err != nil {
+					return err
+				}
+			case "#labels":
+				var evt comatproto.LabelSubscribeLabels_Labels
+				if err := evt.UnmarshalCBOR(r); err != nil {
+					return fmt.Errorf("reading Labels event: %w", err)
+				}
+
+				if evt.Seq < lastSeq {
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
+				}
+
+				lastSeq = evt.Seq
+
+				if err := sched.AddWork(ctx, "", &XRPCStreamEvent{
+					LabelLabels: &evt,
+				}); err != nil {
+					return err
+				}
+			}
+
+		case EvtKindErrorFrame:
+			var errframe ErrorFrame
+			if err := errframe.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+
+			if err := sched.AddWork(ctx, "", &XRPCStreamEvent{
+				Error: &errframe,
+			}); err != nil {
+				return err
+			}
+
+		default:
+			return fmt.Errorf("unrecognized event stream type: %d", header.Op)
+		}
+
+	}
+}

cmd/relay/events/diskpersist/diskpersist.go

···
+package diskpersist
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"github.com/bluesky-social/indigo/cmd/relay/events"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/cmd/relay/models"
+	arc "github.com/hashicorp/golang-lru/arc/v2"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"gorm.io/gorm"
+)
+
+type DiskPersistence struct {
+	primaryDir      string
+	archiveDir      string
+	eventsPerFile   int64
+	writeBufferSize int
+	retention       time.Duration
+
+	meta *gorm.DB
+
+	broadcast func(*events.XRPCStreamEvent)
+
+	logfi *os.File
+
+	eventCounter int64
+	curSeq       int64
+	timeSequence bool
+
+	uids     UidSource
+	uidCache *arc.ARCCache[models.Uid, string] // TODO: unused
+	didCache *arc.ARCCache[string, models.Uid]
+
+	writers *sync.Pool
+	buffers *sync.Pool
+	scratch []byte
+
+	outbuf *bytes.Buffer
+	evtbuf []persistJob
+
+	shutdown chan struct{}
+
+	log *slog.Logger
+
+	lk sync.Mutex
+}
+
+type persistJob struct {
+	Bytes  []byte
+	Evt    *events.XRPCStreamEvent
+	Buffer *bytes.Buffer // so we can put it back in the pool when we're done
+}
+
+type jobResult struct {
+	Err error
+	Seq int64
+}
+
+const (
+	EvtFlagTakedown = 1 << iota
+	EvtFlagRebased
+)
+
+var _ (events.EventPersistence) = (*DiskPersistence)(nil)
+
+type DiskPersistOptions struct {
+	UIDCacheSize    int
+	DIDCacheSize    int
+	EventsPerFile   int64
+	WriteBufferSize int
+	Retention       time.Duration
+
+	Logger *slog.Logger
+
+	TimeSequence bool
+}
+
+func DefaultDiskPersistOptions() *DiskPersistOptions {
+	return &DiskPersistOptions{
+		EventsPerFile:   10_000,
+		UIDCacheSize:    1_000_000,
+		DIDCacheSize:    1_000_000,
+		WriteBufferSize: 50,
+		Retention:       time.Hour * 24 * 3, // 3 days
+	}
+}
+
+type UidSource interface {
+	DidToUid(ctx context.Context, did string) (models.Uid, error)
+}
+
+func NewDiskPersistence(primaryDir, archiveDir string, db *gorm.DB, opts *DiskPersistOptions) (*DiskPersistence, error) {
+	if opts == nil {
+		opts = DefaultDiskPersistOptions()
+	}
+
+	uidCache, err := arc.NewARC[models.Uid, string](opts.UIDCacheSize)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create uid cache: %w", err)
+	}
+
+	didCache, err := arc.NewARC[string, models.Uid](opts.DIDCacheSize)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create did cache: %w", err)
+	}
+
+	db.AutoMigrate(&LogFileRef{})
+
+	bufpool := &sync.Pool{
+		New: func() any {
+			return new(bytes.Buffer)
+		},
+	}
+
+	wrpool := &sync.Pool{
+		New: func() any {
+			return cbg.NewCborWriter(nil)
+		},
+	}
+
+	dp := &DiskPersistence{
+		meta:            db,
+		primaryDir:      primaryDir,
+		archiveDir:      archiveDir,
+		buffers:         bufpool,
+		retention:       opts.Retention,
+		writers:         wrpool,
+		uidCache:        uidCache,
+		didCache:        didCache,
+		eventsPerFile:   opts.EventsPerFile,
+		scratch:         make([]byte, headerSize),
+		outbuf:          new(bytes.Buffer),
+		writeBufferSize: opts.WriteBufferSize,
+		shutdown:        make(chan struct{}),
+		timeSequence:    opts.TimeSequence,
+		log:             opts.Logger,
+	}
+	if dp.log == nil {
+		dp.log = slog.Default().With("system", "diskpersist")
+	}
+
+	if err := dp.resumeLog(); err != nil {
+		return nil, err
+	}
+
+	go dp.flushRoutine()
+
+	go dp.garbageCollectRoutine()
+
+	return dp, nil
+}
+
+type LogFileRef struct {
+	gorm.Model
+	Path     string
+	Archived bool
+	SeqStart int64
+}
+
+func (dp *DiskPersistence) SetUidSource(uids UidSource) {
+	dp.uids = uids
+}
+
+func (dp *DiskPersistence) resumeLog() error {
+	var lfr LogFileRef
+	if err := dp.meta.Order("seq_start desc").Limit(1).Find(&lfr).Error; err != nil {
+		return err
+	}
+
+	if lfr.ID == 0 {
+		// no files, start anew!
+		return dp.initLogFile()
+	}
+
+	// 0 for the mode is fine since that is only used if O_CREAT is passed
+	fi, err := os.OpenFile(filepath.Join(dp.primaryDir, lfr.Path), os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+
+	seq, err := scanForLastSeq(fi, -1)
+	if err != nil {
+		return fmt.Errorf("failed to scan log file for last seqno: %w", err)
+	}
+
+	dp.log.Info("loaded seq", "seq", seq, "now", time.Now().UnixMicro(), "time-seq", dp.timeSequence)
+
+	dp.curSeq = seq + 1
+	dp.logfi = fi
+
+	return nil
+}
+
+func (dp *DiskPersistence) initLogFile() error {
+	if err := os.MkdirAll(dp.primaryDir, 0775); err != nil {
+		return err
+	}
+
+	p := filepath.Join(dp.primaryDir, "evts-0")
+	fi, err := os.Create(p)
+	if err != nil {
+		return err
+	}
+
+	if err := dp.meta.Create(&LogFileRef{
+		Path:     "evts-0",
+		SeqStart: 0,
+	}).Error; err != nil {
+		return err
+	}
+
+	dp.logfi = fi
+	dp.curSeq = 1
+	return nil
+}
+
+// swapLog swaps the current log file out for a new empty one
+// must only be called while holding dp.lk
+func (dp *DiskPersistence) swapLog(ctx context.Context) error {
+	if err := dp.logfi.Close(); err != nil {
+		return fmt.Errorf("failed to close current log file: %w", err)
+	}
+
+	fname := fmt.Sprintf("evts-%d", dp.curSeq)
+	nextp := filepath.Join(dp.primaryDir, fname)
+
+	fi, err := os.Create(nextp)
+	if err != nil {
+		return err
+	}
+
+	if err := dp.meta.Create(&LogFileRef{
+		Path:     fname,
+		SeqStart: dp.curSeq,
+	}).Error; err != nil {
+		return err
+	}
+
+	dp.logfi = fi
+	return nil
+}
+
+func scanForLastSeq(fi *os.File, end int64) (int64, error) {
+	scratch := make([]byte, headerSize)
+
+	var lastSeq int64 = -1
+	var offset int64
+	for {
+		eh, err := readHeader(fi, scratch)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return lastSeq, nil
+			}
+			return 0, err
+		}
+
+		if end > 0 && eh.Seq > end {
+			// return to beginning of offset
+			n, err := fi.Seek(offset, io.SeekStart)
+			if err != nil {
+				return 0, err
+			}
+
+			if n != offset {
+				return 0, fmt.Errorf("rewind seek failed")
+			}
+
+			return eh.Seq, nil
+		}
+
+		lastSeq = eh.Seq
+
+		noff, err := fi.Seek(int64(eh.Len), io.SeekCurrent)
+		if err != nil {
+			return 0, err
+		}
+
+		if noff != offset+headerSize+int64(eh.Len) {
+			// TODO: must recover from this
+			return 0, fmt.Errorf("did not seek to next event properly")
+		}
+
+		offset = noff
+	}
+}
+
+const (
+	evtKindCommit    = 1
+	evtKindHandle    = 2
+	evtKindTombstone = 3
+	evtKindIdentity  = 4
+	evtKindAccount   = 5
+	evtKindSync      = 6
+)
+
+var emptyHeader = make([]byte, headerSize)
+
+func (dp *DiskPersistence) addJobToQueue(ctx context.Context, job persistJob) error {
+	dp.lk.Lock()
+	defer dp.lk.Unlock()
+
+	if err := dp.doPersist(ctx, job); err != nil {
+		return err
+	}
+
+	// TODO: for some reason replacing this constant with p.writeBufferSize dramatically reduces perf...
+	if len(dp.evtbuf) > 400 {
+		if err := dp.flushLog(ctx); err != nil {
+			return fmt.Errorf("failed to flush disk log: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (dp *DiskPersistence) flushRoutine() {
+	t := time.NewTicker(time.Millisecond * 100)
+
+	for {
+		ctx := context.Background()
+		select {
+		case <-dp.shutdown:
+			return
+		case <-t.C:
+			dp.lk.Lock()
+			if err := dp.flushLog(ctx); err != nil {
+				// TODO: this happening is quite bad. Need a recovery strategy
+				dp.log.Error("failed to flush disk log", "err", err)
+			}
+			dp.lk.Unlock()
+		}
+	}
+}
+
+func (dp *DiskPersistence) flushLog(ctx context.Context) error {
+	if len(dp.evtbuf) == 0 {
+		return nil
+	}
+
+	_, err := io.Copy(dp.logfi, dp.outbuf)
+	if err != nil {
+		return err
+	}
+
+	dp.outbuf.Truncate(0)
+
+	for _, ej := range dp.evtbuf {
+		dp.broadcast(ej.Evt)
+		ej.Buffer.Truncate(0)
+		dp.buffers.Put(ej.Buffer)
+	}
+
+	dp.evtbuf = dp.evtbuf[:0]
+
+	return nil
+}
+
+func (dp *DiskPersistence) garbageCollectRoutine() {
+	t := time.NewTicker(time.Hour)
+
+	for {
+		ctx := context.Background()
+		select {
+		// Closing a channel can be listened to with multiple routines: https://goplay.tools/snippet/UcwbC0CeJAL
+		case <-dp.shutdown:
+			return
+		case <-t.C:
+			if errs := dp.garbageCollect(ctx); len(errs) > 0 {
+				for _, err := range errs {
+					dp.log.Error("garbage collection error", "err", err)
+				}
+			}
+		}
+	}
+}
+
+var garbageCollectionsExecuted = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "disk_persister_garbage_collections_executed",
+	Help: "Number of garbage collections executed",
+}, []string{})
+
+var garbageCollectionErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "disk_persister_garbage_collections_errors",
+	Help: "Number of errors encountered during garbage collection",
+}, []string{})
+
+var refsGarbageCollected = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "disk_persister_garbage_collections_refs_collected",
+	Help: "Number of refs collected during garbage collection",
+}, []string{})
+
+var filesGarbageCollected = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "disk_persister_garbage_collections_files_collected",
+	Help: "Number of files collected during garbage collection",
+}, []string{})
+
+func (dp *DiskPersistence) garbageCollect(ctx context.Context) []error {
+	garbageCollectionsExecuted.WithLabelValues().Inc()
+
+	// Grab refs created before the retention period
+	var refs []LogFileRef
+	var errs []error
+
+	defer func() {
+		garbageCollectionErrors.WithLabelValues().Add(float64(len(errs)))
+	}()
+
+	if err := dp.meta.WithContext(ctx).Find(&refs, "created_at < ?", time.Now().Add(-dp.retention)).Error; err != nil {
+		return []error{err}
+	}
+
+	oldRefsFound := len(refs)
+	refsDeleted := 0
+	filesDeleted := 0
+
+	// In the future if we want to support Archiving, we could do that here instead of deleting
+	for _, r := range refs {
+		dp.lk.Lock()
+		currentLogfile := dp.logfi.Name()
+		dp.lk.Unlock()
+
+		if filepath.Join(dp.primaryDir, r.Path) == currentLogfile {
+			// Don't delete the current log file
+			dp.log.Info("skipping deletion of current log file")
+			continue
+		}
+
+		// Delete the ref in the database to prevent playback from finding it
+		if err := dp.meta.WithContext(ctx).Delete(&r).Error; err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		refsDeleted++
+
+		// Delete the file from disk
+		if err := os.Remove(filepath.Join(dp.primaryDir, r.Path)); err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		filesDeleted++
+	}
+
+	refsGarbageCollected.WithLabelValues().Add(float64(refsDeleted))
+	filesGarbageCollected.WithLabelValues().Add(float64(filesDeleted))
+
+	dp.log.Info("garbage collection complete",
+		"filesDeleted", filesDeleted,
+		"refsDeleted", refsDeleted,
+		"oldRefsFound", oldRefsFound,
+	)
+
+	return errs
+}
+
+func (dp *DiskPersistence) doPersist(ctx context.Context, pjob persistJob) error {
+	seq := dp.curSeq
+	if dp.timeSequence {
+		seq = time.Now().UnixMicro()
+		if seq < dp.curSeq {
+			seq = dp.curSeq
+		}
+		dp.curSeq = seq + 1
+	} else {
+		dp.curSeq++
+	}
+
+	// Set sequence number in event header
+	// the rest of the header is set in DiskPersistence.Persist()
+	binary.LittleEndian.PutUint64(pjob.Bytes[20:], uint64(seq))
+
+	// update the seq in the message
+	// copy the message from outside to a new object, clobber the seq, add it back to the event
+	switch {
+	case pjob.Evt.RepoCommit != nil:
+		pjob.Evt.RepoCommit.Seq = seq
+	case pjob.Evt.RepoSync != nil:
+		pjob.Evt.RepoSync.Seq = seq
+	case pjob.Evt.RepoHandle != nil:
+		pjob.Evt.RepoHandle.Seq = seq
+	case pjob.Evt.RepoIdentity != nil:
+		pjob.Evt.RepoIdentity.Seq = seq
+	case pjob.Evt.RepoAccount != nil:
+		pjob.Evt.RepoAccount.Seq = seq
+	case pjob.Evt.RepoTombstone != nil:
+		pjob.Evt.RepoTombstone.Seq = seq
+	default:
+		// only those three get peristed right now
+		// we should not actually ever get here...
+		return nil
+	}
+
+	_, err := dp.outbuf.Write(pjob.Bytes)
+	if err != nil {
+		return err
+	}
+
+	dp.evtbuf = append(dp.evtbuf, pjob)
+
+	dp.eventCounter++
+	if dp.eventCounter%dp.eventsPerFile == 0 {
+		if err := dp.flushLog(ctx); err != nil {
+			return err
+		}
+
+		// time to roll the log file
+		if err := dp.swapLog(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Persist implements events.EventPersistence
+// Persist may mutate contents of xevt and what it points to
+func (dp *DiskPersistence) Persist(ctx context.Context, xevt *events.XRPCStreamEvent) error {
+	buffer := dp.buffers.Get().(*bytes.Buffer)
+	cw := dp.writers.Get().(*cbg.CborWriter)
+	defer dp.writers.Put(cw)
+	cw.SetWriter(buffer)
+
+	buffer.Truncate(0)
+
+	buffer.Write(emptyHeader)
+
+	var did string
+	var evtKind uint32
+	switch {
+	case xevt.RepoCommit != nil:
+		evtKind = evtKindCommit
+		did = xevt.RepoCommit.Repo
+		if err := xevt.RepoCommit.MarshalCBOR(cw); err != nil {
+			return fmt.Errorf("failed to marshal: %w", err)
+		}
+	case xevt.RepoSync != nil:
+		evtKind = evtKindSync
+		did = xevt.RepoSync.Did
+		if err := xevt.RepoSync.MarshalCBOR(cw); err != nil {
+			return fmt.Errorf("failed to marshal: %w", err)
+		}
+	case xevt.RepoHandle != nil:
+		evtKind = evtKindHandle
+		did = xevt.RepoHandle.Did
+		if err := xevt.RepoHandle.MarshalCBOR(cw); err != nil {
+			return fmt.Errorf("failed to marshal: %w", err)
+		}
+	case xevt.RepoIdentity != nil:
+		evtKind = evtKindIdentity
+		did = xevt.RepoIdentity.Did
+		if err := xevt.RepoIdentity.MarshalCBOR(cw); err != nil {
+			return fmt.Errorf("failed to marshal: %w", err)
+		}
+	case xevt.RepoAccount != nil:
+		evtKind = evtKindAccount
+		did = xevt.RepoAccount.Did
+		if err := xevt.RepoAccount.MarshalCBOR(cw); err != nil {
+			return fmt.Errorf("failed to marshal: %w", err)
+		}
+	case xevt.RepoTombstone != nil:
+		evtKind = evtKindTombstone
+		did = xevt.RepoTombstone.Did
+		if err := xevt.RepoTombstone.MarshalCBOR(cw); err != nil {
+			return fmt.Errorf("failed to marshal: %w", err)
+		}
+	default:
+		return nil
+		// only those two get peristed right now
+	}
+
+	usr, err := dp.uidForDid(ctx, did)
+	if err != nil {
+		return err
+	}
+
+	b := buffer.Bytes()
+
+	// Set flags in header (no flags for now)
+	binary.LittleEndian.PutUint32(b, 0)
+	// Set event kind in header
+	binary.LittleEndian.PutUint32(b[4:], evtKind)
+	// Set event length in header
+	binary.LittleEndian.PutUint32(b[8:], uint32(len(b)-headerSize))
+	// Set user UID in header
+	binary.LittleEndian.PutUint64(b[12:], uint64(usr))
+	// set seq at [20:] inside mutex section inside doPersist
+
+	return dp.addJobToQueue(ctx, persistJob{
+		Bytes:  b,
+		Evt:    xevt,
+		Buffer: buffer,
+	})
+}
+
+type evtHeader struct {
+	Flags uint32
+	Kind  uint32
+	Seq   int64
+	Usr   models.Uid
+	Len   uint32
+}
+
+func (eh *evtHeader) Len64() int64 {
+	return int64(eh.Len)
+}
+
+const headerSize = 4 + 4 + 4 + 8 + 8
+
+func readHeader(r io.Reader, scratch []byte) (*evtHeader, error) {
+	if len(scratch) < headerSize {
+		return nil, fmt.Errorf("must pass scratch buffer of at least %d bytes", headerSize)
+	}
+
+	scratch = scratch[:headerSize]
+	_, err := io.ReadFull(r, scratch)
+	if err != nil {
+		return nil, fmt.Errorf("reading header: %w", err)
+	}
+
+	flags := binary.LittleEndian.Uint32(scratch[:4])
+	kind := binary.LittleEndian.Uint32(scratch[4:8])
+	l := binary.LittleEndian.Uint32(scratch[8:12])
+	usr := binary.LittleEndian.Uint64(scratch[12:20])
+	seq := binary.LittleEndian.Uint64(scratch[20:28])
+
+	return &evtHeader{
+		Flags: flags,
+		Kind:  kind,
+		Len:   l,
+		Usr:   models.Uid(usr),
+		Seq:   int64(seq),
+	}, nil
+}
+
+func (dp *DiskPersistence) writeHeader(ctx context.Context, flags uint32, kind uint32, l uint32, usr uint64, seq int64) error {
+	binary.LittleEndian.PutUint32(dp.scratch, flags)
+	binary.LittleEndian.PutUint32(dp.scratch[4:], kind)
+	binary.LittleEndian.PutUint32(dp.scratch[8:], l)
+	binary.LittleEndian.PutUint64(dp.scratch[12:], usr)
+	binary.LittleEndian.PutUint64(dp.scratch[20:], uint64(seq))
+
+	nw, err := dp.logfi.Write(dp.scratch)
+	if err != nil {
+		return err
+	}
+
+	if nw != headerSize {
+		return fmt.Errorf("only wrote %d bytes for header", nw)
+	}
+
+	return nil
+}
+
+func (dp *DiskPersistence) uidForDid(ctx context.Context, did string) (models.Uid, error) {
+	if uid, ok := dp.didCache.Get(did); ok {
+		return uid, nil
+	}
+
+	uid, err := dp.uids.DidToUid(ctx, did)
+	if err != nil {
+		return 0, err
+	}
+
+	dp.didCache.Add(did, uid)
+
+	return uid, nil
+}
+
+func (dp *DiskPersistence) Playback(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error) error {
+	var logs []LogFileRef
+	needslogs := true
+	if since != 0 {
+		// find the log file that starts before our since
+		result := dp.meta.Debug().Order("seq_start desc").Where("seq_start < ?", since).Limit(1).Find(&logs)
+		if result.Error != nil {
+			return result.Error
+		}
+		if result.RowsAffected != 0 {
+			needslogs = false
+		}
+	}
+
+	// playback data from all the log files we found, then check the db to see if more were written during playback.
+	// repeat a few times but not unboundedly.
+	// don't decrease '10' below 2 because we should always do two passes through this if the above before-chunk query was used.
+	for i := 0; i < 10; i++ {
+		if needslogs {
+			if err := dp.meta.Debug().Order("seq_start asc").Find(&logs, "seq_start >= ?", since).Error; err != nil {
+				return err
+			}
+		}
+
+		lastSeq, err := dp.PlaybackLogfiles(ctx, since, cb, logs)
+		if err != nil {
+			return err
+		}
+
+		// No lastSeq implies that we read until the end of known events
+		if lastSeq == nil {
+			break
+		}
+
+		since = *lastSeq
+		needslogs = true
+	}
+
+	return nil
+}
+
+func (dp *DiskPersistence) PlaybackLogfiles(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error, logFiles []LogFileRef) (*int64, error) {
+	for i, lf := range logFiles {
+		lastSeq, err := dp.readEventsFrom(ctx, since, filepath.Join(dp.primaryDir, lf.Path), cb)
+		if err != nil {
+			return nil, err
+		}
+		since = 0
+		if i == len(logFiles)-1 &&
+			lastSeq != nil &&
+			(*lastSeq-lf.SeqStart) == dp.eventsPerFile-1 {
+			// There may be more log files to read since the last one was full
+			return lastSeq, nil
+		}
+	}
+
+	return nil, nil
+}
+
+func postDoNotEmit(flags uint32) bool {
+	if flags&(EvtFlagRebased|EvtFlagTakedown) != 0 {
+		return true
+	}
+
+	return false
+}
+
+func (dp *DiskPersistence) readEventsFrom(ctx context.Context, since int64, fn string, cb func(*events.XRPCStreamEvent) error) (*int64, error) {
+	fi, err := os.OpenFile(fn, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	if since != 0 {
+		lastSeq, err := scanForLastSeq(fi, since)
+		if err != nil {
+			return nil, err
+		}
+		if since > lastSeq {
+			dp.log.Error("playback cursor is greater than last seq of file checked",
+				"since", since,
+				"lastSeq", lastSeq,
+				"filename", fn,
+			)
+			return nil, nil
+		}
+	}
+
+	bufr := bufio.NewReader(fi)
+
+	lastSeq := int64(0)
+
+	scratch := make([]byte, headerSize)
+	for {
+		h, err := readHeader(bufr, scratch)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return &lastSeq, nil
+			}
+
+			return nil, err
+		}
+
+		lastSeq = h.Seq
+
+		if postDoNotEmit(h.Flags) {
+			// event taken down, skip
+			_, err := io.CopyN(io.Discard, bufr, h.Len64()) // would be really nice if the buffered reader had a 'skip' method that does a seek under the hood
+			if err != nil {
+				return nil, fmt.Errorf("failed while skipping event (seq: %d, fn: %q): %w", h.Seq, fn, err)
+			}
+			continue
+		}
+
+		switch h.Kind {
+		case evtKindCommit:
+			var evt atproto.SyncSubscribeRepos_Commit
+			if err := evt.UnmarshalCBOR(io.LimitReader(bufr, h.Len64())); err != nil {
+				return nil, err
+			}
+			evt.Seq = h.Seq
+			if err := cb(&events.XRPCStreamEvent{RepoCommit: &evt}); err != nil {
+				return nil, err
+			}
+		case evtKindSync:
+			var evt atproto.SyncSubscribeRepos_Sync
+			if err := evt.UnmarshalCBOR(io.LimitReader(bufr, h.Len64())); err != nil {
+				return nil, err
+			}
+			evt.Seq = h.Seq
+			if err := cb(&events.XRPCStreamEvent{RepoSync: &evt}); err != nil {
+				return nil, err
+			}
+		case evtKindHandle:
+			var evt atproto.SyncSubscribeRepos_Handle
+			if err := evt.UnmarshalCBOR(io.LimitReader(bufr, h.Len64())); err != nil {
+				return nil, err
+			}
+			evt.Seq = h.Seq
+			if err := cb(&events.XRPCStreamEvent{RepoHandle: &evt}); err != nil {
+				return nil, err
+			}
+		case evtKindIdentity:
+			var evt atproto.SyncSubscribeRepos_Identity
+			if err := evt.UnmarshalCBOR(io.LimitReader(bufr, h.Len64())); err != nil {
+				return nil, err
+			}
+			evt.Seq = h.Seq
+			if err := cb(&events.XRPCStreamEvent{RepoIdentity: &evt}); err != nil {
+				return nil, err
+			}
+		case evtKindAccount:
+			var evt atproto.SyncSubscribeRepos_Account
+			if err := evt.UnmarshalCBOR(io.LimitReader(bufr, h.Len64())); err != nil {
+				return nil, err
+			}
+			evt.Seq = h.Seq
+			if err := cb(&events.XRPCStreamEvent{RepoAccount: &evt}); err != nil {
+				return nil, err
+			}
+		case evtKindTombstone:
+			var evt atproto.SyncSubscribeRepos_Tombstone
+			if err := evt.UnmarshalCBOR(io.LimitReader(bufr, h.Len64())); err != nil {
+				return nil, err
+			}
+			evt.Seq = h.Seq
+			if err := cb(&events.XRPCStreamEvent{RepoTombstone: &evt}); err != nil {
+				return nil, err
+			}
+		default:
+			dp.log.Warn("unrecognized event kind coming from log file", "seq", h.Seq, "kind", h.Kind)
+			return nil, fmt.Errorf("halting on unrecognized event kind")
+		}
+	}
+}
+
+type UserAction struct {
+	gorm.Model
+
+	Usr      models.Uid
+	RebaseAt int64
+	Takedown bool
+}
+
+func (dp *DiskPersistence) TakeDownRepo(ctx context.Context, usr models.Uid) error {
+	/*
+		if err := p.meta.Create(&UserAction{
+			Usr:      usr,
+			Takedown: true,
+		}).Error; err != nil {
+			return err
+		}
+	*/
+
+	return dp.forEachShardWithUserEvents(ctx, usr, func(ctx context.Context, fn string) error {
+		if err := dp.deleteEventsForUser(ctx, usr, fn); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func (dp *DiskPersistence) forEachShardWithUserEvents(ctx context.Context, usr models.Uid, cb func(context.Context, string) error) error {
+	var refs []LogFileRef
+	if err := dp.meta.Order("created_at desc").Find(&refs).Error; err != nil {
+		return err
+	}
+
+	for _, r := range refs {
+		mhas, err := dp.refMaybeHasUserEvents(ctx, usr, r)
+		if err != nil {
+			return err
+		}
+
+		if mhas {
+			var path string
+			if r.Archived {
+				path = filepath.Join(dp.archiveDir, r.Path)
+			} else {
+				path = filepath.Join(dp.primaryDir, r.Path)
+			}
+
+			if err := cb(ctx, path); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (dp *DiskPersistence) refMaybeHasUserEvents(ctx context.Context, usr models.Uid, ref LogFileRef) (bool, error) {
+	// TODO: lazily computed bloom filters for users in each logfile
+	return true, nil
+}
+
+type zeroReader struct{}
+
+func (zr *zeroReader) Read(p []byte) (n int, err error) {
+	for i := range p {
+		p[i] = 0
+	}
+	return len(p), nil
+}
+
+func (dp *DiskPersistence) deleteEventsForUser(ctx context.Context, usr models.Uid, fn string) error {
+	return dp.mutateUserEventsInLog(ctx, usr, fn, EvtFlagTakedown, true)
+}
+
+func (dp *DiskPersistence) mutateUserEventsInLog(ctx context.Context, usr models.Uid, fn string, flag uint32, zeroEvts bool) error {
+	fi, err := os.OpenFile(fn, os.O_RDWR, 0)
+	if err != nil {
+		return fmt.Errorf("failed to open log file: %w", err)
+	}
+	defer fi.Close()
+	defer fi.Sync()
+
+	scratch := make([]byte, headerSize)
+	var offset int64
+	for {
+		h, err := readHeader(fi, scratch)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return nil
+			}
+
+			return err
+		}
+
+		if h.Usr == usr && h.Flags&flag == 0 {
+			nflag := h.Flags | flag
+
+			binary.LittleEndian.PutUint32(scratch, nflag)
+
+			if _, err := fi.WriteAt(scratch[:4], offset); err != nil {
+				return fmt.Errorf("failed to write updated flag value: %w", err)
+			}
+
+			if zeroEvts {
+				// sync that write before blanking the event data
+				if err := fi.Sync(); err != nil {
+					return err
+				}
+
+				if _, err := fi.Seek(offset+headerSize, io.SeekStart); err != nil {
+					return fmt.Errorf("failed to seek: %w", err)
+				}
+
+				_, err := io.CopyN(fi, &zeroReader{}, h.Len64())
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		offset += headerSize + h.Len64()
+		_, err = fi.Seek(offset, io.SeekStart)
+		if err != nil {
+			return fmt.Errorf("failed to seek: %w", err)
+		}
+	}
+}
+
+func (dp *DiskPersistence) Flush(ctx context.Context) error {
+	dp.lk.Lock()
+	defer dp.lk.Unlock()
+	if len(dp.evtbuf) > 0 {
+		return dp.flushLog(ctx)
+	}
+	return nil
+}
+
+func (dp *DiskPersistence) Shutdown(ctx context.Context) error {
+	close(dp.shutdown)
+	if err := dp.Flush(ctx); err != nil {
+		return err
+	}
+
+	dp.logfi.Close()
+	return nil
+}
+
+func (dp *DiskPersistence) SetEventBroadcaster(f func(*events.XRPCStreamEvent)) {
+	dp.broadcast = f
+}

cmd/relay/events/events.go

···
+package events
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"sync"
+	"time"
+
+	comatproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/cmd/relay/models"
+	lexutil "github.com/bluesky-social/indigo/lex/util"
+	"github.com/prometheus/client_golang/prometheus"
+
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"go.opentelemetry.io/otel"
+)
+
+var log = slog.Default().With("system", "events")
+
+type Scheduler interface {
+	AddWork(ctx context.Context, repo string, val *XRPCStreamEvent) error
+	Shutdown()
+}
+
+type EventManager struct {
+	subs   []*Subscriber
+	subsLk sync.Mutex
+
+	bufferSize          int
+	crossoverBufferSize int
+
+	persister EventPersistence
+
+	log *slog.Logger
+}
+
+func NewEventManager(persister EventPersistence) *EventManager {
+	em := &EventManager{
+		bufferSize:          16 << 10,
+		crossoverBufferSize: 512,
+		persister:           persister,
+		log:                 slog.Default().With("system", "events"),
+	}
+
+	persister.SetEventBroadcaster(em.broadcastEvent)
+
+	return em
+}
+
+const (
+	opSubscribe = iota
+	opUnsubscribe
+	opSend
+)
+
+type Operation struct {
+	op  int
+	sub *Subscriber
+	evt *XRPCStreamEvent
+}
+
+func (em *EventManager) Shutdown(ctx context.Context) error {
+	return em.persister.Shutdown(ctx)
+}
+
+// broadcastEvent is the target for EventPersistence.SetEventBroadcaster()
+func (em *EventManager) broadcastEvent(evt *XRPCStreamEvent) {
+	// the main thing we do is send it out, so MarshalCBOR once
+	if err := evt.Preserialize(); err != nil {
+		em.log.Error("broadcast serialize failed", "err", err)
+		// serialize isn't going to go better later, this event is cursed
+		return
+	}
+
+	em.subsLk.Lock()
+	defer em.subsLk.Unlock()
+
+	// TODO: for a larger fanout we should probably have dedicated goroutines
+	// for subsets of the subscriber set, and tiered channels to distribute
+	// events out to them, or some similar architecture
+	// Alternatively, we might just want to not allow too many subscribers
+	// directly to the bgs, and have rebroadcasting proxies instead
+	for _, s := range em.subs {
+		if s.filter(evt) {
+			s.enqueuedCounter.Inc()
+			select {
+			case s.outgoing <- evt:
+				// sent evt on this subscriber's chan! yay!
+			case <-s.done:
+				// this subscriber is closing, quickly do nothing
+			default:
+				// filter out all future messages that would be
+				// sent to this subscriber, but wait for it to
+				// actually be removed by the correct bit of
+				// code
+				s.filter = func(*XRPCStreamEvent) bool { return false }
+
+				em.log.Warn("dropping slow consumer due to event overflow", "bufferSize", len(s.outgoing), "ident", s.ident)
+				go func(torem *Subscriber) {
+					torem.lk.Lock()
+					if !torem.cleanedUp {
+						select {
+						case torem.outgoing <- &XRPCStreamEvent{
+							Error: &ErrorFrame{
+								Error: "ConsumerTooSlow",
+							},
+						}:
+						case <-time.After(time.Second * 5):
+							em.log.Warn("failed to send error frame to backed up consumer", "ident", torem.ident)
+						}
+					}
+					torem.lk.Unlock()
+					torem.cleanup()
+				}(s)
+			}
+			s.broadcastCounter.Inc()
+		}
+	}
+}
+
+func (em *EventManager) persistAndSendEvent(ctx context.Context, evt *XRPCStreamEvent) {
+	// TODO: can cut 5-10% off of disk persister benchmarks by making this function
+	// accept a uid. The lookup inside the persister is notably expensive (despite
+	// being an lru cache?)
+	if err := em.persister.Persist(ctx, evt); err != nil {
+		em.log.Error("failed to persist outbound event", "err", err)
+	}
+}
+
+type Subscriber struct {
+	outgoing chan *XRPCStreamEvent
+
+	filter func(*XRPCStreamEvent) bool
+
+	done chan struct{}
+
+	cleanup func()
+
+	lk        sync.Mutex
+	cleanedUp bool
+
+	ident            string
+	enqueuedCounter  prometheus.Counter
+	broadcastCounter prometheus.Counter
+}
+
+const (
+	EvtKindErrorFrame = -1
+	EvtKindMessage    = 1
+)
+
+type EventHeader struct {
+	Op      int64  `cborgen:"op"`
+	MsgType string `cborgen:"t,omitempty"`
+}
+
+var (
+	// AccountStatusActive is not in the spec but used internally
+	// the alternative would be an additional SQL column for "active" or status="" to imply active
+	AccountStatusActive = "active"
+
+	AccountStatusDeactivated    = "deactivated"
+	AccountStatusDeleted        = "deleted"
+	AccountStatusDesynchronized = "desynchronized"
+	AccountStatusSuspended      = "suspended"
+	AccountStatusTakendown      = "takendown"
+	AccountStatusThrottled      = "throttled"
+)
+
+var AccountStatusList = []string{
+	AccountStatusActive,
+	AccountStatusDeactivated,
+	AccountStatusDeleted,
+	AccountStatusDesynchronized,
+	AccountStatusSuspended,
+	AccountStatusTakendown,
+	AccountStatusThrottled,
+}
+var AccountStatuses map[string]bool
+
+func init() {
+	AccountStatuses = make(map[string]bool, len(AccountStatusList))
+	for _, status := range AccountStatusList {
+		AccountStatuses[status] = true
+	}
+}
+
+type XRPCStreamEvent struct {
+	Error         *ErrorFrame
+	RepoCommit    *comatproto.SyncSubscribeRepos_Commit
+	RepoSync      *comatproto.SyncSubscribeRepos_Sync
+	RepoHandle    *comatproto.SyncSubscribeRepos_Handle // DEPRECATED
+	RepoIdentity  *comatproto.SyncSubscribeRepos_Identity
+	RepoInfo      *comatproto.SyncSubscribeRepos_Info
+	RepoMigrate   *comatproto.SyncSubscribeRepos_Migrate   // DEPRECATED
+	RepoTombstone *comatproto.SyncSubscribeRepos_Tombstone // DEPRECATED
+	RepoAccount   *comatproto.SyncSubscribeRepos_Account
+	LabelLabels   *comatproto.LabelSubscribeLabels_Labels
+	LabelInfo     *comatproto.LabelSubscribeLabels_Info
+
+	// some private fields for internal routing perf
+	PrivUid         models.Uid `json:"-" cborgen:"-"`
+	PrivPdsId       uint       `json:"-" cborgen:"-"`
+	PrivRelevantPds []uint     `json:"-" cborgen:"-"`
+	Preserialized   []byte     `json:"-" cborgen:"-"`
+}
+
+func (evt *XRPCStreamEvent) Serialize(wc io.Writer) error {
+	header := EventHeader{Op: EvtKindMessage}
+	var obj lexutil.CBOR
+
+	switch {
+	case evt.Error != nil:
+		header.Op = EvtKindErrorFrame
+		obj = evt.Error
+	case evt.RepoCommit != nil:
+		header.MsgType = "#commit"
+		obj = evt.RepoCommit
+	case evt.RepoSync != nil:
+		header.MsgType = "#sync"
+		obj = evt.RepoSync
+	case evt.RepoHandle != nil:
+		header.MsgType = "#handle"
+		obj = evt.RepoHandle
+	case evt.RepoIdentity != nil:
+		header.MsgType = "#identity"
+		obj = evt.RepoIdentity
+	case evt.RepoAccount != nil:
+		header.MsgType = "#account"
+		obj = evt.RepoAccount
+	case evt.RepoInfo != nil:
+		header.MsgType = "#info"
+		obj = evt.RepoInfo
+	case evt.RepoMigrate != nil:
+		header.MsgType = "#migrate"
+		obj = evt.RepoMigrate
+	case evt.RepoTombstone != nil:
+		header.MsgType = "#tombstone"
+		obj = evt.RepoTombstone
+	default:
+		return fmt.Errorf("unrecognized event kind")
+	}
+
+	cborWriter := cbg.NewCborWriter(wc)
+	if err := header.MarshalCBOR(cborWriter); err != nil {
+		return fmt.Errorf("failed to write header: %w", err)
+	}
+	return obj.MarshalCBOR(cborWriter)
+}
+
+func (xevt *XRPCStreamEvent) Deserialize(r io.Reader) error {
+	var header EventHeader
+	if err := header.UnmarshalCBOR(r); err != nil {
+		return fmt.Errorf("reading header: %w", err)
+	}
+	switch header.Op {
+	case EvtKindMessage:
+		switch header.MsgType {
+		case "#commit":
+			var evt comatproto.SyncSubscribeRepos_Commit
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return fmt.Errorf("reading repoCommit event: %w", err)
+			}
+			xevt.RepoCommit = &evt
+		case "#sync":
+			var evt comatproto.SyncSubscribeRepos_Sync
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return fmt.Errorf("reading repoSync event: %w", err)
+			}
+			xevt.RepoSync = &evt
+		case "#handle":
+			// TODO: DEPRECATED message; warning/counter; drop message
+			var evt comatproto.SyncSubscribeRepos_Handle
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoHandle = &evt
+		case "#identity":
+			var evt comatproto.SyncSubscribeRepos_Identity
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoIdentity = &evt
+		case "#account":
+			var evt comatproto.SyncSubscribeRepos_Account
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoAccount = &evt
+		case "#info":
+			// TODO: this might also be a LabelInfo (as opposed to RepoInfo)
+			var evt comatproto.SyncSubscribeRepos_Info
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoInfo = &evt
+		case "#migrate":
+			// TODO: DEPRECATED message; warning/counter; drop message
+			var evt comatproto.SyncSubscribeRepos_Migrate
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoMigrate = &evt
+		case "#tombstone":
+			// TODO: DEPRECATED message; warning/counter; drop message
+			var evt comatproto.SyncSubscribeRepos_Tombstone
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoTombstone = &evt
+		case "#labels":
+			var evt comatproto.LabelSubscribeLabels_Labels
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return fmt.Errorf("reading Labels event: %w", err)
+			}
+			xevt.LabelLabels = &evt
+		}
+	case EvtKindErrorFrame:
+		var errframe ErrorFrame
+		if err := errframe.UnmarshalCBOR(r); err != nil {
+			return err
+		}
+		xevt.Error = &errframe
+	default:
+		return fmt.Errorf("unrecognized event stream type: %d", header.Op)
+	}
+	return nil
+}
+
+var ErrNoSeq = errors.New("event has no sequence number")
+
+// serialize content into Preserialized cache
+func (evt *XRPCStreamEvent) Preserialize() error {
+	if evt.Preserialized != nil {
+		return nil
+	}
+	var buf bytes.Buffer
+	err := evt.Serialize(&buf)
+	if err != nil {
+		return err
+	}
+	evt.Preserialized = buf.Bytes()
+	return nil
+}
+
+type ErrorFrame struct {
+	Error   string `cborgen:"error"`
+	Message string `cborgen:"message"`
+}
+
+func (em *EventManager) AddEvent(ctx context.Context, ev *XRPCStreamEvent) error {
+	ctx, span := otel.Tracer("events").Start(ctx, "AddEvent")
+	defer span.End()
+
+	em.persistAndSendEvent(ctx, ev)
+	return nil
+}
+
+var (
+	ErrPlaybackShutdown = fmt.Errorf("playback shutting down")
+	ErrCaughtUp         = fmt.Errorf("caught up")
+)
+
+func (em *EventManager) Subscribe(ctx context.Context, ident string, filter func(*XRPCStreamEvent) bool, since *int64) (<-chan *XRPCStreamEvent, func(), error) {
+	// TODO: the only known filters are 'true' and 'false', replace the function pointer with a bool
+	if filter == nil {
+		filter = func(*XRPCStreamEvent) bool { return true }
+	}
+
+	done := make(chan struct{})
+	sub := &Subscriber{
+		ident:            ident,
+		outgoing:         make(chan *XRPCStreamEvent, em.bufferSize),
+		filter:           filter,
+		done:             done,
+		enqueuedCounter:  eventsEnqueued.WithLabelValues(ident),
+		broadcastCounter: eventsBroadcast.WithLabelValues(ident),
+	}
+
+	sub.cleanup = sync.OnceFunc(func() {
+		sub.lk.Lock()
+		defer sub.lk.Unlock()
+		close(done)
+		em.rmSubscriber(sub)
+		close(sub.outgoing)
+		sub.cleanedUp = true
+	})
+
+	if since == nil {
+		em.addSubscriber(sub)
+		return sub.outgoing, sub.cleanup, nil
+	}
+
+	out := make(chan *XRPCStreamEvent, em.crossoverBufferSize)
+
+	go func() {
+		lastSeq := *since
+		// run playback to get through *most* of the events, getting our current cursor close to realtime
+		if err := em.persister.Playback(ctx, *since, func(e *XRPCStreamEvent) error {
+			select {
+			case <-done:
+				return ErrPlaybackShutdown
+			case out <- e:
+				seq := SequenceForEvent(e)
+				if seq > 0 {
+					lastSeq = seq
+				}
+				return nil
+			}
+		}); err != nil {
+			if errors.Is(err, ErrPlaybackShutdown) {
+				em.log.Warn("events playback", "err", err)
+			} else {
+				em.log.Error("events playback", "err", err)
+			}
+
+			// TODO: send an error frame or something?
+			close(out)
+			return
+		}
+
+		// now, start buffering events from the live stream
+		em.addSubscriber(sub)
+
+		first := <-sub.outgoing
+
+		// run playback again to get us to the events that have started buffering
+		if err := em.persister.Playback(ctx, lastSeq, func(e *XRPCStreamEvent) error {
+			seq := SequenceForEvent(e)
+			if seq > SequenceForEvent(first) {
+				return ErrCaughtUp
+			}
+
+			select {
+			case <-done:
+				return ErrPlaybackShutdown
+			case out <- e:
+				return nil
+			}
+		}); err != nil {
+			if !errors.Is(err, ErrCaughtUp) {
+				em.log.Error("events playback", "err", err)
+
+				// TODO: send an error frame or something?
+				close(out)
+				em.rmSubscriber(sub)
+				return
+			}
+		}
+
+		// now that we are caught up, just copy events from the channel over
+		for evt := range sub.outgoing {
+			select {
+			case out <- evt:
+			case <-done:
+				em.rmSubscriber(sub)
+				return
+			}
+		}
+	}()
+
+	return out, sub.cleanup, nil
+}
+
+func SequenceForEvent(evt *XRPCStreamEvent) int64 {
+	return evt.Sequence()
+}
+
+func (evt *XRPCStreamEvent) Sequence() int64 {
+	switch {
+	case evt == nil:
+		return -1
+	case evt.RepoCommit != nil:
+		return evt.RepoCommit.Seq
+	case evt.RepoSync != nil:
+		return evt.RepoSync.Seq
+	case evt.RepoHandle != nil:
+		return evt.RepoHandle.Seq
+	case evt.RepoMigrate != nil:
+		return evt.RepoMigrate.Seq
+	case evt.RepoTombstone != nil:
+		return evt.RepoTombstone.Seq
+	case evt.RepoIdentity != nil:
+		return evt.RepoIdentity.Seq
+	case evt.RepoAccount != nil:
+		return evt.RepoAccount.Seq
+	case evt.RepoInfo != nil:
+		return -1
+	case evt.Error != nil:
+		return -1
+	default:
+		return -1
+	}
+}
+
+func (evt *XRPCStreamEvent) GetSequence() (int64, bool) {
+	switch {
+	case evt == nil:
+		return -1, false
+	case evt.RepoCommit != nil:
+		return evt.RepoCommit.Seq, true
+	case evt.RepoSync != nil:
+		return evt.RepoSync.Seq, true
+	case evt.RepoHandle != nil:
+		return evt.RepoHandle.Seq, true
+	case evt.RepoMigrate != nil:
+		return evt.RepoMigrate.Seq, true
+	case evt.RepoTombstone != nil:
+		return evt.RepoTombstone.Seq, true
+	case evt.RepoIdentity != nil:
+		return evt.RepoIdentity.Seq, true
+	case evt.RepoAccount != nil:
+		return evt.RepoAccount.Seq, true
+	case evt.RepoInfo != nil:
+		return -1, false
+	case evt.Error != nil:
+		return -1, false
+	default:
+		return -1, false
+	}
+}
+
+func (em *EventManager) rmSubscriber(sub *Subscriber) {
+	em.subsLk.Lock()
+	defer em.subsLk.Unlock()
+
+	for i, s := range em.subs {
+		if s == sub {
+			em.subs[i] = em.subs[len(em.subs)-1]
+			em.subs = em.subs[:len(em.subs)-1]
+			break
+		}
+	}
+}
+
+func (em *EventManager) addSubscriber(sub *Subscriber) {
+	em.subsLk.Lock()
+	defer em.subsLk.Unlock()
+
+	em.subs = append(em.subs, sub)
+}
+
+func (em *EventManager) TakeDownRepo(ctx context.Context, user models.Uid) error {
+	return em.persister.TakeDownRepo(ctx, user)
+}

cmd/relay/events/metrics.go

···
+package events
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var eventsFromStreamCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indigo_repo_stream_events_received_total",
+	Help: "Total number of events received from the stream",
+}, []string{"remote_addr"})
+
+var bytesFromStreamCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indigo_repo_stream_bytes_total",
+	Help: "Total bytes received from the stream",
+}, []string{"remote_addr"})
+
+var eventsEnqueued = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indigo_events_enqueued_for_broadcast_total",
+	Help: "Total number of events enqueued to broadcast to subscribers",
+}, []string{"pool"})
+
+var eventsBroadcast = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indigo_events_broadcast_total",
+	Help: "Total number of events broadcast to subscribers",
+}, []string{"pool"})

cmd/relay/events/persist.go

···
+package events
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/bluesky-social/indigo/cmd/relay/models"
+)
+
+// Note that this interface looks generic, but some persisters might only work with RepoAppend or LabelLabels
+type EventPersistence interface {
+	Persist(ctx context.Context, e *XRPCStreamEvent) error
+	Playback(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error) error
+	TakeDownRepo(ctx context.Context, usr models.Uid) error
+	Flush(context.Context) error
+	Shutdown(context.Context) error
+
+	SetEventBroadcaster(func(*XRPCStreamEvent))
+}
+
+// MemPersister is the most naive implementation of event persistence
+// This EventPersistence option works fine with all event types
+// ill do better later
+type MemPersister struct {
+	buf []*XRPCStreamEvent
+	lk  sync.Mutex
+	seq int64
+
+	broadcast func(*XRPCStreamEvent)
+}
+
+func NewMemPersister() *MemPersister {
+	return &MemPersister{}
+}
+
+func (mp *MemPersister) Persist(ctx context.Context, e *XRPCStreamEvent) error {
+	mp.lk.Lock()
+	defer mp.lk.Unlock()
+	mp.seq++
+	switch {
+	case e.RepoCommit != nil:
+		e.RepoCommit.Seq = mp.seq
+	case e.RepoHandle != nil:
+		e.RepoHandle.Seq = mp.seq
+	case e.RepoIdentity != nil:
+		e.RepoIdentity.Seq = mp.seq
+	case e.RepoAccount != nil:
+		e.RepoAccount.Seq = mp.seq
+	case e.RepoMigrate != nil:
+		e.RepoMigrate.Seq = mp.seq
+	case e.RepoTombstone != nil:
+		e.RepoTombstone.Seq = mp.seq
+	case e.LabelLabels != nil:
+		e.LabelLabels.Seq = mp.seq
+	default:
+		panic("no event in persist call")
+	}
+	mp.buf = append(mp.buf, e)
+
+	mp.broadcast(e)
+
+	return nil
+}
+
+func (mp *MemPersister) Playback(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error) error {
+	mp.lk.Lock()
+	l := len(mp.buf)
+	mp.lk.Unlock()
+
+	if since >= int64(l) {
+		return nil
+	}
+
+	// TODO: abusing the fact that buf[0].seq is currently always 1
+	for _, e := range mp.buf[since:l] {
+		if err := cb(e); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (mp *MemPersister) TakeDownRepo(ctx context.Context, uid models.Uid) error {
+	return fmt.Errorf("repo takedowns not currently supported by memory persister, test usage only")
+}
+
+func (mp *MemPersister) Flush(ctx context.Context) error {
+	return nil
+}
+
+func (mp *MemPersister) SetEventBroadcaster(brc func(*XRPCStreamEvent)) {
+	mp.broadcast = brc
+}
+
+func (mp *MemPersister) Shutdown(context.Context) error {
+	return nil
+}

cmd/relay/events/schedulers/metrics.go

···
+package schedulers
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var WorkItemsAdded = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indigo_scheduler_work_items_added_total",
+	Help: "Total number of work items added to the consumer pool",
+}, []string{"pool", "scheduler_type"})
+
+var WorkItemsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indigo_scheduler_work_items_processed_total",
+	Help: "Total number of work items processed by the consumer pool",
+}, []string{"pool", "scheduler_type"})
+
+var WorkItemsActive = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indigo_scheduler_work_items_active_total",
+	Help: "Total number of work items passed into a worker",
+}, []string{"pool", "scheduler_type"})
+
+var WorkersActive = promauto.NewGaugeVec(prometheus.GaugeOpts{
+	Name: "indigo_scheduler_workers_active",
+	Help: "Number of workers currently active",
+}, []string{"pool", "scheduler_type"})

cmd/relay/events/schedulers/parallel/parallel.go

···
+package parallel
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+
+	"github.com/bluesky-social/indigo/cmd/relay/events"
+	"github.com/bluesky-social/indigo/events/schedulers"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Scheduler is a parallel scheduler that will run work on a fixed number of workers
+type Scheduler struct {
+	maxConcurrency int
+	maxQueue       int
+
+	do func(context.Context, *events.XRPCStreamEvent) error
+
+	feeder chan *consumerTask
+	out    chan struct{}
+
+	lk     sync.Mutex
+	active map[string][]*consumerTask
+
+	ident string
+
+	// metrics
+	itemsAdded     prometheus.Counter
+	itemsProcessed prometheus.Counter
+	itemsActive    prometheus.Counter
+	workesActive   prometheus.Gauge
+
+	log *slog.Logger
+}
+
+func NewScheduler(maxC, maxQ int, ident string, do func(context.Context, *events.XRPCStreamEvent) error) *Scheduler {
+	p := &Scheduler{
+		maxConcurrency: maxC,
+		maxQueue:       maxQ,
+
+		do: do,
+
+		feeder: make(chan *consumerTask),
+		active: make(map[string][]*consumerTask),
+		out:    make(chan struct{}),
+
+		ident: ident,
+
+		itemsAdded:     schedulers.WorkItemsAdded.WithLabelValues(ident, "parallel"),
+		itemsProcessed: schedulers.WorkItemsProcessed.WithLabelValues(ident, "parallel"),
+		itemsActive:    schedulers.WorkItemsActive.WithLabelValues(ident, "parallel"),
+		workesActive:   schedulers.WorkersActive.WithLabelValues(ident, "parallel"),
+
+		log: slog.Default().With("system", "parallel-scheduler"),
+	}
+
+	for i := 0; i < maxC; i++ {
+		go p.worker()
+	}
+
+	p.workesActive.Set(float64(maxC))
+
+	return p
+}
+
+func (p *Scheduler) Shutdown() {
+	p.log.Info("shutting down parallel scheduler", "ident", p.ident)
+
+	for i := 0; i < p.maxConcurrency; i++ {
+		p.feeder <- &consumerTask{
+			control: "stop",
+		}
+	}
+
+	close(p.feeder)
+
+	for i := 0; i < p.maxConcurrency; i++ {
+		<-p.out
+	}
+
+	p.log.Info("parallel scheduler shutdown complete")
+}
+
+type consumerTask struct {
+	repo    string
+	val     *events.XRPCStreamEvent
+	control string
+}
+
+func (p *Scheduler) AddWork(ctx context.Context, repo string, val *events.XRPCStreamEvent) error {
+	p.itemsAdded.Inc()
+	t := &consumerTask{
+		repo: repo,
+		val:  val,
+	}
+	p.lk.Lock()
+
+	a, ok := p.active[repo]
+	if ok {
+		p.active[repo] = append(a, t)
+		p.lk.Unlock()
+		return nil
+	}
+
+	p.active[repo] = []*consumerTask{}
+	p.lk.Unlock()
+
+	select {
+	case p.feeder <- t:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (p *Scheduler) worker() {
+	for work := range p.feeder {
+		for work != nil {
+			if work.control == "stop" {
+				p.out <- struct{}{}
+				return
+			}
+
+			p.itemsActive.Inc()
+			if err := p.do(context.TODO(), work.val); err != nil {
+				p.log.Error("event handler failed", "err", err)
+			}
+			p.itemsProcessed.Inc()
+
+			p.lk.Lock()
+			rem, ok := p.active[work.repo]
+			if !ok {
+				p.log.Error("should always have an 'active' entry if a worker is processing a job")
+			}
+
+			if len(rem) == 0 {
+				delete(p.active, work.repo)
+				work = nil
+			} else {
+				work = rem[0]
+				p.active[work.repo] = rem[1:]
+			}
+			p.lk.Unlock()
+		}
+	}
+}

cmd/relay/events/schedulers/scheduler.go

···
+package schedulers

cmd/relay/main.go

···
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"github.com/bluesky-social/indigo/atproto/identity"
+	"github.com/bluesky-social/indigo/cmd/relay/events/diskpersist"
+	"gorm.io/gorm"
+	"io"
+	"log/slog"
+	_ "net/http/pprof"
+	"net/url"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	libbgs "github.com/bluesky-social/indigo/cmd/relay/bgs"
+	"github.com/bluesky-social/indigo/cmd/relay/events"
+	"github.com/bluesky-social/indigo/util"
+	"github.com/bluesky-social/indigo/util/cliutil"
+	"github.com/bluesky-social/indigo/xrpc"
+
+	_ "github.com/joho/godotenv/autoload"
+	_ "go.uber.org/automaxprocs"
+
+	"github.com/carlmjohnson/versioninfo"
+	"github.com/urfave/cli/v2"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/sdk/resource"
+	tracesdk "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	"gorm.io/plugin/opentelemetry/tracing"
+)
+
+func main() {
+	if err := run(os.Args); err != nil {
+		slog.Error(err.Error())
+		os.Exit(1)
+	}
+}
+
+func run(args []string) error {
+
+	app := cli.App{
+		Name:    "relay",
+		Usage:   "atproto Relay daemon",
+		Version: versioninfo.Short(),
+	}
+
+	app.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name: "jaeger",
+		},
+		&cli.StringFlag{
+			Name:    "db-url",
+			Usage:   "database connection string for BGS database",
+			Value:   "sqlite://./data/bigsky/bgs.sqlite",
+			EnvVars: []string{"DATABASE_URL"},
+		},
+		&cli.BoolFlag{
+			Name: "db-tracing",
+		},
+		&cli.StringFlag{
+			Name:    "plc-host",
+			Usage:   "method, hostname, and port of PLC registry",
+			Value:   "https://plc.directory",
+			EnvVars: []string{"ATP_PLC_HOST"},
+		},
+		&cli.BoolFlag{
+			Name:  "crawl-insecure-ws",
+			Usage: "when connecting to PDS instances, use ws:// instead of wss://",
+		},
+		&cli.StringFlag{
+			Name:    "api-listen",
+			Value:   ":2470",
+			EnvVars: []string{"RELAY_API_LISTEN"},
+		},
+		&cli.StringFlag{
+			Name:    "metrics-listen",
+			Value:   ":2471",
+			EnvVars: []string{"RELAY_METRICS_LISTEN", "BGS_METRICS_LISTEN"},
+		},
+		&cli.StringFlag{
+			Name:    "disk-persister-dir",
+			Usage:   "set directory for disk persister (implicitly enables disk persister)",
+			EnvVars: []string{"RELAY_PERSISTER_DIR"},
+		},
+		&cli.StringFlag{
+			Name:    "admin-key",
+			EnvVars: []string{"RELAY_ADMIN_KEY", "BGS_ADMIN_KEY"},
+		},
+		&cli.IntFlag{
+			Name:    "max-metadb-connections",
+			EnvVars: []string{"MAX_METADB_CONNECTIONS"},
+			Value:   40,
+		},
+		&cli.StringFlag{
+			Name:    "env",
+			Value:   "dev",
+			EnvVars: []string{"ENVIRONMENT"},
+			Usage:   "declared hosting environment (prod, qa, etc); used in metrics",
+		},
+		&cli.StringFlag{
+			Name:    "otel-exporter-otlp-endpoint",
+			EnvVars: []string{"OTEL_EXPORTER_OTLP_ENDPOINT"},
+		},
+		&cli.StringFlag{
+			Name:    "bsky-social-rate-limit-skip",
+			EnvVars: []string{"BSKY_SOCIAL_RATE_LIMIT_SKIP"},
+			Usage:   "ratelimit bypass secret token for *.bsky.social domains",
+		},
+		&cli.IntFlag{
+			Name:    "default-repo-limit",
+			Value:   100,
+			EnvVars: []string{"RELAY_DEFAULT_REPO_LIMIT"},
+		},
+		&cli.IntFlag{
+			Name:    "concurrency-per-pds",
+			EnvVars: []string{"RELAY_CONCURRENCY_PER_PDS"},
+			Value:   100,
+		},
+		&cli.IntFlag{
+			Name:    "max-queue-per-pds",
+			EnvVars: []string{"RELAY_MAX_QUEUE_PER_PDS"},
+			Value:   1_000,
+		},
+		&cli.IntFlag{
+			Name:    "did-cache-size",
+			Usage:   "in-process cache by number of Did documents",
+			EnvVars: []string{"RELAY_DID_CACHE_SIZE"},
+			Value:   5_000_000,
+		},
+		&cli.DurationFlag{
+			Name:    "event-playback-ttl",
+			Usage:   "time to live for event playback buffering (only applies to disk persister)",
+			EnvVars: []string{"RELAY_EVENT_PLAYBACK_TTL"},
+			Value:   72 * time.Hour,
+		},
+		&cli.StringSliceFlag{
+			Name:    "next-crawler",
+			Usage:   "forward POST requestCrawl to this url, should be machine root url and not xrpc/requestCrawl, comma separated list",
+			EnvVars: []string{"RELAY_NEXT_CRAWLER"},
+		},
+		&cli.StringFlag{
+			Name:    "trace-induction",
+			Usage:   "file path to log debug trace stuff about induction firehose",
+			EnvVars: []string{"RELAY_TRACE_INDUCTION"},
+		},
+		&cli.BoolFlag{
+			Name:    "time-seq",
+			EnvVars: []string{"RELAY_TIME_SEQUENCE"},
+			Value:   false,
+			Usage:   "make outbound firehose sequence number approximately unix microseconds",
+		},
+	}
+
+	app.Action = runRelay
+	return app.Run(os.Args)
+}
+
+func setupOTEL(cctx *cli.Context) error {
+
+	env := cctx.String("env")
+	if env == "" {
+		env = "dev"
+	}
+	if cctx.Bool("jaeger") {
+		jaegerUrl := "http://localhost:14268/api/traces"
+		exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerUrl)))
+		if err != nil {
+			return err
+		}
+		tp := tracesdk.NewTracerProvider(
+			// Always be sure to batch in production.
+			tracesdk.WithBatcher(exp),
+			// Record information about this application in a Resource.
+			tracesdk.WithResource(resource.NewWithAttributes(
+				semconv.SchemaURL,
+				semconv.ServiceNameKey.String("bgs"),
+				attribute.String("env", env),         // DataDog
+				attribute.String("environment", env), // Others
+				attribute.Int64("ID", 1),
+			)),
+		)
+
+		otel.SetTracerProvider(tp)
+	}
+
+	// Enable OTLP HTTP exporter
+	// For relevant environment variables:
+	// https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables
+	// At a minimum, you need to set
+	// OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+	if ep := cctx.String("otel-exporter-otlp-endpoint"); ep != "" {
+		slog.Info("setting up trace exporter", "endpoint", ep)
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		exp, err := otlptracehttp.New(ctx)
+		if err != nil {
+			slog.Error("failed to create trace exporter", "error", err)
+			os.Exit(1)
+		}
+		defer func() {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+			defer cancel()
+			if err := exp.Shutdown(ctx); err != nil {
+				slog.Error("failed to shutdown trace exporter", "error", err)
+			}
+		}()
+
+		tp := tracesdk.NewTracerProvider(
+			tracesdk.WithBatcher(exp),
+			tracesdk.WithResource(resource.NewWithAttributes(
+				semconv.SchemaURL,
+				semconv.ServiceNameKey.String("bgs"),
+				attribute.String("env", env),         // DataDog
+				attribute.String("environment", env), // Others
+				attribute.Int64("ID", 1),
+			)),
+		)
+		otel.SetTracerProvider(tp)
+	}
+
+	return nil
+}
+
+func runRelay(cctx *cli.Context) error {
+	// Trap SIGINT to trigger a shutdown.
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
+
+	logger, logWriter, err := cliutil.SetupSlog(cliutil.LogOptions{})
+	if err != nil {
+		return err
+	}
+
+	var inductionTraceLog *slog.Logger
+
+	if cctx.IsSet("trace-induction") {
+		traceFname := cctx.String("trace-induction")
+		traceFout, err := os.OpenFile(traceFname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return fmt.Errorf("%s: could not open trace file: %w", traceFname, err)
+		}
+		defer traceFout.Close()
+		if traceFname != "" {
+			inductionTraceLog = slog.New(slog.NewJSONHandler(traceFout, &slog.HandlerOptions{Level: slog.LevelDebug}))
+		}
+	} else {
+		inductionTraceLog = slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.Level(999)}))
+	}
+
+	// start observability/tracing (OTEL and jaeger)
+	if err := setupOTEL(cctx); err != nil {
+		return err
+	}
+
+	dburl := cctx.String("db-url")
+	logger.Info("setting up main database", "url", dburl)
+	db, err := cliutil.SetupDatabase(dburl, cctx.Int("max-metadb-connections"))
+	if err != nil {
+		return err
+	}
+	if cctx.Bool("db-tracing") {
+		if err := db.Use(tracing.NewPlugin()); err != nil {
+			return err
+		}
+	}
+	if err := db.AutoMigrate(RelaySetting{}); err != nil {
+		panic(err)
+	}
+
+	// TODO: add shared external cache
+	baseDir := identity.BaseDirectory{
+		SkipHandleVerification: true,
+		SkipDNSDomainSuffixes:  []string{".bsky.social"},
+		TryAuthoritativeDNS:    true,
+	}
+	cacheDir := identity.NewCacheDirectory(&baseDir, cctx.Int("did-cache-size"), time.Hour*24, time.Minute*2, time.Minute*5)
+
+	// TODO: rename repoman
+	repoman := libbgs.NewValidator(&cacheDir, inductionTraceLog)
+
+	var persister events.EventPersistence
+
+	dpd := cctx.String("disk-persister-dir")
+	if dpd == "" {
+		logger.Info("empty disk-persister-dir, use current working directory")
+		cwd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+		dpd = filepath.Join(cwd, "relay-persist")
+	}
+	logger.Info("setting up disk persister", "dir", dpd)
+
+	pOpts := diskpersist.DefaultDiskPersistOptions()
+	pOpts.Retention = cctx.Duration("event-playback-ttl")
+	pOpts.TimeSequence = cctx.Bool("time-seq")
+
+	// ensure that time-ish sequence stays consistent within a server context
+	storedTimeSeq, hadStoredTimeSeq, err := getRelaySettingBool(db, "time-seq")
+	if err != nil {
+		return err
+	}
+	if !hadStoredTimeSeq {
+		if err := setRelaySettingBool(db, "time-seq", pOpts.TimeSequence); err != nil {
+			return err
+		}
+	} else {
+		if pOpts.TimeSequence != storedTimeSeq {
+			return fmt.Errorf("time-seq stored as %v but param/env set as %v", storedTimeSeq, pOpts.TimeSequence)
+		}
+	}
+
+	dp, err := diskpersist.NewDiskPersistence(dpd, "", db, pOpts)
+	if err != nil {
+		return fmt.Errorf("setting up disk persister: %w", err)
+	}
+	persister = dp
+
+	evtman := events.NewEventManager(persister)
+
+	ratelimitBypass := cctx.String("bsky-social-rate-limit-skip")
+
+	logger.Info("constructing bgs")
+	bgsConfig := libbgs.DefaultBGSConfig()
+	bgsConfig.SSL = !cctx.Bool("crawl-insecure-ws")
+	bgsConfig.ConcurrencyPerPDS = cctx.Int64("concurrency-per-pds")
+	bgsConfig.MaxQueuePerPDS = cctx.Int64("max-queue-per-pds")
+	bgsConfig.DefaultRepoLimit = cctx.Int64("default-repo-limit")
+	bgsConfig.ApplyPDSClientSettings = makePdsClientSetup(ratelimitBypass)
+	bgsConfig.InductionTraceLog = inductionTraceLog
+	nextCrawlers := cctx.StringSlice("next-crawler")
+	if len(nextCrawlers) != 0 {
+		nextCrawlerUrls := make([]*url.URL, len(nextCrawlers))
+		for i, tu := range nextCrawlers {
+			var err error
+			nextCrawlerUrls[i], err = url.Parse(tu)
+			if err != nil {
+				return fmt.Errorf("failed to parse next-crawler url: %w", err)
+			}
+			logger.Info("configuring relay for requestCrawl", "host", nextCrawlerUrls[i])
+		}
+		bgsConfig.NextCrawlers = nextCrawlerUrls
+	}
+	if cctx.IsSet("admin-key") {
+		bgsConfig.AdminToken = cctx.String("admin-key")
+	} else {
+		var rblob [10]byte
+		_, _ = rand.Read(rblob[:])
+		bgsConfig.AdminToken = base64.URLEncoding.EncodeToString(rblob[:])
+		logger.Info("generated random admin key", "header", "Authorization: Bearer "+bgsConfig.AdminToken)
+	}
+	bgs, err := libbgs.NewBGS(db, repoman, evtman, &cacheDir, bgsConfig)
+	if err != nil {
+		return err
+	}
+	dp.SetUidSource(bgs)
+
+	// set up metrics endpoint
+	go func() {
+		if err := bgs.StartMetrics(cctx.String("metrics-listen")); err != nil {
+			logger.Error("failed to start metrics endpoint", "err", err)
+			os.Exit(1)
+		}
+	}()
+
+	bgsErr := make(chan error, 1)
+
+	go func() {
+		err := bgs.Start(cctx.String("api-listen"), logWriter)
+		bgsErr <- err
+	}()
+
+	logger.Info("startup complete")
+	select {
+	case <-signals:
+		logger.Info("received shutdown signal")
+		errs := bgs.Shutdown()
+		for err := range errs {
+			logger.Error("error during BGS shutdown", "err", err)
+		}
+	case err := <-bgsErr:
+		if err != nil {
+			logger.Error("error during BGS startup", "err", err)
+		}
+		logger.Info("shutting down")
+		errs := bgs.Shutdown()
+		for err := range errs {
+			logger.Error("error during BGS shutdown", "err", err)
+		}
+	}
+
+	logger.Info("shutdown complete")
+
+	return nil
+}
+
+func makePdsClientSetup(ratelimitBypass string) func(c *xrpc.Client) {
+	return func(c *xrpc.Client) {
+		if c.Client == nil {
+			c.Client = util.RobustHTTPClient()
+		}
+		if strings.HasSuffix(c.Host, ".bsky.network") {
+			c.Client.Timeout = time.Minute * 30
+			if ratelimitBypass != "" {
+				c.Headers = map[string]string{
+					"x-ratelimit-bypass": ratelimitBypass,
+				}
+			}
+		} else {
+			// Generic PDS timeout
+			c.Client.Timeout = time.Minute * 1
+		}
+	}
+}
+
+// RelaySetting is a gorm model
+type RelaySetting struct {
+	Name  string `gorm:"primarykey"`
+	Value string
+}
+
+func getRelaySetting(db *gorm.DB, name string) (value string, found bool, err error) {
+	var setting RelaySetting
+	dbResult := db.First(&setting, "name = ?", name)
+	if errors.Is(dbResult.Error, gorm.ErrRecordNotFound) {
+		return "", false, nil
+	}
+	if dbResult.Error != nil {
+		return "", false, dbResult.Error
+	}
+	return setting.Value, true, nil
+}
+
+func setRelaySetting(db *gorm.DB, name string, value string) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		var setting RelaySetting
+		found := tx.First(&setting, "name = ?", name)
+		if errors.Is(found.Error, gorm.ErrRecordNotFound) {
+			// ok! create it
+			setting.Name = name
+			setting.Value = value
+			return tx.Create(&setting).Error
+		} else if found.Error != nil {
+			return found.Error
+		}
+		setting.Value = value
+		return tx.Save(&setting).Error
+	})
+}
+
+func getRelaySettingBool(db *gorm.DB, name string) (value bool, found bool, err error) {
+	strval, found, err := getRelaySetting(db, name)
+	if err != nil || !found {
+		return false, found, err
+	}
+	value, err = strconv.ParseBool(strval)
+	if err != nil {
+		return false, false, err
+	}
+	return value, true, nil
+}
+func setRelaySettingBool(db *gorm.DB, name string, value bool) error {
+	return setRelaySetting(db, name, strconv.FormatBool(value))
+}

cmd/relay/models/models.go

···
+package models
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"github.com/ipfs/go-cid"
+	"gorm.io/gorm"
+)
+
+type Uid uint64
+
+type DbCID struct {
+	CID cid.Cid
+}
+
+func (dbc *DbCID) Scan(v interface{}) error {
+	b, ok := v.([]byte)
+	if !ok {
+		return fmt.Errorf("dbcids must get bytes!")
+	}
+
+	if len(b) == 0 {
+		return nil
+	}
+
+	c, err := cid.Cast(b)
+	if err != nil {
+		return err
+	}
+
+	dbc.CID = c
+	return nil
+}
+
+func (dbc DbCID) Value() (driver.Value, error) {
+	if !dbc.CID.Defined() {
+		return nil, fmt.Errorf("cannot serialize undefined cid to database")
+	}
+	return dbc.CID.Bytes(), nil
+}
+
+func (dbc DbCID) MarshalJSON() ([]byte, error) {
+	return json.Marshal(dbc.CID.String())
+}
+
+func (dbc *DbCID) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+
+	c, err := cid.Decode(s)
+	if err != nil {
+		return err
+	}
+
+	dbc.CID = c
+	return nil
+}
+
+func (dbc *DbCID) GormDataType() string {
+	return "bytes"
+}
+
+type PDS struct {
+	gorm.Model
+
+	Host       string `gorm:"unique"`
+	SSL        bool
+	Cursor     int64
+	Registered bool
+	Blocked    bool
+
+	RateLimit float64
+
+	RepoCount int64
+	RepoLimit int64
+
+	HourlyEventLimit int64
+	DailyEventLimit  int64
+}

events/consumer.go

···
 	switch {
 	case xev.RepoCommit != nil && rsc.RepoCommit != nil:
 		return rsc.RepoCommit(xev.RepoCommit)
-	case xev.RepoSync != nil && rsc.RepoCommit != nil:
+	case xev.RepoSync != nil && rsc.RepoSync != nil:
 		return rsc.RepoSync(xev.RepoSync)
 	case xev.RepoHandle != nil && rsc.RepoHandle != nil:
 		return rsc.RepoHandle(xev.RepoHandle)
···
 	go func() {
 		t := time.NewTicker(time.Second * 30)
 		defer t.Stop()
+		failcount := 0
 
 		for {
 
···
 			case <-t.C:
 				if err := con.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second*10)); err != nil {
 					log.Warn("failed to ping", "err", err)
+					failcount++
+					if failcount >= 4 {
+						log.Error("too many ping fails", "count", failcount)
+						con.Close()
+						return
+					}
+				} else {
+					failcount = 0 // ok ping
 				}
 			case <-ctx.Done():
 				con.Close()
···
 
 		mt, rawReader, err := con.NextReader()
 		if err != nil {
-			return err
+			return fmt.Errorf("con err at read: %w", err)
 		}
 
 		switch mt {
···
 					return err
 				}
 			case "#handle":
+				// TODO: DEPRECATED message; warning/counter; drop message
 				var evt comatproto.SyncSubscribeRepos_Handle
 				if err := evt.UnmarshalCBOR(r); err != nil {
 					return err
···
 					return err
 				}
 			case "#migrate":
+				// TODO: DEPRECATED message; warning/counter; drop message
 				var evt comatproto.SyncSubscribeRepos_Migrate
 				if err := evt.UnmarshalCBOR(r); err != nil {
 					return err
···
 					return err
 				}
 			case "#tombstone":
+				// TODO: DEPRECATED message; warning/counter; drop message
 				var evt comatproto.SyncSubscribeRepos_Tombstone
 				if err := evt.UnmarshalCBOR(r); err != nil {
 					return err

events/events.go

···
 }
 
 var (
-	AccountStatusActive      = "active"
-	AccountStatusTakendown   = "takendown"
-	AccountStatusSuspended   = "suspended"
-	AccountStatusDeleted     = "deleted"
-	AccountStatusDeactivated = "deactivated"
+	// AccountStatusActive is not in the spec but used internally
+	// the alternative would be an additional SQL column for "active" or status="" to imply active
+	AccountStatusActive = "active"
+
+	AccountStatusDeactivated    = "deactivated"
+	AccountStatusDeleted        = "deleted"
+	AccountStatusDesynchronized = "desynchronized"
+	AccountStatusSuspended      = "suspended"
+	AccountStatusTakendown      = "takendown"
+	AccountStatusThrottled      = "throttled"
 )
 
+var AccountStatusList = []string{
+	AccountStatusActive,
+	AccountStatusDeactivated,
+	AccountStatusDeleted,
+	AccountStatusDesynchronized,
+	AccountStatusSuspended,
+	AccountStatusTakendown,
+	AccountStatusThrottled,
+}
+var AccountStatuses map[string]bool
+
+func init() {
+	AccountStatuses = make(map[string]bool, len(AccountStatusList))
+	for _, status := range AccountStatusList {
+		AccountStatuses[status] = true
+	}
+}
+
 type XRPCStreamEvent struct {
 	Error         *ErrorFrame
 	RepoCommit    *comatproto.SyncSubscribeRepos_Commit
 	RepoSync      *comatproto.SyncSubscribeRepos_Sync
-	RepoHandle    *comatproto.SyncSubscribeRepos_Handle
+	RepoHandle    *comatproto.SyncSubscribeRepos_Handle // DEPRECATED
 	RepoIdentity  *comatproto.SyncSubscribeRepos_Identity
 	RepoInfo      *comatproto.SyncSubscribeRepos_Info
-	RepoMigrate   *comatproto.SyncSubscribeRepos_Migrate
-	RepoTombstone *comatproto.SyncSubscribeRepos_Tombstone
+	RepoMigrate   *comatproto.SyncSubscribeRepos_Migrate   // DEPRECATED
+	RepoTombstone *comatproto.SyncSubscribeRepos_Tombstone // DEPRECATED
 	RepoAccount   *comatproto.SyncSubscribeRepos_Account
 	LabelLabels   *comatproto.LabelSubscribeLabels_Labels
 	LabelInfo     *comatproto.LabelSubscribeLabels_Info
···
 			}
 			xevt.RepoSync = &evt
 		case "#handle":
+			// TODO: DEPRECATED message; warning/counter; drop message
 			var evt comatproto.SyncSubscribeRepos_Handle
 			if err := evt.UnmarshalCBOR(r); err != nil {
 				return err
···
 			}
 			xevt.RepoInfo = &evt
 		case "#migrate":
+			// TODO: DEPRECATED message; warning/counter; drop message
 			var evt comatproto.SyncSubscribeRepos_Migrate
 			if err := evt.UnmarshalCBOR(r); err != nil {
 				return err
 			}
 			xevt.RepoMigrate = &evt
 		case "#tombstone":
+			// TODO: DEPRECATED message; warning/counter; drop message
 			var evt comatproto.SyncSubscribeRepos_Tombstone
 			if err := evt.UnmarshalCBOR(r); err != nil {
 				return err

events/persist.go

···
 
 // Note that this interface looks generic, but some persisters might only work with RepoAppend or LabelLabels
 type EventPersistence interface {
+	// Persist may mutate contents of *XRPCStreamEvent and what it points to
 	Persist(ctx context.Context, e *XRPCStreamEvent) error
 	Playback(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error) error
 	TakeDownRepo(ctx context.Context, usr models.Uid) error

ts/bgs-dash/src/components/Dash/Dash.tsx

···
     useState<PDS | null>(null);
   const [editingPerDayRateLimit, setEditingPerDayRateLimit] =
     useState<PDS | null>(null);
-  const [editingCrawlRateLimit, setEditingCrawlRateLimit] =
-    useState<PDS | null>(null);
   const [editingRepoLimit, setEditingRepoLimit] =
     useState<PDS | null>(null);
 
···
           per_second: pds.PerSecondEventRate.Max,
           per_hour: pds.PerHourEventRate.Max,
           per_day: pds.PerDayEventRate.Max,
-          crawl_rate: pds.CrawlRate.Max,
           repo_limit: pds.RepoLimit,
         }),
       }
···
                   className="px-3 py-3.5 text-right text-sm font-semibold text-gray-900 pr-6 whitespace-nowrap"
                 >
                   <a href="#" className="group inline-flex">
-                    Crawl Limit
-                  </a>
-                </th>
-                <th
-                  scope="col"
-                  className="px-3 py-3.5 text-right text-sm font-semibold text-gray-900 pr-6 whitespace-nowrap"
-                >
-                  <a href="#" className="group inline-flex">
                     Repo Limit
                   </a>
                 </th>
···
                           className={
                             "rounded-md p-2  ml-1 hover:text-green-600 hover:bg-green-100 focus:outline-none focus:ring-2 focus:ring-green-600 focus:ring-offset-2 focus:ring-offset-green-50" +
                             (editingPerDayRateLimit?.ID === pds.ID
-                              ? ""
-                              : " hidden")
-                          }
-                        >
-                          <CheckIcon
-                            className="h-5 w-5 text-green-500 inline-block align-sub"
-                            aria-hidden="true"
-                          />
-                        </a>
-                      </td>
-                      <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6">
-                        <span
-                          className={
-                            editingCrawlRateLimit?.ID === pds.ID
-                              ? "hidden"
-                              : ""
-                          }
-                        >
-                          {pds.CrawlRate.Max?.toLocaleString()}
-                          /sec
-                        </span>
-                        <input
-                          type="number"
-                          name={`crawl-rate-limit-${pds.ID}`}
-                          id={`crawl-rate-limit-${pds.ID}`}
-                          className={
-                            `inline-block w-24 rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6` +
-                            (editingCrawlRateLimit?.ID === pds.ID
-                              ? ""
-                              : " hidden")
-                          }
-                          defaultValue={pds.CrawlRate.Max?.toLocaleString()}
-                        />
-                        <a
-                          href="#"
-                          onClick={() => setEditingCrawlRateLimit(pds)}
-                          className={editingCrawlRateLimit ? "hidden" : ""}
-                        >
-                          <PencilSquareIcon
-                            className="h-5 w-5 text-gray-500 ml-1 inline-block align-sub"
-                            aria-hidden="true"
-                          />
-                        </a>
-                        <a
-                          href="#"
-                          onClick={() => {
-                            const newRateLimit = document.getElementById(
-                              `crawl-rate-limit-${pds.ID}`
-                            ) as HTMLInputElement;
-                            if (newRateLimit) {
-                              pds.CrawlRate.Max = +newRateLimit.value;
-                              updateRateLimits(pds);
-                            }
-                            setEditingCrawlRateLimit(null);
-                          }}
-                          className={
-                            "rounded-md p-2  ml-1 hover:text-green-600 hover:bg-green-100 focus:outline-none focus:ring-2 focus:ring-green-600 focus:ring-offset-2 focus:ring-offset-green-50" +
-                            (editingCrawlRateLimit?.ID === pds.ID
                               ? ""
                               : " hidden")
                           }

ts/bgs-dash/src/models/pds.ts

···
   Blocked: boolean;
   HasActiveConnection: boolean;
   EventsSeenSinceStartup?: number;
-  CrawlRate: RateLimit;
   PerSecondEventRate: RateLimit;
   PerHourEventRate: RateLimit;
   PerDayEventRate: RateLimit;

util/cliutil/util.go

···
 // The env vars were derived from ipfs logging library, and also respond to some GOLOG_ vars from that library,
 // but BSKYLOG_ variables are preferred because imported code still using the ipfs log library may misbehave
 // if some GOLOG values are set, especially GOLOG_FILE.
-func SetupSlog(options LogOptions) (*slog.Logger, error) {
+func SetupSlog(options LogOptions) (*slog.Logger, io.Writer, error) {
 	fmt.Fprintf(os.Stderr, "SetupSlog\n")
 	var hopts slog.HandlerOptions
 	hopts.Level = slog.LevelInfo
···
 		case "error":
 			hopts.Level = slog.LevelError
 		default:
-			return nil, fmt.Errorf("unknown log level: %#v", options.LogLevel)
+			return nil, nil, fmt.Errorf("unknown log level: %#v", options.LogLevel)
 		}
 	}
 	if options.LogFormat == "" {
···
 		if format == "json" || format == "text" {
 			// ok
 		} else {
-			return nil, fmt.Errorf("invalid log format: %#v", options.LogFormat)
+			return nil, nil, fmt.Errorf("invalid log format: %#v", options.LogFormat)
 		}
 		options.LogFormat = format
 	}
···
 		if rotateBytesStr != "" {
 			rotateBytes, err := strconv.ParseInt(rotateBytesStr, 10, 64)
 			if err != nil {
-				return nil, fmt.Errorf("invalid BSKYLOG_ROTATE_BYTES value: %w", err)
+				return nil, nil, fmt.Errorf("invalid BSKYLOG_ROTATE_BYTES value: %w", err)
 			}
 			options.LogRotateBytes = rotateBytes
 		}
···
 		if keepOldStr != "" {
 			keepOld, err := strconv.ParseInt(keepOldStr, 10, 64)
 			if err != nil {
-				return nil, fmt.Errorf("invalid BSKYLOG_ROTATE_KEEP value: %w", err)
+				return nil, nil, fmt.Errorf("invalid BSKYLOG_ROTATE_KEEP value: %w", err)
 			}
 			keepOldUnset = false
 			options.KeepOld = int(keepOld)
···
 		var err error
 		out, err = os.Create(options.LogPath)
 		if err != nil {
-			return nil, fmt.Errorf("%s: %w", options.LogPath, err)
+			return nil, nil, fmt.Errorf("%s: %w", options.LogPath, err)
 		}
 		fmt.Fprintf(os.Stderr, "SetupSlog create %#v\n", options.LogPath)
 	}
···
 	case "json":
 		handler = slog.NewJSONHandler(out, &hopts)
 	default:
-		return nil, fmt.Errorf("unknown log format: %#v", options.LogFormat)
+		return nil, nil, fmt.Errorf("unknown log format: %#v", options.LogFormat)
 	}
 	logger := slog.New(handler)
 	slog.SetDefault(logger)
···
 		fmt.Fprintf(os.Stdout, "%s\n", filepath.Join(templateDirPart, ent.Name()))
 	}
 	SetIpfsWriter(out, options.LogFormat, options.LogLevel)
-	return logger, nil
+	return logger, out, nil
 }
 
 type logRotateWriter struct {