Merge branch 'main' into escalate-on-high-churn

.github/workflows/container-rainbow-aws.yaml

···
+name: container-rainbow-aws
+on: [push]
+env:
+  REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
+  USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}
+  PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }}
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: rainbow
+
+jobs:
+  container-rainbow-aws:
+    if: github.repository == 'bluesky-social/indigo'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME }}
+          password: ${{ env.PASSWORD }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,enable=true,priority=100,prefix=,suffix=,format=long
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./cmd/rainbow/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/golang.yml

···
       - name: Set up Go tooling
         uses: actions/setup-go@v4
         with:
-          go-version: "1.22"
+          go-version: "1.23"
       - name: Build
         run: make build
       - name: Test
···
       - name: Set up Go tooling
         uses: actions/setup-go@v4
         with:
-          go-version: "1.22"
+          go-version: "1.23"
       - name: Lint
         run: make lint

HACKING.md

···
 - `cmd/supercollider`: event stream load generation tool
 - `cmd/sonar`: event stream monitoring tool
 - `cmd/hepa`: auto-moderation rule engine service
+- `cmd/rainbow`: firehose fanout service
 - `gen`: dev tool to run CBOR type codegen
 
 Packages:

api/agnostic/doc.go

···
+// Package indigo/api/agnositc provides schema-agnostic helpers for fetching records from the network.
+//
+// These are variants of endpoints in indigo/api/atproto.
+package agnostic

api/agnostic/repoapplyWrites.go

···
+// Copied from indigo:api/atproto/repoapplyWrites.go
+
+package agnostic
+
+// schema: com.atproto.repo.applyWrites
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/bluesky-social/indigo/lex/util"
+	"github.com/bluesky-social/indigo/xrpc"
+)
+
+// RepoApplyWrites_Create is a "create" in the com.atproto.repo.applyWrites schema.
+//
+// Operation which creates a new record.
+//
+// RECORDTYPE: RepoApplyWrites_Create
+type RepoApplyWrites_Create struct {
+	LexiconTypeID string           `json:"$type,const=com.atproto.repo.applyWrites#create" cborgen:"$type,const=com.atproto.repo.applyWrites#create"`
+	Collection    string           `json:"collection" cborgen:"collection"`
+	Rkey          *string          `json:"rkey,omitempty" cborgen:"rkey,omitempty"`
+	Value         *json.RawMessage `json:"value" cborgen:"value"`
+}
+
+// RepoApplyWrites_CreateResult is a "createResult" in the com.atproto.repo.applyWrites schema.
+//
+// RECORDTYPE: RepoApplyWrites_CreateResult
+type RepoApplyWrites_CreateResult struct {
+	LexiconTypeID    string  `json:"$type,const=com.atproto.repo.applyWrites#createResult" cborgen:"$type,const=com.atproto.repo.applyWrites#createResult"`
+	Cid              string  `json:"cid" cborgen:"cid"`
+	Uri              string  `json:"uri" cborgen:"uri"`
+	ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"`
+}
+
+// RepoApplyWrites_Delete is a "delete" in the com.atproto.repo.applyWrites schema.
+//
+// Operation which deletes an existing record.
+//
+// RECORDTYPE: RepoApplyWrites_Delete
+type RepoApplyWrites_Delete struct {
+	LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#delete" cborgen:"$type,const=com.atproto.repo.applyWrites#delete"`
+	Collection    string `json:"collection" cborgen:"collection"`
+	Rkey          string `json:"rkey" cborgen:"rkey"`
+}
+
+// RepoApplyWrites_DeleteResult is a "deleteResult" in the com.atproto.repo.applyWrites schema.
+//
+// RECORDTYPE: RepoApplyWrites_DeleteResult
+type RepoApplyWrites_DeleteResult struct {
+	LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#deleteResult" cborgen:"$type,const=com.atproto.repo.applyWrites#deleteResult"`
+}
+
+// RepoApplyWrites_Input is the input argument to a com.atproto.repo.applyWrites call.
+type RepoApplyWrites_Input struct {
+	// repo: The handle or DID of the repo (aka, current account).
+	Repo string `json:"repo" cborgen:"repo"`
+	// swapCommit: If provided, the entire operation will fail if the current repo commit CID does not match this value. Used to prevent conflicting repo mutations.
+	SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"`
+	// validate: Can be set to 'false' to skip Lexicon schema validation of record data across all operations, 'true' to require it, or leave unset to validate only for known Lexicons.
+	Validate *bool                                `json:"validate,omitempty" cborgen:"validate,omitempty"`
+	Writes   []*RepoApplyWrites_Input_Writes_Elem `json:"writes" cborgen:"writes"`
+}
+
+type RepoApplyWrites_Input_Writes_Elem struct {
+	RepoApplyWrites_Create *RepoApplyWrites_Create
+	RepoApplyWrites_Update *RepoApplyWrites_Update
+	RepoApplyWrites_Delete *RepoApplyWrites_Delete
+}
+
+func (t *RepoApplyWrites_Input_Writes_Elem) MarshalJSON() ([]byte, error) {
+	if t.RepoApplyWrites_Create != nil {
+		t.RepoApplyWrites_Create.LexiconTypeID = "com.atproto.repo.applyWrites#create"
+		return json.Marshal(t.RepoApplyWrites_Create)
+	}
+	if t.RepoApplyWrites_Update != nil {
+		t.RepoApplyWrites_Update.LexiconTypeID = "com.atproto.repo.applyWrites#update"
+		return json.Marshal(t.RepoApplyWrites_Update)
+	}
+	if t.RepoApplyWrites_Delete != nil {
+		t.RepoApplyWrites_Delete.LexiconTypeID = "com.atproto.repo.applyWrites#delete"
+		return json.Marshal(t.RepoApplyWrites_Delete)
+	}
+	return nil, fmt.Errorf("cannot marshal empty enum")
+}
+func (t *RepoApplyWrites_Input_Writes_Elem) UnmarshalJSON(b []byte) error {
+	typ, err := util.TypeExtract(b)
+	if err != nil {
+		return err
+	}
+
+	switch typ {
+	case "com.atproto.repo.applyWrites#create":
+		t.RepoApplyWrites_Create = new(RepoApplyWrites_Create)
+		return json.Unmarshal(b, t.RepoApplyWrites_Create)
+	case "com.atproto.repo.applyWrites#update":
+		t.RepoApplyWrites_Update = new(RepoApplyWrites_Update)
+		return json.Unmarshal(b, t.RepoApplyWrites_Update)
+	case "com.atproto.repo.applyWrites#delete":
+		t.RepoApplyWrites_Delete = new(RepoApplyWrites_Delete)
+		return json.Unmarshal(b, t.RepoApplyWrites_Delete)
+
+	default:
+		return fmt.Errorf("closed enums must have a matching value")
+	}
+}
+
+// RepoApplyWrites_Output is the output of a com.atproto.repo.applyWrites call.
+type RepoApplyWrites_Output struct {
+	Commit  *RepoDefs_CommitMeta                   `json:"commit,omitempty" cborgen:"commit,omitempty"`
+	Results []*RepoApplyWrites_Output_Results_Elem `json:"results,omitempty" cborgen:"results,omitempty"`
+}
+
+type RepoApplyWrites_Output_Results_Elem struct {
+	RepoApplyWrites_CreateResult *RepoApplyWrites_CreateResult
+	RepoApplyWrites_UpdateResult *RepoApplyWrites_UpdateResult
+	RepoApplyWrites_DeleteResult *RepoApplyWrites_DeleteResult
+}
+
+func (t *RepoApplyWrites_Output_Results_Elem) MarshalJSON() ([]byte, error) {
+	if t.RepoApplyWrites_CreateResult != nil {
+		t.RepoApplyWrites_CreateResult.LexiconTypeID = "com.atproto.repo.applyWrites#createResult"
+		return json.Marshal(t.RepoApplyWrites_CreateResult)
+	}
+	if t.RepoApplyWrites_UpdateResult != nil {
+		t.RepoApplyWrites_UpdateResult.LexiconTypeID = "com.atproto.repo.applyWrites#updateResult"
+		return json.Marshal(t.RepoApplyWrites_UpdateResult)
+	}
+	if t.RepoApplyWrites_DeleteResult != nil {
+		t.RepoApplyWrites_DeleteResult.LexiconTypeID = "com.atproto.repo.applyWrites#deleteResult"
+		return json.Marshal(t.RepoApplyWrites_DeleteResult)
+	}
+	return nil, fmt.Errorf("cannot marshal empty enum")
+}
+func (t *RepoApplyWrites_Output_Results_Elem) UnmarshalJSON(b []byte) error {
+	typ, err := util.TypeExtract(b)
+	if err != nil {
+		return err
+	}
+
+	switch typ {
+	case "com.atproto.repo.applyWrites#createResult":
+		t.RepoApplyWrites_CreateResult = new(RepoApplyWrites_CreateResult)
+		return json.Unmarshal(b, t.RepoApplyWrites_CreateResult)
+	case "com.atproto.repo.applyWrites#updateResult":
+		t.RepoApplyWrites_UpdateResult = new(RepoApplyWrites_UpdateResult)
+		return json.Unmarshal(b, t.RepoApplyWrites_UpdateResult)
+	case "com.atproto.repo.applyWrites#deleteResult":
+		t.RepoApplyWrites_DeleteResult = new(RepoApplyWrites_DeleteResult)
+		return json.Unmarshal(b, t.RepoApplyWrites_DeleteResult)
+
+	default:
+		return fmt.Errorf("closed enums must have a matching value")
+	}
+}
+
+// RepoApplyWrites_Update is a "update" in the com.atproto.repo.applyWrites schema.
+//
+// Operation which updates an existing record.
+//
+// RECORDTYPE: RepoApplyWrites_Update
+type RepoApplyWrites_Update struct {
+	LexiconTypeID string           `json:"$type,const=com.atproto.repo.applyWrites#update" cborgen:"$type,const=com.atproto.repo.applyWrites#update"`
+	Collection    string           `json:"collection" cborgen:"collection"`
+	Rkey          string           `json:"rkey" cborgen:"rkey"`
+	Value         *json.RawMessage `json:"value" cborgen:"value"`
+}
+
+// RepoApplyWrites_UpdateResult is a "updateResult" in the com.atproto.repo.applyWrites schema.
+//
+// RECORDTYPE: RepoApplyWrites_UpdateResult
+type RepoApplyWrites_UpdateResult struct {
+	LexiconTypeID    string  `json:"$type,const=com.atproto.repo.applyWrites#updateResult" cborgen:"$type,const=com.atproto.repo.applyWrites#updateResult"`
+	Cid              string  `json:"cid" cborgen:"cid"`
+	Uri              string  `json:"uri" cborgen:"uri"`
+	ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"`
+}
+
+// RepoApplyWrites calls the XRPC method "com.atproto.repo.applyWrites".
+func RepoApplyWrites(ctx context.Context, c *xrpc.Client, input *RepoApplyWrites_Input) (*RepoApplyWrites_Output, error) {
+	var out RepoApplyWrites_Output
+	if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.applyWrites", nil, input, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}

api/atproto/servercreateSession.go

···
 
 // ServerCreateSession_Input is the input argument to a com.atproto.server.createSession call.
 type ServerCreateSession_Input struct {
+	// allowTakendown: When true, instead of throwing error for takendown accounts, a valid response with a narrow scoped token will be returned
+	AllowTakendown  *bool   `json:"allowTakendown,omitempty" cborgen:"allowTakendown,omitempty"`
 	AuthFactorToken *string `json:"authFactorToken,omitempty" cborgen:"authFactorToken,omitempty"`
 	// identifier: Handle or other identifier supported by the server for the authenticating user.
 	Identifier string `json:"identifier" cborgen:"identifier"`

api/atproto/tempaddReservedHandle.go

···
+// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
+
+package atproto
+
+// schema: com.atproto.temp.addReservedHandle
+
+import (
+	"context"
+
+	"github.com/bluesky-social/indigo/xrpc"
+)
+
+// TempAddReservedHandle_Input is the input argument to a com.atproto.temp.addReservedHandle call.
+type TempAddReservedHandle_Input struct {
+	Handle string `json:"handle" cborgen:"handle"`
+}
+
+// TempAddReservedHandle_Output is the output of a com.atproto.temp.addReservedHandle call.
+type TempAddReservedHandle_Output struct {
+}
+
+// TempAddReservedHandle calls the XRPC method "com.atproto.temp.addReservedHandle".
+func TempAddReservedHandle(ctx context.Context, c *xrpc.Client, input *TempAddReservedHandle_Input) (*TempAddReservedHandle_Output, error) {
+	var out TempAddReservedHandle_Output
+	if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.temp.addReservedHandle", nil, input, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}

api/bsky/actorgetSuggestions.go

···
 type ActorGetSuggestions_Output struct {
 	Actors []*ActorDefs_ProfileView `json:"actors" cborgen:"actors"`
 	Cursor *string                  `json:"cursor,omitempty" cborgen:"cursor,omitempty"`
+	// recId: Snowflake for this recommendation, use when submitting recommendation events.
+	RecId *int64 `json:"recId,omitempty" cborgen:"recId,omitempty"`
 }
 
 // ActorGetSuggestions calls the XRPC method "app.bsky.actor.getSuggestions".

api/bsky/graphgetSuggestedFollowsByActor.go

···
 // GraphGetSuggestedFollowsByActor_Output is the output of a app.bsky.graph.getSuggestedFollowsByActor call.
 type GraphGetSuggestedFollowsByActor_Output struct {
 	// isFallback: If true, response has fallen-back to generic results, and is not scoped using relativeToDid
-	IsFallback  *bool                    `json:"isFallback,omitempty" cborgen:"isFallback,omitempty"`
+	IsFallback *bool `json:"isFallback,omitempty" cborgen:"isFallback,omitempty"`
+	// recId: Snowflake for this recommendation, use when submitting recommendation events.
+	RecId       *int64                   `json:"recId,omitempty" cborgen:"recId,omitempty"`
 	Suggestions []*ActorDefs_ProfileView `json:"suggestions" cborgen:"suggestions"`
 }
 

api/bsky/graphsearchStarterPacks.go

···
+// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
+
+package bsky
+
+// schema: app.bsky.graph.searchStarterPacks
+
+import (
+	"context"
+
+	"github.com/bluesky-social/indigo/xrpc"
+)
+
+// GraphSearchStarterPacks_Output is the output of a app.bsky.graph.searchStarterPacks call.
+type GraphSearchStarterPacks_Output struct {
+	Cursor       *string                           `json:"cursor,omitempty" cborgen:"cursor,omitempty"`
+	StarterPacks []*GraphDefs_StarterPackViewBasic `json:"starterPacks" cborgen:"starterPacks"`
+}
+
+// GraphSearchStarterPacks calls the XRPC method "app.bsky.graph.searchStarterPacks".
+//
+// q: Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.
+func GraphSearchStarterPacks(ctx context.Context, c *xrpc.Client, cursor string, limit int64, q string) (*GraphSearchStarterPacks_Output, error) {
+	var out GraphSearchStarterPacks_Output
+
+	params := map[string]interface{}{
+		"cursor": cursor,
+		"limit":  limit,
+		"q":      q,
+	}
+	if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.searchStarterPacks", params, nil, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}

api/bsky/notificationlistNotifications.go

···
 }
 
 // NotificationListNotifications calls the XRPC method "app.bsky.notification.listNotifications".
-func NotificationListNotifications(ctx context.Context, c *xrpc.Client, cursor string, limit int64, priority bool, seenAt string) (*NotificationListNotifications_Output, error) {
+//
+// reasons: Notification reasons to include in response.
+func NotificationListNotifications(ctx context.Context, c *xrpc.Client, cursor string, limit int64, priority bool, reasons []string, seenAt string) (*NotificationListNotifications_Output, error) {
 	var out NotificationListNotifications_Output
 
 	params := map[string]interface{}{
 		"cursor":   cursor,
 		"limit":    limit,
 		"priority": priority,
+		"reasons":  reasons,
 		"seenAt":   seenAt,
 	}
 	if err := c.Do(ctx, xrpc.Query, "", "app.bsky.notification.listNotifications", params, nil, &out); err != nil {

api/bsky/unspecceddefs.go

···
 type UnspeccedDefs_SkeletonSearchPost struct {
 	Uri string `json:"uri" cborgen:"uri"`
 }
+
+// UnspeccedDefs_SkeletonSearchStarterPack is a "skeletonSearchStarterPack" in the app.bsky.unspecced.defs schema.
+type UnspeccedDefs_SkeletonSearchStarterPack struct {
+	Uri string `json:"uri" cborgen:"uri"`
+}
+
+// UnspeccedDefs_TrendingTopic is a "trendingTopic" in the app.bsky.unspecced.defs schema.
+type UnspeccedDefs_TrendingTopic struct {
+	Description *string `json:"description,omitempty" cborgen:"description,omitempty"`
+	DisplayName *string `json:"displayName,omitempty" cborgen:"displayName,omitempty"`
+	Link        string  `json:"link" cborgen:"link"`
+	Topic       string  `json:"topic" cborgen:"topic"`
+}

api/bsky/unspeccedgetSuggestionsSkeleton.go

···
 type UnspeccedGetSuggestionsSkeleton_Output struct {
 	Actors []*UnspeccedDefs_SkeletonSearchActor `json:"actors" cborgen:"actors"`
 	Cursor *string                              `json:"cursor,omitempty" cborgen:"cursor,omitempty"`
+	// recId: Snowflake for this recommendation, use when submitting recommendation events.
+	RecId *int64 `json:"recId,omitempty" cborgen:"recId,omitempty"`
 	// relativeToDid: DID of the account these suggestions are relative to. If this is returned undefined, suggestions are based on the viewer.
 	RelativeToDid *string `json:"relativeToDid,omitempty" cborgen:"relativeToDid,omitempty"`
 }

api/bsky/unspeccedgetTrendingTopics.go

···
+// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
+
+package bsky
+
+// schema: app.bsky.unspecced.getTrendingTopics
+
+import (
+	"context"
+
+	"github.com/bluesky-social/indigo/xrpc"
+)
+
+// UnspeccedGetTrendingTopics_Output is the output of a app.bsky.unspecced.getTrendingTopics call.
+type UnspeccedGetTrendingTopics_Output struct {
+	Suggested []*UnspeccedDefs_TrendingTopic `json:"suggested" cborgen:"suggested"`
+	Topics    []*UnspeccedDefs_TrendingTopic `json:"topics" cborgen:"topics"`
+}
+
+// UnspeccedGetTrendingTopics calls the XRPC method "app.bsky.unspecced.getTrendingTopics".
+//
+// viewer: DID of the account making the request (not included for public/unauthenticated queries). Used to boost followed accounts in ranking.
+func UnspeccedGetTrendingTopics(ctx context.Context, c *xrpc.Client, limit int64, viewer string) (*UnspeccedGetTrendingTopics_Output, error) {
+	var out UnspeccedGetTrendingTopics_Output
+
+	params := map[string]interface{}{
+		"limit":  limit,
+		"viewer": viewer,
+	}
+	if err := c.Do(ctx, xrpc.Query, "", "app.bsky.unspecced.getTrendingTopics", params, nil, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}

api/bsky/unspeccedsearchStarterPacksSkeleton.go

···
+// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
+
+package bsky
+
+// schema: app.bsky.unspecced.searchStarterPacksSkeleton
+
+import (
+	"context"
+
+	"github.com/bluesky-social/indigo/xrpc"
+)
+
+// UnspeccedSearchStarterPacksSkeleton_Output is the output of a app.bsky.unspecced.searchStarterPacksSkeleton call.
+type UnspeccedSearchStarterPacksSkeleton_Output struct {
+	Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"`
+	// hitsTotal: Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits.
+	HitsTotal    *int64                                     `json:"hitsTotal,omitempty" cborgen:"hitsTotal,omitempty"`
+	StarterPacks []*UnspeccedDefs_SkeletonSearchStarterPack `json:"starterPacks" cborgen:"starterPacks"`
+}
+
+// UnspeccedSearchStarterPacksSkeleton calls the XRPC method "app.bsky.unspecced.searchStarterPacksSkeleton".
+//
+// cursor: Optional pagination mechanism; may not necessarily allow scrolling through entire result set.
+// q: Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.
+// viewer: DID of the account making the request (not included for public/unauthenticated queries).
+func UnspeccedSearchStarterPacksSkeleton(ctx context.Context, c *xrpc.Client, cursor string, limit int64, q string, viewer string) (*UnspeccedSearchStarterPacksSkeleton_Output, error) {
+	var out UnspeccedSearchStarterPacksSkeleton_Output
+
+	params := map[string]interface{}{
+		"cursor": cursor,
+		"limit":  limit,
+		"q":      q,
+		"viewer": viewer,
+	}
+	if err := c.Do(ctx, xrpc.Query, "", "app.bsky.unspecced.searchStarterPacksSkeleton", params, nil, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}

api/chat/convodefs.go

···
 	LastMessage *ConvoDefs_ConvoView_LastMessage `json:"lastMessage,omitempty" cborgen:"lastMessage,omitempty"`
 	Members     []*ActorDefs_ProfileViewBasic    `json:"members" cborgen:"members"`
 	Muted       bool                             `json:"muted" cborgen:"muted"`
+	Opened      *bool                            `json:"opened,omitempty" cborgen:"opened,omitempty"`
 	Rev         string                           `json:"rev" cborgen:"rev"`
 	UnreadCount int64                            `json:"unreadCount" cborgen:"unreadCount"`
 }

api/ozone/moderationdefs.go

···
 	"github.com/bluesky-social/indigo/lex/util"
 )
 
+// ModerationDefs_AccountEvent is a "accountEvent" in the tools.ozone.moderation.defs schema.
+//
+// Logs account status related events on a repo subject. Normally captured by automod from the firehose and emitted to ozone for historical tracking.
+//
+// RECORDTYPE: ModerationDefs_AccountEvent
+type ModerationDefs_AccountEvent struct {
+	LexiconTypeID string `json:"$type,const=tools.ozone.moderation.defs#accountEvent" cborgen:"$type,const=tools.ozone.moderation.defs#accountEvent"`
+	// active: Indicates that the account has a repository which can be fetched from the host that emitted this event.
+	Active    bool    `json:"active" cborgen:"active"`
+	Comment   *string `json:"comment,omitempty" cborgen:"comment,omitempty"`
+	Status    *string `json:"status,omitempty" cborgen:"status,omitempty"`
+	Timestamp string  `json:"timestamp" cborgen:"timestamp"`
+}
+
+// ModerationDefs_AccountHosting is a "accountHosting" in the tools.ozone.moderation.defs schema.
+//
+// RECORDTYPE: ModerationDefs_AccountHosting
+type ModerationDefs_AccountHosting struct {
+	LexiconTypeID string  `json:"$type,const=tools.ozone.moderation.defs#accountHosting" cborgen:"$type,const=tools.ozone.moderation.defs#accountHosting"`
+	CreatedAt     *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"`
+	DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"`
+	DeletedAt     *string `json:"deletedAt,omitempty" cborgen:"deletedAt,omitempty"`
+	ReactivatedAt *string `json:"reactivatedAt,omitempty" cborgen:"reactivatedAt,omitempty"`
+	Status        string  `json:"status" cborgen:"status"`
+	UpdatedAt     *string `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"`
+}
+
 // ModerationDefs_BlobView is a "blobView" in the tools.ozone.moderation.defs schema.
 type ModerationDefs_BlobView struct {
 	Cid        string                           `json:"cid" cborgen:"cid"`
···
 	}
 }
 
+// ModerationDefs_IdentityEvent is a "identityEvent" in the tools.ozone.moderation.defs schema.
+//
+// Logs identity related events on a repo subject. Normally captured by automod from the firehose and emitted to ozone for historical tracking.
+//
+// RECORDTYPE: ModerationDefs_IdentityEvent
+type ModerationDefs_IdentityEvent struct {
+	LexiconTypeID string  `json:"$type,const=tools.ozone.moderation.defs#identityEvent" cborgen:"$type,const=tools.ozone.moderation.defs#identityEvent"`
+	Comment       *string `json:"comment,omitempty" cborgen:"comment,omitempty"`
+	Handle        *string `json:"handle,omitempty" cborgen:"handle,omitempty"`
+	PdsHost       *string `json:"pdsHost,omitempty" cborgen:"pdsHost,omitempty"`
+	Timestamp     string  `json:"timestamp" cborgen:"timestamp"`
+	Tombstone     *bool   `json:"tombstone,omitempty" cborgen:"tombstone,omitempty"`
+}
+
 // ModerationDefs_ImageDetails is a "imageDetails" in the tools.ozone.moderation.defs schema.
 //
 // RECORDTYPE: ModerationDefs_ImageDetails
···
 //
 // RECORDTYPE: ModerationDefs_ModEventAcknowledge
 type ModerationDefs_ModEventAcknowledge struct {
-	LexiconTypeID string  `json:"$type,const=tools.ozone.moderation.defs#modEventAcknowledge" cborgen:"$type,const=tools.ozone.moderation.defs#modEventAcknowledge"`
-	Comment       *string `json:"comment,omitempty" cborgen:"comment,omitempty"`
+	LexiconTypeID string `json:"$type,const=tools.ozone.moderation.defs#modEventAcknowledge" cborgen:"$type,const=tools.ozone.moderation.defs#modEventAcknowledge"`
+	// acknowledgeAccountSubjects: If true, all other reports on content authored by this account will be resolved (acknowledged).
+	AcknowledgeAccountSubjects *bool   `json:"acknowledgeAccountSubjects,omitempty" cborgen:"acknowledgeAccountSubjects,omitempty"`
+	Comment                    *string `json:"comment,omitempty" cborgen:"comment,omitempty"`
 }
 
 // ModerationDefs_ModEventComment is a "modEventComment" in the tools.ozone.moderation.defs schema.
···
 type ModerationDefs_ModEventMuteReporter struct {
 	LexiconTypeID string  `json:"$type,const=tools.ozone.moderation.defs#modEventMuteReporter" cborgen:"$type,const=tools.ozone.moderation.defs#modEventMuteReporter"`
 	Comment       *string `json:"comment,omitempty" cborgen:"comment,omitempty"`
-	// durationInHours: Indicates how long the account should remain muted.
-	DurationInHours int64 `json:"durationInHours" cborgen:"durationInHours"`
+	// durationInHours: Indicates how long the account should remain muted. Falsy value here means a permanent mute.
+	DurationInHours *int64 `json:"durationInHours,omitempty" cborgen:"durationInHours,omitempty"`
 }
 
 // ModerationDefs_ModEventReport is a "modEventReport" in the tools.ozone.moderation.defs schema.
···
 	Comment                    *string `json:"comment,omitempty" cborgen:"comment,omitempty"`
 	// durationInHours: Indicates how long the takedown should be in effect before automatically expiring.
 	DurationInHours *int64 `json:"durationInHours,omitempty" cborgen:"durationInHours,omitempty"`
+	// policies: Names/Keywords of the policies that drove the decision.
+	Policies []string `json:"policies,omitempty" cborgen:"policies,omitempty"`
 }
 
 // ModerationDefs_ModEventUnmute is a "modEventUnmute" in the tools.ozone.moderation.defs schema.
···
 	ModerationDefs_ModEventResolveAppeal   *ModerationDefs_ModEventResolveAppeal
 	ModerationDefs_ModEventDivert          *ModerationDefs_ModEventDivert
 	ModerationDefs_ModEventTag             *ModerationDefs_ModEventTag
+	ModerationDefs_AccountEvent            *ModerationDefs_AccountEvent
+	ModerationDefs_IdentityEvent           *ModerationDefs_IdentityEvent
+	ModerationDefs_RecordEvent             *ModerationDefs_RecordEvent
 }
 
 func (t *ModerationDefs_ModEventViewDetail_Event) MarshalJSON() ([]byte, error) {
···
 		t.ModerationDefs_ModEventTag.LexiconTypeID = "tools.ozone.moderation.defs#modEventTag"
 		return json.Marshal(t.ModerationDefs_ModEventTag)
 	}
+	if t.ModerationDefs_AccountEvent != nil {
+		t.ModerationDefs_AccountEvent.LexiconTypeID = "tools.ozone.moderation.defs#accountEvent"
+		return json.Marshal(t.ModerationDefs_AccountEvent)
+	}
+	if t.ModerationDefs_IdentityEvent != nil {
+		t.ModerationDefs_IdentityEvent.LexiconTypeID = "tools.ozone.moderation.defs#identityEvent"
+		return json.Marshal(t.ModerationDefs_IdentityEvent)
+	}
+	if t.ModerationDefs_RecordEvent != nil {
+		t.ModerationDefs_RecordEvent.LexiconTypeID = "tools.ozone.moderation.defs#recordEvent"
+		return json.Marshal(t.ModerationDefs_RecordEvent)
+	}
 	return nil, fmt.Errorf("cannot marshal empty enum")
 }
 func (t *ModerationDefs_ModEventViewDetail_Event) UnmarshalJSON(b []byte) error {
···
 	case "tools.ozone.moderation.defs#modEventTag":
 		t.ModerationDefs_ModEventTag = new(ModerationDefs_ModEventTag)
 		return json.Unmarshal(b, t.ModerationDefs_ModEventTag)
+	case "tools.ozone.moderation.defs#accountEvent":
+		t.ModerationDefs_AccountEvent = new(ModerationDefs_AccountEvent)
+		return json.Unmarshal(b, t.ModerationDefs_AccountEvent)
+	case "tools.ozone.moderation.defs#identityEvent":
+		t.ModerationDefs_IdentityEvent = new(ModerationDefs_IdentityEvent)
+		return json.Unmarshal(b, t.ModerationDefs_IdentityEvent)
+	case "tools.ozone.moderation.defs#recordEvent":
+		t.ModerationDefs_RecordEvent = new(ModerationDefs_RecordEvent)
+		return json.Unmarshal(b, t.ModerationDefs_RecordEvent)
 
 	default:
 		return nil
···
 	ModerationDefs_ModEventResolveAppeal   *ModerationDefs_ModEventResolveAppeal
 	ModerationDefs_ModEventDivert          *ModerationDefs_ModEventDivert
 	ModerationDefs_ModEventTag             *ModerationDefs_ModEventTag
+	ModerationDefs_AccountEvent            *ModerationDefs_AccountEvent
+	ModerationDefs_IdentityEvent           *ModerationDefs_IdentityEvent
+	ModerationDefs_RecordEvent             *ModerationDefs_RecordEvent
 }
 
 func (t *ModerationDefs_ModEventView_Event) MarshalJSON() ([]byte, error) {
···
 		t.ModerationDefs_ModEventTag.LexiconTypeID = "tools.ozone.moderation.defs#modEventTag"
 		return json.Marshal(t.ModerationDefs_ModEventTag)
 	}
+	if t.ModerationDefs_AccountEvent != nil {
+		t.ModerationDefs_AccountEvent.LexiconTypeID = "tools.ozone.moderation.defs#accountEvent"
+		return json.Marshal(t.ModerationDefs_AccountEvent)
+	}
+	if t.ModerationDefs_IdentityEvent != nil {
+		t.ModerationDefs_IdentityEvent.LexiconTypeID = "tools.ozone.moderation.defs#identityEvent"
+		return json.Marshal(t.ModerationDefs_IdentityEvent)
+	}
+	if t.ModerationDefs_RecordEvent != nil {
+		t.ModerationDefs_RecordEvent.LexiconTypeID = "tools.ozone.moderation.defs#recordEvent"
+		return json.Marshal(t.ModerationDefs_RecordEvent)
+	}
 	return nil, fmt.Errorf("cannot marshal empty enum")
 }
 func (t *ModerationDefs_ModEventView_Event) UnmarshalJSON(b []byte) error {
···
 	case "tools.ozone.moderation.defs#modEventTag":
 		t.ModerationDefs_ModEventTag = new(ModerationDefs_ModEventTag)
 		return json.Unmarshal(b, t.ModerationDefs_ModEventTag)
+	case "tools.ozone.moderation.defs#accountEvent":
+		t.ModerationDefs_AccountEvent = new(ModerationDefs_AccountEvent)
+		return json.Unmarshal(b, t.ModerationDefs_AccountEvent)
+	case "tools.ozone.moderation.defs#identityEvent":
+		t.ModerationDefs_IdentityEvent = new(ModerationDefs_IdentityEvent)
+		return json.Unmarshal(b, t.ModerationDefs_IdentityEvent)
+	case "tools.ozone.moderation.defs#recordEvent":
+		t.ModerationDefs_RecordEvent = new(ModerationDefs_RecordEvent)
+		return json.Unmarshal(b, t.ModerationDefs_RecordEvent)
 
 	default:
 		return nil
···
 	SubjectStatus *ModerationDefs_SubjectStatusView `json:"subjectStatus,omitempty" cborgen:"subjectStatus,omitempty"`
 }
 
+// ModerationDefs_RecordEvent is a "recordEvent" in the tools.ozone.moderation.defs schema.
+//
+// Logs lifecycle event on a record subject. Normally captured by automod from the firehose and emitted to ozone for historical tracking.
+//
+// RECORDTYPE: ModerationDefs_RecordEvent
+type ModerationDefs_RecordEvent struct {
+	LexiconTypeID string  `json:"$type,const=tools.ozone.moderation.defs#recordEvent" cborgen:"$type,const=tools.ozone.moderation.defs#recordEvent"`
+	Cid           *string `json:"cid,omitempty" cborgen:"cid,omitempty"`
+	Comment       *string `json:"comment,omitempty" cborgen:"comment,omitempty"`
+	Op            string  `json:"op" cborgen:"op"`
+	Timestamp     string  `json:"timestamp" cborgen:"timestamp"`
+}
+
+// ModerationDefs_RecordHosting is a "recordHosting" in the tools.ozone.moderation.defs schema.
+//
+// RECORDTYPE: ModerationDefs_RecordHosting
+type ModerationDefs_RecordHosting struct {
+	LexiconTypeID string  `json:"$type,const=tools.ozone.moderation.defs#recordHosting" cborgen:"$type,const=tools.ozone.moderation.defs#recordHosting"`
+	CreatedAt     *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"`
+	DeletedAt     *string `json:"deletedAt,omitempty" cborgen:"deletedAt,omitempty"`
+	Status        string  `json:"status" cborgen:"status"`
+	UpdatedAt     *string `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"`
+}
+
 // ModerationDefs_RecordView is a "recordView" in the tools.ozone.moderation.defs schema.
 //
 // RECORDTYPE: ModerationDefs_RecordView
···
 	// comment: Sticky comment on the subject.
 	Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"`
 	// createdAt: Timestamp referencing the first moderation status impacting event was emitted on the subject
-	CreatedAt string `json:"createdAt" cborgen:"createdAt"`
-	Id        int64  `json:"id" cborgen:"id"`
+	CreatedAt string                                    `json:"createdAt" cborgen:"createdAt"`
+	Hosting   *ModerationDefs_SubjectStatusView_Hosting `json:"hosting,omitempty" cborgen:"hosting,omitempty"`
+	Id        int64                                     `json:"id" cborgen:"id"`
 	// lastAppealedAt: Timestamp referencing when the author of the subject appealed a moderation action
 	LastAppealedAt     *string                                   `json:"lastAppealedAt,omitempty" cborgen:"lastAppealedAt,omitempty"`
 	LastReportedAt     *string                                   `json:"lastReportedAt,omitempty" cborgen:"lastReportedAt,omitempty"`
···
 	Takendown          *bool                                     `json:"takendown,omitempty" cborgen:"takendown,omitempty"`
 	// updatedAt: Timestamp referencing when the last update was made to the moderation status of the subject
 	UpdatedAt string `json:"updatedAt" cborgen:"updatedAt"`
+}
+
+type ModerationDefs_SubjectStatusView_Hosting struct {
+	ModerationDefs_AccountHosting *ModerationDefs_AccountHosting
+	ModerationDefs_RecordHosting  *ModerationDefs_RecordHosting
+}
+
+func (t *ModerationDefs_SubjectStatusView_Hosting) MarshalJSON() ([]byte, error) {
+	if t.ModerationDefs_AccountHosting != nil {
+		t.ModerationDefs_AccountHosting.LexiconTypeID = "tools.ozone.moderation.defs#accountHosting"
+		return json.Marshal(t.ModerationDefs_AccountHosting)
+	}
+	if t.ModerationDefs_RecordHosting != nil {
+		t.ModerationDefs_RecordHosting.LexiconTypeID = "tools.ozone.moderation.defs#recordHosting"
+		return json.Marshal(t.ModerationDefs_RecordHosting)
+	}
+	return nil, fmt.Errorf("cannot marshal empty enum")
+}
+func (t *ModerationDefs_SubjectStatusView_Hosting) UnmarshalJSON(b []byte) error {
+	typ, err := util.TypeExtract(b)
+	if err != nil {
+		return err
+	}
+
+	switch typ {
+	case "tools.ozone.moderation.defs#accountHosting":
+		t.ModerationDefs_AccountHosting = new(ModerationDefs_AccountHosting)
+		return json.Unmarshal(b, t.ModerationDefs_AccountHosting)
+	case "tools.ozone.moderation.defs#recordHosting":
+		t.ModerationDefs_RecordHosting = new(ModerationDefs_RecordHosting)
+		return json.Unmarshal(b, t.ModerationDefs_RecordHosting)
+
+	default:
+		return nil
+	}
 }
 
 type ModerationDefs_SubjectStatusView_Subject struct {

api/ozone/moderationemitEvent.go

···
 	ModerationDefs_ModEventResolveAppeal   *ModerationDefs_ModEventResolveAppeal
 	ModerationDefs_ModEventEmail           *ModerationDefs_ModEventEmail
 	ModerationDefs_ModEventTag             *ModerationDefs_ModEventTag
+	ModerationDefs_AccountEvent            *ModerationDefs_AccountEvent
+	ModerationDefs_IdentityEvent           *ModerationDefs_IdentityEvent
+	ModerationDefs_RecordEvent             *ModerationDefs_RecordEvent
 }
 
 func (t *ModerationEmitEvent_Input_Event) MarshalJSON() ([]byte, error) {
···
 		t.ModerationDefs_ModEventTag.LexiconTypeID = "tools.ozone.moderation.defs#modEventTag"
 		return json.Marshal(t.ModerationDefs_ModEventTag)
 	}
+	if t.ModerationDefs_AccountEvent != nil {
+		t.ModerationDefs_AccountEvent.LexiconTypeID = "tools.ozone.moderation.defs#accountEvent"
+		return json.Marshal(t.ModerationDefs_AccountEvent)
+	}
+	if t.ModerationDefs_IdentityEvent != nil {
+		t.ModerationDefs_IdentityEvent.LexiconTypeID = "tools.ozone.moderation.defs#identityEvent"
+		return json.Marshal(t.ModerationDefs_IdentityEvent)
+	}
+	if t.ModerationDefs_RecordEvent != nil {
+		t.ModerationDefs_RecordEvent.LexiconTypeID = "tools.ozone.moderation.defs#recordEvent"
+		return json.Marshal(t.ModerationDefs_RecordEvent)
+	}
 	return nil, fmt.Errorf("cannot marshal empty enum")
 }
 func (t *ModerationEmitEvent_Input_Event) UnmarshalJSON(b []byte) error {
···
 	case "tools.ozone.moderation.defs#modEventTag":
 		t.ModerationDefs_ModEventTag = new(ModerationDefs_ModEventTag)
 		return json.Unmarshal(b, t.ModerationDefs_ModEventTag)
+	case "tools.ozone.moderation.defs#accountEvent":
+		t.ModerationDefs_AccountEvent = new(ModerationDefs_AccountEvent)
+		return json.Unmarshal(b, t.ModerationDefs_AccountEvent)
+	case "tools.ozone.moderation.defs#identityEvent":
+		t.ModerationDefs_IdentityEvent = new(ModerationDefs_IdentityEvent)
+		return json.Unmarshal(b, t.ModerationDefs_IdentityEvent)
+	case "tools.ozone.moderation.defs#recordEvent":
+		t.ModerationDefs_RecordEvent = new(ModerationDefs_RecordEvent)
+		return json.Unmarshal(b, t.ModerationDefs_RecordEvent)
 
 	default:
 		return nil

api/ozone/moderationqueryEvents.go

···
 // addedLabels: If specified, only events where all of these labels were added are returned
 // addedTags: If specified, only events where all of these tags were added are returned
 // collections: If specified, only events where the subject belongs to the given collections will be returned. When subjectType is set to 'account', this will be ignored.
-// comment: If specified, only events with comments containing the keyword are returned
+// comment: If specified, only events with comments containing the keyword are returned. Apply || separator to use multiple keywords and match using OR condition.
 // createdAfter: Retrieve events created after a given timestamp
 // createdBefore: Retrieve events created before a given timestamp
 // hasComment: If true, only events with comments are returned
···
 // sortDirection: Sort direction for the events. Defaults to descending order of created at timestamp.
 // subjectType: If specified, only events where the subject is of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.
 // types: The types of events (fully qualified string in the format of tools.ozone.moderation.defs#modEvent<name>) to filter by. If not specified, all events are returned.
-func ModerationQueryEvents(ctx context.Context, c *xrpc.Client, addedLabels []string, addedTags []string, collections []string, comment string, createdAfter string, createdBefore string, createdBy string, cursor string, hasComment bool, includeAllUserRecords bool, limit int64, removedLabels []string, removedTags []string, reportTypes []string, sortDirection string, subject string, subjectType string, types []string) (*ModerationQueryEvents_Output, error) {
+func ModerationQueryEvents(ctx context.Context, c *xrpc.Client, addedLabels []string, addedTags []string, collections []string, comment string, createdAfter string, createdBefore string, createdBy string, cursor string, hasComment bool, includeAllUserRecords bool, limit int64, policies []string, removedLabels []string, removedTags []string, reportTypes []string, sortDirection string, subject string, subjectType string, types []string) (*ModerationQueryEvents_Output, error) {
 	var out ModerationQueryEvents_Output
 
 	params := map[string]interface{}{
···
 		"hasComment":            hasComment,
 		"includeAllUserRecords": includeAllUserRecords,
 		"limit":                 limit,
+		"policies":              policies,
 		"removedLabels":         removedLabels,
 		"removedTags":           removedTags,
 		"reportTypes":           reportTypes,

api/ozone/moderationqueryStatuses.go

···
 // appealed: Get subjects in unresolved appealed status
 // collections: If specified, subjects belonging to the given collections will be returned. When subjectType is set to 'account', this will be ignored.
 // comment: Search subjects by keyword from comments
+// hostingDeletedAfter: Search subjects where the associated record/account was deleted after a given timestamp
+// hostingDeletedBefore: Search subjects where the associated record/account was deleted before a given timestamp
+// hostingStatuses: Search subjects by the status of the associated record/account
+// hostingUpdatedAfter: Search subjects where the associated record/account was updated after a given timestamp
+// hostingUpdatedBefore: Search subjects where the associated record/account was updated before a given timestamp
 // includeAllUserRecords: All subjects, or subjects from given 'collections' param, belonging to the account specified in the 'subject' param will be returned.
 // includeMuted: By default, we don't include muted subjects in the results. Set this to true to include them.
 // lastReviewedBy: Get all subject statuses that were reviewed by a specific moderator
 // onlyMuted: When set to true, only muted subjects and reporters will be returned.
+// queueCount: Number of queues being used by moderators. Subjects will be split among all queues.
+// queueIndex: Index of the queue to fetch subjects from. Works only when queueCount value is specified.
+// queueSeed: A seeder to shuffle/balance the queue items.
 // reportedAfter: Search subjects reported after a given timestamp
 // reportedBefore: Search subjects reported before a given timestamp
 // reviewState: Specify when fetching subjects in a certain state
···
 // subject: The subject to get the status for.
 // subjectType: If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.
 // takendown: Get subjects that were taken down
-func ModerationQueryStatuses(ctx context.Context, c *xrpc.Client, appealed bool, collections []string, comment string, cursor string, excludeTags []string, ignoreSubjects []string, includeAllUserRecords bool, includeMuted bool, lastReviewedBy string, limit int64, onlyMuted bool, reportedAfter string, reportedBefore string, reviewState string, reviewedAfter string, reviewedBefore string, sortDirection string, sortField string, subject string, subjectType string, tags []string, takendown bool) (*ModerationQueryStatuses_Output, error) {
+func ModerationQueryStatuses(ctx context.Context, c *xrpc.Client, appealed bool, collections []string, comment string, cursor string, excludeTags []string, hostingDeletedAfter string, hostingDeletedBefore string, hostingStatuses []string, hostingUpdatedAfter string, hostingUpdatedBefore string, ignoreSubjects []string, includeAllUserRecords bool, includeMuted bool, lastReviewedBy string, limit int64, onlyMuted bool, queueCount int64, queueIndex int64, queueSeed string, reportedAfter string, reportedBefore string, reviewState string, reviewedAfter string, reviewedBefore string, sortDirection string, sortField string, subject string, subjectType string, tags []string, takendown bool) (*ModerationQueryStatuses_Output, error) {
 	var out ModerationQueryStatuses_Output
 
 	params := map[string]interface{}{
···
 		"comment":               comment,
 		"cursor":                cursor,
 		"excludeTags":           excludeTags,
+		"hostingDeletedAfter":   hostingDeletedAfter,
+		"hostingDeletedBefore":  hostingDeletedBefore,
+		"hostingStatuses":       hostingStatuses,
+		"hostingUpdatedAfter":   hostingUpdatedAfter,
+		"hostingUpdatedBefore":  hostingUpdatedBefore,
 		"ignoreSubjects":        ignoreSubjects,
 		"includeAllUserRecords": includeAllUserRecords,
 		"includeMuted":          includeMuted,
 		"lastReviewedBy":        lastReviewedBy,
 		"limit":                 limit,
 		"onlyMuted":             onlyMuted,
+		"queueCount":            queueCount,
+		"queueIndex":            queueIndex,
+		"queueSeed":             queueSeed,
 		"reportedAfter":         reportedAfter,
 		"reportedBefore":        reportedBefore,
 		"reviewState":           reviewState,

api/ozone/settingdefs.go

···
+// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
+
+package ozone
+
+// schema: tools.ozone.setting.defs
+
+import (
+	"github.com/bluesky-social/indigo/lex/util"
+)
+
+// SettingDefs_Option is a "option" in the tools.ozone.setting.defs schema.
+type SettingDefs_Option struct {
+	CreatedAt     *string                  `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"`
+	CreatedBy     string                   `json:"createdBy" cborgen:"createdBy"`
+	Description   *string                  `json:"description,omitempty" cborgen:"description,omitempty"`
+	Did           string                   `json:"did" cborgen:"did"`
+	Key           string                   `json:"key" cborgen:"key"`
+	LastUpdatedBy string                   `json:"lastUpdatedBy" cborgen:"lastUpdatedBy"`
+	ManagerRole   *string                  `json:"managerRole,omitempty" cborgen:"managerRole,omitempty"`
+	Scope         string                   `json:"scope" cborgen:"scope"`
+	UpdatedAt     *string                  `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"`
+	Value         *util.LexiconTypeDecoder `json:"value" cborgen:"value"`
+}

api/ozone/settinglistOptions.go

···
+// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
+
+package ozone
+
+// schema: tools.ozone.setting.listOptions
+
+import (
+	"context"
+
+	"github.com/bluesky-social/indigo/xrpc"
+)
+
+// SettingListOptions_Output is the output of a tools.ozone.setting.listOptions call.
+type SettingListOptions_Output struct {
+	Cursor  *string               `json:"cursor,omitempty" cborgen:"cursor,omitempty"`
+	Options []*SettingDefs_Option `json:"options" cborgen:"options"`
+}
+
+// SettingListOptions calls the XRPC method "tools.ozone.setting.listOptions".
+//
+// keys: Filter for only the specified keys. Ignored if prefix is provided
+// prefix: Filter keys by prefix
+func SettingListOptions(ctx context.Context, c *xrpc.Client, cursor string, keys []string, limit int64, prefix string, scope string) (*SettingListOptions_Output, error) {
+	var out SettingListOptions_Output
+
+	params := map[string]interface{}{
+		"cursor": cursor,
+		"keys":   keys,
+		"limit":  limit,
+		"prefix": prefix,
+		"scope":  scope,
+	}
+	if err := c.Do(ctx, xrpc.Query, "", "tools.ozone.setting.listOptions", params, nil, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}

api/ozone/settingremoveOptions.go

···
+// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
+
+package ozone
+
+// schema: tools.ozone.setting.removeOptions
+
+import (
+	"context"
+
+	"github.com/bluesky-social/indigo/xrpc"
+)
+
+// SettingRemoveOptions_Input is the input argument to a tools.ozone.setting.removeOptions call.
+type SettingRemoveOptions_Input struct {
+	Keys  []string `json:"keys" cborgen:"keys"`
+	Scope string   `json:"scope" cborgen:"scope"`
+}
+
+// SettingRemoveOptions_Output is the output of a tools.ozone.setting.removeOptions call.
+type SettingRemoveOptions_Output struct {
+}
+
+// SettingRemoveOptions calls the XRPC method "tools.ozone.setting.removeOptions".
+func SettingRemoveOptions(ctx context.Context, c *xrpc.Client, input *SettingRemoveOptions_Input) (*SettingRemoveOptions_Output, error) {
+	var out SettingRemoveOptions_Output
+	if err := c.Do(ctx, xrpc.Procedure, "application/json", "tools.ozone.setting.removeOptions", nil, input, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}

api/ozone/settingupsertOption.go

···
+// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.
+
+package ozone
+
+// schema: tools.ozone.setting.upsertOption
+
+import (
+	"context"
+
+	"github.com/bluesky-social/indigo/lex/util"
+	"github.com/bluesky-social/indigo/xrpc"
+)
+
+// SettingUpsertOption_Input is the input argument to a tools.ozone.setting.upsertOption call.
+type SettingUpsertOption_Input struct {
+	Description *string                  `json:"description,omitempty" cborgen:"description,omitempty"`
+	Key         string                   `json:"key" cborgen:"key"`
+	ManagerRole *string                  `json:"managerRole,omitempty" cborgen:"managerRole,omitempty"`
+	Scope       string                   `json:"scope" cborgen:"scope"`
+	Value       *util.LexiconTypeDecoder `json:"value" cborgen:"value"`
+}
+
+// SettingUpsertOption_Output is the output of a tools.ozone.setting.upsertOption call.
+type SettingUpsertOption_Output struct {
+	Option *SettingDefs_Option `json:"option" cborgen:"option"`
+}
+
+// SettingUpsertOption calls the XRPC method "tools.ozone.setting.upsertOption".
+func SettingUpsertOption(ctx context.Context, c *xrpc.Client, input *SettingUpsertOption_Input) (*SettingUpsertOption_Output, error) {
+	var out SettingUpsertOption_Output
+	if err := c.Do(ctx, xrpc.Procedure, "application/json", "tools.ozone.setting.upsertOption", nil, input, &out); err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}

atproto/data/data.go

···
 		out = append(out, v)
 	case []any:
 		for _, el := range v {
-			down := extractBlobsAtom(el)
-			for _, d := range down {
-				out = append(out, d)
-			}
+			out = append(out, extractBlobsAtom(el)...)
 		}
 	case map[string]any:
 		for _, val := range v {
-			down := extractBlobsAtom(val)
-			for _, d := range down {
-				out = append(out, d)
-			}
+			out = append(out, extractBlobsAtom(val)...)
 		}
 	default:
 	}

atproto/identity/base_directory.go

···
 	ident := ParseIdentity(doc)
 	declared, err := ident.DeclaredHandle()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not verify handle/DID match: %w", err)
 	}
 	if declared != h {
-		return nil, ErrHandleMismatch
+		return nil, fmt.Errorf("%w: %s != %s", ErrHandleMismatch, declared, h)
 	}
 	ident.Handle = declared
 
···
 	if errors.Is(err, ErrHandleNotDeclared) {
 		ident.Handle = syntax.HandleInvalid
 	} else if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not parse handle from DID document: %w", err)
 	} else {
 		// if a handle was declared, resolve it
 		resolvedDID, err := d.ResolveHandle(ctx, declared)
···
 }
 
 func (d *BaseDirectory) Purge(ctx context.Context, a syntax.AtIdentifier) error {
+	// BaseDirectory itself does not implement caching
 	return nil
 }

atproto/identity/cache_directory.go

···
 
 func (d *CacheDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) {
 	if h.IsInvalidHandle() {
-		return "", fmt.Errorf("invalid handle")
+		return "", fmt.Errorf("can not resolve handle: %w", ErrInvalidHandle)
 	}
 	entry, ok := d.handleCache.Get(h)
 	if ok && !d.IsHandleStale(&entry) {
···
 
 	declared, err := ident.DeclaredHandle()
 	if err != nil {
-		return nil, hit, err
+		return nil, hit, fmt.Errorf("could not verify handle/DID mapping: %w", err)
 	}
 	if declared != h {
-		return nil, hit, ErrHandleMismatch
+		return nil, hit, fmt.Errorf("%w: %s != %s", ErrHandleMismatch, declared, h)
 	}
 	return ident, hit, nil
 }

atproto/identity/did.go

···
 	"github.com/bluesky-social/indigo/atproto/syntax"
 )
 
-type DIDDocument struct {
-	DID                syntax.DID              `json:"id"`
-	AlsoKnownAs        []string                `json:"alsoKnownAs,omitempty"`
-	VerificationMethod []DocVerificationMethod `json:"verificationMethod,omitempty"`
-	Service            []DocService            `json:"service,omitempty"`
-}
-
-type DocVerificationMethod struct {
-	ID                 string `json:"id"`
-	Type               string `json:"type"`
-	Controller         string `json:"controller"`
-	PublicKeyMultibase string `json:"publicKeyMultibase"`
-}
-
-type DocService struct {
-	ID              string `json:"id"`
-	Type            string `json:"type"`
-	ServiceEndpoint string `json:"serviceEndpoint"`
-}
-
 // WARNING: this does *not* bi-directionally verify account metadata; it only implements direct DID-to-DID-document lookup for the supported DID methods, and parses the resulting DID Doc into an Identity struct
 func (d *BaseDirectory) ResolveDID(ctx context.Context, did syntax.DID) (*DIDDocument, error) {
 	start := time.Now()

atproto/identity/diddoc.go

···
+package identity
+
+import (
+	"github.com/bluesky-social/indigo/atproto/syntax"
+)
+
+type DIDDocument struct {
+	DID                syntax.DID              `json:"id"`
+	AlsoKnownAs        []string                `json:"alsoKnownAs,omitempty"`
+	VerificationMethod []DocVerificationMethod `json:"verificationMethod,omitempty"`
+	Service            []DocService            `json:"service,omitempty"`
+}
+
+type DocVerificationMethod struct {
+	ID                 string `json:"id"`
+	Type               string `json:"type"`
+	Controller         string `json:"controller"`
+	PublicKeyMultibase string `json:"publicKeyMultibase"`
+}
+
+type DocService struct {
+	ID              string `json:"id"`
+	Type            string `json:"type"`
+	ServiceEndpoint string `json:"serviceEndpoint"`
+}

atproto/identity/directory.go

···
+package identity
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/bluesky-social/indigo/atproto/syntax"
+)
+
+// API for doing account lookups by DID or handle, with bi-directional verification handled automatically. Almost all atproto services and clients should use an implementation of this interface instead of resolving handles or DIDs separately
+//
+// Handles which fail to resolve, or don't match DID alsoKnownAs, are an error. DIDs which resolve but the handle does not resolve back to the DID return an Identity where the Handle is the special `handle.invalid` value.
+//
+// Some example implementations of this interface could be:
+//   - basic direct resolution on every call
+//   - local in-memory caching layer to reduce network hits
+//   - API client, which just makes requests to PDS (or other remote service)
+//   - client for shared network cache (eg, Redis)
+type Directory interface {
+	LookupHandle(ctx context.Context, h syntax.Handle) (*Identity, error)
+	LookupDID(ctx context.Context, d syntax.DID) (*Identity, error)
+	Lookup(ctx context.Context, i syntax.AtIdentifier) (*Identity, error)
+
+	// Flushes any cache of the indicated identifier. If directory is not using caching, can ignore this.
+	Purge(ctx context.Context, i syntax.AtIdentifier) error
+}
+
+// Indicates that handle resolution failed. A wrapped error may provide more context. This is only returned when looking up a handle, not when looking up a DID.
+var ErrHandleResolutionFailed = errors.New("handle resolution failed")
+
+// Indicates that resolution process completed successfully, but handle does not exist. This is only returned when looking up a handle, not when looking up a DID.
+var ErrHandleNotFound = errors.New("handle not found")
+
+// Indicates that resolution process completed successfully, handle mapped to a different DID. This is only returned when looking up a handle, not when looking up a DID.
+var ErrHandleMismatch = errors.New("handle/DID mismatch")
+
+// Indicates that DID document did not include any handle ("alsoKnownAs"). This is only returned when looking up a handle, not when looking up a DID.
+var ErrHandleNotDeclared = errors.New("DID document did not declare a handle")
+
+// Handle top-level domain (TLD) is one of the special "Reserved" suffixes, and not allowed for atproto use
+var ErrHandleReservedTLD = errors.New("handle top-level domain is disallowed")
+
+// Indicates that resolution process completed successfully, but the DID does not exist.
+var ErrDIDNotFound = errors.New("DID not found")
+
+// Indicates that DID resolution process failed. A wrapped error may provide more context.
+var ErrDIDResolutionFailed = errors.New("DID resolution failed")
+
+// Indicates that DID document did not include a public key with the specified ID
+var ErrKeyNotDeclared = errors.New("DID document did not declare a relevant public key")
+
+// Handle was invalid, in a situation where a valid handle is required.
+var ErrInvalidHandle = errors.New("Invalid Handle")
+
+var DefaultPLCURL = "https://plc.directory"
+
+// Returns a reasonable Directory implementation for applications
+func DefaultDirectory() Directory {
+	base := BaseDirectory{
+		PLCURL: DefaultPLCURL,
+		HTTPClient: http.Client{
+			Timeout: time.Second * 10,
+			Transport: &http.Transport{
+				// would want this around 100ms for services doing lots of handle resolution. Impacts PLC connections as well, but not too bad.
+				IdleConnTimeout: time.Millisecond * 1000,
+				MaxIdleConns:    100,
+			},
+		},
+		Resolver: net.Resolver{
+			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+				d := net.Dialer{Timeout: time.Second * 3}
+				return d.DialContext(ctx, network, address)
+			},
+		},
+		TryAuthoritativeDNS: true,
+		// primary Bluesky PDS instance only supports HTTP resolution method
+		SkipDNSDomainSuffixes: []string{".bsky.social"},
+	}
+	cached := NewCacheDirectory(&base, 250_000, time.Hour*24, time.Minute*2, time.Minute*5)
+	return &cached
+}

atproto/identity/handle.go

···
 	var dnsErr *net.DNSError
 	if errors.As(err, &dnsErr) {
 		if dnsErr.IsNotFound {
-			return "", ErrHandleNotFound
+			return "", fmt.Errorf("%w: %s", ErrHandleNotFound, handle)
 		}
 	}
 	if err != nil {
···
 		var dnsErr *net.DNSError
 		if errors.As(err, &dnsErr) {
 			if dnsErr.IsNotFound {
-				return "", fmt.Errorf("%w: DNS NXDOMAIN for %s", ErrHandleNotFound, handle)
+				return "", fmt.Errorf("%w: DNS NXDOMAIN for HTTP well-known resolution of %s", ErrHandleNotFound, handle)
 			}
 		}
 		return "", fmt.Errorf("%w: HTTP well-known request error: %w", ErrHandleResolutionFailed, err)
···
 		return "", fmt.Errorf("%w: HTTP well-known body read for %s: %w", ErrHandleResolutionFailed, handle, err)
 	}
 	line := strings.TrimSpace(string(b))
-	return syntax.ParseDID(line)
+	outDid, err := syntax.ParseDID(line)
+	if err != nil {
+		return outDid, fmt.Errorf("%w: invalid DID in HTTP well-known for %s", ErrHandleResolutionFailed, handle)
+	}
+	return outDid, err
 }
 
 func (d *BaseDirectory) ResolveHandle(ctx context.Context, handle syntax.Handle) (syntax.DID, error) {
···
 	var did syntax.DID
 
 	if handle.IsInvalidHandle() {
-		return "", fmt.Errorf("invalid handle")
+		return "", fmt.Errorf("can not resolve handle: %w", ErrInvalidHandle)
 	}
 
 	if !handle.AllowedTLD() {

atproto/identity/identity.go

···
 package identity
 
 import (
-	"context"
-	"errors"
 	"fmt"
-	"net"
-	"net/http"
 	"net/url"
 	"strings"
-	"time"
 
 	"github.com/bluesky-social/indigo/atproto/crypto"
 	"github.com/bluesky-social/indigo/atproto/syntax"
 
 	"github.com/mr-tron/base58"
 )
-
-// API for doing account lookups by DID or handle, with bi-directional verification handled automatically. Almost all atproto services and clients should use an implementation of this interface instead of resolving handles or DIDs separately
-//
-// Handles which fail to resolve, or don't match DID alsoKnownAs, are an error. DIDs which resolve but the handle does not resolve back to the DID return an Identity where the Handle is the special `handle.invalid` value.
-//
-// Some example implementations of this interface could be:
-//   - basic direct resolution on every call
-//   - local in-memory caching layer to reduce network hits
-//   - API client, which just makes requests to PDS (or other remote service)
-//   - client for shared network cache (eg, Redis)
-type Directory interface {
-	LookupHandle(ctx context.Context, h syntax.Handle) (*Identity, error)
-	LookupDID(ctx context.Context, d syntax.DID) (*Identity, error)
-	Lookup(ctx context.Context, i syntax.AtIdentifier) (*Identity, error)
-
-	// Flushes any cache of the indicated identifier. If directory is not using caching, can ignore this.
-	Purge(ctx context.Context, i syntax.AtIdentifier) error
-}
-
-// Indicates that handle resolution failed. A wrapped error may provide more context. This is only returned when looking up a handle, not when looking up a DID.
-var ErrHandleResolutionFailed = errors.New("handle resolution failed")
-
-// Indicates that resolution process completed successfully, but handle does not exist. This is only returned when looking up a handle, not when looking up a DID.
-var ErrHandleNotFound = errors.New("handle not found")
-
-// Indicates that resolution process completed successfully, handle mapped to a different DID. This is only returned when looking up a handle, not when looking up a DID.
-var ErrHandleMismatch = errors.New("handle/DID mismatch")
-
-// Indicates that DID document did not include any handle ("alsoKnownAs"). This is only returned when looking up a handle, not when looking up a DID.
-var ErrHandleNotDeclared = errors.New("DID document did not declare a handle")
-
-// Handle top-level domain (TLD) is one of the special "Reserved" suffixes, and not allowed for atproto use
-var ErrHandleReservedTLD = errors.New("handle top-level domain is disallowed")
-
-// Indicates that resolution process completed successfully, but the DID does not exist.
-var ErrDIDNotFound = errors.New("DID not found")
-
-// Indicates that DID resolution process failed. A wrapped error may provide more context.
-var ErrDIDResolutionFailed = errors.New("DID resolution failed")
-
-// Indicates that DID document did not include a public key with the specified ID
-var ErrKeyNotDeclared = errors.New("DID document did not declare a relevant public key")
-
-var DefaultPLCURL = "https://plc.directory"
-
-// Returns a reasonable Directory implementation for applications
-func DefaultDirectory() Directory {
-	base := BaseDirectory{
-		PLCURL: DefaultPLCURL,
-		HTTPClient: http.Client{
-			Timeout: time.Second * 10,
-			Transport: &http.Transport{
-				// would want this around 100ms for services doing lots of handle resolution. Impacts PLC connections as well, but not too bad.
-				IdleConnTimeout: time.Millisecond * 1000,
-				MaxIdleConns:    100,
-			},
-		},
-		Resolver: net.Resolver{
-			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
-				d := net.Dialer{Timeout: time.Second * 3}
-				return d.DialContext(ctx, network, address)
-			},
-		},
-		TryAuthoritativeDNS: true,
-		// primary Bluesky PDS instance only supports HTTP resolution method
-		SkipDNSDomainSuffixes: []string{".bsky.social"},
-	}
-	cached := NewCacheDirectory(&base, 250_000, time.Hour*24, time.Minute*2, time.Minute*5)
-	return &cached
-}
 
 // Represents an atproto identity. Could be a regular user account, or a service account (eg, feed generator)
 type Identity struct {

atproto/identity/redisdir/redis_directory.go

···
 func NewRedisDirectory(inner identity.Directory, redisURL string, hitTTL, errTTL, invalidHandleTTL time.Duration, lruSize int) (*RedisDirectory, error) {
 	opt, err := redis.ParseURL(redisURL)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not configure redis identity cache: %w", err)
 	}
 	rdb := redis.NewClient(opt)
 	// check redis connection
 	_, err = rdb.Ping(context.TODO()).Result()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not connect to redis identity cache: %w", err)
 	}
 	handleCache := cache.New(&cache.Options{
 		Redis:      rdb,
···
 		})
 		if err != nil {
 			he.DID = nil
-			he.Err = fmt.Errorf("identity cache write: %w", err)
+			he.Err = fmt.Errorf("identity cache write failed: %w", err)
 			return he
 		}
 		return he
···
 	})
 	if err != nil {
 		he.DID = nil
-		he.Err = fmt.Errorf("identity cache write: %w", err)
+		he.Err = fmt.Errorf("identity cache write failed: %w", err)
 		return he
 	}
 	err = d.handleCache.Set(&cache.Item{
···
 	})
 	if err != nil {
 		he.DID = nil
-		he.Err = fmt.Errorf("identity cache write: %w", err)
+		he.Err = fmt.Errorf("identity cache write failed: %w", err)
 		return he
 	}
 	return he
···
 
 func (d *RedisDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) {
 	if h.IsInvalidHandle() {
-		return "", errors.New("invalid handle")
+		return "", fmt.Errorf("can not resolve handle: %w", identity.ErrInvalidHandle)
 	}
 	var entry handleEntry
 	err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), &entry)
 	if err != nil && err != cache.ErrCacheMiss {
-		return "", fmt.Errorf("identity cache read: %w", err)
+		return "", fmt.Errorf("identity cache read failed: %w", err)
 	}
 	if err == nil && !d.isHandleStale(&entry) { // if no error...
 		handleCacheHits.Inc()
···
 			// The result should now be in the cache
 			err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), entry)
 			if err != nil && err != cache.ErrCacheMiss {
-				return "", fmt.Errorf("identity cache read: %w", err)
+				return "", fmt.Errorf("identity cache read failed: %w", err)
 			}
 			if err == nil && !d.isHandleStale(&entry) { // if no error...
 				if entry.Err != nil {
···
 	})
 	if err != nil {
 		entry.Identity = nil
-		entry.Err = fmt.Errorf("identity cache write: %v", err)
+		entry.Err = fmt.Errorf("identity cache write failed: %w", err)
 		return entry
 	}
 	if he != nil {
···
 		})
 		if err != nil {
 			entry.Identity = nil
-			entry.Err = fmt.Errorf("identity cache write: %v", err)
+			entry.Err = fmt.Errorf("identity cache write failed: %w", err)
 			return entry
 		}
 	}
···
 	var entry identityEntry
 	err := d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry)
 	if err != nil && err != cache.ErrCacheMiss {
-		return nil, false, fmt.Errorf("identity cache read: %v", err)
+		return nil, false, fmt.Errorf("identity cache read failed: %w", err)
 	}
 	if err == nil && !d.isIdentityStale(&entry) { // if no error...
 		identityCacheHits.Inc()
···
 			// The result should now be in the cache
 			err = d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry)
 			if err != nil && err != cache.ErrCacheMiss {
-				return nil, false, fmt.Errorf("identity cache read: %v", err)
+				return nil, false, fmt.Errorf("identity cache read failed: %w", err)
 			}
 			if err == nil && !d.isIdentityStale(&entry) { // if no error...
 				return entry.Identity, false, entry.Err

atproto/lexicon/language.go

···
 
 func (s *SchemaRecord) CheckSchema() error {
 	switch s.Key {
-	case "tid", "any":
+	case "tid", "nsid", "any":
 		// pass
 	default:
 		if !strings.HasPrefix(s.Key, "literal:") {

atproto/syntax/cmd/atp-syntax/main.go

···
 			ArgsUsage: "<tid>",
 			Action:    runParseTID,
 		},
+		&cli.Command{
+			Name:      "parse-did",
+			Usage:     "parse a DID",
+			ArgsUsage: "<did>",
+			Action:    runParseDID,
+		},
 	}
 	h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
 	slog.SetDefault(slog.New(h))
···
 
 	return nil
 }
+
+func runParseDID(cctx *cli.Context) error {
+	s := cctx.Args().First()
+	if s == "" {
+		return fmt.Errorf("need to provide identifier as an argument")
+	}
+
+	did, err := syntax.ParseDID(s)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%s\n", did)
+
+	return nil
+}

atproto/syntax/did.go

···
 type DID string
 
 var didRegex = regexp.MustCompile(`^did:[a-z]+:[a-zA-Z0-9._:%-]*[a-zA-Z0-9._-]$`)
+var plcChars = ""
+
+func isASCIIAlphaNum(c rune) bool {
+	if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') {
+		return true
+	}
+	return false
+}
 
 func ParseDID(raw string) (DID, error) {
+	// fast-path for did:plc, avoiding regex
+	if len(raw) == 32 && strings.HasPrefix(raw, "did:plc:") {
+		// NOTE: this doesn't really check base32, just broader alphanumberic. might pass invalid PLC DIDs, but they still have overall valid DID syntax
+		isPlc := true
+		for _, c := range raw[8:32] {
+			if !isASCIIAlphaNum(c) {
+				isPlc = false
+				break
+			}
+		}
+		if isPlc {
+			return DID(raw), nil
+		}
+	}
 	if raw == "" {
 		return "", errors.New("expected DID, got empty string")
 	}

atproto/syntax/path.go

···
+package syntax
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// Parses an atproto repo path string in to "collection" (NSID) and record key parts.
+//
+// Does not return partial success: either both collection and record key are complete (and error is nil), or both are empty string (and error is not nil)
+func ParseRepoPath(raw string) (NSID, RecordKey, error) {
+	parts := strings.SplitN(raw, "/", 3)
+	if len(parts) != 2 {
+		return "", "", errors.New("expected path to have two parts, separated by single slash")
+	}
+	nsid, err := ParseNSID(parts[0])
+	if err != nil {
+		return "", "", fmt.Errorf("collection part of path not a valid NSID: %w", err)
+	}
+	rkey, err := ParseRecordKey(parts[1])
+	if err != nil {
+		return "", "", fmt.Errorf("record key part of path not valid: %w", err)
+	}
+	return nsid, rkey, nil
+}

atproto/syntax/path_test.go

···
+package syntax
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRepoPath(t *testing.T) {
+	assert := assert.New(t)
+
+	testValid := [][]string{
+		{"app.bsky.feed.post/asdf", "app.bsky.feed.post", "asdf"},
+	}
+
+	testErr := []string{
+		"",
+		"/",
+		"/app.bsky.feed.post/asdf",
+		"/asdf",
+		"./app.bsky.feed.post",
+		"blob/asdf",
+		"app.bsky.feed.post/",
+		"app.bsky.feed.post/.",
+		"app.bsky.feed.post/!",
+	}
+
+	for _, parts := range testValid {
+		nsid, rkey, err := ParseRepoPath(parts[0])
+		assert.NoError(err)
+		assert.Equal(parts[1], nsid.String())
+		assert.Equal(parts[2], rkey.String())
+	}
+
+	for _, raw := range testErr {
+		nsid, rkey, err := ParseRepoPath(raw)
+		assert.Error(err)
+		assert.Equal("", nsid.String())
+		assert.Equal("", rkey.String())
+	}
+}

automod/consumer/firehose.go

···
 		fc.Logger.Info("hepa scheduler configured", "scheduler", "autoscaling", "initial", scaleSettings.Concurrency, "max", scaleSettings.MaxConcurrency)
 	}
 
-	return events.HandleRepoStream(ctx, con, scheduler)
+	return events.HandleRepoStream(ctx, con, scheduler, fc.Logger)
 }
 
 // NOTE: for now, this function basically never errors, just logs and returns nil. Should think through error processing better.
···
 
 	for _, op := range evt.Ops {
 		logger = logger.With("eventKind", op.Action, "path", op.Path)
-		collection, rkey, err := splitRepoPath(op.Path)
+		collection, rkey, err := syntax.ParseRepoPath(op.Path)
 		if err != nil {
-			logger.Error("invalid path in repo op")
+			logger.Error("invalid path in repo op", "err", err)
 			return nil
 		}
 

automod/consumer/ozone.go

···
 			false,          // hasComment bool
 			true,           // includeAllUserRecords bool
 			limit,          // limit int64
+			nil,            // policies []string
 			nil,            // removedLabels []string
 			nil,            // removedTags []string
 			nil,            // reportTypes []string

automod/consumer/util.go

···
-package consumer
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/bluesky-social/indigo/atproto/syntax"
-)
-
-// TODO: move this to a "ParsePath" helper in syntax package?
-func splitRepoPath(path string) (syntax.NSID, syntax.RecordKey, error) {
-	parts := strings.SplitN(path, "/", 3)
-	if len(parts) != 2 {
-		return "", "", fmt.Errorf("invalid record path: %s", path)
-	}
-	collection, err := syntax.ParseNSID(parts[0])
-	if err != nil {
-		return "", "", err
-	}
-	rkey, err := syntax.ParseRecordKey(parts[1])
-	if err != nil {
-		return "", "", err
-	}
-	return collection, rkey, nil
-}

automod/engine/circuit_breaker_test.go

···
 	p1cbor := p1buf.Bytes()
 
 	// generate double the quote of events; expect to only count the quote worth of actions
-	for i := 0; i < 2*QuotaModTakedownDay; i++ {
+	for i := 0; i < 2*eng.Config.QuotaModTakedownDay; i++ {
 		ident := identity.Identity{
 			DID:    syntax.DID(fmt.Sprintf("did:plc:abc%d", i)),
 			Handle: syntax.Handle("handle.example.com"),
···
 
 	takedowns, err := eng.Counters.GetCount(ctx, "automod-quota", "takedown", countstore.PeriodDay)
 	assert.NoError(err)
-	assert.Equal(QuotaModTakedownDay, takedowns)
+	assert.Equal(eng.Config.QuotaModTakedownDay, takedowns)
 
 	reports, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay)
 	assert.NoError(err)
···
 	p1cbor := p1buf.Bytes()
 
 	// generate double the quota of events; expect to only count the quota worth of actions
-	for i := 0; i < 2*QuotaModReportDay; i++ {
+	for i := 0; i < 2*eng.Config.QuotaModReportDay; i++ {
 		ident := identity.Identity{
 			DID:    syntax.DID(fmt.Sprintf("did:plc:abc%d", i)),
 			Handle: syntax.Handle("handle.example.com"),
···
 
 	reports, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay)
 	assert.NoError(err)
-	assert.Equal(QuotaModReportDay, reports)
+	assert.Equal(eng.Config.QuotaModReportDay, reports)
 }

automod/engine/effects.go

···
 
 import (
 	"sync"
-	"time"
-)
-
-var (
-	// time period within which automod will not re-report an account for the same reasonType
-	ReportDupePeriod = 1 * 24 * time.Hour
-	// number of reports automod can file per day, for all subjects and types combined (circuit breaker)
-	QuotaModReportDay = 2000
-	// number of takedowns automod can action per day, for all subjects combined (circuit breaker)
-	QuotaModTakedownDay = 200
-	// number of misc actions automod can do per day, for all subjects combined (circuit breaker)
-	QuotaModActionDay = 1000
 )
 
 type CounterRef struct {

automod/engine/engine.go

···
 type EngineConfig struct {
 	// if enabled, account metadata is not hydrated for every event by default
 	SkipAccountMeta bool
+	// time period within which automod will not re-report an account for the same reasonType
+	ReportDupePeriod time.Duration
+	// number of reports automod can file per day, for all subjects and types combined (circuit breaker)
+	QuotaModReportDay int
+	// number of takedowns automod can action per day, for all subjects combined (circuit breaker)
+	QuotaModTakedownDay int
+	// number of misc actions automod can do per day, for all subjects combined (circuit breaker)
+	QuotaModActionDay int
 }
 
 // Entrypoint for external code pushing #identity events in to the engine.

automod/engine/fetch_account_meta.go

···
 				ap.AccountTags = dedupeStrings(rd.Moderation.SubjectStatus.Tags)
 				if rd.Moderation.SubjectStatus.ReviewState != nil {
 					switch *rd.Moderation.SubjectStatus.ReviewState {
-					case "#reviewOpen":
+					case "tools.ozone.moderation.defs#reviewOpen":
 						ap.ReviewState = ReviewStateOpen
-					case "#reviewEscalated":
+					case "tools.ozone.moderation.defs#reviewEscalated":
 						ap.ReviewState = ReviewStateEscalated
-					case "#reviewClosed":
+					case "tools.ozone.moderation.defs#reviewClosed":
 						ap.ReviewState = ReviewStateClosed
-					case "#reviewNonde":
+					case "tools.ozone.moderation.defs#reviewNone":
 						ap.ReviewState = ReviewStateNone
+					default:
+						logger.Warn("unexpected ozone moderation review state", "state", rd.Moderation.SubjectStatus.ReviewState, "did", ident.DID)
 					}
 				}
 			}

automod/engine/persisthelpers.go

···
 	if err != nil {
 		return nil, fmt.Errorf("checking report action quota: %w", err)
 	}
-	if c >= QuotaModReportDay {
+
+	quotaModReportDay := eng.Config.QuotaModReportDay
+	if quotaModReportDay == 0 {
+		quotaModReportDay = 10000
+	}
+	if c >= quotaModReportDay {
 		eng.Logger.Warn("CIRCUIT BREAKER: automod reports")
 		return []ModReport{}, nil
 	}
···
 	if err != nil {
 		return false, fmt.Errorf("checking takedown action quota: %w", err)
 	}
-	if c >= QuotaModTakedownDay {
+	quotaModTakedownDay := eng.Config.QuotaModTakedownDay
+	if quotaModTakedownDay == 0 {
+		quotaModTakedownDay = 200
+	}
+	if c >= quotaModTakedownDay {
 		eng.Logger.Warn("CIRCUIT BREAKER: automod takedowns")
 		return false, nil
 	}
···
 	if err != nil {
 		return false, fmt.Errorf("checking mod action quota: %w", err)
 	}
-	if c >= QuotaModActionDay {
+	quotaModActionDay := eng.Config.QuotaModActionDay
+	if quotaModActionDay == 0 {
+		quotaModActionDay = 2000
+	}
+	if c >= quotaModActionDay {
 		eng.Logger.Warn("CIRCUIT BREAKER: automod action")
 		return false, nil
 	}
···
 		false,          // hasComment bool
 		false,          // includeAllUserRecords bool
 		5,              // limit int64
+		nil,            // policies []string
 		nil,            // removedLabels []string
 		nil,            // removedTags []string
 		nil,            // reportTypes []string
···
 		if err != nil {
 			return false, err
 		}
-		if time.Since(created.Time()) > ReportDupePeriod {
+		reportDupePeriod := eng.Config.ReportDupePeriod
+		if reportDupePeriod == 0 {
+			reportDupePeriod = 1 * 24 * time.Hour
+		}
+		if time.Since(created.Time()) > reportDupePeriod {
 			continue
 		}
 
···
 		false,          // hasComment bool
 		false,          // includeAllUserRecords bool
 		5,              // limit int64
+		nil,            // policies []string
 		nil,            // removedLabels []string
 		nil,            // removedTags []string
 		nil,            // reportTypes []string
···
 		if err != nil {
 			return false, err
 		}
-		if time.Since(created.Time()) > ReportDupePeriod {
+		reportDupePeriod := eng.Config.ReportDupePeriod
+		if reportDupePeriod == 0 {
+			reportDupePeriod = 1 * 24 * time.Hour
+		}
+		if time.Since(created.Time()) > reportDupePeriod {
 			continue
 		}
 

automod/engine/slack.go

···
 		msg += fmt.Sprintf("Report `%s`: %s\n", rep.ReasonType, rep.Comment)
 	}
 	if newTakedown {
-		msg += fmt.Sprintf("Takedown!\n")
+		msg += "Takedown!\n"
 	}
 	return msg
 }

automod/flagstore/flagstore_mem.go

···
 	if !ok {
 		v = []string{}
 	}
-	for _, f := range flags {
-		v = append(v, f)
-	}
+	v = append(v, flags...)
 	v = dedupeStrings(v)
 	s.Data[key] = v
 	return nil

automod/helpers/bsky.go

···
 
 func ExtractHashtagsPost(post *appbsky.FeedPost) []string {
 	var tags []string
-	for _, tag := range post.Tags {
-		tags = append(tags, tag)
-	}
+	tags = append(tags, post.Tags...)
 	for _, facet := range post.Facets {
 		for _, feat := range facet.Features {
 			if feat.RichtextFacet_Tag != nil {

automod/rules/harassment.go

···
 
 	if count > 5 {
 		//c.AddRecordFlag("trivial-harassing-post")
-		c.ReportAccount(automod.ReportReasonOther, fmt.Sprintf("possible targetted harassment (also labeled; remove label if this isn't harassment!)"))
+		c.ReportAccount(automod.ReportReasonOther, "possible targetted harassment (also labeled; remove label if this isn't harassment!)")
 		c.AddAccountLabel("!hide")
 		c.Notify("slack")
 	}

automod/rules/nostr.go

···
 package rules
 
 import (
-	"fmt"
 	"strings"
 	"time"
 
···
 		return nil
 	}
 
-	c.ReportAccount(automod.ReportReasonOther, fmt.Sprintf("likely nostr spam account (also labeled; remove label if this isn't spam!)"))
+	c.ReportAccount(automod.ReportReasonOther, "likely nostr spam account (also labeled; remove label if this isn't spam!)")
 	c.AddAccountLabel("!hide")
 	c.Notify("slack")
 	return nil

automod/rules/promo.go

···
 package rules
 
 import (
-	"fmt"
 	"net/url"
 	"strings"
 	"time"
···
 	uniqueReplies := c.GetCountDistinct("reply-to", did, countstore.PeriodDay)
 	if uniqueReplies >= 10 {
 		c.AddAccountFlag("promo-multi-reply")
-		c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("possible aggressive self-promotion"))
+		c.ReportAccount(automod.ReportReasonSpam, "possible aggressive self-promotion")
 		c.Notify("slack")
 	}
 

automod/rules/quick.go

···
 		}
 		if strings.Contains(*profile.Description, "🏈🍕🌀") {
 			c.AddAccountFlag("profile-bot-string")
-			c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("possible bot based on string in profile"))
+			c.ReportAccount(automod.ReportReasonSpam, "possible bot based on string in profile")
 			c.Notify("slack")
 			return nil
 		}
···
 		return nil
 	}
 
-	c.ReportAccount(automod.ReportReasonOther, fmt.Sprintf("trivial spam account (also labeled; remove label if this isn't spam!)"))
+	c.ReportAccount(automod.ReportReasonOther, "trivial spam account (also labeled; remove label if this isn't spam!)")
 	c.AddAccountLabel("!hide")
 	c.Notify("slack")
 	return nil

backfill/backfill.go

···
 	"time"
 
 	"github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/atproto/identity"
+	"github.com/bluesky-social/indigo/atproto/syntax"
 	"github.com/bluesky-social/indigo/repo"
 	"github.com/bluesky-social/indigo/repomgr"
+
 	"github.com/ipfs/go-cid"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.opentelemetry.io/otel"
···
 	ParallelRecordCreates int
 	// Prefix match for records to backfill i.e. app.bsky.feed.app/
 	// If empty, all records will be backfilled
-	NSIDFilter   string
-	CheckoutPath string
+	NSIDFilter string
+	RelayHost  string
 
 	syncLimiter *rate.Limiter
 
···
 	magicHeaderVal string
 
 	stop chan chan struct{}
+
+	Directory identity.Directory
 }
 
 var (
···
 	ParallelRecordCreates int
 	NSIDFilter            string
 	SyncRequestsPerSecond int
-	CheckoutPath          string
+	RelayHost             string
 }
 
 func DefaultBackfillOptions() *BackfillOptions {
···
 		ParallelRecordCreates: 100,
 		NSIDFilter:            "",
 		SyncRequestsPerSecond: 2,
-		CheckoutPath:          "https://bsky.network/xrpc/com.atproto.sync.getRepo",
+		RelayHost:             "https://bsky.network",
 	}
 }
 
···
 	if opts == nil {
 		opts = DefaultBackfillOptions()
 	}
+
+	// Convert wss:// or ws:// to https:// or http://
+	if strings.HasPrefix(opts.RelayHost, "wss://") {
+		opts.RelayHost = "https://" + opts.RelayHost[6:]
+	} else if strings.HasPrefix(opts.RelayHost, "ws://") {
+		opts.RelayHost = "http://" + opts.RelayHost[5:]
+	}
+
 	return &Backfiller{
 		Name:                  name,
 		Store:                 store,
···
 		ParallelRecordCreates: opts.ParallelRecordCreates,
 		NSIDFilter:            opts.NSIDFilter,
 		syncLimiter:           rate.NewLimiter(rate.Limit(opts.SyncRequestsPerSecond), 1),
-		CheckoutPath:          opts.CheckoutPath,
+		RelayHost:             opts.RelayHost,
 		stop:                  make(chan chan struct{}, 1),
+		Directory:             identity.DefaultDirectory(),
 	}
 }
 
···
 	err        error
 }
 
-// BackfillRepo backfills a repo
-func (b *Backfiller) BackfillRepo(ctx context.Context, job Job) (string, error) {
-	ctx, span := tracer.Start(ctx, "BackfillRepo")
-	defer span.End()
+// Fetches a repo CAR file over HTTP from the indicated host. If successful, parses the CAR and returns repo.Repo
+func (b *Backfiller) fetchRepo(ctx context.Context, did, since, host string) (*repo.Repo, error) {
+	url := fmt.Sprintf("%s/xrpc/com.atproto.sync.getRepo?did=%s", host, did)
 
-	start := time.Now()
-
-	repoDid := job.Repo()
-
-	log := slog.With("source", "backfiller_backfill_repo", "repo", repoDid)
-	if job.RetryCount() > 0 {
-		log = log.With("retry_count", job.RetryCount())
-	}
-	log.Info(fmt.Sprintf("processing backfill for %s", repoDid))
-
-	url := fmt.Sprintf("%s?did=%s", b.CheckoutPath, repoDid)
-
-	if job.Rev() != "" {
-		url = url + fmt.Sprintf("&since=%s", job.Rev())
+	if since != "" {
+		url = url + fmt.Sprintf("&since=%s", since)
 	}
 
 	// GET and CAR decode the body
···
 	}
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
-		state := fmt.Sprintf("failed (create request: %s)", err.Error())
-		return state, fmt.Errorf("failed to create request: %w", err)
+		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
 
 	req.Header.Set("Accept", "application/vnd.ipld.car")
···
 
 	resp, err := client.Do(req)
 	if err != nil {
-		state := fmt.Sprintf("failed (do request: %s)", err.Error())
-		return state, fmt.Errorf("failed to send request: %w", err)
+		return nil, fmt.Errorf("failed to send request: %w", err)
 	}
 
 	if resp.StatusCode != http.StatusOK {
···
 		} else {
 			reason = resp.Status
 		}
-		state := fmt.Sprintf("failed (%s)", reason)
-		return state, fmt.Errorf("failed to get repo: %s", reason)
+		return nil, fmt.Errorf("failed to get repo: %s", reason)
 	}
 
 	instrumentedReader := instrumentedReader{
···
 
 	defer instrumentedReader.Close()
 
-	r, err := repo.ReadRepoFromCar(ctx, instrumentedReader)
+	repo, err := repo.ReadRepoFromCar(ctx, instrumentedReader)
 	if err != nil {
-		state := "failed (couldn't read repo CAR from response body)"
-		return state, fmt.Errorf("failed to read repo from car: %w", err)
+		return nil, fmt.Errorf("failed to parse repo from CAR file: %w", err)
+	}
+	return repo, nil
+}
+
+// BackfillRepo backfills a repo
+func (b *Backfiller) BackfillRepo(ctx context.Context, job Job) (string, error) {
+	ctx, span := tracer.Start(ctx, "BackfillRepo")
+	defer span.End()
+
+	start := time.Now()
+
+	repoDID := job.Repo()
+
+	log := slog.With("source", "backfiller_backfill_repo", "repo", repoDID)
+	if job.RetryCount() > 0 {
+		log = log.With("retry_count", job.RetryCount())
+	}
+	log.Info(fmt.Sprintf("processing backfill for %s", repoDID))
+
+	// first try with Relay endpoint
+	r, err := b.fetchRepo(ctx, repoDID, job.Rev(), b.RelayHost)
+	if err != nil {
+		slog.Warn("repo CAR fetch from relay failed", "did", repoDID, "since", job.Rev(), "relayHost", b.RelayHost, "err", err)
+		// fallback to direct PDS fetch
+		ident, err := b.Directory.LookupDID(ctx, syntax.DID(repoDID))
+		if err != nil {
+			return "failed resolving DID to PDS repo", fmt.Errorf("resolving DID for PDS repo fetch: %w", err)
+		}
+		pdsHost := ident.PDSEndpoint()
+		if pdsHost == "" {
+			return "DID document missing PDS endpoint", fmt.Errorf("no PDS endpoint for DID: %s", repoDID)
+		}
+		r, err = b.fetchRepo(ctx, repoDID, job.Rev(), pdsHost)
+		if err != nil {
+			slog.Warn("repo CAR fetch from PDS failed", "did", repoDID, "since", job.Rev(), "pdsHost", pdsHost, "err", err)
+			return "repo CAR fetch from PDS failed", err
+		}
+		slog.Info("repo CAR fetch from PDS successful", "did", repoDID, "since", job.Rev(), "pdsHost", pdsHost, "err", err)
 	}
 
 	numRecords := 0
···
 
 				raw := blk.RawData()
 
-				err = b.HandleCreateRecord(ctx, repoDid, rev, item.recordPath, &raw, &item.nodeCid)
+				err = b.HandleCreateRecord(ctx, repoDID, rev, item.recordPath, &raw, &item.nodeCid)
 				if err != nil {
 					recordResults <- recordResult{recordPath: item.recordPath, err: fmt.Errorf("failed to handle create record: %w", err)}
 					continue

bgs/admin.go

···
 		return err
 	}
 
+	// don't care if this errors, but we should try to disconnect something we just blocked
+	_ = bgs.slurper.KillUpstreamConnection(host, false)
+
 	return e.JSON(200, map[string]any{
 		"success": "true",
 	})
···
 		ctx := context.Background()
 		err := bgs.ResyncPDS(ctx, pds)
 		if err != nil {
-			log.Errorw("failed to resync PDS", "err", err, "pds", pds.Host)
+			log.Error("failed to resync PDS", "err", err, "pds", pds.Host)
 		}
 	}()
 

bgs/bgs.go

···
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log/slog"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
 	"net/url"
+	"reflect"
 	"strconv"
 	"strings"
 	"sync"
···
 	"github.com/bluesky-social/indigo/models"
 	"github.com/bluesky-social/indigo/repomgr"
 	"github.com/bluesky-social/indigo/xrpc"
+	lru "github.com/hashicorp/golang-lru/v2"
 	"golang.org/x/sync/semaphore"
 	"golang.org/x/time/rate"
 
 	"github.com/gorilla/websocket"
 	"github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
-	logging "github.com/ipfs/go-log"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	promclient "github.com/prometheus/client_golang/prometheus"
···
 	"gorm.io/gorm"
 )
 
-var log = logging.Logger("bgs")
 var tracer = otel.Tracer("bgs")
 
 // serverListenerBootTimeout is how long to wait for the requested server socket
···
 
 	// Management of Compaction
 	compactor *Compactor
+
+	// User cache
+	userCache *lru.Cache[string, *User]
+
+	// nextCrawlers gets forwarded POST /xrpc/com.atproto.sync.requestCrawl
+	nextCrawlers []*url.URL
+	httpClient   http.Client
+
+	log *slog.Logger
 }
 
 type PDSResync struct {
···
 	ConcurrencyPerPDS    int64
 	MaxQueuePerPDS       int64
 	NumCompactionWorkers int
+
+	// NextCrawlers gets forwarded POST /xrpc/com.atproto.sync.requestCrawl
+	NextCrawlers []*url.URL
 }
 
 func DefaultBGSConfig() *BGSConfig {
···
 	db.AutoMigrate(models.PDS{})
 	db.AutoMigrate(models.DomainBan{})
 
+	uc, _ := lru.New[string, *User](1_000_000)
+
 	bgs := &BGS{
 		Index:       ix,
 		db:          db,
···
 		consumers:   make(map[uint64]*SocketConsumer),
 
 		pdsResyncs: make(map[uint]*PDSResync),
+
+		userCache: uc,
+
+		log: slog.Default().With("system", "bgs"),
 	}
 
 	ix.CreateExternalUser = bgs.createExternalUser
···
 	compactor.Start(bgs)
 	bgs.compactor = compactor
 
+	bgs.nextCrawlers = config.NextCrawlers
+	bgs.httpClient.Timeout = time.Second * 5
+
 	return bgs, nil
 }
 
···
 		act, err := bgs.Index.GetUserOrMissing(ctx, did)
 		if err != nil {
 			w.WriteHeader(500)
-			log.Errorf("failed to get user: %s", err)
+			bgs.log.Error("failed to get user", "err", err)
 			return
 		}
 
 		if err := bgs.Index.Crawler.Crawl(ctx, act); err != nil {
 			w.WriteHeader(500)
-			log.Errorf("failed to add user to crawler: %s", err)
+			bgs.log.Error("failed to add user to crawler", "err", err)
 			return
 		}
 	})
···
 			if err2 := ctx.JSON(err.Code, map[string]any{
 				"error": err.Message,
 			}); err2 != nil {
-				log.Errorf("Failed to write http error: %s", err2)
+				bgs.log.Error("Failed to write http error", "err", err2)
 			}
 		default:
 			sendHeader := true
···
 				sendHeader = false
 			}
 
-			log.Warnf("HANDLER ERROR: (%s) %s", ctx.Path(), err)
+			bgs.log.Warn("HANDLER ERROR: (%s) %s", ctx.Path(), err)
 
 			if strings.HasPrefix(ctx.Path(), "/admin/") {
 				ctx.JSON(500, map[string]any{
···
 
 func (bgs *BGS) HandleHealthCheck(c echo.Context) error {
 	if err := bgs.db.Exec("SELECT 1").Error; err != nil {
-		log.Errorf("healthcheck can't connect to database: %v", err)
+		bgs.log.Error("healthcheck can't connect to database", "err", err)
 		return c.JSON(500, HealthStatus{Status: "error", Message: "can't connect to database"})
 	} else {
 		return c.JSON(200, HealthStatus{Status: "ok"})
···
 
 	// UpstreamStatus is the state of the user as reported by the upstream PDS
 	UpstreamStatus string `gorm:"index"`
+
+	lk sync.Mutex
+}
+
+func (u *User) SetTakenDown(v bool) {
+	u.lk.Lock()
+	defer u.lk.Unlock()
+	u.TakenDown = v
+}
+
+func (u *User) GetTakenDown() bool {
+	u.lk.Lock()
+	defer u.lk.Unlock()
+	return u.TakenDown
+}
+
+func (u *User) SetTombstoned(v bool) {
+	u.lk.Lock()
+	defer u.lk.Unlock()
+	u.Tombstoned = v
+}
+
+func (u *User) GetTombstoned() bool {
+	u.lk.Lock()
+	defer u.lk.Unlock()
+	return u.Tombstoned
+}
+
+func (u *User) SetUpstreamStatus(v string) {
+	u.lk.Lock()
+	defer u.lk.Unlock()
+	u.UpstreamStatus = v
+}
+
+func (u *User) GetUpstreamStatus() string {
+	u.lk.Lock()
+	defer u.lk.Unlock()
+	return u.UpstreamStatus
 }
 
 type addTargetBody struct {
···
 
 	var m = &dto.Metric{}
 	if err := c.EventsSent.Write(m); err != nil {
-		log.Errorf("failed to get sent counter: %s", err)
+		bgs.log.Error("failed to get sent counter", "err", err)
 	}
 
-	log.Infow("consumer disconnected",
+	bgs.log.Info("consumer disconnected",
 		"consumer_id", id,
 		"remote_addr", c.RemoteAddr,
 		"user_agent", c.UserAgent,
···
 				}
 
 				if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(5*time.Second)); err != nil {
-					log.Warnf("failed to ping client: %s", err)
+					bgs.log.Warn("failed to ping client", "err", err)
 					cancel()
 					return
 				}
···
 		for {
 			_, _, err := conn.ReadMessage()
 			if err != nil {
-				log.Warnf("failed to read message from client: %s", err)
+				bgs.log.Warn("failed to read message from client", "err", err)
 				cancel()
 				return
 			}
···
 	consumerID := bgs.registerConsumer(&consumer)
 	defer bgs.cleanupConsumer(consumerID)
 
-	logger := log.With(
+	logger := bgs.log.With(
 		"consumer_id", consumerID,
 		"remote_addr", consumer.RemoteAddr,
 		"user_agent", consumer.UserAgent,
 	)
 
-	logger.Infow("new consumer", "cursor", since)
+	logger.Info("new consumer", "cursor", since)
 
 	for {
 		select {
···
 
 			wc, err := conn.NextWriter(websocket.BinaryMessage)
 			if err != nil {
-				logger.Errorf("failed to get next writer: %s", err)
+				logger.Error("failed to get next writer", "err", err)
 				return err
 			}
 
···
 			}
 
 			if err := wc.Close(); err != nil {
-				logger.Warnf("failed to flush-close our event write: %s", err)
+				logger.Warn("failed to flush-close our event write", "err", err)
 				return nil
 			}
 
···
 	// defensive in case things change under the hood.
 	registry, ok := promclient.DefaultRegisterer.(*promclient.Registry)
 	if !ok {
-		log.Warnf("failed to export default prometheus registry; some metrics will be unavailable; unexpected type: %T", promclient.DefaultRegisterer)
+		slog.Warn("failed to export default prometheus registry; some metrics will be unavailable; unexpected type", "type", reflect.TypeOf(promclient.DefaultRegisterer))
 	}
 	exporter, err := prometheus.NewExporter(prometheus.Options{
 		Registry:  registry,
 		Namespace: "bigsky",
 	})
 	if err != nil {
-		log.Errorf("could not create the prometheus stats exporter: %v", err)
+		slog.Error("could not create the prometheus stats exporter", "err", err, "system", "bgs")
 	}
 
 	return exporter
···
 	ctx, span := tracer.Start(ctx, "lookupUserByDid")
 	defer span.End()
 
+	cu, ok := bgs.userCache.Get(did)
+	if ok {
+		return cu, nil
+	}
+
 	var u User
 	if err := bgs.db.Find(&u, "did = ?", did).Error; err != nil {
 		return nil, err
···
 	if u.ID == 0 {
 		return nil, gorm.ErrRecordNotFound
 	}
+
+	bgs.userCache.Add(did, &u)
 
 	return &u, nil
 }
···
 	case env.RepoCommit != nil:
 		repoCommitsReceivedCounter.WithLabelValues(host.Host).Add(1)
 		evt := env.RepoCommit
-		log.Debugw("bgs got repo append event", "seq", evt.Seq, "pdsHost", host.Host, "repo", evt.Repo)
+		bgs.log.Debug("bgs got repo append event", "seq", evt.Seq, "pdsHost", host.Host, "repo", evt.Repo)
+
+		s := time.Now()
 		u, err := bgs.lookupUserByDid(ctx, evt.Repo)
+		userLookupDuration.Observe(time.Since(s).Seconds())
 		if err != nil {
 			if !errors.Is(err, gorm.ErrRecordNotFound) {
+				repoCommitsResultCounter.WithLabelValues(host.Host, "nou").Inc()
 				return fmt.Errorf("looking up event user: %w", err)
 			}
 
 			newUsersDiscovered.Inc()
+			start := time.Now()
 			subj, err := bgs.createExternalUser(ctx, evt.Repo)
+			newUserDiscoveryDuration.Observe(time.Since(start).Seconds())
 			if err != nil {
+				repoCommitsResultCounter.WithLabelValues(host.Host, "uerr").Inc()
 				return fmt.Errorf("fed event create external user: %w", err)
 			}
 
···
 			u.Did = evt.Repo
 		}
 
-		span.SetAttributes(attribute.String("upstream_status", u.UpstreamStatus))
+		ustatus := u.GetUpstreamStatus()
+		span.SetAttributes(attribute.String("upstream_status", ustatus))
 
-		if u.TakenDown || u.UpstreamStatus == events.AccountStatusTakendown {
-			span.SetAttributes(attribute.Bool("taken_down_by_relay_admin", u.TakenDown))
-			log.Debugw("dropping commit event from taken down user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+		if u.GetTakenDown() || ustatus == events.AccountStatusTakendown {
+			span.SetAttributes(attribute.Bool("taken_down_by_relay_admin", u.GetTakenDown()))
+			bgs.log.Debug("dropping commit event from taken down user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+			repoCommitsResultCounter.WithLabelValues(host.Host, "tdu").Inc()
 			return nil
 		}
 
-		if u.UpstreamStatus == events.AccountStatusSuspended {
-			log.Debugw("dropping commit event from suspended user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+		if ustatus == events.AccountStatusSuspended {
+			bgs.log.Debug("dropping commit event from suspended user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+			repoCommitsResultCounter.WithLabelValues(host.Host, "susu").Inc()
 			return nil
 		}
 
-		if u.UpstreamStatus == events.AccountStatusDeactivated {
-			log.Debugw("dropping commit event from deactivated user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+		if ustatus == events.AccountStatusDeactivated {
+			bgs.log.Debug("dropping commit event from deactivated user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host)
+			repoCommitsResultCounter.WithLabelValues(host.Host, "du").Inc()
 			return nil
 		}
 
 		if evt.Rebase {
+			repoCommitsResultCounter.WithLabelValues(host.Host, "rebase").Inc()
 			return fmt.Errorf("rebase was true in event seq:%d,host:%s", evt.Seq, host.Host)
 		}
 
 		if host.ID != u.PDS && u.PDS != 0 {
-			log.Warnw("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", u.PDS, "gotPds", host.Host)
+			bgs.log.Warn("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", u.PDS, "gotPds", host.Host)
 			// Flush any cached DID documents for this user
 			bgs.didr.FlushCacheFor(env.RepoCommit.Repo)
 
 			subj, err := bgs.createExternalUser(ctx, evt.Repo)
 			if err != nil {
+				repoCommitsResultCounter.WithLabelValues(host.Host, "uerr2").Inc()
 				return err
 			}
 
 			if subj.PDS != host.ID {
+				repoCommitsResultCounter.WithLabelValues(host.Host, "noauth").Inc()
 				return fmt.Errorf("event from non-authoritative pds")
 			}
 		}
 
-		if u.Tombstoned {
+		if u.GetTombstoned() {
 			span.SetAttributes(attribute.Bool("tombstoned", true))
 			// we've checked the authority of the users PDS, so reinstate the account
 			if err := bgs.db.Model(&User{}).Where("id = ?", u.ID).UpdateColumn("tombstoned", false).Error; err != nil {
+				repoCommitsResultCounter.WithLabelValues(host.Host, "tomb").Inc()
 				return fmt.Errorf("failed to un-tombstone a user: %w", err)
 			}
+			u.SetTombstoned(false)
 
 			ai, err := bgs.Index.LookupUser(ctx, u.ID)
 			if err != nil {
+				repoCommitsResultCounter.WithLabelValues(host.Host, "nou2").Inc()
 				return fmt.Errorf("failed to look up user (tombstone recover): %w", err)
 			}
 
 			// Now a simple re-crawl should suffice to bring the user back online
+			repoCommitsResultCounter.WithLabelValues(host.Host, "catchupt").Inc()
 			return bgs.Index.Crawler.AddToCatchupQueue(ctx, host, ai, evt)
 		}
 
···
 			rebasesCounter.WithLabelValues(host.Host).Add(1)
 			ai, err := bgs.Index.LookupUser(ctx, u.ID)
 			if err != nil {
+				repoCommitsResultCounter.WithLabelValues(host.Host, "nou3").Inc()
 				return fmt.Errorf("failed to look up user (slow path): %w", err)
 			}
 
···
 			// processor coming off of the pds stream, we should investigate
 			// whether or not we even need this 'slow path' logic, as it makes
 			// accounting for which events have been processed much harder
+			repoCommitsResultCounter.WithLabelValues(host.Host, "catchup").Inc()
 			return bgs.Index.Crawler.AddToCatchupQueue(ctx, host, ai, evt)
 		}
 
 		if err := bgs.repoman.HandleExternalUserEvent(ctx, host.ID, u.ID, u.Did, evt.Since, evt.Rev, evt.Blocks, evt.Ops); err != nil {
-			log.Warnw("failed handling event", "err", err, "pdsHost", host.Host, "seq", evt.Seq, "repo", u.Did, "prev", stringLink(evt.Prev), "commit", evt.Commit.String())
 
 			if errors.Is(err, carstore.ErrRepoBaseMismatch) || ipld.IsNotFound(err) {
 				ai, lerr := bgs.Index.LookupUser(ctx, u.ID)
 				if lerr != nil {
+					log.Warn("failed handling event, no user", "err", err, "pdsHost", host.Host, "seq", evt.Seq, "repo", u.Did, "prev", stringLink(evt.Prev), "commit", evt.Commit.String())
+					repoCommitsResultCounter.WithLabelValues(host.Host, "nou4").Inc()
 					return fmt.Errorf("failed to look up user %s (%d) (err case: %s): %w", u.Did, u.ID, err, lerr)
 				}
 
 				span.SetAttributes(attribute.Bool("catchup_queue", true))
 
+				log.Info("failed handling event, catchup", "err", err, "pdsHost", host.Host, "seq", evt.Seq, "repo", u.Did, "prev", stringLink(evt.Prev), "commit", evt.Commit.String())
+				repoCommitsResultCounter.WithLabelValues(host.Host, "catchup2").Inc()
 				return bgs.Index.Crawler.AddToCatchupQueue(ctx, host, ai, evt)
 			}
 
+			log.Warn("failed handling event", "err", err, "pdsHost", host.Host, "seq", evt.Seq, "repo", u.Did, "prev", stringLink(evt.Prev), "commit", evt.Commit.String())
+			repoCommitsResultCounter.WithLabelValues(host.Host, "err").Inc()
 			return fmt.Errorf("handle user event failed: %w", err)
 		}
 
+		repoCommitsResultCounter.WithLabelValues(host.Host, "ok").Inc()
 		return nil
 	case env.RepoHandle != nil:
-		log.Infow("bgs got repo handle event", "did", env.RepoHandle.Did, "handle", env.RepoHandle.Handle)
+		bgs.log.Info("bgs got repo handle event", "did", env.RepoHandle.Did, "handle", env.RepoHandle.Handle)
 		// Flush any cached DID documents for this user
 		bgs.didr.FlushCacheFor(env.RepoHandle.Did)
 
···
 		}
 
 		if act.Handle.String != env.RepoHandle.Handle {
-			log.Warnw("handle update did not update handle to asserted value", "did", env.RepoHandle.Did, "expected", env.RepoHandle.Handle, "actual", act.Handle)
+			bgs.log.Warn("handle update did not update handle to asserted value", "did", env.RepoHandle.Did, "expected", env.RepoHandle.Handle, "actual", act.Handle)
 		}
 
 		// TODO: Update the ReposHandle event type to include "verified" or something
···
 			},
 		})
 		if err != nil {
-			log.Errorw("failed to broadcast RepoHandle event", "error", err, "did", env.RepoHandle.Did, "handle", env.RepoHandle.Handle)
+			bgs.log.Error("failed to broadcast RepoHandle event", "error", err, "did", env.RepoHandle.Did, "handle", env.RepoHandle.Handle)
 			return fmt.Errorf("failed to broadcast RepoHandle event: %w", err)
 		}
 
 		return nil
 	case env.RepoIdentity != nil:
-		log.Infow("bgs got identity event", "did", env.RepoIdentity.Did)
+		bgs.log.Info("bgs got identity event", "did", env.RepoIdentity.Did)
 		// Flush any cached DID documents for this user
 		bgs.didr.FlushCacheFor(env.RepoIdentity.Did)
 
···
 			},
 		})
 		if err != nil {
-			log.Errorw("failed to broadcast Identity event", "error", err, "did", env.RepoIdentity.Did)
+			bgs.log.Error("failed to broadcast Identity event", "error", err, "did", env.RepoIdentity.Did)
 			return fmt.Errorf("failed to broadcast Identity event: %w", err)
 		}
 
···
 			span.SetAttributes(attribute.String("repo_status", *env.RepoAccount.Status))
 		}
 
-		log.Infow("bgs got account event", "did", env.RepoAccount.Did)
+		bgs.log.Info("bgs got account event", "did", env.RepoAccount.Did)
 		// Flush any cached DID documents for this user
 		bgs.didr.FlushCacheFor(env.RepoAccount.Did)
 
···
 		// Check if the PDS is still authoritative
 		// if not we don't want to be propagating this account event
 		if ai.PDS != host.ID {
-			log.Errorw("account event from non-authoritative pds",
+			bgs.log.Error("account event from non-authoritative pds",
 				"seq", env.RepoAccount.Seq,
 				"did", env.RepoAccount.Did,
 				"event_from", host.Host,
···
 			return fmt.Errorf("failed to look up user by did: %w", err)
 		}
 
-		if u.TakenDown {
+		if u.GetTakenDown() {
 			shouldBeActive = false
 			status = &events.AccountStatusTakendown
 		}
···
 			},
 		})
 		if err != nil {
-			log.Errorw("failed to broadcast Account event", "error", err, "did", env.RepoAccount.Did)
+			bgs.log.Error("failed to broadcast Account event", "error", err, "did", env.RepoAccount.Did)
 			return fmt.Errorf("failed to broadcast Account event: %w", err)
 		}
 
···
 	}).Error; err != nil {
 		return err
 	}
+	u.SetTombstoned(true)
 
 	if err := bgs.db.Model(&models.ActorInfo{}).Where("uid = ?", u.ID).UpdateColumns(map[string]any{
 		"handle": nil,
···
 	// delete data from carstore
 	if err := bgs.repoman.TakeDownRepo(ctx, u.ID); err != nil {
 		// don't let a failure here prevent us from propagating this event
-		log.Errorf("failed to delete user data from carstore: %s", err)
+		bgs.log.Error("failed to delete user data from carstore", "err", err)
 	}
 
 	return bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{
···
 
 	externalUserCreationAttempts.Inc()
 
-	log.Debugf("create external user: %s", did)
+	s.log.Debug("create external user", "did", did)
 	doc, err := s.didr.GetDocument(ctx, did)
 	if err != nil {
 		return nil, fmt.Errorf("could not locate DID document for followed user (%s): %w", did, err)
···
 	// TODO: the PDS's DID should also be in the service, we could use that to look up?
 	var peering models.PDS
 	if err := s.db.Find(&peering, "host = ?", durl.Host).Error; err != nil {
-		log.Error("failed to find pds", durl.Host)
+		s.log.Error("failed to find pds", "host", durl.Host)
 		return nil, err
 	}
 
···
 	defer func() {
 		if !successfullyCreated {
 			if err := s.db.Model(&models.PDS{}).Where("id = ?", peering.ID).Update("repo_count", gorm.Expr("repo_count - 1")).Error; err != nil {
-				log.Errorf("failed to decrement repo count for pds: %s", err)
+				s.log.Error("failed to decrement repo count for pds", "err", err)
 			}
 		}
 	}()
···
 		return nil, err
 	}
 
-	log.Debugw("creating external user", "did", did, "handle", hurl.Host, "pds", peering.ID)
+	s.log.Debug("creating external user", "did", did, "handle", hurl.Host, "pds", peering.ID)
 
 	handle := hurl.Host
 
···
 
 	resdid, err := s.hr.ResolveHandleToDid(ctx, handle)
 	if err != nil {
-		log.Errorf("failed to resolve users claimed handle (%q) on pds: %s", handle, err)
+		s.log.Error("failed to resolve users claimed handle on pds", "handle", handle, "err", err)
 		validHandle = false
 	}
 
 	if resdid != did {
-		log.Errorf("claimed handle did not match servers response (%s != %s)", resdid, did)
+		s.log.Error("claimed handle did not match servers response", "resdid", resdid, "did", did)
 		validHandle = false
 	}
 
···
 
 	exu, err := s.Index.LookupUserByDid(ctx, did)
 	if err == nil {
-		log.Debugw("lost the race to create a new user", "did", did, "handle", handle, "existing_hand", exu.Handle)
+		s.log.Debug("lost the race to create a new user", "did", did, "handle", handle, "existing_hand", exu.Handle)
 		if exu.PDS != peering.ID {
 			// User is now on a different PDS, update
 			if err := s.db.Model(User{}).Where("id = ?", exu.Uid).Update("pds", peering.ID).Error; err != nil {
···
 			if err := s.db.Create(&u).Error; err != nil {
 				return nil, fmt.Errorf("failed to create user after handle conflict: %w", err)
 			}
+
+			s.userCache.Remove(did)
 		} else {
 			return nil, fmt.Errorf("failed to create other pds user: %w", err)
 		}
···
 		if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("upstream_status", events.AccountStatusActive).Error; err != nil {
 			return fmt.Errorf("failed to set user active status: %w", err)
 		}
+		u.SetUpstreamStatus(events.AccountStatusActive)
 	case events.AccountStatusDeactivated:
 		if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("upstream_status", events.AccountStatusDeactivated).Error; err != nil {
 			return fmt.Errorf("failed to set user deactivation status: %w", err)
 		}
+		u.SetUpstreamStatus(events.AccountStatusDeactivated)
 	case events.AccountStatusSuspended:
 		if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("upstream_status", events.AccountStatusSuspended).Error; err != nil {
 			return fmt.Errorf("failed to set user suspension status: %w", err)
 		}
+		u.SetUpstreamStatus(events.AccountStatusSuspended)
 	case events.AccountStatusTakendown:
 		if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("upstream_status", events.AccountStatusTakendown).Error; err != nil {
 			return fmt.Errorf("failed to set user taken down status: %w", err)
 		}
+		u.SetUpstreamStatus(events.AccountStatusTakendown)
 
 		if err := bgs.db.Model(&models.ActorInfo{}).Where("uid = ?", u.ID).UpdateColumns(map[string]any{
 			"handle": nil,
···
 		}).Error; err != nil {
 			return err
 		}
+		u.SetUpstreamStatus(events.AccountStatusDeleted)
 
 		if err := bgs.db.Model(&models.ActorInfo{}).Where("uid = ?", u.ID).UpdateColumns(map[string]any{
 			"handle": nil,
···
 		// delete data from carstore
 		if err := bgs.repoman.TakeDownRepo(ctx, u.ID); err != nil {
 			// don't let a failure here prevent us from propagating this event
-			log.Errorf("failed to delete user data from carstore: %s", err)
+			bgs.log.Error("failed to delete user data from carstore", "err", err)
 		}
 	}
 
···
 	if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("taken_down", true).Error; err != nil {
 		return err
 	}
+	u.SetTakenDown(true)
 
 	if err := bgs.repoman.TakeDownRepo(ctx, u.ID); err != nil {
 		return err
···
 	if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("taken_down", false).Error; err != nil {
 		return err
 	}
+	u.SetTakenDown(false)
 
 	return nil
 }
···
 func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error {
 	ctx, span := tracer.Start(ctx, "ResyncPDS")
 	defer span.End()
-	log := log.With("pds", pds.Host, "source", "resync_pds")
+	log := bgs.log.With("pds", pds.Host, "source", "resync_pds")
 	resync, found := bgs.LoadOrStoreResync(pds)
 	if found {
 		return fmt.Errorf("resync already in progress")
···
 	for {
 		pages++
 		if pages%10 == 0 {
-			log.Warnw("fetching PDS page during resync", "pages", pages, "total_repos", len(repos))
+			log.Warn("fetching PDS page during resync", "pages", pages, "total_repos", len(repos))
 			resync.NumRepoPages = pages
 			resync.NumRepos = len(repos)
 			bgs.UpdateResync(resync)
 		}
 		if err := limiter.Wait(ctx); err != nil {
-			log.Errorw("failed to wait for rate limiter", "error", err)
+			log.Error("failed to wait for rate limiter", "error", err)
 			return fmt.Errorf("failed to wait for rate limiter: %w", err)
 		}
 		repoList, err := comatproto.SyncListRepos(ctx, &xrpcc, cursor, limit)
 		if err != nil {
-			log.Errorw("failed to list repos", "error", err)
+			log.Error("failed to list repos", "error", err)
 			return fmt.Errorf("failed to list repos: %w", err)
 		}
 
···
 
 	repolistDone := time.Now()
 
-	log.Warnw("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start))
+	log.Warn("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start))
 	resync = bgs.SetResyncStatus(pds.ID, "checking revs")
 
-	// Create a buffered channel for collecting results
-	results := make(chan revCheckResult, len(repos))
+	// run loop over repos with some concurrency
 	sem := semaphore.NewWeighted(40)
 
 	// Check repo revs against our local copy and enqueue crawls for any that are out of date
-	for _, r := range repos {
+	for i, r := range repos {
 		if err := sem.Acquire(ctx, 1); err != nil {
-			log.Errorw("failed to acquire semaphore", "error", err)
-			results <- revCheckResult{err: err}
+			log.Error("failed to acquire semaphore", "error", err)
 			continue
 		}
 		go func(r comatproto.SyncListRepos_Repo) {
 			defer sem.Release(1)
-			log := log.With("did", r.Did, "remote_rev", r.Rev)
+			log := bgs.log.With("did", r.Did, "remote_rev", r.Rev)
 			// Fetches the user if we have it, otherwise automatically enqueues it for crawling
 			ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did)
 			if err != nil {
-				log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err)
-				results <- revCheckResult{err: err}
+				log.Error("failed to get user while resyncing PDS, we can't recrawl it", "error", err)
 				return
 			}
 
 			rev, err := bgs.repoman.GetRepoRev(ctx, ai.Uid)
 			if err != nil {
-				log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid)
-				results <- revCheckResult{ai: ai}
+				log.Warn("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid)
+				err := bgs.Index.Crawler.Crawl(ctx, ai)
+				if err != nil {
+					log.Error("failed to enqueue crawl for repo during resync", "error", err, "uid", ai.Uid, "did", ai.Did)
+				}
 				return
 			}
 
 			if rev == "" || rev < r.Rev {
-				log.Warnw("recrawling because the repo rev from the PDS is newer than our local repo rev", "local_rev", rev)
-				results <- revCheckResult{ai: ai}
+				log.Warn("recrawling because the repo rev from the PDS is newer than our local repo rev", "local_rev", rev)
+				err := bgs.Index.Crawler.Crawl(ctx, ai)
+				if err != nil {
+					log.Error("failed to enqueue crawl for repo during resync", "error", err, "uid", ai.Uid, "did", ai.Did)
+				}
 				return
 			}
-
-			results <- revCheckResult{}
 		}(r)
-	}
-
-	var numReposToResync int
-	for i := 0; i < len(repos); i++ {
-		res := <-results
-		if res.err != nil {
-			log.Errorw("failed to process repo during resync", "error", res.err)
-
-		}
-		if res.ai != nil {
-			numReposToResync++
-			err := bgs.Index.Crawler.Crawl(ctx, res.ai)
-			if err != nil {
-				log.Errorw("failed to enqueue crawl for repo during resync", "error", err, "uid", res.ai.Uid, "did", res.ai.Did)
-			}
-		}
 		if i%100 == 0 {
 			if i%10_000 == 0 {
-				log.Warnw("checked revs during resync", "num_repos_checked", i, "num_repos_to_crawl", numReposToResync, "took", time.Now().Sub(resync.StatusChangedAt))
+				log.Warn("checked revs during resync", "num_repos_checked", i, "num_repos_to_crawl", -1, "took", time.Now().Sub(resync.StatusChangedAt))
 			}
 			resync.NumReposChecked = i
-			resync.NumReposToResync = numReposToResync
 			bgs.UpdateResync(resync)
 		}
 	}
 
 	resync.NumReposChecked = len(repos)
-	resync.NumReposToResync = numReposToResync
 	bgs.UpdateResync(resync)
 
-	log.Warnw("enqueued all crawls, exiting resync", "took", time.Now().Sub(start), "num_repos_to_crawl", numReposToResync)
+	bgs.log.Warn("enqueued all crawls, exiting resync", "took", time.Now().Sub(start), "num_repos_to_crawl", -1)
 
 	return nil
 }

bgs/compactor.go

···
 	}
 	if c.requeueInterval > 0 {
 		go func() {
-			log.Infow("starting compactor requeue routine",
+			log.Info("starting compactor requeue routine",
 				"interval", c.requeueInterval,
 				"limit", c.requeueLimit,
 				"shardCount", c.requeueShardCount,
···
 					ctx := context.Background()
 					ctx, span := otel.Tracer("compactor").Start(ctx, "RequeueRoutine")
 					if err := c.EnqueueAllRepos(ctx, bgs, c.requeueLimit, c.requeueShardCount, c.requeueFast); err != nil {
-						log.Errorw("failed to enqueue all repos", "err", err)
+						log.Error("failed to enqueue all repos", "err", err)
 					}
 					span.End()
 				}
···
 				time.Sleep(time.Second * 5)
 				continue
 			}
-			log.Errorw("failed to compact repo",
+			log.Error("failed to compact repo",
 				"err", err,
 				"uid", state.latestUID,
 				"repo", state.latestDID,
···
 			// Pause for a bit to avoid spamming failed compactions
 			time.Sleep(time.Millisecond * 100)
 		} else {
-			log.Infow("compacted repo",
+			log.Info("compacted repo",
 				"uid", state.latestUID,
 				"repo", state.latestDID,
 				"status", state.status,
···
 	return state, nil
 }
 
-func (c *Compactor) EnqueueRepo(ctx context.Context, user User, fast bool) {
+func (c *Compactor) EnqueueRepo(ctx context.Context, user *User, fast bool) {
 	ctx, span := otel.Tracer("compactor").Start(ctx, "EnqueueRepo")
 	defer span.End()
-	log.Infow("enqueueing compaction for repo", "repo", user.Did, "uid", user.ID, "fast", fast)
+	log.Info("enqueueing compaction for repo", "repo", user.Did, "uid", user.ID, "fast", fast)
 	c.q.Append(user.ID, fast)
 }
 
···
 		c.q.Append(r.Usr, fast)
 	}
 
-	log.Infow("done enqueueing all repos", "repos_enqueued", len(repos))
+	log.Info("done enqueueing all repos", "repos_enqueued", len(repos))
 
 	return nil
 }

bgs/fedmgr.go

···
 	"context"
 	"errors"
 	"fmt"
+	"log/slog"
 	"math/rand"
 	"strings"
 	"sync"
···
 	pq "github.com/lib/pq"
 	"gorm.io/gorm"
 )
+
+var log = slog.Default().With("system", "bgs")
 
 type IndexCallback func(context.Context, *models.PDS, *events.XRPCStreamEvent) error
 
···
 				var errs []error
 				if errs = s.flushCursors(ctx); len(errs) > 0 {
 					for _, err := range errs {
-						log.Errorf("failed to flush cursors on shutdown: %s", err)
+						log.Error("failed to flush cursors on shutdown", "err", err)
 					}
 				}
 				log.Info("done flushing PDS cursors on shutdown")
···
 				defer span.End()
 				if errs := s.flushCursors(ctx); len(errs) > 0 {
 					for _, err := range errs {
-						log.Errorf("failed to flush cursors: %s", err)
+						log.Error("failed to flush cursors", "err", err)
 					}
 				}
 				log.Debug("done flushing PDS cursors")
···
 	errs := <-s.shutdownResult
 	if len(errs) > 0 {
 		for _, err := range errs {
-			log.Errorf("shutdown error: %s", err)
+			log.Error("shutdown error", "err", err)
 		}
 	}
 	log.Info("slurper shutdown complete")
···
 		protocol = "wss"
 	}
 
+	// Special case `.host.bsky.network` PDSs to rewind cursor by 200 events to smooth over unclean shutdowns
+	if strings.HasSuffix(host.Host, ".host.bsky.network") && host.Cursor > 200 {
+		host.Cursor -= 200
+	}
+
 	cursor := host.Cursor
+
+	connectedInbound.Inc()
+	defer connectedInbound.Dec()
+	// TODO:? maybe keep a gauge of 'in retry backoff' sources?
 
 	var backoff int
 	for {
···
 		url := fmt.Sprintf("%s://%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", protocol, host.Host, cursor)
 		con, res, err := d.DialContext(ctx, url, nil)
 		if err != nil {
-			log.Warnw("dialing failed", "pdsHost", host.Host, "err", err, "backoff", backoff)
+			log.Warn("dialing failed", "pdsHost", host.Host, "err", err, "backoff", backoff)
 			time.Sleep(sleepForBackoff(backoff))
 			backoff++
 
 			if backoff > 15 {
-				log.Warnw("pds does not appear to be online, disabling for now", "pdsHost", host.Host)
+				log.Warn("pds does not appear to be online, disabling for now", "pdsHost", host.Host)
 				if err := s.db.Model(&models.PDS{}).Where("id = ?", host.ID).Update("registered", false).Error; err != nil {
-					log.Errorf("failed to unregister failing pds: %w", err)
+					log.Error("failed to unregister failing pds", "err", err)
 				}
 
 				return
···
 			continue
 		}
 
-		log.Info("event subscription response code: ", res.StatusCode)
+		log.Info("event subscription response", "code", res.StatusCode)
 
 		curCursor := cursor
 		if err := s.handleConnection(ctx, host, con, &cursor, sub); err != nil {
 			if errors.Is(err, ErrTimeoutShutdown) {
-				log.Infof("shutting down pds subscription to %s, no activity after %s", host.Host, EventsTimeout)
+				log.Info("shutting down pds subscription after timeout", "host", host.Host, "time", EventsTimeout)
 				return
 			}
-			log.Warnf("connection to %q failed: %s", host.Host, err)
+			log.Warn("connection to failed", "host", host.Host, "err", err)
 		}
 
 		if cursor > curCursor {
···
 
 	rsc := &events.RepoStreamCallbacks{
 		RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error {
-			log.Debugw("got remote repo event", "pdsHost", host.Host, "repo", evt.Repo, "seq", evt.Seq)
+			log.Debug("got remote repo event", "pdsHost", host.Host, "repo", evt.Repo, "seq", evt.Seq)
 			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
 				RepoCommit: evt,
 			}); err != nil {
-				log.Errorf("failed handling event from %q (%d): %s", host.Host, evt.Seq, err)
+				log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
 			}
 			*lastCursor = evt.Seq
 
···
 			return nil
 		},
 		RepoHandle: func(evt *comatproto.SyncSubscribeRepos_Handle) error {
-			log.Infow("got remote handle update event", "pdsHost", host.Host, "did", evt.Did, "handle", evt.Handle)
+			log.Info("got remote handle update event", "pdsHost", host.Host, "did", evt.Did, "handle", evt.Handle)
 			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
 				RepoHandle: evt,
 			}); err != nil {
-				log.Errorf("failed handling event from %q (%d): %s", host.Host, evt.Seq, err)
+				log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
 			}
 			*lastCursor = evt.Seq
 
···
 			return nil
 		},
 		RepoMigrate: func(evt *comatproto.SyncSubscribeRepos_Migrate) error {
-			log.Infow("got remote repo migrate event", "pdsHost", host.Host, "did", evt.Did, "migrateTo", evt.MigrateTo)
+			log.Info("got remote repo migrate event", "pdsHost", host.Host, "did", evt.Did, "migrateTo", evt.MigrateTo)
 			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
 				RepoMigrate: evt,
 			}); err != nil {
-				log.Errorf("failed handling event from %q (%d): %s", host.Host, evt.Seq, err)
+				log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
 			}
 			*lastCursor = evt.Seq
 
···
 			return nil
 		},
 		RepoTombstone: func(evt *comatproto.SyncSubscribeRepos_Tombstone) error {
-			log.Infow("got remote repo tombstone event", "pdsHost", host.Host, "did", evt.Did)
+			log.Info("got remote repo tombstone event", "pdsHost", host.Host, "did", evt.Did)
 			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
 				RepoTombstone: evt,
 			}); err != nil {
-				log.Errorf("failed handling event from %q (%d): %s", host.Host, evt.Seq, err)
+				log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err)
 			}
 			*lastCursor = evt.Seq
 
···
 			return nil
 		},
 		RepoInfo: func(info *comatproto.SyncSubscribeRepos_Info) error {
-			log.Infow("info event", "name", info.Name, "message", info.Message, "pdsHost", host.Host)
+			log.Info("info event", "name", info.Name, "message", info.Message, "pdsHost", host.Host)
 			return nil
 		},
 		RepoIdentity: func(ident *comatproto.SyncSubscribeRepos_Identity) error {
-			log.Infow("identity event", "did", ident.Did)
+			log.Info("identity event", "did", ident.Did)
 			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
 				RepoIdentity: ident,
 			}); err != nil {
-				log.Errorf("failed handling event from %q (%d): %s", host.Host, ident.Seq, err)
+				log.Error("failed handling event", "host", host.Host, "seq", ident.Seq, "err", err)
 			}
 			*lastCursor = ident.Seq
 
···
 			return nil
 		},
 		RepoAccount: func(acct *comatproto.SyncSubscribeRepos_Account) error {
-			log.Infow("account event", "did", acct.Did, "status", acct.Status)
+			log.Info("account event", "did", acct.Did, "status", acct.Status)
 			if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{
 				RepoAccount: acct,
 			}); err != nil {
-				log.Errorf("failed handling event from %q (%d): %s", host.Host, acct.Seq, err)
+				log.Error("failed handling event", "host", host.Host, "seq", acct.Seq, "err", err)
 			}
 			*lastCursor = acct.Seq
 
···
 		con.RemoteAddr().String(),
 		instrumentedRSC.EventHandler,
 	)
-	return events.HandleRepoStream(ctx, con, pool)
+	return events.HandleRepoStream(ctx, con, pool, nil)
 }
 
 func (s *Slurper) updateCursor(sub *activeSub, curs int64) error {
···
 		return fmt.Errorf("killing connection %q: %w", host, ErrNoActiveConnection)
 	}
 	ac.cancel()
+	// cleanup in the run thread subscribeWithRedialer() will delete(s.active, host)
 
 	if block {
 		if err := s.db.Model(models.PDS{}).Where("id = ?", ac.pds.ID).UpdateColumn("blocked", true).Error; err != nil {

bgs/handlers.go

···
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
···
 		if errors.Is(err, gorm.ErrRecordNotFound) {
 			return nil, echo.NewHTTPError(http.StatusNotFound, "user not found")
 		}
-		log.Errorw("failed to lookup user", "err", err, "did", did)
+		log.Error("failed to lookup user", "err", err, "did", did)
 		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user")
 	}
 
-	if u.Tombstoned {
+	if u.GetTombstoned() {
 		return nil, fmt.Errorf("account was deleted")
 	}
 
-	if u.TakenDown {
+	if u.GetTakenDown() {
 		return nil, fmt.Errorf("account was taken down by the Relay")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusTakendown {
+	ustatus := u.GetUpstreamStatus()
+	if ustatus == events.AccountStatusTakendown {
 		return nil, fmt.Errorf("account was taken down by its PDS")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusDeactivated {
+	if ustatus == events.AccountStatusDeactivated {
 		return nil, fmt.Errorf("account is temporarily deactivated")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusSuspended {
+	if ustatus == events.AccountStatusSuspended {
 		return nil, fmt.Errorf("account is suspended by its PDS")
 	}
 
···
 		if errors.Is(err, mst.ErrNotFound) {
 			return nil, echo.NewHTTPError(http.StatusNotFound, "record not found in repo")
 		}
-		log.Errorw("failed to get record from repo", "err", err, "did", did, "collection", collection, "rkey", rkey)
+		log.Error("failed to get record from repo", "err", err, "did", did, "collection", collection, "rkey", rkey)
 		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get record from repo")
 	}
 
···
 		if errors.Is(err, gorm.ErrRecordNotFound) {
 			return nil, echo.NewHTTPError(http.StatusNotFound, "user not found")
 		}
-		log.Errorw("failed to lookup user", "err", err, "did", did)
+		log.Error("failed to lookup user", "err", err, "did", did)
 		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user")
 	}
 
-	if u.Tombstoned {
+	if u.GetTombstoned() {
 		return nil, fmt.Errorf("account was deleted")
 	}
 
-	if u.TakenDown {
+	if u.GetTakenDown() {
 		return nil, fmt.Errorf("account was taken down by the Relay")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusTakendown {
+	ustatus := u.GetUpstreamStatus()
+	if ustatus == events.AccountStatusTakendown {
 		return nil, fmt.Errorf("account was taken down by its PDS")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusDeactivated {
+	if ustatus == events.AccountStatusDeactivated {
 		return nil, fmt.Errorf("account is temporarily deactivated")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusSuspended {
+	if ustatus == events.AccountStatusSuspended {
 		return nil, fmt.Errorf("account is suspended by its PDS")
 	}
 
 	// TODO: stream the response
 	buf := new(bytes.Buffer)
 	if err := s.repoman.ReadRepo(ctx, u.ID, since, buf); err != nil {
-		log.Errorw("failed to read repo into buffer", "err", err, "did", did)
+		log.Error("failed to read repo into buffer", "err", err, "did", did)
 		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to read repo into buffer")
 	}
 
···
 		return echo.NewHTTPError(http.StatusUnauthorized, "domain is banned")
 	}
 
-	log.Warnf("TODO: better host validation for crawl requests")
+	log.Warn("TODO: better host validation for crawl requests")
 
 	clientHost := fmt.Sprintf("%s://%s", u.Scheme, host)
 
···
 	// Maybe we could do something with this response later
 	_ = desc
 
+	if len(s.nextCrawlers) != 0 {
+		blob, err := json.Marshal(body)
+		if err != nil {
+			log.Warn("could not forward requestCrawl, json err", "err", err)
+		} else {
+			go func(bodyBlob []byte) {
+				for _, rpu := range s.nextCrawlers {
+					pu := rpu.JoinPath("/xrpc/com.atproto.sync.requestCrawl")
+					response, err := s.httpClient.Post(pu.String(), "application/json", bytes.NewReader(bodyBlob))
+					if response != nil && response.Body != nil {
+						response.Body.Close()
+					}
+					if err != nil || response == nil {
+						log.Warn("requestCrawl forward failed", "host", rpu, "err", err)
+					} else if response.StatusCode != http.StatusOK {
+						log.Warn("requestCrawl forward failed", "host", rpu, "status", response.Status)
+					} else {
+						log.Info("requestCrawl forward successful", "host", rpu)
+					}
+				}
+			}(blob)
+		}
+	}
+
 	return s.slurper.SubscribeToPds(ctx, host, true, false)
 }
 
···
 		if err == gorm.ErrRecordNotFound {
 			return &comatprototypes.SyncListRepos_Output{}, nil
 		}
-		log.Errorw("failed to query users", "err", err)
+		log.Error("failed to query users", "err", err)
 		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to query users")
 	}
 
···
 
 		root, err := s.repoman.GetRepoRoot(ctx, user.ID)
 		if err != nil {
-			log.Errorw("failed to get repo root", "err", err, "did", user.Did)
+			log.Error("failed to get repo root", "err", err, "did", user.Did)
 			return nil, echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("failed to get repo root for (%s): %v", user.Did, err.Error()))
 		}
 
···
 		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user")
 	}
 
-	if u.Tombstoned {
+	if u.GetTombstoned() {
 		return nil, fmt.Errorf("account was deleted")
 	}
 
-	if u.TakenDown {
+	if u.GetTakenDown() {
 		return nil, fmt.Errorf("account was taken down by the Relay")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusTakendown {
+	ustatus := u.GetUpstreamStatus()
+	if ustatus == events.AccountStatusTakendown {
 		return nil, fmt.Errorf("account was taken down by its PDS")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusDeactivated {
+	if ustatus == events.AccountStatusDeactivated {
 		return nil, fmt.Errorf("account is temporarily deactivated")
 	}
 
-	if u.UpstreamStatus == events.AccountStatusSuspended {
+	if ustatus == events.AccountStatusSuspended {
 		return nil, fmt.Errorf("account is suspended by its PDS")
 	}
 
 	root, err := s.repoman.GetRepoRoot(ctx, u.ID)
 	if err != nil {
-		log.Errorw("failed to get repo root", "err", err, "did", u.Did)
+		log.Error("failed to get repo root", "err", err, "did", u.Did)
 		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get repo root")
 	}
 
 	rev, err := s.repoman.GetRepoRev(ctx, u.ID)
 	if err != nil {
-		log.Errorw("failed to get repo rev", "err", err, "did", u.Did)
+		log.Error("failed to get repo rev", "err", err, "did", u.Did)
 		return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get repo rev")
 	}
 

bgs/metrics.go

···
 	Help: "The total number of events received",
 }, []string{"pds"})
 
+var repoCommitsResultCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "repo_commits_result_counter",
+	Help: "The results of commit events received",
+}, []string{"pds", "status"})
+
 var rebasesCounter = promauto.NewCounterVec(prometheus.CounterOpts{
 	Name: "event_rebases",
 	Help: "The total number of rebase events received",
···
 var externalUserCreationAttempts = promauto.NewCounter(prometheus.CounterOpts{
 	Name: "bgs_external_user_creation_attempts",
 	Help: "The total number of external users created",
+})
+
+var connectedInbound = promauto.NewGauge(prometheus.GaugeOpts{
+	Name: "bgs_connected_inbound",
+	Help: "Number of inbound firehoses we are consuming",
 })
 
 var compactionDuration = promauto.NewHistogram(prometheus.HistogramOpts{
···
 	Help:    "A histogram of response sizes for requests.",
 	Buckets: prometheus.ExponentialBuckets(100, 10, 8),
 }, []string{"code", "method", "path"})
+
+var userLookupDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "relay_user_lookup_duration",
+	Help:    "A histogram of user lookup latencies",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+})
+
+var newUserDiscoveryDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "relay_new_user_discovery_duration",
+	Help:    "A histogram of new user discovery latencies",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+})
 
 // MetricsMiddleware defines handler function for metrics middleware
 func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc {

carstore/bs.go

···
 	"context"
 	"fmt"
 	"io"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"sort"
···
 	cbor "github.com/ipfs/go-ipld-cbor"
 	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/ipfs/go-libipfs/blocks"
-	logging "github.com/ipfs/go-log"
 	car "github.com/ipld/go-car"
 	carutil "github.com/ipld/go-car/util"
 	cbg "github.com/whyrusleeping/cbor-gen"
···
 var blockGetTotalCounterUsrskip = blockGetTotalCounter.WithLabelValues("true", "miss")
 var blockGetTotalCounterCached = blockGetTotalCounter.WithLabelValues("false", "hit")
 var blockGetTotalCounterNormal = blockGetTotalCounter.WithLabelValues("false", "miss")
-
-var log = logging.Logger("carstore")
 
 const MaxSliceLength = 2 << 20
 
···
 }
 
 type FileCarStore struct {
-	meta    *CarStoreGormMeta
-	rootDir string
+	meta     *CarStoreGormMeta
+	rootDirs []string
 
 	lscLk          sync.Mutex
 	lastShardCache map[models.Uid]*CarShard
+
+	log *slog.Logger
 }
 
-func NewCarStore(meta *gorm.DB, root string) (CarStore, error) {
-	if _, err := os.Stat(root); err != nil {
-		if !os.IsNotExist(err) {
-			return nil, err
-		}
+func NewCarStore(meta *gorm.DB, roots []string) (CarStore, error) {
+	for _, root := range roots {
+		if _, err := os.Stat(root); err != nil {
+			if !os.IsNotExist(err) {
+				return nil, err
+			}
 
-		if err := os.Mkdir(root, 0775); err != nil {
-			return nil, err
+			if err := os.Mkdir(root, 0775); err != nil {
+				return nil, err
+			}
 		}
 	}
 	if err := meta.AutoMigrate(&CarShard{}, &blockRef{}); err != nil {
···
 
 	return &FileCarStore{
 		meta:           &CarStoreGormMeta{meta: meta},
-		rootDir:        root,
+		rootDirs:       roots,
 		lastShardCache: make(map[models.Uid]*CarShard),
+		log:            slog.Default().With("system", "carstore"),
 	}, nil
 }
 
 type userView struct {
-	cs   *FileCarStore
+	cs   CarStore
 	user models.Uid
 
 	cache    map[cid.Cid]blockformat.Block
···
 }
 
 func (uv *userView) Has(ctx context.Context, k cid.Cid) (bool, error) {
-	return uv.cs.meta.HasUidCid(ctx, uv.user, k)
+	_, have := uv.cache[k]
+	if have {
+		return have, nil
+	}
+
+	fcd, ok := uv.cs.(*FileCarStore)
+	if !ok {
+		return false, nil
+	}
+
+	return fcd.meta.HasUidCid(ctx, uv.user, k)
 }
 
 var CacheHits int64
 var CacheMiss int64
 
 func (uv *userView) Get(ctx context.Context, k cid.Cid) (blockformat.Block, error) {
+
 	if !k.Defined() {
 		return nil, fmt.Errorf("attempted to 'get' undefined cid")
 	}
···
 	}
 	atomic.AddInt64(&CacheMiss, 1)
 
-	path, offset, user, err := uv.cs.meta.LookupBlockRef(ctx, k)
+	fcd, ok := uv.cs.(*FileCarStore)
+	if !ok {
+		return nil, ipld.ErrNotFound{Cid: k}
+	}
+
+	path, offset, user, err := fcd.meta.LookupBlockRef(ctx, k)
 	if err != nil {
 		return nil, err
 	}
···
 	baseCid  cid.Cid
 	seq      int
 	readonly bool
-	cs       *FileCarStore
+	cs       CarStore
 	lastRev  string
 }
 
···
 func fnameForShard(user models.Uid, seq int) string {
 	return fmt.Sprintf("sh-%d-%d", user, seq)
 }
+
+func (cs *FileCarStore) dirForUser(user models.Uid) string {
+	return cs.rootDirs[int(user)%len(cs.rootDirs)]
+}
+
 func (cs *FileCarStore) openNewShardFile(ctx context.Context, user models.Uid, seq int) (*os.File, string, error) {
 	// TODO: some overwrite protections
-	fname := filepath.Join(cs.rootDir, fnameForShard(user, seq))
+	fname := filepath.Join(cs.dirForUser(user), fnameForShard(user, seq))
 	fi, err := os.Create(fname)
 	if err != nil {
 		return nil, "", err
···
 	defer span.End()
 
 	// TODO: some overwrite protections
-	fname := filepath.Join(cs.rootDir, fnameForShard(user, seq))
+	fname := filepath.Join(cs.dirForUser(user), fnameForShard(user, seq))
 	if err := os.WriteFile(fname, data, 0664); err != nil {
 		return "", err
 	}
···
 		return nil, fmt.Errorf("cannot write to readonly deltaSession")
 	}
 
-	return ds.cs.writeNewShard(ctx, root, rev, ds.user, ds.seq, ds.blks, ds.rmcids)
+	switch ocs := ds.cs.(type) {
+	case *FileCarStore:
+		return ocs.writeNewShard(ctx, root, rev, ds.user, ds.seq, ds.blks, ds.rmcids)
+	case *NonArchivalCarstore:
+		slice, err := blocksToCar(ctx, root, rev, ds.blks)
+		if err != nil {
+			return nil, err
+		}
+		return slice, ocs.updateLastCommit(ctx, ds.user, rev, root)
+	default:
+		return nil, fmt.Errorf("unsupported carstore type")
+	}
 }
 
 func WriteCarHeader(w io.Writer, root cid.Cid) (int64, error) {
···
 	return hnw, nil
 }
 
+func blocksToCar(ctx context.Context, root cid.Cid, rev string, blks map[cid.Cid]blockformat.Block) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	_, err := WriteCarHeader(buf, root)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write car header: %w", err)
+	}
+
+	for k, blk := range blks {
+		_, err := LdWrite(buf, k.Bytes(), blk.RawData())
+		if err != nil {
+			return nil, fmt.Errorf("failed to write block: %w", err)
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
 func (cs *FileCarStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) {
 
 	buf := new(bytes.Buffer)
···
 		offset += nw
 	}
 
+	start := time.Now()
 	path, err := cs.writeNewShardFile(ctx, user, seq, buf.Bytes())
 	if err != nil {
 		return nil, fmt.Errorf("failed to write shard file: %w", err)
 	}
+	writeShardFileDuration.Observe(time.Since(start).Seconds())
 
 	shard := CarShard{
 		Root:      models.DbCID{CID: root},
···
 		Rev:       rev,
 	}
 
+	start = time.Now()
 	if err := cs.putShard(ctx, &shard, brefs, rmcids, false); err != nil {
 		return nil, err
 	}
+	writeShardMetadataDuration.Observe(time.Since(start).Seconds())
 
 	return buf.Bytes(), nil
 }
···
 				if !os.IsNotExist(err) {
 					return err
 				}
-				log.Warnw("shard file we tried to delete did not exist", "shard", sh.ID, "path", sh.Path)
+				cs.log.Warn("shard file we tried to delete did not exist", "shard", sh.ID, "path", sh.Path)
 			}
 		}
 
···
 	// TODO: some overwrite protections
 	// NOTE CreateTemp is used for creating a non-colliding file, but we keep it and don't delete it so don't think of it as "temporary".
 	// This creates "sh-%d-%d%s" with some random stuff in the last position
-	fi, err := os.CreateTemp(cs.rootDir, fnameForShard(user, seq))
+	fi, err := os.CreateTemp(cs.dirForUser(user), fnameForShard(user, seq))
 	if err != nil {
 		return nil, "", err
 	}
···
 	st, err := os.Stat(sh.Path)
 	if err != nil {
 		if os.IsNotExist(err) {
-			log.Warnw("missing shard, return size of zero", "path", sh.Path, "shard", sh.ID)
+			slog.Warn("missing shard, return size of zero", "path", sh.Path, "shard", sh.ID, "system", "carstore")
 			return 0, nil
 		}
 		return 0, fmt.Errorf("stat %q: %w", sh.Path, err)
···
 		// still around but we're doing that anyways since compaction isn't a
 		// perfect process
 
-		log.Debugw("repo has dirty dupes", "count", len(dupes), "uid", user, "staleRefs", len(staleRefs), "blockRefs", len(brefs))
+		cs.log.Debug("repo has dirty dupes", "count", len(dupes), "uid", user, "staleRefs", len(staleRefs), "blockRefs", len(brefs))
 
 		//return nil, fmt.Errorf("WIP: not currently handling this case")
 	}
···
 		}); err != nil {
 			// If we ever fail to iterate a shard file because its
 			// corrupted, just log an error and skip the shard
-			log.Errorw("iterating blocks in shard", "shard", s.ID, "err", err, "uid", user)
+			cs.log.Error("iterating blocks in shard", "shard", s.ID, "err", err, "uid", user)
 		}
 	}
 
···
 		_ = fi.Close()
 
 		if err2 := os.Remove(fi.Name()); err2 != nil {
-			log.Errorf("failed to remove shard file (%s) after failed db transaction: %w", fi.Name(), err2)
+			cs.log.Error("failed to remove shard file after failed db transaction", "path", fi.Name(), "err", err2)
 		}
 
 		return err

carstore/metrics.go

···
+package carstore
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var writeShardFileDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "carstore_write_shard_file_duration",
+	Help:    "Duration of writing shard file to disk",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+})
+
+var writeShardMetadataDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "carstore_write_shard_metadata_duration",
+	Help:    "Duration of writing shard metadata to DB",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+})

carstore/nonarchive.go

···
+package carstore
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"sync"
+
+	"github.com/bluesky-social/indigo/models"
+	blockformat "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-datastore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	car "github.com/ipld/go-car"
+	"go.opentelemetry.io/otel"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+type NonArchivalCarstore struct {
+	db *gorm.DB
+
+	lk              sync.Mutex
+	lastCommitCache map[models.Uid]*commitRefInfo
+
+	log *slog.Logger
+}
+
+func NewNonArchivalCarstore(db *gorm.DB) (*NonArchivalCarstore, error) {
+	if err := db.AutoMigrate(&commitRefInfo{}); err != nil {
+		return nil, err
+	}
+
+	return &NonArchivalCarstore{
+		db:              db,
+		lastCommitCache: make(map[models.Uid]*commitRefInfo),
+		log:             slog.Default().With("system", "carstorena"),
+	}, nil
+}
+
+type commitRefInfo struct {
+	ID   uint       `gorm:"primarykey"`
+	Uid  models.Uid `gorm:"uniqueIndex"`
+	Rev  string
+	Root models.DbCID
+}
+
+func (cs *NonArchivalCarstore) checkLastShardCache(user models.Uid) *commitRefInfo {
+	cs.lk.Lock()
+	defer cs.lk.Unlock()
+
+	ls, ok := cs.lastCommitCache[user]
+	if ok {
+		return ls
+	}
+
+	return nil
+}
+
+func (cs *NonArchivalCarstore) removeLastShardCache(user models.Uid) {
+	cs.lk.Lock()
+	defer cs.lk.Unlock()
+
+	delete(cs.lastCommitCache, user)
+}
+
+func (cs *NonArchivalCarstore) putLastShardCache(ls *commitRefInfo) {
+	cs.lk.Lock()
+	defer cs.lk.Unlock()
+
+	cs.lastCommitCache[ls.Uid] = ls
+}
+
+func (cs *NonArchivalCarstore) loadCommitRefInfo(ctx context.Context, user models.Uid) (*commitRefInfo, error) {
+	var out commitRefInfo
+	if err := cs.db.Find(&out, "uid = ?", user).Error; err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}
+
+func (cs *NonArchivalCarstore) getCommitRefInfo(ctx context.Context, user models.Uid) (*commitRefInfo, error) {
+	ctx, span := otel.Tracer("carstore").Start(ctx, "getCommitRefInfo")
+	defer span.End()
+
+	maybeLs := cs.checkLastShardCache(user)
+	if maybeLs != nil {
+		return maybeLs, nil
+	}
+
+	lastShard, err := cs.loadCommitRefInfo(ctx, user)
+	if err != nil {
+		return nil, err
+	}
+
+	cs.putLastShardCache(lastShard)
+	return lastShard, nil
+}
+
+func (cs *NonArchivalCarstore) updateLastCommit(ctx context.Context, uid models.Uid, rev string, cid cid.Cid) error {
+	cri := &commitRefInfo{
+		Uid:  uid,
+		Rev:  rev,
+		Root: models.DbCID{CID: cid},
+	}
+
+	if err := cs.db.Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "uid"}},
+		UpdateAll: true,
+	}).Create(cri).Error; err != nil {
+		return fmt.Errorf("update or set last commit info: %w", err)
+	}
+
+	cs.putLastShardCache(cri)
+
+	return nil
+}
+
+func (cs *NonArchivalCarstore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) {
+	ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession")
+	defer span.End()
+
+	// TODO: ensure that we don't write updates on top of the wrong head
+	// this needs to be a compare and swap type operation
+	lastShard, err := cs.getCommitRefInfo(ctx, user)
+	if err != nil {
+		return nil, err
+	}
+
+	if since != nil && *since != lastShard.Rev {
+		cs.log.Warn("revision mismatch", "commitSince", since, "lastRev", lastShard.Rev, "err", ErrRepoBaseMismatch)
+	}
+
+	return &DeltaSession{
+		fresh: blockstore.NewBlockstore(datastore.NewMapDatastore()),
+		blks:  make(map[cid.Cid]blockformat.Block),
+		base: &userView{
+			user:     user,
+			cs:       cs,
+			prefetch: true,
+			cache:    make(map[cid.Cid]blockformat.Block),
+		},
+		user:    user,
+		baseCid: lastShard.Root.CID,
+		cs:      cs,
+		seq:     0,
+		lastRev: lastShard.Rev,
+	}, nil
+}
+
+func (cs *NonArchivalCarstore) ReadOnlySession(user models.Uid) (*DeltaSession, error) {
+	return &DeltaSession{
+		base: &userView{
+			user:     user,
+			cs:       cs,
+			prefetch: false,
+			cache:    make(map[cid.Cid]blockformat.Block),
+		},
+		readonly: true,
+		user:     user,
+		cs:       cs,
+	}, nil
+}
+
+// TODO: incremental is only ever called true, remove the param
+func (cs *NonArchivalCarstore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error {
+	return fmt.Errorf("not supported in non-archival mode")
+}
+
+func (cs *NonArchivalCarstore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) {
+	ctx, span := otel.Tracer("carstore").Start(ctx, "ImportSlice")
+	defer span.End()
+
+	carr, err := car.NewCarReader(bytes.NewReader(carslice))
+	if err != nil {
+		return cid.Undef, nil, err
+	}
+
+	if len(carr.Header.Roots) != 1 {
+		return cid.Undef, nil, fmt.Errorf("invalid car file, header must have a single root (has %d)", len(carr.Header.Roots))
+	}
+
+	ds, err := cs.NewDeltaSession(ctx, uid, since)
+	if err != nil {
+		return cid.Undef, nil, fmt.Errorf("new delta session failed: %w", err)
+	}
+
+	var cids []cid.Cid
+	for {
+		blk, err := carr.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return cid.Undef, nil, err
+		}
+
+		cids = append(cids, blk.Cid())
+
+		if err := ds.Put(ctx, blk); err != nil {
+			return cid.Undef, nil, err
+		}
+	}
+
+	return carr.Header.Roots[0], ds, nil
+}
+
+func (cs *NonArchivalCarstore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) {
+	lastShard, err := cs.getCommitRefInfo(ctx, user)
+	if err != nil {
+		return cid.Undef, err
+	}
+	if lastShard.ID == 0 {
+		return cid.Undef, nil
+	}
+
+	return lastShard.Root.CID, nil
+}
+
+func (cs *NonArchivalCarstore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) {
+	lastShard, err := cs.getCommitRefInfo(ctx, user)
+	if err != nil {
+		return "", err
+	}
+	if lastShard.ID == 0 {
+		return "", nil
+	}
+
+	return lastShard.Rev, nil
+}
+
+func (cs *NonArchivalCarstore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) {
+	return nil, nil
+}
+
+func (cs *NonArchivalCarstore) WipeUserData(ctx context.Context, user models.Uid) error {
+	if err := cs.db.Raw("DELETE from commit_ref_infos WHERE uid = ?", user).Error; err != nil {
+		return err
+	}
+
+	cs.removeLastShardCache(user)
+	return nil
+}
+
+func (cs *NonArchivalCarstore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) {
+	return nil, fmt.Errorf("compaction not supported on non-archival")
+}
+
+func (cs *NonArchivalCarstore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) {
+	return nil, fmt.Errorf("compaction not supported in non-archival")
+}

carstore/repo_test.go

···
 		return nil, nil, err
 	}
 
-	sharddir := filepath.Join(tempdir, "shards")
-	if err := os.MkdirAll(sharddir, 0775); err != nil {
+	sharddir1 := filepath.Join(tempdir, "shards1")
+	if err := os.MkdirAll(sharddir1, 0775); err != nil {
+		return nil, nil, err
+	}
+
+	sharddir2 := filepath.Join(tempdir, "shards2")
+	if err := os.MkdirAll(sharddir2, 0775); err != nil {
 		return nil, nil, err
 	}
 
···
 		return nil, nil, err
 	}
 
-	cs, err := NewCarStore(db, sharddir)
+	cs, err := NewCarStore(db, []string{sharddir1, sharddir2})
 	if err != nil {
 		return nil, nil, err
 	}

cmd/astrolabe/handlers.go

···
 	"net/http"
 	"strings"
 
+	"github.com/bluesky-social/indigo/api/agnostic"
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	_ "github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/atproto/data"
···
 
 	atid, err := syntax.ParseAtIdentifier(c.Param("atid"))
 	if err != nil {
-		return echo.NewHTTPError(404, fmt.Sprintf("failed to parse handle or DID"))
+		return echo.NewHTTPError(404, "failed to parse handle or DID")
 	}
 
 	ident, err := srv.dir.Lookup(ctx, *atid)
···
 
 	atid, err := syntax.ParseAtIdentifier(c.Param("atid"))
 	if err != nil {
-		return echo.NewHTTPError(400, fmt.Sprintf("failed to parse handle or DID"))
+		return echo.NewHTTPError(400, "failed to parse handle or DID")
 	}
 
 	ident, err := srv.dir.Lookup(ctx, *atid)
···
 
 	atid, err := syntax.ParseAtIdentifier(c.Param("atid"))
 	if err != nil {
-		return echo.NewHTTPError(400, fmt.Sprintf("failed to parse handle or DID"))
+		return echo.NewHTTPError(400, "failed to parse handle or DID")
 	}
 
 	collection, err := syntax.ParseNSID(c.Param("collection"))
 	if err != nil {
-		return echo.NewHTTPError(400, fmt.Sprintf("failed to parse collection NSID"))
+		return echo.NewHTTPError(400, "failed to parse collection NSID")
 	}
 
 	ident, err := srv.dir.Lookup(ctx, *atid)
···
 
 	cursor := c.QueryParam("cursor")
 	// collection string, cursor string, limit int64, repo string, reverse bool, rkeyEnd string, rkeyStart string
-	resp, err := RepoListRecords(ctx, &xrpcc, collection.String(), cursor, 100, ident.DID.String(), false, "", "")
+	resp, err := agnostic.RepoListRecords(ctx, &xrpcc, collection.String(), cursor, 100, ident.DID.String(), false, "", "")
 	if err != nil {
 		return err
 	}
···
 
 	atid, err := syntax.ParseAtIdentifier(c.Param("atid"))
 	if err != nil {
-		return echo.NewHTTPError(400, fmt.Sprintf("failed to parse handle or DID"))
+		return echo.NewHTTPError(400, "failed to parse handle or DID")
 	}
 
 	collection, err := syntax.ParseNSID(c.Param("collection"))
 	if err != nil {
-		return echo.NewHTTPError(400, fmt.Sprintf("failed to parse collection NSID"))
+		return echo.NewHTTPError(400, "failed to parse collection NSID")
 	}
 
 	rkey, err := syntax.ParseRecordKey(c.Param("rkey"))
 	if err != nil {
-		return echo.NewHTTPError(400, fmt.Sprintf("failed to parse record key"))
+		return echo.NewHTTPError(400, "failed to parse record key")
 	}
 
 	ident, err := srv.dir.Lookup(ctx, *atid)
···
 	xrpcc := xrpc.Client{
 		Host: ident.PDSEndpoint(),
 	}
-	resp, err := RepoGetRecord(ctx, &xrpcc, "", collection.String(), ident.DID.String(), rkey.String())
+	resp, err := agnostic.RepoGetRecord(ctx, &xrpcc, "", collection.String(), ident.DID.String(), rkey.String())
 	if err != nil {
 		return echo.NewHTTPError(400, fmt.Sprintf("failed to load record: %s", err))
 	}

cmd/astrolabe/repogetRecord.go

···
-// Copied from indigo:api/atproto/repolistRecords.go
-
-package main
-
-// schema: com.atproto.repo.getRecord
-
-import (
-	"context"
-	"encoding/json"
-
-	"github.com/bluesky-social/indigo/xrpc"
-)
-
-// RepoGetRecord_Output is the output of a com.atproto.repo.getRecord call.
-type RepoGetRecord_Output struct {
-	Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"`
-	Uri string  `json:"uri" cborgen:"uri"`
-	//  NOTE: changed from lex decoder to json.RawMessage
-	Value *json.RawMessage `json:"value" cborgen:"value"`
-}
-
-// RepoGetRecord calls the XRPC method "com.atproto.repo.getRecord".
-//
-// cid: The CID of the version of the record. If not specified, then return the most recent version.
-// collection: The NSID of the record collection.
-// repo: The handle or DID of the repo.
-// rkey: The Record Key.
-func RepoGetRecord(ctx context.Context, c *xrpc.Client, cid string, collection string, repo string, rkey string) (*RepoGetRecord_Output, error) {
-	var out RepoGetRecord_Output
-
-	params := map[string]interface{}{
-		"cid":        cid,
-		"collection": collection,
-		"repo":       repo,
-		"rkey":       rkey,
-	}
-	if err := c.Do(ctx, xrpc.Query, "", "com.atproto.repo.getRecord", params, nil, &out); err != nil {
-		return nil, err
-	}
-
-	return &out, nil
-}

cmd/astrolabe/repolistRecords.go

···
-// Copied from indigo:api/atproto/repolistRecords.go
-
-package main
-
-// schema: com.atproto.repo.listRecords
-
-import (
-	"context"
-	"encoding/json"
-
-	"github.com/bluesky-social/indigo/xrpc"
-)
-
-// RepoListRecords_Output is the output of a com.atproto.repo.listRecords call.
-type RepoListRecords_Output struct {
-	Cursor  *string                   `json:"cursor,omitempty" cborgen:"cursor,omitempty"`
-	Records []*RepoListRecords_Record `json:"records" cborgen:"records"`
-}
-
-// RepoListRecords_Record is a "record" in the com.atproto.repo.listRecords schema.
-type RepoListRecords_Record struct {
-	Cid string `json:"cid" cborgen:"cid"`
-	Uri string `json:"uri" cborgen:"uri"`
-	//  NOTE: changed from lex decoder to json.RawMessage
-	Value *json.RawMessage `json:"value" cborgen:"value"`
-}
-
-// RepoListRecords calls the XRPC method "com.atproto.repo.listRecords".
-//
-// collection: The NSID of the record type.
-// limit: The number of records to return.
-// repo: The handle or DID of the repo.
-// reverse: Flag to reverse the order of the returned records.
-// rkeyEnd: DEPRECATED: The highest sort-ordered rkey to stop at (exclusive)
-// rkeyStart: DEPRECATED: The lowest sort-ordered rkey to start from (exclusive)
-func RepoListRecords(ctx context.Context, c *xrpc.Client, collection string, cursor string, limit int64, repo string, reverse bool, rkeyEnd string, rkeyStart string) (*RepoListRecords_Output, error) {
-	var out RepoListRecords_Output
-
-	params := map[string]interface{}{
-		"collection": collection,
-		"cursor":     cursor,
-		"limit":      limit,
-		"repo":       repo,
-		"reverse":    reverse,
-		"rkeyEnd":    rkeyEnd,
-		"rkeyStart":  rkeyStart,
-	}
-	if err := c.Do(ctx, xrpc.Query, "", "com.atproto.repo.listRecords", params, nil, &out); err != nil {
-		return nil, err
-	}
-
-	return &out, nil
-}

cmd/beemo/Dockerfile

···
 #   podman build -f ./cmd/beemo/Dockerfile -t beemo .
 
 ### Compile stage
-FROM golang:1.22-alpine3.19 AS build-env
+FROM golang:1.23-alpine3.20 AS build-env
 RUN apk add --no-cache build-base make git
 
 ADD . /dockerbuild
···
     go build -tags timetzdata -o /beemo ./cmd/beemo
 
 ### Run stage
-FROM alpine:3.19
+FROM alpine:3.20
 
 RUN apk add --no-cache --update dumb-init ca-certificates
 ENTRYPOINT ["dumb-init", "--"]

cmd/beemo/firehose_consumer.go

···
 	"log/slog"
 	"net/http"
 	"net/url"
-	"strings"
 
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	appbsky "github.com/bluesky-social/indigo/api/bsky"
···
 	)
 	logger.Info("beemo firehose scheduler configured", "scheduler", "parallel", "workers", parallelism)
 
-	return events.HandleRepoStream(ctx, con, scheduler)
-}
-
-// TODO: move this to a "ParsePath" helper in syntax package?
-func splitRepoPath(path string) (syntax.NSID, syntax.RecordKey, error) {
-	parts := strings.SplitN(path, "/", 3)
-	if len(parts) != 2 {
-		return "", "", fmt.Errorf("invalid record path: %s", path)
-	}
-	collection, err := syntax.ParseNSID(parts[0])
-	if err != nil {
-		return "", "", err
-	}
-	rkey, err := syntax.ParseRecordKey(parts[1])
-	if err != nil {
-		return "", "", err
-	}
-	return collection, rkey, nil
+	return events.HandleRepoStream(ctx, con, scheduler, logger)
 }
 
 // NOTE: for now, this function basically never errors, just logs and returns nil. Should think through error processing better.
···
 
 	for _, op := range evt.Ops {
 		logger = logger.With("eventKind", op.Action, "path", op.Path)
-		collection, rkey, err := splitRepoPath(op.Path)
+		collection, rkey, err := syntax.ParseRepoPath(op.Path)
 		if err != nil {
 			logger.Error("invalid path in repo op")
 			return nil

cmd/beemo/notify_reports.go

···
 			false, // hasComment bool
 			true,  // includeAllUserRecords bool
 			limit, // limit int64
+			nil,   // policies []string
 			nil,   // removedLabels []string
 			nil,   // removedTags []string
 			nil,   // reportTypes []string

cmd/bigsky/Dockerfile

···
 #   podman build -f ./cmd/bigsky/Dockerfile -t bigsky .
 
 ### Compile stage
-FROM golang:1.22-alpine3.19 AS build-env
+FROM golang:1.23-alpine3.20 AS build-env
 RUN apk add --no-cache build-base make git
 
 ADD . /dockerbuild
···
 RUN yarn build
 
 ### Run stage
-FROM alpine:3.19
+FROM alpine:3.20
 
 RUN apk add --no-cache --update dumb-init ca-certificates runit
 ENTRYPOINT ["dumb-init", "--"]

cmd/bigsky/copy_pdses.py

···
+#!/usr/bin/env python3
+#
+# pip install requests
+#
+# python3 copy_pdses.py --admin-key hunter2 --source-url http://srcrelay:2470 --dest-url http://destrelay:2470
+
+import json
+import logging
+import sys
+import urllib.parse
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+class relay:
+    def __init__(self, rooturl, headers=None, session=None):
+        "rooturl string, headers dict or None, session requests.Session() or None"
+        self.rooturl = rooturl
+        self.headers = headers or dict()
+        self.session = session or requests.Session()
+
+    def crawl(self, host):
+        pheaders = dict(self.headers)
+        pheaders['Content-Type'] = 'application/json'
+        url = urllib.parse.urljoin(self.rooturl, '/admin/pds/requestCrawl')
+        response = self.session.post(url, headers=pheaders, data=json.dumps({"hostname": host}))
+        if response.status_code != 200:
+            return False
+        return True
+
+    def crawlAndSetLimits(self, host, limits):
+        "host string, limits dict"
+        if not self.crawl(host):
+            logger.error("%s %s : %d %r", url, host, response.status_code, response.text)
+            return
+        if limits is None:
+            logger.debug("requestCrawl %s OK", host)
+        if self.setLimits(host, limits):
+            logger.debug("requestCrawl + changeLimits %s OK", host)
+    def setLimits(self, host, limits):
+        url = urllib.parse.urljoin(self.rooturl, '/admin/pds/changeLimits')
+        plimits = dict(limits)
+        plimits["host"] = host
+        pheaders = dict(self.headers)
+        pheaders['Content-Type'] = 'application/json'
+        response = self.session.post(url, headers=pheaders, data=json.dumps(plimits))
+        if response.status_code != 200:
+            logger.error("%s %s : %d %r", url, host, response.status_code, response.text)
+            return False
+        return True
+
+    def crawlAndBlock(self, host):
+        "make relay aware of PDS, and block it"
+        if not self.crawl(host):
+            logger.error("%s %s : %d %r", url, host, response.status_code, response.text)
+            return
+        if self.block(host):
+            logger.debug("requestCrawl + block %s OK", host)
+
+    def block(self, host):
+        url = urllib.parse.urljoin(self.rooturl, '/admin/pds/block')
+        response = self.session.post(url, headers=self.headers, data='', params={"host":host})
+        if response.status_code != 200:
+            logger.error("%s %s : %d %r", url, host, response.status_code, response.text)
+            return False
+        return True
+
+    def unblock(self, host):
+        url = urllib.parse.urljoin(self.rooturl, '/admin/pds/unblock')
+        response = self.session.post(url, headers=self.headers, data='', params={"host":host})
+        if response.status_code != 200:
+            logger.error("%s %s : %d %r", url, host, response.status_code, response.text)
+            return False
+        return True
+
+    def pdsList(self):
+        "GET /admin/pds/list"
+        url = urllib.parse.urljoin(self.rooturl, '/admin/pds/list')
+        response = self.session.get(url, headers=self.headers)
+        if response.status_code != 200:
+            logger.error("%s : %d %r", url, response.status_code, response.text)
+            return None
+        return response.json()
+
+def makeByHost(they):
+    out = dict()
+    for rec in they:
+        out[rec['Host']] = rec
+    return out
+
+def makeLimits(rec):
+    "for submitting to changeLimits"
+    return {
+        "host": rec['Host'],
+        "per_second":rec['RateLimit'],
+        "per_hour":rec['HourlyEventLimit'],
+        "per_day":rec['DailyEventLimit'],
+        "crawl_rate":rec['CrawlRateLimit'],
+        "repo_limit":rec['RepoLimit'],
+    }
+
+def makeRequestCrawl(rec):
+    "for submitting to requestCrawl"
+    return {"hostname":rec["Host"]}
+
+def de(a,b):
+    # dict equal
+    for ka, va in a.items():
+        vb = b[ka]
+        if (va is None) and (vb is None):
+            continue
+        if va == vb:
+            continue
+        return False
+    for kb in b.keys():
+        if kb not in a:
+            return False
+    return True
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--admin-key', default=None, help='relay auth bearer token', required=True)
+    ap.add_argument('--source-url', default=None, help='base url to GET /admin/pds/list')
+    ap.add_argument('--source-json', default=None, help='load /admin/pds/list json from file')
+    ap.add_argument('--dest-url', default=None, help='dest URL to POST requestCrawl etc to')
+    ap.add_argument('--dry-run', default=False, action='store_true')
+    ap.add_argument('--verbose', default=False, action='store_true')
+    args = ap.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    headers = {'Authorization': 'Bearer ' + args.admin_key}
+
+    if args.source_json:
+        with open(args.source_json, 'rt') as fin:
+            sourceList = json.load(fin)
+    elif args.source_url:
+        relaySession = relay(args.source_url, headers)
+        sourceList = relaySession.pdsList()
+    else:
+        sys.stdout.write("need --source-url or --source-json\n")
+        sys.exit(1)
+
+    r2 = relay(args.dest_url, headers)
+    destList = r2.pdsList()
+
+    source = makeByHost(sourceList)
+    dests = makeByHost(destList)
+
+    snotd = []
+    dnots = []
+    diflim = []
+    difblock = []
+    recrawl = []
+
+    for k1, v1 in source.items():
+        v2 = dests.get(k1)
+        if v2 is None:
+            snotd.append(v1)
+            continue
+        lim1 = makeLimits(v1)
+        lim2 = makeLimits(v2)
+        if v1["Blocked"] != v2["Blocked"]:
+            difblock.append((k1,v1["Blocked"]))
+        if v1["Blocked"]:
+            continue
+        if not de(lim1, lim2):
+            diflim.append(lim1)
+        if v1["HasActiveConnection"] and not v2["HasActiveConnection"]:
+            recrawl.append(k1)
+    for k2 in dests.keys():
+        if k2 not in source:
+            dnots.append(k2)
+
+    logger.debug("%d source not dest", len(snotd))
+    for rec in snotd:
+        if rec["Blocked"]:
+            if args.dry_run:
+                sys.stdout.write("crawl and block: {!r}\n".format(rec["Host"]))
+            else:
+                r2.crawlAndBlock(rec["Host"])
+        else:
+            limits = makeLimits(rec)
+            if args.dry_run:
+                sys.stdout.write("crawl and limit: {}\n".format(json.dumps(limits)))
+            else:
+                r2.crawlAndSetLimits(rec["Host"], limits)
+    logger.debug("adjust limits: %d", len(diflim))
+    for limits in diflim:
+        if args.dry_run:
+            sys.stdout.write("set limits: {}\n".format(json.dumps(limits)))
+        else:
+            r2.setLimits(limits["host"], limits)
+    logger.debug("adjust block status: %d", len(difblock))
+    for host, blocked in difblock:
+        if args.dry_run:
+            sys.stdout.write("{} block={}\n".format(host, blocked))
+        else:
+            if blocked:
+                r2.block(host)
+            else:
+                r2.unblock(host)
+    logger.debug("restart requestCrawl: %d", len(recrawl))
+    for host in recrawl:
+        if args.dry_run:
+            logger.info("requestCrawl %s", host)
+        else:
+            if r2.crawl(host):
+                logger.debug("requestCrawl %s OK", host)
+    logger.info("%d in dest but not source", len(dnots))
+    for k2 in dnots:
+        logger.debug("%s", k2)
+
+
+
+
+
+if __name__ == '__main__':
+    main()

cmd/bigsky/main.go

···
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"net/http"
 	_ "net/http/pprof"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
···
 	_ "go.uber.org/automaxprocs"
 
 	"github.com/carlmjohnson/versioninfo"
-	logging "github.com/ipfs/go-log"
 	"github.com/urfave/cli/v2"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
···
 	"gorm.io/plugin/opentelemetry/tracing"
 )
 
-var log = logging.Logger("bigsky")
+var log = slog.Default().With("system", "bigsky")
 
 func init() {
 	// control log level using, eg, GOLOG_LOG_LEVEL=debug
···
 
 func main() {
 	if err := run(os.Args); err != nil {
-		log.Fatal(err)
+		slog.Error(err.Error())
+		os.Exit(1)
 	}
 }
 
···
 			EnvVars: []string{"RELAY_DID_CACHE_SIZE"},
 			Value:   5_000_000,
 		},
+		&cli.StringSliceFlag{
+			Name:    "did-memcached",
+			EnvVars: []string{"RELAY_DID_MEMCACHED"},
+		},
 		&cli.DurationFlag{
 			Name:    "event-playback-ttl",
 			Usage:   "time to live for event playback buffering (only applies to disk persister)",
···
 			EnvVars: []string{"RELAY_NUM_COMPACTION_WORKERS"},
 			Value:   2,
 		},
+		&cli.StringSliceFlag{
+			Name:    "carstore-shard-dirs",
+			Usage:   "specify list of shard directories for carstore storage, overrides default storage within datadir",
+			EnvVars: []string{"RELAY_CARSTORE_SHARD_DIRS"},
+		},
+		&cli.StringSliceFlag{
+			Name:    "next-crawler",
+			Usage:   "forward POST requestCrawl to this url, should be machine root url and not xrpc/requestCrawl, comma separated list",
+			EnvVars: []string{"RELAY_NEXT_CRAWLER"},
+		},
+		&cli.BoolFlag{
+			Name:    "non-archival",
+			EnvVars: []string{"RELAY_NON_ARCHIVAL"},
+			Value:   false,
+		},
 	}
 
 	app.Action = runBigsky
···
 		env = "dev"
 	}
 	if cctx.Bool("jaeger") {
-		url := "http://localhost:14268/api/traces"
-		exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url)))
+		jaegerUrl := "http://localhost:14268/api/traces"
+		exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerUrl)))
 		if err != nil {
 			return err
 		}
···
 	// At a minimum, you need to set
 	// OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
 	if ep := cctx.String("otel-exporter-otlp-endpoint"); ep != "" {
-		log.Infow("setting up trace exporter", "endpoint", ep)
+		slog.Info("setting up trace exporter", "endpoint", ep)
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 
 		exp, err := otlptracehttp.New(ctx)
 		if err != nil {
-			log.Fatalw("failed to create trace exporter", "error", err)
+			slog.Error("failed to create trace exporter", "error", err)
+			os.Exit(1)
 		}
 		defer func() {
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 			defer cancel()
 			if err := exp.Shutdown(ctx); err != nil {
-				log.Errorw("failed to shutdown trace exporter", "error", err)
+				slog.Error("failed to shutdown trace exporter", "error", err)
 			}
 		}()
 
···
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
 
+	_, err := cliutil.SetupSlog(cliutil.LogOptions{})
+	if err != nil {
+		return err
+	}
+
 	// start observability/tracing (OTEL and jaeger)
 	if err := setupOTEL(cctx); err != nil {
 		return err
···
 		return err
 	}
 
-	log.Infow("setting up main database")
+	slog.Info("setting up main database")
 	dburl := cctx.String("db-url")
 	db, err := cliutil.SetupDatabase(dburl, cctx.Int("max-metadb-connections"))
 	if err != nil {
 		return err
 	}
 
-	log.Infow("setting up carstore database")
+	slog.Info("setting up carstore database")
 	csdburl := cctx.String("carstore-db-url")
 	csdb, err := cliutil.SetupDatabase(csdburl, cctx.Int("max-carstore-connections"))
 	if err != nil {
···
 		}
 	}
 
-	os.MkdirAll(filepath.Dir(csdir), os.ModePerm)
-	cstore, err := carstore.NewCarStore(csdb, csdir)
-	if err != nil {
-		return err
+	csdirs := []string{csdir}
+	if paramDirs := cctx.StringSlice("carstore-shard-dirs"); len(paramDirs) > 0 {
+		csdirs = paramDirs
 	}
 
-	mr := did.NewMultiResolver()
+	for _, csd := range csdirs {
+		if err := os.MkdirAll(filepath.Dir(csd), os.ModePerm); err != nil {
+			return err
+		}
+	}
 
-	didr := &api.PLCServer{Host: cctx.String("plc-host")}
-	mr.AddHandler("plc", didr)
+	var cstore carstore.CarStore
 
-	webr := did.WebResolver{}
-	if cctx.Bool("crawl-insecure-ws") {
-		webr.Insecure = true
+	if cctx.Bool("non-archival") {
+		cs, err := carstore.NewNonArchivalCarstore(csdb)
+		if err != nil {
+			return err
+		}
+
+		cstore = cs
+	} else {
+		cs, err := carstore.NewCarStore(csdb, csdirs)
+		if err != nil {
+			return err
+		}
+
+		cstore = cs
 	}
-	mr.AddHandler("web", &webr)
+
+	// DID RESOLUTION
+	// 1. the outside world, PLCSerever or Web
+	// 2. (maybe memcached)
+	// 3. in-process cache
+	var cachedidr did.Resolver
+	{
+		mr := did.NewMultiResolver()
 
-	cachedidr := plc.NewCachingDidResolver(mr, time.Hour*24, cctx.Int("did-cache-size"))
+		didr := &api.PLCServer{Host: cctx.String("plc-host")}
+		mr.AddHandler("plc", didr)
+
+		webr := did.WebResolver{}
+		if cctx.Bool("crawl-insecure-ws") {
+			webr.Insecure = true
+		}
+		mr.AddHandler("web", &webr)
+
+		var prevResolver did.Resolver
+		memcachedServers := cctx.StringSlice("did-memcached")
+		if len(memcachedServers) > 0 {
+			prevResolver = plc.NewMemcachedDidResolver(mr, time.Hour*24, memcachedServers)
+		} else {
+			prevResolver = mr
+		}
+
+		cachedidr = plc.NewCachingDidResolver(prevResolver, time.Hour*24, cctx.Int("did-cache-size"))
+	}
 
 	kmgr := indexer.NewKeyManager(cachedidr, nil)
 
···
 	var persister events.EventPersistence
 
 	if dpd := cctx.String("disk-persister-dir"); dpd != "" {
-		log.Infow("setting up disk persister")
+		slog.Info("setting up disk persister")
 
 		pOpts := events.DefaultDiskPersistOptions()
 		pOpts.Retention = cctx.Duration("event-playback-ttl")
···
 
 	rf := indexer.NewRepoFetcher(db, repoman, cctx.Int("max-fetch-concurrency"))
 
-	ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, rf, true, cctx.Bool("spidering"), false)
+	ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, rf, true, false, cctx.Bool("spidering"))
 	if err != nil {
 		return err
 	}
+	defer ix.Shutdown()
 
 	rlskip := cctx.String("bsky-social-rate-limit-skip")
 	ix.ApplyPDSClientSettings = func(c *xrpc.Client) {
···
 
 	repoman.SetEventHandler(func(ctx context.Context, evt *repomgr.RepoEvent) {
 		if err := ix.HandleRepoEvent(ctx, evt); err != nil {
-			log.Errorw("failed to handle repo event", "err", err)
+			slog.Error("failed to handle repo event", "err", err)
 		}
 	}, false)
 
···
 		}
 	}
 
-	log.Infow("constructing bgs")
+	slog.Info("constructing bgs")
 	bgsConfig := libbgs.DefaultBGSConfig()
 	bgsConfig.SSL = !cctx.Bool("crawl-insecure-ws")
 	bgsConfig.CompactInterval = cctx.Duration("compact-interval")
···
 	bgsConfig.MaxQueuePerPDS = cctx.Int64("max-queue-per-pds")
 	bgsConfig.DefaultRepoLimit = cctx.Int64("default-repo-limit")
 	bgsConfig.NumCompactionWorkers = cctx.Int("num-compaction-workers")
+	nextCrawlers := cctx.StringSlice("next-crawler")
+	if len(nextCrawlers) != 0 {
+		nextCrawlerUrls := make([]*url.URL, len(nextCrawlers))
+		for i, tu := range nextCrawlers {
+			var err error
+			nextCrawlerUrls[i], err = url.Parse(tu)
+			if err != nil {
+				return fmt.Errorf("failed to parse next-crawler url: %w", err)
+			}
+			slog.Info("configuring relay for requestCrawl", "host", nextCrawlerUrls[i])
+		}
+		bgsConfig.NextCrawlers = nextCrawlerUrls
+	}
 	bgs, err := libbgs.NewBGS(db, ix, repoman, evtman, cachedidr, rf, hr, bgsConfig)
 	if err != nil {
 		return err
···
 	// set up metrics endpoint
 	go func() {
 		if err := bgs.StartMetrics(cctx.String("metrics-listen")); err != nil {
-			log.Fatalf("failed to start metrics endpoint: %s", err)
+			log.Error("failed to start metrics endpoint", "err", err)
+			os.Exit(1)
 		}
 	}()
 
···
 		bgsErr <- err
 	}()
 
-	log.Infow("startup complete")
+	slog.Info("startup complete")
 	select {
 	case <-signals:
 		log.Info("received shutdown signal")
 		errs := bgs.Shutdown()
 		for err := range errs {
-			log.Errorw("error during BGS shutdown", "err", err)
+			slog.Error("error during BGS shutdown", "err", err)
 		}
 	case err := <-bgsErr:
 		if err != nil {
-			log.Errorw("error during BGS startup", "err", err)
+			slog.Error("error during BGS startup", "err", err)
 		}
 		log.Info("shutting down")
 		errs := bgs.Shutdown()
 		for err := range errs {
-			log.Errorw("error during BGS shutdown", "err", err)
+			slog.Error("error during BGS shutdown", "err", err)
 		}
 	}
 

cmd/bigsky/resync_pdses.py

···
+#!/usr/bin/env python3
+#
+# pip install requests
+#
+# python3 resync_pdses.py --admin-key hunter2 --url http://myrelay:2470 host_per_line.txt
+
+import json
+import sys
+import urllib.parse
+
+import requests
+
+
+# pds limits for POST /admin/pds/changeLimits
+# {"host":"", "per_second": int, "per_hour": int, "per_day": int, "crawl_rate": int, "repo_limit": int}
+
+limitsKeys = ('per_second', 'per_hour', 'per_day', 'crawl_rate', 'repo_limit')
+
+def checkLimits(limits):
+    for k in limits.keys():
+        if k not in limitsKeys:
+            raise Exception(f"unknown pds rate limits key {k!r}")
+    return True
+
+class relay:
+    def __init__(self, rooturl, headers=None, session=None):
+        "rooturl string, headers dict or None, session requests.Session() or None"
+        self.rooturl = rooturl
+        self.headers = headers or dict()
+        self.session = session or requests.Session()
+
+    def resync(self, host):
+        "host string"
+        url = urllib.parse.urljoin(self.rooturl, '/admin/pds/resync')
+        response = self.session.post(url, params={"host": host}, headers=self.headers, data='')
+        if response.status_code != 200:
+            sys.stderr.write(f"{url}?host={host} : ({response.status_code}) ({response.text!r})\n")
+        else:
+            sys.stderr.write(f"{url}?host={host} : OK\n")
+
+    def crawlAndSetLimits(self, host, limits):
+        "host string, limits dict"
+        pheaders = dict(self.headers)
+        pheaders['Content-Type'] = 'application/json'
+        url = urllib.parse.urljoin(self.rooturl, '/admin/pds/requestCrawl')
+        response = self.session.post(url, headers=pheaders, data=json.dumps({"hostname": host}))
+        if response.status_code != 200:
+            sys.stderr.write(f"{url} {host} : {response.status_code} {response.text!r}\n")
+            return
+        if limits is None:
+            sys.stderr.write(f"requestCrawl {host} OK\n")
+        url = urllib.parse.urljoin(self.rooturl, '/admin/pds/changeLimits')
+        plimits = dict(limits)
+        plimits["host"] = host
+        response = self.session.post(url, headers=pheaders, data=json.dumps(plimits))
+        if response.status_code != 200:
+            sys.stderr.write(f"{url} {host} : {response.status_code} {response.text!r}\n")
+            return
+        sys.stderr.write(f"requestCrawl + changeLimits {host} OK\n")
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument('input', default='-', help='host per line text file to read, - for stdin')
+    ap.add_argument('--admin-key', default=None, help='relay auth bearer token', required=True)
+    ap.add_argument('--url', default=None, help='base url to POST /admin/pds/resync', required=True)
+    ap.add_argument('--resync', default=False, action='store_true', help='resync selected PDSes')
+    ap.add_argument('--limits', default=None, help='json pds rate limits')
+    ap.add_argument('--crawl', default=False, action='store_true', help='crawl & set limits')
+    args = ap.parse_args()
+
+    headers = {'Authorization': 'Bearer ' + args.admin_key}
+
+    relaySession = relay(args.url, headers)
+
+    #url = urllib.parse.urljoin(args.url, '/admin/pds/resync')
+
+    #sess = requests.Session()
+    if args.crawl and args.resync:
+        sys.stderr.write("should only specify one of --resync --crawl")
+        sys.exit(1)
+    if (not args.crawl) and (not args.resync):
+        sys.stderr.write("should specify one of --resync --crawl")
+        sys.exit(1)
+
+    limits = None
+    if args.limits:
+        limits = json.loads(args.limits)
+        checkLimits(limits)
+
+    if args.input == '-':
+        fin = sys.stdin
+    else:
+        fin = open(args.input, 'rt')
+    for line in fin:
+        if not line:
+            continue
+        line = line.strip()
+        if not line:
+            continue
+        if line[0] == '#':
+            continue
+        host = line
+        if args.crawl:
+            relaySession.crawlAndSetLimits(host, limits)
+        elif args.resync:
+            relaySession.resync(host)
+        # response = sess.post(url, params={"host": line}, headers=headers)
+        # if response.status_code != 200:
+        #     sys.stderr.write(f"{url}?host={line} : ({response.status_code}) ({response.text!r})\n")
+        # else:
+        #     sys.stderr.write(f"{url}?host={line} : OK\n")
+
+if __name__ == '__main__':
+    main()

cmd/goat/account.go

···
 					EnvVars:  []string{"ATP_AUTH_PASSWORD"},
 				},
 				&cli.StringFlag{
+					Name:    "auth-factor-token",
+					Usage:   "token required if password is used and 2fa is required",
+					EnvVars: []string{"ATP_AUTH_FACTOR_TOKEN"},
+				},
+				&cli.StringFlag{
 					Name:    "pds-host",
 					Usage:   "URL of the PDS to create account on (overrides DID doc)",
 					EnvVars: []string{"ATP_PDS_HOST"},
···
 		return err
 	}
 
-	_, err = refreshAuthSession(ctx, *username, cctx.String("app-password"), cctx.String("pds-host"))
+	_, err = refreshAuthSession(ctx, *username, cctx.String("app-password"), cctx.String("pds-host"), cctx.String("auth-factor-token"))
 	return err
 }
 

cmd/goat/account_migrate.go

···
 	"strings"
 	"time"
 
+	"github.com/bluesky-social/indigo/api/agnostic"
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/atproto/syntax"
 	"github.com/bluesky-social/indigo/xrpc"
···
 
 	slog.Info("migrating preferences")
 	// TODO: service proxy header for AppView?
-	prefResp, err := ActorGetPreferences(ctx, oldClient)
+	prefResp, err := agnostic.ActorGetPreferences(ctx, oldClient)
 	if err != nil {
 		return fmt.Errorf("failed fetching old preferences: %w", err)
 	}
-	err = ActorPutPreferences(ctx, &newClient, &ActorPutPreferences_Input{
+	err = agnostic.ActorPutPreferences(ctx, &newClient, &agnostic.ActorPutPreferences_Input{
 		Preferences: prefResp.Preferences,
 	})
 	if err != nil {
···
 	// NOTE: to work with did:web or non-PDS-managed did:plc, need to do manual migraiton process
 	slog.Info("updating identity to new host")
 
-	credsResp, err := IdentityGetRecommendedDidCredentials(ctx, &newClient)
+	credsResp, err := agnostic.IdentityGetRecommendedDidCredentials(ctx, &newClient)
 	if err != nil {
 		return fmt.Errorf("failed fetching new credentials: %w", err)
 	}
···
 		return nil
 	}
 
-	var unsignedOp IdentitySignPlcOperation_Input
+	var unsignedOp agnostic.IdentitySignPlcOperation_Input
 	if err = json.Unmarshal(credsBytes, &unsignedOp); err != nil {
 		return fmt.Errorf("failed parsing PLC op: %w", err)
 	}
···
 
 	// NOTE: could add additional sanity checks here that any extra rotation keys were retained, and that old alsoKnownAs and service entries are retained? The stakes aren't super high for the later, as PLC has the full history. PLC and the new PDS already implement some basic sanity checks.
 
-	signedPlcOpResp, err := IdentitySignPlcOperation(ctx, oldClient, &unsignedOp)
+	signedPlcOpResp, err := agnostic.IdentitySignPlcOperation(ctx, oldClient, &unsignedOp)
 	if err != nil {
 		return fmt.Errorf("failed requesting PLC operation signature: %w", err)
 	}
 
-	err = IdentitySubmitPlcOperation(ctx, &newClient, &IdentitySubmitPlcOperation_Input{
+	err = agnostic.IdentitySubmitPlcOperation(ctx, &newClient, &agnostic.IdentitySubmitPlcOperation_Input{
 		Operation: signedPlcOpResp.Operation,
 	})
 	if err != nil {

cmd/goat/account_plc.go

···
 	"fmt"
 	"os"
 
+	"github.com/bluesky-social/indigo/api/agnostic"
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 
 	"github.com/urfave/cli/v2"
···
 		return err
 	}
 
-	resp, err := IdentityGetRecommendedDidCredentials(ctx, xrpcc)
+	resp, err := agnostic.IdentityGetRecommendedDidCredentials(ctx, xrpcc)
 	if err != nil {
 		return err
 	}
···
 		return err
 	}
 
-	var body IdentitySignPlcOperation_Input
+	var body agnostic.IdentitySignPlcOperation_Input
 	if err = json.Unmarshal(fileBytes, &body); err != nil {
 		return fmt.Errorf("failed decoding PLC op JSON: %w", err)
 	}
···
 		body.Token = &token
 	}
 
-	resp, err := IdentitySignPlcOperation(ctx, xrpcc, &body)
+	resp, err := agnostic.IdentitySignPlcOperation(ctx, xrpcc, &body)
 	if err != nil {
 		return err
 	}
···
 		return fmt.Errorf("failed decoding PLC op JSON: %w", err)
 	}
 
-	err = IdentitySubmitPlcOperation(ctx, xrpcc, &IdentitySubmitPlcOperation_Input{
+	err = agnostic.IdentitySubmitPlcOperation(ctx, xrpcc, &agnostic.IdentitySubmitPlcOperation_Input{
 		Operation: &op,
 	})
 	if err != nil {

cmd/goat/actorgetPreferences.go api/agnostic/actorgetPreferences.go

···
-// Copied from indigo:api/atproto/actorgetPreferences.go
+// Copied from indigo:api/bsky/actorgetPreferences.go
 
-package main
+package agnostic
 
 // schema: app.bsky.actor.getPreferences
 

cmd/goat/actorputPreferences.go api/agnostic/actorputPreferences.go

···
-// Copied from indigo:api/atproto/actorputPreferences.go
+// Copied from indigo:api/bsky/actorputPreferences.go
 
-package main
+package agnostic
 
 // schema: app.bsky.actor.putPreferences
 

cmd/goat/auth.go

···
 	}
 	resp, err := comatproto.ServerRefreshSession(ctx, &client)
 	if err != nil {
-		// TODO: if failure, try creating a new session from password
+		// TODO: if failure, try creating a new session from password (2fa tokens are only valid once, so not reused)
 		fmt.Println("trying to refresh auth from password...")
-		as, err := refreshAuthSession(ctx, sess.DID.AtIdentifier(), sess.Password, sess.PDS)
+		as, err := refreshAuthSession(ctx, sess.DID.AtIdentifier(), sess.Password, sess.PDS, "")
 		if err != nil {
 			return nil, err
 		}
···
 	return &client, nil
 }
 
-func refreshAuthSession(ctx context.Context, username syntax.AtIdentifier, password, pdsURL string) (*AuthSession, error) {
+func refreshAuthSession(ctx context.Context, username syntax.AtIdentifier, password, pdsURL, authFactorToken string) (*AuthSession, error) {
 
 	var did syntax.DID
 	if pdsURL == "" {
···
 	client := xrpc.Client{
 		Host: pdsURL,
 	}
+	var token *string
+	if authFactorToken != "" {
+		token = &authFactorToken
+	}
 	sess, err := comatproto.ServerCreateSession(ctx, &client, &comatproto.ServerCreateSession_Input{
-		Identifier: username.String(),
-		Password:   password,
+		Identifier:      username.String(),
+		Password:        password,
+		AuthFactorToken: token,
 	})
 	if err != nil {
 		return nil, err

cmd/goat/blob.go

···
 					Aliases: []string{"o"},
 					Usage:   "directory to store blobs in",
 				},
+				&cli.StringFlag{
+					Name:  "pds-host",
+					Usage: "URL of the PDS to export blobs from (overrides DID doc)",
+				},
 			},
 			Action: runBlobExport,
 		},
···
 		return err
 	}
 
+	pdsHost := cctx.String("pds-host")
+	if pdsHost == "" {
+		pdsHost = ident.PDSEndpoint()
+	}
+
 	// create a new API client to connect to the account's PDS
 	xrpcc := xrpc.Client{
-		Host: ident.PDSEndpoint(),
+		Host: pdsHost,
 	}
 	if xrpcc.Host == "" {
 		return fmt.Errorf("no PDS endpoint for identity")

cmd/goat/bsky_prefs.go

···
 	"fmt"
 	"os"
 
+	"github.com/bluesky-social/indigo/api/agnostic"
+
 	"github.com/urfave/cli/v2"
 )
 
···
 	}
 
 	// TODO: does indigo API code crash with unsupported preference '$type'? Eg "Lexicon decoder" with unsupported type.
-	resp, err := ActorGetPreferences(ctx, xrpcc)
+	resp, err := agnostic.ActorGetPreferences(ctx, xrpcc)
 	if err != nil {
 		return fmt.Errorf("failed fetching old preferences: %w", err)
 	}
···
 		return err
 	}
 
-	err = ActorPutPreferences(ctx, xrpcc, &ActorPutPreferences_Input{
+	err = agnostic.ActorPutPreferences(ctx, xrpcc, &agnostic.ActorPutPreferences_Input{
 		Preferences: prefsArray,
 	})
 	if err != nil {

cmd/goat/firehose.go

···
 		rsc.EventHandler,
 	)
 	slog.Info("starting firehose consumer", "relayHost", relayHost)
-	return events.HandleRepoStream(ctx, con, scheduler)
-}
-
-// TODO: move this to a "ParsePath" helper in syntax package?
-func splitRepoPath(path string) (syntax.NSID, syntax.RecordKey, error) {
-	parts := strings.SplitN(path, "/", 3)
-	if len(parts) != 2 {
-		return "", "", fmt.Errorf("invalid record path: %s", path)
-	}
-	collection, err := syntax.ParseNSID(parts[0])
-	if err != nil {
-		return "", "", err
-	}
-	rkey, err := syntax.ParseRecordKey(parts[1])
-	if err != nil {
-		return "", "", err
-	}
-	return collection, rkey, nil
+	return events.HandleRepoStream(ctx, con, scheduler, nil)
 }
 
 func (gfc *GoatFirehoseConsumer) handleIdentityEvent(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Identity) error {
···
 	}
 
 	for _, op := range evt.Ops {
-		collection, rkey, err := splitRepoPath(op.Path)
+		collection, rkey, err := syntax.ParseRepoPath(op.Path)
 		if err != nil {
 			logger.Error("invalid path in repo op", "eventKind", op.Action, "path", op.Path)
 			return nil

cmd/goat/identity.go

···
 	Name:      "resolve",
 	Usage:     "lookup identity metadata",
 	ArgsUsage: `<at-identifier>`,
-	Flags:     []cli.Flag{},
-	Action:    runResolve,
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "did",
+			Usage: "just resolve to DID",
+		},
+	},
+	Action: runResolve,
 }
 
 func runResolve(cctx *cli.Context) error {
···
 	dir := identity.BaseDirectory{}
 	var doc *identity.DIDDocument
 
+	if cctx.Bool("did") {
+		if atid.IsDID() {
+		}
+	}
+
 	if atid.IsDID() {
 		did, err := atid.AsDID()
 		if err != nil {
 			return err
 		}
+		if cctx.Bool("did") {
+			fmt.Println(did)
+			return nil
+		}
 		doc, err = dir.ResolveDID(ctx, did)
 		if err != nil {
 			return err
···
 		did, err := dir.ResolveHandle(ctx, handle)
 		if err != nil {
 			return err
+		}
+		if cctx.Bool("did") {
+			fmt.Println(did)
+			return nil
 		}
 		doc, err = dir.ResolveDID(ctx, did)
 		if err != nil {

cmd/goat/identitygetRecommendedDidCredentials.go api/agnostic/identitygetRecommendedDidCredentials.go

···
 // Copied from indigo:api/atproto/identitygetRecommendedDidCredentials.go
 
-package main
+package agnostic
 
 // schema: com.atproto.identity.getRecommendedDidCredentials
 

cmd/goat/identitysignPlcOperation.go api/agnostic/identitysignPlcOperation.go

···
 // Copied from indigo:api/atproto/identitysignPlcOperation.go
 
-package main
+package agnostic
 
 // schema: com.atproto.identity.signPlcOperation
 

cmd/goat/identitysubmitPlcOperation.go api/agnostic/identitysubmitPlcOperation.go

···
 // Copied from indigo:api/atproto/identitysubmitPlcOperation.go
 
-package main
+package agnostic
 
 // schema: com.atproto.identity.submitPlcOperation
 

cmd/goat/net.go

···
 	"fmt"
 	"log/slog"
 
+	"github.com/bluesky-social/indigo/api/agnostic"
 	"github.com/bluesky-social/indigo/atproto/data"
 	"github.com/bluesky-social/indigo/atproto/identity"
 	"github.com/bluesky-social/indigo/atproto/syntax"
···
 	xrpcc := xrpc.Client{
 		Host: ident.PDSEndpoint(),
 	}
-	resp, err := RepoGetRecord(ctx, &xrpcc, "", aturi.Collection().String(), ident.DID.String(), aturi.RecordKey().String())
+	resp, err := agnostic.RepoGetRecord(ctx, &xrpcc, "", aturi.Collection().String(), ident.DID.String(), aturi.RecordKey().String())
 	if err != nil {
 		return nil, err
 	}

cmd/goat/record.go

···
 	"fmt"
 	"os"
 
+	"github.com/bluesky-social/indigo/api/agnostic"
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/atproto/data"
 	"github.com/bluesky-social/indigo/atproto/identity"
···
 		cursor := ""
 		for {
 			// collection string, cursor string, limit int64, repo string, reverse bool, rkeyEnd string, rkeyStart string
-			resp, err := RepoListRecords(ctx, &xrpcc, nsid, cursor, 100, ident.DID.String(), false, "", "")
+			resp, err := agnostic.RepoListRecords(ctx, &xrpcc, nsid, cursor, 100, ident.DID.String(), false, "", "")
 			if err != nil {
 				return err
 			}
···
 	}
 	validate := !cctx.Bool("no-validate")
 
-	resp, err := RepoCreateRecord(ctx, xrpcc, &RepoCreateRecord_Input{
+	resp, err := agnostic.RepoCreateRecord(ctx, xrpcc, &agnostic.RepoCreateRecord_Input{
 		Collection: nsid,
 		Repo:       xrpcc.Auth.Did,
 		Record:     recordVal,
···
 	rkey := cctx.String("rkey")
 
 	// NOTE: need to fetch existing record CID to perform swap. this is optional in theory, but golang can't deal with "optional" and "nullable", so we always need to set this (?)
-	existing, err := RepoGetRecord(ctx, xrpcc, "", nsid, xrpcc.Auth.Did, rkey)
+	existing, err := agnostic.RepoGetRecord(ctx, xrpcc, "", nsid, xrpcc.Auth.Did, rkey)
 	if err != nil {
 		return err
 	}
···
 
 	validate := !cctx.Bool("no-validate")
 
-	resp, err := RepoPutRecord(ctx, xrpcc, &RepoPutRecord_Input{
+	resp, err := agnostic.RepoPutRecord(ctx, xrpcc, &agnostic.RepoPutRecord_Input{
 		Collection: nsid,
 		Repo:       xrpcc.Auth.Did,
 		Record:     recordVal,

cmd/goat/repo.go

···
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/atproto/data"
 	"github.com/bluesky-social/indigo/atproto/syntax"
+	"github.com/bluesky-social/indigo/mst"
 	"github.com/bluesky-social/indigo/repo"
+	"github.com/bluesky-social/indigo/util"
 	"github.com/bluesky-social/indigo/xrpc"
 
 	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/urfave/cli/v2"
+	"github.com/xlab/treeprint"
 )
 
 var cmdRepo = &cli.Command{
···
 			Action:    runRepoInspect,
 		},
 		&cli.Command{
+			Name:      "mst",
+			Usage:     "show repo MST structure",
+			ArgsUsage: `<car-file>`,
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:    "full-cid",
+					Aliases: []string{"f"},
+					Usage:   "display full CIDs",
+				},
+				&cli.StringFlag{
+					Name:    "root",
+					Aliases: []string{"r"},
+					Usage:   "CID of root block",
+				},
+			},
+			Action: runRepoMST,
+		},
+		&cli.Command{
 			Name:      "unpack",
 			Usage:     "extract records from CAR file as directory of JSON files",
 			ArgsUsage: `<car-file>`,
···
 		return fmt.Errorf("no PDS endpoint for identity")
 	}
 
+	// set longer timeout, for large CAR files
+	xrpcc.Client = util.RobustHTTPClient()
+	xrpcc.Client.Timeout = 600 * time.Second
+
 	carPath := cctx.String("output")
 	if carPath == "" {
 		// NOTE: having the rev in the the path might be nice
 		now := time.Now().Format("20060102150405")
 		carPath = fmt.Sprintf("%s.%s.car", username, now)
 	}
-	// NOTE: there is a race condition, but nice to give a friendly error earlier before downloading
-	if _, err := os.Stat(carPath); err == nil {
-		return fmt.Errorf("file already exists: %s", carPath)
+	output, err := getFileOrStdout(carPath)
+	if err != nil {
+		if errors.Is(err, os.ErrExist) {
+			return fmt.Errorf("file already exists: %s", carPath)
+		}
+		return err
 	}
-	fmt.Printf("downloading from %s to: %s\n", xrpcc.Host, carPath)
+	defer output.Close()
+	if carPath != stdIOPath {
+		fmt.Printf("downloading from %s to: %s\n", xrpcc.Host, carPath)
+	}
 	repoBytes, err := comatproto.SyncGetRepo(ctx, &xrpcc, ident.DID.String(), "")
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(carPath, repoBytes, 0666)
+	if _, err := output.Write(repoBytes); err != nil {
+		return err
+	}
+	return nil
 }
 
 func runRepoImport(cctx *cli.Context) error {
···
 	// TODO: Signature?
 
 	return nil
+}
+
+func runRepoMST(cctx *cli.Context) error {
+	ctx := context.Background()
+	opts := repoMSTOptions{
+		carPath: cctx.Args().First(),
+		fullCID: cctx.Bool("full-cid"),
+		root:    cctx.String("root"),
+	}
+	// read from file or stdin
+	if opts.carPath == "" {
+		return fmt.Errorf("need to provide path to CAR file as argument")
+	}
+	inputCAR, err := getFileOrStdin(opts.carPath)
+	if err != nil {
+		return err
+	}
+	// read repository tree in to memory
+	r, err := repo.ReadRepoFromCar(ctx, inputCAR)
+	if err != nil {
+		return err
+	}
+	cst := util.CborStore(r.Blockstore())
+	// determine which root cid to use, defaulting to repo data root
+	rootCID := r.DataCid()
+	if opts.root != "" {
+		optsRootCID, err := cid.Decode(opts.root)
+		if err != nil {
+			return err
+		}
+		rootCID = optsRootCID
+	}
+	// start walking mst
+	exists, err := nodeExists(ctx, cst, rootCID)
+	if err != nil {
+		return err
+	}
+	tree := treeprint.NewWithRoot(displayCID(&rootCID, exists, opts))
+	if exists {
+		if err := walkMST(ctx, cst, rootCID, tree, opts); err != nil {
+			return err
+		}
+	}
+	// print tree
+	fmt.Println(tree.String())
+	return nil
+}
+
+func walkMST(ctx context.Context, cst *cbor.BasicIpldStore, cid cid.Cid, tree treeprint.Tree, opts repoMSTOptions) error {
+	var node mst.NodeData
+	if err := cst.Get(ctx, cid, &node); err != nil {
+		return err
+	}
+	if node.Left != nil {
+		exists, err := nodeExists(ctx, cst, *node.Left)
+		if err != nil {
+			return err
+		}
+		subtree := tree.AddBranch(displayCID(node.Left, exists, opts))
+		if exists {
+			if err := walkMST(ctx, cst, *node.Left, subtree, opts); err != nil {
+				return err
+			}
+		}
+	}
+	for _, entry := range node.Entries {
+		exists, err := nodeExists(ctx, cst, entry.Val)
+		if err != nil {
+			return err
+		}
+		tree.AddNode(displayEntryVal(&entry, exists, opts))
+		if entry.Tree != nil {
+			exists, err := nodeExists(ctx, cst, *entry.Tree)
+			if err != nil {
+				return err
+			}
+			subtree := tree.AddBranch(displayCID(entry.Tree, exists, opts))
+			if exists {
+				if err := walkMST(ctx, cst, *entry.Tree, subtree, opts); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func displayEntryVal(entry *mst.TreeEntry, exists bool, opts repoMSTOptions) string {
+	key := string(entry.KeySuffix)
+	divider := " "
+	if opts.fullCID {
+		divider = "\n"
+	}
+	return strings.Repeat("∙", int(entry.PrefixLen)) + key + divider + displayCID(&entry.Val, exists, opts)
+}
+
+func displayCID(cid *cid.Cid, exists bool, opts repoMSTOptions) string {
+	cidDisplay := cid.String()
+	if !opts.fullCID {
+		cidDisplay = "…" + string(cidDisplay[len(cidDisplay)-7:])
+	}
+	connector := "─◉"
+	if !exists {
+		connector = "─◌"
+	}
+	return "[" + cidDisplay + "]" + connector
+}
+
+type repoMSTOptions struct {
+	carPath string
+	fullCID bool
+	root    string
+}
+
+func nodeExists(ctx context.Context, cst *cbor.BasicIpldStore, cid cid.Cid) (bool, error) {
+	if _, err := cst.Blocks.Get(ctx, cid); err != nil {
+		if errors.Is(err, ipld.ErrNotFound{}) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
 }
 
 func runRepoUnpack(cctx *cli.Context) error {

cmd/goat/repocreateRecord.go api/agnostic/repocreateRecord.go

···
 // Copied from indigo:api/atproto/repocreateRecords.go
 
-package main
+package agnostic
 
 // schema: com.atproto.repo.createRecord
 

cmd/goat/repogetRecord.go api/agnostic/repogetRecord.go

···
 // Copied from indigo:api/atproto/repolistRecords.go
 
-package main
+package agnostic
 
 // schema: com.atproto.repo.getRecord
 

cmd/goat/repolistRecords.go api/agnostic/repolistRecords.go

···
 // Copied from indigo:api/atproto/repolistRecords.go
 
-package main
+package agnostic
 
 // schema: com.atproto.repo.listRecords
 

cmd/goat/repoputRecord.go api/agnostic/repoputRecord.go

···
 // Copied from indigo:api/atproto/repoputRecords.go
 
-package main
+package agnostic
 
 // schema: com.atproto.repo.putRecord
 

cmd/goat/util.go

···
 
 import (
 	"context"
+	"io"
+	"os"
 
 	"github.com/bluesky-social/indigo/atproto/identity"
 	"github.com/bluesky-social/indigo/atproto/syntax"
···
 	dir := identity.DefaultDirectory()
 	return dir.Lookup(ctx, *id)
 }
+
+const stdIOPath = "-"
+
+func getFileOrStdin(path string) (io.Reader, error) {
+	if path == stdIOPath {
+		return os.Stdin, nil
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func getFileOrStdout(path string) (io.WriteCloser, error) {
+	if path == stdIOPath {
+		return os.Stdout, nil
+	}
+	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}

cmd/gosky/admin.go

···
 			false, // hasComment bool
 			false, // includeAllUserRecords bool
 			100,   // limit int64
+			nil,   // policies []string
 			nil,   // removedLabels []string
 			nil,   // removedTags []string
 			nil,   // reportTypes []string
···
 			false, // hasComment bool
 			false, // includeAllUserRecords bool
 			100,   // limit int64
+			nil,   // policies []string
 			nil,   // removedLabels []string
 			nil,   // removedTags []string
 			nil,   // reportTypes []string

cmd/gosky/bsky.go

···
 			return err
 		}
 
-		notifs, err := appbsky.NotificationListNotifications(ctx, xrpcc, "", 50, false, "")
+		notifs, err := appbsky.NotificationListNotifications(ctx, xrpcc, "", 50, false, nil, "")
 		if err != nil {
 			return err
 		}

cmd/gosky/car.go

···
 		if topDir == "" {
 			topDir = did.String()
 		}
-		log.Infof("writing output to: %s", topDir)
+		log.Info("writing output", "topDir", topDir)
 
 		commitPath := topDir + "/_commit"
 		os.MkdirAll(filepath.Dir(commitPath), os.ModePerm)
···
 			if err != nil {
 				return err
 			}
-			log.Debugf("processing record: %s", k)
+			log.Debug("processing record", "rec", k)
 
 			// TODO: check if path is safe more carefully
 			recPath := topDir + "/" + k

cmd/gosky/debug.go

···
 		}
 
 		seqScheduler := sequential.NewScheduler("debug-inspect-event", rsc.EventHandler)
-		err = events.HandleRepoStream(ctx, con, seqScheduler)
+		err = events.HandleRepoStream(ctx, con, seqScheduler, nil)
 		if err != errFoundIt {
 			return err
 		}
···
 			},
 		}
 		seqScheduler := sequential.NewScheduler("debug-stream", rsc.EventHandler)
-		err = events.HandleRepoStream(ctx, con, seqScheduler)
+		err = events.HandleRepoStream(ctx, con, seqScheduler, nil)
 		if err != nil {
 			return err
 		}
···
 			go func(i int, url string) {
 				con, _, err := d.Dial(url, http.Header{})
 				if err != nil {
-					log.Fatalf("Dial failure on url%d: %s", i+1, err)
+					log.Error("Dial failure", "i", i, "url", url, "err", err)
+					os.Exit(1)
 				}
 
 				ctx := context.TODO()
···
 					},
 				}
 				seqScheduler := sequential.NewScheduler(fmt.Sprintf("debug-stream-%d", i+1), rsc.EventHandler)
-				if err := events.HandleRepoStream(ctx, con, seqScheduler); err != nil {
-					log.Fatalf("HandleRepoStream failure on url%d: %s", i+1, err)
+				if err := events.HandleRepoStream(ctx, con, seqScheduler, nil); err != nil {
+					log.Error("HandleRepoStream failure", "i", i, "url", url, "err", err)
+					os.Exit(1)
 				}
 			}(i, url)
 		}
···
 			logger := log.With("host", cctx.String("host-1"))
 			repo1bytes, err := comatproto.SyncGetRepo(ctx, &xrpc1, did.String(), "")
 			if err != nil {
-				logger.Fatalf("getting repo: %s", err)
+				logger.Error("getting repo", "err", err)
+				os.Exit(1)
 				return
 			}
 
 			rep1, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo1bytes))
 			if err != nil {
-				logger.Fatalf("reading repo: %s", err)
+				logger.Error("reading repo", "err", err)
+				os.Exit(1)
 				return
 			}
 		}()
···
 			logger := log.With("host", cctx.String("host-2"))
 			repo2bytes, err := comatproto.SyncGetRepo(ctx, &xrpc2, did.String(), "")
 			if err != nil {
-				logger.Fatalf("getting repo: %s", err)
+				logger.Error("getting repo", "err", err)
+				os.Exit(1)
 				return
 			}
 
 			rep2, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo2bytes))
 			if err != nil {
-				logger.Fatalf("reading repo: %s", err)
+				logger.Error("reading repo", "err", err)
+				os.Exit(1)
 				return
 			}
 		}()

cmd/gosky/main.go

···
 	"encoding/json"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
···
 	"github.com/bluesky-social/indigo/api/atproto"
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/api/bsky"
+	"github.com/bluesky-social/indigo/atproto/identity"
 	"github.com/bluesky-social/indigo/atproto/syntax"
 	"github.com/bluesky-social/indigo/events"
 	"github.com/bluesky-social/indigo/events/schedulers/sequential"
···
 	_ "github.com/joho/godotenv/autoload"
 
 	"github.com/carlmjohnson/versioninfo"
-	logging "github.com/ipfs/go-log"
 	"github.com/polydawn/refmt/cbor"
 	rejson "github.com/polydawn/refmt/json"
 	"github.com/polydawn/refmt/shared"
 	cli "github.com/urfave/cli/v2"
 )
 
-var log = logging.Logger("gosky")
+var log = slog.Default().With("system", "gosky")
 
 func main() {
 	run(os.Args)
···
 			EnvVars: []string{"ATP_PLC_HOST"},
 		},
 	}
+
+	_, err := cliutil.SetupSlog(cliutil.LogOptions{})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "logging setup error: %s\n", err.Error())
+		os.Exit(1)
+		return
+	}
+
 	app.Commands = []*cli.Command{
 		accountCmd,
 		adminCmd,
···
 			},
 		}
 		seqScheduler := sequential.NewScheduler(con.RemoteAddr().String(), rsc.EventHandler)
-		return events.HandleRepoStream(ctx, con, seqScheduler)
+		return events.HandleRepoStream(ctx, con, seqScheduler, log)
 	},
 }
 
···
 				return fmt.Errorf("unrecognized link")
 			}
 
+			atid, err := syntax.ParseAtIdentifier(did)
+			if err != nil {
+				return err
+			}
+
+			resp, err := identity.DefaultDirectory().Lookup(ctx, *atid)
+			if err != nil {
+				return err
+			}
+
+			xrpcc.Host = resp.PDSEndpoint()
+
 			out, err := comatproto.RepoGetRecord(ctx, xrpcc, "", collection, did, rkey)
 			if err != nil {
 				return err
···
 
 		rc, rec, err := rr.GetRecord(ctx, cctx.Args().First())
 		if err != nil {
-			return err
+			return fmt.Errorf("get record failed: %w", err)
 		}
 
 		if cctx.Bool("raw") {

cmd/gosky/streamdiff.go

···
 				},
 			}
 			seqScheduler := sequential.NewScheduler("streamA", rsc.EventHandler)
-			err = events.HandleRepoStream(ctx, cona, seqScheduler)
+			err = events.HandleRepoStream(ctx, cona, seqScheduler, log)
 			if err != nil {
-				log.Errorf("stream A failed: %s", err)
+				log.Error("stream A failed", "err", err)
 			}
 		}()
 
···
 			}
 
 			seqScheduler := sequential.NewScheduler("streamB", rsc.EventHandler)
-			err = events.HandleRepoStream(ctx, conb, seqScheduler)
+			err = events.HandleRepoStream(ctx, conb, seqScheduler, log)
 			if err != nil {
-				log.Errorf("stream B failed: %s", err)
+				log.Error("stream B failed", "err", err)
 			}
 		}()
 

cmd/gosky/sync.go

···
 			xrpcc.Host = h
 		}
 
-		log.Infof("downloading from %s to: %s", xrpcc.Host, carPath)
+		log.Info("downloading", "from", xrpcc.Host, "to", carPath)
 		repoBytes, err := comatproto.SyncGetRepo(ctx, xrpcc, ident.DID.String(), "")
 		if err != nil {
 			return err

cmd/hepa/Dockerfile

···
 #   podman build -f ./cmd/hepa/Dockerfile -t hepa .
 
 ### Compile stage
-FROM golang:1.22-alpine3.19 AS build-env
+FROM golang:1.23-alpine3.20 AS build-env
 RUN apk add --no-cache build-base make git
 
 ADD . /dockerbuild
···
     go build -tags timetzdata -o /hepa ./cmd/hepa
 
 ### Run stage
-FROM alpine:3.19
+FROM alpine:3.20
 
 RUN apk add --no-cache --update dumb-init ca-certificates
 ENTRYPOINT ["dumb-init", "--"]

cmd/hepa/main.go

···
 			Usage:   "secret token for prescreen server",
 			EnvVars: []string{"HEPA_PRESCREEN_TOKEN"},
 		},
+		&cli.DurationFlag{
+			Name:    "report-dupe-period",
+			Usage:   "time period within which automod will not re-report an account for the same reasonType",
+			EnvVars: []string{"HEPA_REPORT_DUPE_PERIOD"},
+			Value:   1 * 24 * time.Hour,
+		},
+		&cli.IntFlag{
+			Name:    "quota-mod-report-day",
+			Usage:   "number of reports automod can file per day, for all subjects and types combined (circuit breaker)",
+			EnvVars: []string{"HEPA_QUOTA_MOD_REPORT_DAY"},
+			Value:   10000,
+		},
+		&cli.IntFlag{
+			Name:    "quota-mod-takedown-day",
+			Usage:   "number of takedowns automod can action per day, for all subjects combined (circuit breaker)",
+			EnvVars: []string{"HEPA_QUOTA_MOD_TAKEDOWN_DAY"},
+			Value:   200,
+		},
+		&cli.IntFlag{
+			Name:    "quota-mod-action-day",
+			Usage:   "number of misc actions automod can do per day, for all subjects combined (circuit breaker)",
+			EnvVars: []string{"HEPA_QUOTA_MOD_ACTION_DAY"},
+			Value:   2000,
+		},
 	}
 
 	app.Commands = []*cli.Command{
···
 			dir,
 			Config{
 				Logger:              logger,
-				RelayHost:           cctx.String("atp-relay-host"), // DEPRECATED
 				BskyHost:            cctx.String("atp-bsky-host"),
 				OzoneHost:           cctx.String("atp-ozone-host"),
 				OzoneDID:            cctx.String("ozone-did"),
···
 				AbyssPassword:       cctx.String("abyss-password"),
 				RatelimitBypass:     cctx.String("ratelimit-bypass"),
 				RulesetName:         cctx.String("ruleset"),
-				FirehoseParallelism: cctx.Int("firehose-parallelism"), // DEPRECATED
 				PreScreenHost:       cctx.String("prescreen-host"),
 				PreScreenToken:      cctx.String("prescreen-token"),
+				ReportDupePeriod:    cctx.Duration("report-dupe-period"),
+				QuotaModReportDay:   cctx.Int("quota-mod-report-day"),
+				QuotaModTakedownDay: cctx.Int("quota-mod-takedown-day"),
+				QuotaModActionDay:   cctx.Int("quota-mod-action-day"),
 			},
 		)
 		if err != nil {
···
 	return NewServer(
 		dir,
 		Config{
-			Logger:              logger,
-			RelayHost:           cctx.String("atp-relay-host"),
-			BskyHost:            cctx.String("atp-bsky-host"),
-			OzoneHost:           cctx.String("atp-ozone-host"),
-			OzoneDID:            cctx.String("ozone-did"),
-			OzoneAdminToken:     cctx.String("ozone-admin-token"),
-			PDSHost:             cctx.String("atp-pds-host"),
-			PDSAdminToken:       cctx.String("pds-admin-token"),
-			SetsFileJSON:        cctx.String("sets-json-path"),
-			RedisURL:            cctx.String("redis-url"),
-			HiveAPIToken:        cctx.String("hiveai-api-token"),
-			AbyssHost:           cctx.String("abyss-host"),
-			AbyssPassword:       cctx.String("abyss-password"),
-			RatelimitBypass:     cctx.String("ratelimit-bypass"),
-			RulesetName:         cctx.String("ruleset"),
-			FirehoseParallelism: cctx.Int("firehose-parallelism"),
-			PreScreenHost:       cctx.String("prescreen-host"),
-			PreScreenToken:      cctx.String("prescreen-token"),
+			Logger:          logger,
+			BskyHost:        cctx.String("atp-bsky-host"),
+			OzoneHost:       cctx.String("atp-ozone-host"),
+			OzoneDID:        cctx.String("ozone-did"),
+			OzoneAdminToken: cctx.String("ozone-admin-token"),
+			PDSHost:         cctx.String("atp-pds-host"),
+			PDSAdminToken:   cctx.String("pds-admin-token"),
+			SetsFileJSON:    cctx.String("sets-json-path"),
+			RedisURL:        cctx.String("redis-url"),
+			HiveAPIToken:    cctx.String("hiveai-api-token"),
+			AbyssHost:       cctx.String("abyss-host"),
+			AbyssPassword:   cctx.String("abyss-password"),
+			RatelimitBypass: cctx.String("ratelimit-bypass"),
+			RulesetName:     cctx.String("ruleset"),
+			PreScreenHost:   cctx.String("prescreen-host"),
+			PreScreenToken:  cctx.String("prescreen-token"),
 		},
 	)
 }

cmd/hepa/server.go

···
 	"log/slog"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/bluesky-social/indigo/atproto/identity"
···
 	"github.com/bluesky-social/indigo/automod"
 	"github.com/bluesky-social/indigo/automod/cachestore"
 	"github.com/bluesky-social/indigo/automod/countstore"
+	"github.com/bluesky-social/indigo/automod/engine"
 	"github.com/bluesky-social/indigo/automod/flagstore"
 	"github.com/bluesky-social/indigo/automod/rules"
 	"github.com/bluesky-social/indigo/automod/setstore"
···
 	Engine      *automod.Engine
 	RedisClient *redis.Client
 
-	relayHost           string // DEPRECATED
-	firehoseParallelism int    // DEPRECATED
-	logger              *slog.Logger
+	logger *slog.Logger
 }
 
 type Config struct {
 	Logger              *slog.Logger
-	RelayHost           string // DEPRECATED
 	BskyHost            string
 	OzoneHost           string
 	OzoneDID            string
···
 	AbyssPassword       string
 	RulesetName         string
 	RatelimitBypass     string
-	FirehoseParallelism int // DEPRECATED
 	PreScreenHost       string
 	PreScreenToken      string
+	ReportDupePeriod    time.Duration
+	QuotaModReportDay   int
+	QuotaModTakedownDay int
+	QuotaModActionDay   int
 }
 
 func NewServer(dir identity.Directory, config Config) (*Server, error) {
···
 		logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
 			Level: slog.LevelInfo,
 		}))
-	}
-
-	relayws := config.RelayHost
-	if !strings.HasPrefix(relayws, "ws") {
-		return nil, fmt.Errorf("specified relay host must include 'ws://' or 'wss://'")
 	}
 
 	var ozoneClient *xrpc.Client
···
 		bskyClient.Headers["x-ratelimit-bypass"] = config.RatelimitBypass
 	}
 	blobClient := util.RobustHTTPClient()
-	engine := automod.Engine{
+	eng := automod.Engine{
 		Logger:      logger,
 		Directory:   dir,
 		Counters:    counters,
···
 		OzoneClient: ozoneClient,
 		AdminClient: adminClient,
 		BlobClient:  blobClient,
+		Config: engine.EngineConfig{
+			ReportDupePeriod:    config.ReportDupePeriod,
+			QuotaModReportDay:   config.QuotaModReportDay,
+			QuotaModTakedownDay: config.QuotaModTakedownDay,
+			QuotaModActionDay:   config.QuotaModActionDay,
+		},
 	}
 
 	s := &Server{
-		relayHost:           config.RelayHost,
-		firehoseParallelism: config.FirehoseParallelism,
-		logger:              logger,
-		Engine:              &engine,
-		RedisClient:         rdb,
+		logger:      logger,
+		Engine:      &eng,
+		RedisClient: rdb,
 	}
 
 	return s, nil

cmd/laputa/main.go

···
 	_ "go.uber.org/automaxprocs"
 
 	"github.com/carlmjohnson/versioninfo"
-	logging "github.com/ipfs/go-log"
 	"github.com/urfave/cli/v2"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
···
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"gorm.io/plugin/opentelemetry/tracing"
 )
-
-var log = logging.Logger("laputa")
 
 func main() {
 	run(os.Args)
···
 			}
 		}
 
-		cstore, err := carstore.NewCarStore(csdb, csdir)
+		cstore, err := carstore.NewCarStore(csdb, []string{csdir})
 		if err != nil {
 			return err
 		}

cmd/netsync/main.go

···
 		Handler: mux,
 	}
 
+	state.wg.Add(1)
 	go func() {
-		state.wg.Add(1)
 		defer state.wg.Done()
 		if err := metricsServer.ListenAndServe(); err != http.ErrServerClosed {
 			logger.Error("failed to start metrics server", "err", err)
···
 	}
 
 	// Check for empty queue
+	state.wg.Add(1)
 	go func() {
-		state.wg.Add(1)
 		defer state.wg.Done()
 		t := time.NewTicker(30 * time.Second)
 		for {

cmd/palomar/Dockerfile

···
 #   podman build -f ./cmd/palomar/Dockerfile -t palomar .
 
 ### Compile stage
-FROM golang:1.22-alpine3.19 AS build-env
+FROM golang:1.23-alpine3.20 AS build-env
 RUN apk add --no-cache build-base make git
 
 ADD . /dockerbuild
···
     go build -tags timetzdata -o /palomar ./cmd/palomar
 
 ### Run stage
-FROM alpine:3.19
+FROM alpine:3.20
 
 RUN apk add --no-cache --update dumb-init ca-certificates
 ENTRYPOINT ["dumb-init", "--"]

cmd/rainbow/Dockerfile

···
+FROM golang:1.23-bullseye AS build-env
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+ENV GODEBUG="netdns=go"
+ENV GOOS="linux"
+ENV GOARCH="amd64"
+ENV CGO_ENABLED="1"
+
+WORKDIR /usr/src/rainbow
+
+COPY . .
+
+RUN go mod download && \
+  go mod verify
+
+RUN go build \
+    -v  \
+    -trimpath \
+    -tags timetzdata \
+    -o /rainbow-bin \
+    ./cmd/rainbow
+
+FROM debian:bullseye-slim
+
+ENV DEBIAN_FRONTEND="noninteractive"
+ENV TZ=Etc/UTC
+ENV GODEBUG="netdns=go"
+
+RUN apt-get update && apt-get install --yes \
+  dumb-init \
+  ca-certificates \
+  runit
+
+WORKDIR /rainbow
+COPY --from=build-env /rainbow-bin /usr/bin/rainbow
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["/usr/bin/rainbow"]
+
+LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo
+LABEL org.opencontainers.image.description="rainbow atproto firehose fanout service"
+LABEL org.opencontainers.image.licenses=MIT

cmd/rainbow/README.md

···
+
+`rainbow`: atproto Firehose Fanout Service
+==========================================
+
+This is an atproto service which consumes from a firehose (eg, from a relay or PDS) and fans out events to many subscribers.
+
+Features and design points:
+
+- retains "backfill window" on local disk (using [pebble](https://github.com/cockroachdb/pebble))
+- serves the `com.atproto.sync.subscribeRepos` endpoint (WebSocket)
+- retains upstream firehose "sequence numbers"
+- does not validate events (signatures, repo tree, hashes, etc), just passes through
+- does not archive or mirror individual records or entire repositories (or implement related API endpoints)
+- disk I/O intensive: fast NVMe disks are recommended, and RAM is helpful for caching
+- single golang binary for easy deployment
+- observability: logging, prometheus metrics, OTEL traces
+
+## Running 
+
+This is a simple, single-binary Go program. You can also build and run it as a docker container (see `./Dockerfile`).
+
+From the top level of this repo, you can build:
+
+```shell
+go build ./cmd/rainbow -o rainbow-bin
+```
+
+or just run it, and see configuration options:
+
+```shell
+go run ./cmd/rainbow --help
+```

cmd/rainbow/main.go

···
+package main
+
+import (
+	"context"
+	"log/slog"
+	_ "net/http/pprof"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/bluesky-social/indigo/events"
+	"github.com/bluesky-social/indigo/splitter"
+
+	"github.com/carlmjohnson/versioninfo"
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/urfave/cli/v2"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/sdk/resource"
+	tracesdk "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	_ "go.uber.org/automaxprocs"
+)
+
+var log = slog.Default().With("system", "rainbow")
+
+func init() {
+	// control log level using, eg, GOLOG_LOG_LEVEL=debug
+	//logging.SetAllLoggers(logging.LevelDebug)
+}
+
+func main() {
+	run(os.Args)
+}
+
+func run(args []string) {
+	app := cli.App{
+		Name:    "rainbow",
+		Usage:   "atproto firehose fan-out daemon",
+		Version: versioninfo.Short(),
+	}
+
+	app.Flags = []cli.Flag{
+		// TODO: unimplemented, always assumes https:// and wss://
+		//&cli.BoolFlag{
+		//	Name:    "crawl-insecure-ws",
+		//	Usage:   "when connecting to PDS instances, use ws:// instead of wss://",
+		//	EnvVars: []string{"RAINBOW_INSECURE_CRAWL"},
+		//},
+		&cli.StringFlag{
+			Name:    "splitter-host",
+			Value:   "bsky.network",
+			EnvVars: []string{"ATP_RELAY_HOST", "RAINBOW_RELAY_HOST"},
+		},
+		&cli.StringFlag{
+			Name:    "persist-db",
+			Value:   "./rainbow.db",
+			Usage:   "path to persistence db",
+			EnvVars: []string{"RAINBOW_DB_PATH"},
+		},
+		&cli.StringFlag{
+			Name:    "cursor-file",
+			Value:   "./rainbow-cursor",
+			Usage:   "write upstream cursor number to this file",
+			EnvVars: []string{"RAINBOW_CURSOR_PATH"},
+		},
+		&cli.StringFlag{
+			Name:    "api-listen",
+			Value:   ":2480",
+			EnvVars: []string{"RAINBOW_API_LISTEN"},
+		},
+		&cli.StringFlag{
+			Name:    "metrics-listen",
+			Value:   ":2481",
+			EnvVars: []string{"RAINBOW_METRICS_LISTEN", "SPLITTER_METRICS_LISTEN"},
+		},
+		&cli.Float64Flag{
+			Name:    "persist-hours",
+			Value:   24 * 3,
+			EnvVars: []string{"RAINBOW_PERSIST_HOURS", "SPLITTER_PERSIST_HOURS"},
+			Usage:   "hours to buffer (float, may be fractional)",
+		},
+		&cli.Int64Flag{
+			Name:    "persist-bytes",
+			Value:   0,
+			Usage:   "max bytes target for event cache, 0 to disable size target trimming",
+			EnvVars: []string{"RAINBOW_PERSIST_BYTES", "SPLITTER_PERSIST_BYTES"},
+		},
+		&cli.StringSliceFlag{
+			Name:    "next-crawler",
+			Usage:   "forward POST requestCrawl to this url, should be machine root url and not xrpc/requestCrawl, comma separated list",
+			EnvVars: []string{"RELAY_NEXT_CRAWLER"},
+		},
+	}
+
+	// TODO: slog.SetDefault and set module `var log *slog.Logger` based on flags and env
+
+	app.Action = Splitter
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Error(err.Error())
+		os.Exit(1)
+	}
+}
+
+func Splitter(cctx *cli.Context) error {
+	// Trap SIGINT to trigger a shutdown.
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
+
+	// Enable OTLP HTTP exporter
+	// For relevant environment variables:
+	// https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables
+	// At a minimum, you need to set
+	// OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+	if ep := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); ep != "" {
+		log.Info("setting up trace exporter", "endpoint", ep)
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		exp, err := otlptracehttp.New(ctx)
+		if err != nil {
+			log.Error("failed to create trace exporter", "error", err)
+			os.Exit(1)
+		}
+		defer func() {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+			defer cancel()
+			if err := exp.Shutdown(ctx); err != nil {
+				log.Error("failed to shutdown trace exporter", "error", err)
+			}
+		}()
+
+		tp := tracesdk.NewTracerProvider(
+			tracesdk.WithBatcher(exp),
+			tracesdk.WithResource(resource.NewWithAttributes(
+				semconv.SchemaURL,
+				semconv.ServiceNameKey.String("splitter"),
+				attribute.String("env", os.Getenv("ENVIRONMENT")),         // DataDog
+				attribute.String("environment", os.Getenv("ENVIRONMENT")), // Others
+				attribute.Int64("ID", 1),
+			)),
+		)
+		otel.SetTracerProvider(tp)
+	}
+
+	persistPath := cctx.String("persist-db")
+	upstreamHost := cctx.String("splitter-host")
+	nextCrawlers := cctx.StringSlice("next-crawler")
+
+	var spl *splitter.Splitter
+	var err error
+	if persistPath != "" {
+		log.Info("building splitter with storage at", "path", persistPath)
+		ppopts := events.PebblePersistOptions{
+			DbPath:          persistPath,
+			PersistDuration: time.Duration(float64(time.Hour) * cctx.Float64("persist-hours")),
+			GCPeriod:        5 * time.Minute,
+			MaxBytes:        uint64(cctx.Int64("persist-bytes")),
+		}
+		conf := splitter.SplitterConfig{
+			UpstreamHost:  upstreamHost,
+			CursorFile:    cctx.String("cursor-file"),
+			PebbleOptions: &ppopts,
+		}
+		spl, err = splitter.NewSplitter(conf, nextCrawlers)
+	} else {
+		log.Info("building in-memory splitter")
+		conf := splitter.SplitterConfig{
+			UpstreamHost: upstreamHost,
+			CursorFile:   cctx.String("cursor-file"),
+		}
+		spl, err = splitter.NewSplitter(conf, nextCrawlers)
+	}
+	if err != nil {
+		log.Error("failed to create splitter", "path", persistPath, "error", err)
+		os.Exit(1)
+		return err
+	}
+
+	// set up metrics endpoint
+	go func() {
+		if err := spl.StartMetrics(cctx.String("metrics-listen")); err != nil {
+			log.Error("failed to start metrics endpoint", "err", err)
+			os.Exit(1)
+		}
+	}()
+
+	runErr := make(chan error, 1)
+
+	go func() {
+		err := spl.Start(cctx.String("api-listen"))
+		runErr <- err
+	}()
+
+	log.Info("startup complete")
+	select {
+	case <-signals:
+		log.Info("received shutdown signal")
+		if err := spl.Shutdown(); err != nil {
+			log.Error("error during Splitter shutdown", "err", err)
+		}
+	case err := <-runErr:
+		if err != nil {
+			log.Error("error during Splitter startup", "err", err)
+		}
+		log.Info("shutting down")
+		if err := spl.Shutdown(); err != nil {
+			log.Error("error during Splitter shutdown", "err", err)
+		}
+	}
+
+	log.Info("shutdown complete")
+
+	return nil
+}

cmd/sonar/Dockerfile

···
 # Stage 1: Build the Go binary
-FROM golang:1.22-alpine3.19 AS builder
+FROM golang:1.23-alpine3.20 AS builder
 
 # Install SSL ca certificates.
 RUN apk update && apk add --no-cache ca-certificates && update-ca-certificates

cmd/sonar/main.go

···
 	pool := sequential.NewScheduler(u.Host, s.HandleStreamEvent)
 
 	// Start a goroutine to manage the cursor file, saving the current cursor every 5 seconds.
+	wg.Add(1)
 	go func() {
-		wg.Add(1)
 		defer wg.Done()
 		ticker := time.NewTicker(5 * time.Second)
 		logger := logger.With("source", "cursor_file_manager")
···
 	}()
 
 	// Start a goroutine to manage the liveness checker, shutting down if no events are received for 15 seconds
+	wg.Add(1)
 	go func() {
-		wg.Add(1)
 		defer wg.Done()
 		ticker := time.NewTicker(15 * time.Second)
 		lastSeq := int64(0)
···
 	}
 
 	// Startup metrics server
+	wg.Add(1)
 	go func() {
-		wg.Add(1)
 		defer wg.Done()
 		logger = logger.With("source", "metrics_server")
 
···
 	}
 	defer c.Close()
 
+	wg.Add(1)
 	go func() {
-		wg.Add(1)
 		defer wg.Done()
-		err = events.HandleRepoStream(ctx, c, pool)
+		err = events.HandleRepoStream(ctx, c, pool, logger)
 		logger.Info("HandleRepoStream returned unexpectedly", "err", err)
 		cancel()
 	}()

cmd/stress/main.go

···
 	_ "github.com/joho/godotenv/autoload"
 
 	"github.com/carlmjohnson/versioninfo"
-	logging "github.com/ipfs/go-log"
 	"github.com/ipld/go-car"
 	cli "github.com/urfave/cli/v2"
 )
-
-var log = logging.Logger("stress")
 
 func main() {
 	run(os.Args)

cmd/supercollider/Dockerfile

···
 # Stage 1: Build the Go binary
-FROM golang:1.22-alpine3.18 AS builder
+FROM golang:1.23-alpine3.20 AS builder
 
 # Create a directory for the application
 WORKDIR /app

cmd/supercollider/main.go

···
 		return nil, nil, err
 	}
 
-	cs, err := carstore.NewCarStore(cardb, cspath)
+	cs, err := carstore.NewCarStore(cardb, []string{cspath})
 	if err != nil {
 		return nil, nil, err
 	}

did/metrics.go

···
 	Name: "multiresolver_resolved_dids_total",
 	Help: "Total number of DIDs resolved",
 }, []string{"resolver"})
+
+var mrResolveDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "indigo_multiresolver_resolve_duration_seconds",
+	Help:    "A histogram of resolve latencies",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+}, []string{"resolver"})

did/multi.go

···
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/whyrusleeping/go-did"
 )
···
 }
 
 func (mr *MultiResolver) GetDocument(ctx context.Context, didstr string) (*did.Document, error) {
+	s := time.Now()
+
 	pdid, err := did.ParseDID(didstr)
 	if err != nil {
 		return nil, err
 	}
 
 	method := pdid.Protocol()
+	defer func() {
+		mrResolveDuration.WithLabelValues(method).Observe(time.Since(s).Seconds())
+	}()
 
 	res, ok := mr.handlers[method]
 	if !ok {

events/consumer.go

···
 	"context"
 	"fmt"
 	"io"
+	"log/slog"
 	"net"
 	"time"
 
···
 	return n, err
 }
 
-func HandleRepoStream(ctx context.Context, con *websocket.Conn, sched Scheduler) error {
+// HandleRepoStream
+// con is source of events
+// sched gets AddWork for each event
+// log may be nil for default logger
+func HandleRepoStream(ctx context.Context, con *websocket.Conn, sched Scheduler, log *slog.Logger) error {
+	if log == nil {
+		log = slog.Default().With("system", "events")
+	}
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	defer sched.Shutdown()
···
 			select {
 			case <-t.C:
 				if err := con.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second*10)); err != nil {
-					log.Warnf("failed to ping: %s", err)
+					log.Warn("failed to ping", "err", err)
 				}
 			case <-ctx.Done():
 				con.Close()
···
 
 	con.SetPongHandler(func(_ string) error {
 		if err := con.SetReadDeadline(time.Now().Add(time.Minute)); err != nil {
-			log.Errorf("failed to set read deadline: %s", err)
+			log.Error("failed to set read deadline", "err", err)
 		}
 
 		return nil
···
 				}
 
 				if evt.Seq < lastSeq {
-					log.Errorf("Got events out of order from stream (seq = %d, prev = %d)", evt.Seq, lastSeq)
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
 				}
 
 				lastSeq = evt.Seq
···
 				}
 
 				if evt.Seq < lastSeq {
-					log.Errorf("Got events out of order from stream (seq = %d, prev = %d)", evt.Seq, lastSeq)
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
 				}
 				lastSeq = evt.Seq
 
···
 				}
 
 				if evt.Seq < lastSeq {
-					log.Errorf("Got events out of order from stream (seq = %d, prev = %d)", evt.Seq, lastSeq)
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
 				}
 				lastSeq = evt.Seq
 
···
 				}
 
 				if evt.Seq < lastSeq {
-					log.Errorf("Got events out of order from stream (seq = %d, prev = %d)", evt.Seq, lastSeq)
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
 				}
 				lastSeq = evt.Seq
 
···
 				}
 
 				if evt.Seq < lastSeq {
-					log.Errorf("Got events out of order from stream (seq = %d, prev = %d)", evt.Seq, lastSeq)
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
 				}
 				lastSeq = evt.Seq
 
···
 				}
 
 				if evt.Seq < lastSeq {
-					log.Errorf("Got events out of order from stream (seq = %d, prev = %d)", evt.Seq, lastSeq)
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
 				}
 				lastSeq = evt.Seq
 
···
 				}
 
 				if evt.Seq < lastSeq {
-					log.Errorf("Got events out of order from stream (seq = %d, prev = %d)", evt.Seq, lastSeq)
+					log.Error("Got events out of order from stream", "seq", evt.Seq, "prev", lastSeq)
 				}
 
 				lastSeq = evt.Seq

events/dbpersist.go

···
 
 		if needsFlush {
 			if err := p.Flush(context.Background()); err != nil {
-				log.Errorf("failed to flush batch: %s", err)
+				log.Error("failed to flush batch", "err", err)
 			}
 		}
 	}
···
 func (p *DbPersistence) RecordFromRepoCommit(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) (*RepoEventRecord, error) {
 	// TODO: hack hack hack
 	if len(evt.Ops) > 8192 {
-		log.Errorf("(VERY BAD) truncating ops field in outgoing event (len = %d)", len(evt.Ops))
+		log.Error("(VERY BAD) truncating ops field in outgoing event", "len", len(evt.Ops))
 		evt.Ops = evt.Ops[:8192]
 	}
 

events/dbpersist_test.go

···
-package events_test
+package events
 
 import (
 	"context"
···
 	atproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/carstore"
-	"github.com/bluesky-social/indigo/events"
 	lexutil "github.com/bluesky-social/indigo/lex/util"
 	"github.com/bluesky-social/indigo/models"
-	"github.com/bluesky-social/indigo/pds"
+	pds "github.com/bluesky-social/indigo/pds/data"
 	"github.com/bluesky-social/indigo/repomgr"
 	"github.com/bluesky-social/indigo/util"
-	"github.com/ipfs/go-log/v2"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )
 
-func init() {
-	log.SetAllLoggers(log.LevelDebug)
-}
-
 func BenchmarkDBPersist(b *testing.B) {
 	ctx := context.Background()
 
···
 	defer os.RemoveAll(tempPath)
 
 	// Initialize a DBPersister
-	dbp, err := events.NewDbPersistence(db, cs, nil)
+	dbp, err := NewDbPersistence(db, cs, nil)
 	if err != nil {
 		b.Fatal(err)
 	}
 
 	// Create a bunch of events
-	evtman := events.NewEventManager(dbp)
+	evtman := NewEventManager(dbp)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	inEvts := make([]*events.XRPCStreamEvent, b.N)
+	inEvts := make([]*XRPCStreamEvent, b.N)
 	for i := 0; i < b.N; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &events.XRPCStreamEvent{
+		inEvts[i] = &XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 
 	b.StopTimer()
 
-	dbp.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
+	dbp.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
 		outEvtCount++
 		return nil
 	})
···
 	defer os.RemoveAll(tempPath)
 
 	// Initialize a DBPersister
-	dbp, err := events.NewDbPersistence(db, cs, nil)
+	dbp, err := NewDbPersistence(db, cs, nil)
 	if err != nil {
 		b.Fatal(err)
 	}
 
 	// Create a bunch of events
-	evtman := events.NewEventManager(dbp)
+	evtman := NewEventManager(dbp)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	inEvts := make([]*events.XRPCStreamEvent, n)
+	inEvts := make([]*XRPCStreamEvent, n)
 	for i := 0; i < n; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &events.XRPCStreamEvent{
+		inEvts[i] = &XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 
 	b.ResetTimer()
 
-	dbp.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
+	dbp.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
 		outEvtCount++
 		return nil
 	})
···
 		return nil, nil, nil, "", err
 	}
 
-	cs, err := carstore.NewCarStore(cardb, cspath)
+	cs, err := carstore.NewCarStore(cardb, []string{cspath})
 	if err != nil {
 		return nil, nil, nil, "", err
 	}

events/diskpersist.go

···
 			dp.lk.Lock()
 			if err := dp.flushLog(ctx); err != nil {
 				// TODO: this happening is quite bad. Need a recovery strategy
-				log.Errorf("failed to flush disk log: %s", err)
+				log.Error("failed to flush disk log", "err", err)
 			}
 			dp.lk.Unlock()
 		}
···
 		case <-t.C:
 			if errs := dp.garbageCollect(ctx); len(errs) > 0 {
 				for _, err := range errs {
-					log.Errorf("garbage collection error: %s", err)
+					log.Error("garbage collection error", "err", err)
 				}
 			}
 		}
···
 	refsGarbageCollected.WithLabelValues().Add(float64(refsDeleted))
 	filesGarbageCollected.WithLabelValues().Add(float64(filesDeleted))
 
-	log.Infow("garbage collection complete",
+	log.Info("garbage collection complete",
 		"filesDeleted", filesDeleted,
 		"refsDeleted", refsDeleted,
 		"oldRefsFound", oldRefsFound,
···
 			return nil, err
 		}
 		if since > lastSeq {
-			log.Errorw("playback cursor is greater than last seq of file checked",
+			log.Error("playback cursor is greater than last seq of file checked",
 				"since", since,
 				"lastSeq", lastSeq,
 				"filename", fn,
···
 				return nil, err
 			}
 		default:
-			log.Warnw("unrecognized event kind coming from log file", "seq", h.Seq, "kind", h.Kind)
+			log.Warn("unrecognized event kind coming from log file", "seq", h.Seq, "kind", h.Kind)
 			return nil, fmt.Errorf("halting on unrecognized event kind")
 		}
 	}

events/diskpersist_test.go

···
-package events_test
+package events
 
 import (
 	"context"
···
 	atproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/carstore"
-	"github.com/bluesky-social/indigo/events"
 	lexutil "github.com/bluesky-social/indigo/lex/util"
 	"github.com/bluesky-social/indigo/models"
-	"github.com/bluesky-social/indigo/pds"
+	pds "github.com/bluesky-social/indigo/pds/data"
 	"github.com/bluesky-social/indigo/repomgr"
 	"github.com/bluesky-social/indigo/util"
 	"gorm.io/gorm"
 )
 
-func TestDiskPersist(t *testing.T) {
+func testPersister(t *testing.T, perisistenceFactory func(path string, db *gorm.DB) (EventPersistence, error)) {
 	ctx := context.Background()
 
 	db, _, cs, tempPath, err := setupDBs(t)
···
 
 	defer os.RemoveAll(tempPath)
 
-	// Initialize a DBPersister
-
-	dp, err := events.NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &events.DiskPersistOptions{
-		EventsPerFile: 10,
-		UIDCacheSize:  100000,
-		DIDCacheSize:  100000,
-	})
+	// Initialize a persister
+	dp, err := perisistenceFactory(tempPath, db)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Create a bunch of events
-	evtman := events.NewEventManager(dp)
+	evtman := NewEventManager(dp)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
···
 	}
 
 	n := 100
-	inEvts := make([]*events.XRPCStreamEvent, n)
+	inEvts := make([]*XRPCStreamEvent, n)
 	for i := 0; i < n; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &events.XRPCStreamEvent{
+		inEvts[i] = &XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 					},
 				},
 				Time: time.Now().Format(util.ISO8601),
+				Seq:  int64(i),
 			},
 		}
 	}
···
 	outEvtCount := 0
 	expectedEvtCount := n
 
-	dp.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
+	dp.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
 		outEvtCount++
 		return nil
 	})
···
 
 	time.Sleep(time.Millisecond * 100)
 
-	dp2, err := events.NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &events.DiskPersistOptions{
+	dp2, err := NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &DiskPersistOptions{
 		EventsPerFile: 10,
 		UIDCacheSize:  100000,
 		DIDCacheSize:  100000,
···
 		t.Fatal(err)
 	}
 
-	evtman2 := events.NewEventManager(dp2)
+	evtman2 := NewEventManager(dp2)
 
-	inEvts = make([]*events.XRPCStreamEvent, n)
+	inEvts = make([]*XRPCStreamEvent, n)
 	for i := 0; i < n; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &events.XRPCStreamEvent{
+		inEvts[i] = &XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 		}
 	}
 }
+func TestDiskPersist(t *testing.T) {
+	factory := func(tempPath string, db *gorm.DB) (EventPersistence, error) {
+		return NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &DiskPersistOptions{
+			EventsPerFile: 10,
+			UIDCacheSize:  100000,
+			DIDCacheSize:  100000,
+		})
+	}
+	testPersister(t, factory)
+}
 
 func BenchmarkDiskPersist(b *testing.B) {
 	db, _, cs, tempPath, err := setupDBs(b)
···
 
 	// Initialize a DBPersister
 
-	dp, err := events.NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &events.DiskPersistOptions{
+	dp, err := NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &DiskPersistOptions{
 		EventsPerFile: 5000,
 		UIDCacheSize:  100000,
 		DIDCacheSize:  100000,
···
 
 }
 
-func runPersisterBenchmark(b *testing.B, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) {
+func runPersisterBenchmark(b *testing.B, cs carstore.CarStore, db *gorm.DB, p EventPersistence) {
 	ctx := context.Background()
 
 	db.AutoMigrate(&pds.User{})
···
 	}
 
 	// Create a bunch of events
-	evtman := events.NewEventManager(p)
+	evtman := NewEventManager(p)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	inEvts := make([]*events.XRPCStreamEvent, b.N)
+	inEvts := make([]*XRPCStreamEvent, b.N)
 	for i := 0; i < b.N; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &events.XRPCStreamEvent{
+		inEvts[i] = &XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 
 	// Initialize a DBPersister
 
-	dp, err := events.NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &events.DiskPersistOptions{
+	dp, err := NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &DiskPersistOptions{
 		EventsPerFile: 20,
 		UIDCacheSize:  100000,
 		DIDCacheSize:  100000,
···
 	runEventManagerTest(t, cs, db, dp)
 }
 
-func runEventManagerTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) {
+func runEventManagerTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p EventPersistence) {
 	ctx := context.Background()
 
 	db.AutoMigrate(&pds.User{})
···
 		t.Fatal(err)
 	}
 
-	evtman := events.NewEventManager(p)
+	evtman := NewEventManager(p)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
···
 	}
 
 	testSize := 100 // you can adjust this number as needed
-	inEvts := make([]*events.XRPCStreamEvent, testSize)
+	inEvts := make([]*XRPCStreamEvent, testSize)
 	for i := 0; i < testSize; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &events.XRPCStreamEvent{
+		inEvts[i] = &XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 	}
 
 	outEvtCount := 0
-	p.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
+	p.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
 		// Check that the contents of the output events match the input events
 		// Clear cache, don't care if one has it and not the other
 		inEvts[outEvtCount].Preserialized = nil
···
 
 	// Initialize a DBPersister
 
-	dp, err := events.NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &events.DiskPersistOptions{
+	dp, err := NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &DiskPersistOptions{
 		EventsPerFile: 10,
 		UIDCacheSize:  100000,
 		DIDCacheSize:  100000,
···
 	runTakedownTest(t, cs, db, dp)
 }
 
-func runTakedownTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) {
+func runTakedownTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p EventPersistence) {
 	ctx := context.TODO()
 
 	db.AutoMigrate(&pds.User{})
···
 		}
 	}
 
-	evtman := events.NewEventManager(p)
+	evtman := NewEventManager(p)
 
 	testSize := 100 // you can adjust this number as needed
-	inEvts := make([]*events.XRPCStreamEvent, testSize*userCount)
+	inEvts := make([]*XRPCStreamEvent, testSize*userCount)
 	for i := 0; i < testSize*userCount; i++ {
 		user := users[i%userCount]
 		_, cid, err := mgr.CreateRecord(ctx, user.Uid, "app.bsky.feed.post", &bsky.FeedPost{
···
 
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &events.XRPCStreamEvent{
+		inEvts[i] = &XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   user.Did,
 				Commit: headLink,
···
 
 	// Verify that the events of the user have been removed from the event stream
 	var evtsCount int
-	if err := p.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
+	if err := p.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
 		evtsCount++
 		if evt.RepoCommit.Repo == takeDownUser.Did {
 			t.Fatalf("found event for user %d after takedown", takeDownUser.Uid)

events/events.go

···
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"sync"
 	"time"
 
···
 	"github.com/bluesky-social/indigo/models"
 	"github.com/prometheus/client_golang/prometheus"
 
-	logging "github.com/ipfs/go-log"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"go.opentelemetry.io/otel"
 )
 
-var log = logging.Logger("events")
+var log = slog.Default().With("system", "events")
 
 type Scheduler interface {
 	AddWork(ctx context.Context, repo string, val *XRPCStreamEvent) error
···
 	crossoverBufferSize int
 
 	persister EventPersistence
+
+	log *slog.Logger
 }
 
 func NewEventManager(persister EventPersistence) *EventManager {
···
 		bufferSize:          16 << 10,
 		crossoverBufferSize: 512,
 		persister:           persister,
+		log:                 slog.Default().With("system", "events"),
 	}
 
 	persister.SetEventBroadcaster(em.broadcastEvent)
···
 func (em *EventManager) broadcastEvent(evt *XRPCStreamEvent) {
 	// the main thing we do is send it out, so MarshalCBOR once
 	if err := evt.Preserialize(); err != nil {
-		log.Errorf("broadcast serialize failed, %s", err)
+		em.log.Error("broadcast serialize failed", "err", err)
 		// serialize isn't going to go better later, this event is cursed
 		return
 	}
···
 				// code
 				s.filter = func(*XRPCStreamEvent) bool { return false }
 
-				log.Warnw("dropping slow consumer due to event overflow", "bufferSize", len(s.outgoing), "ident", s.ident)
+				em.log.Warn("dropping slow consumer due to event overflow", "bufferSize", len(s.outgoing), "ident", s.ident)
 				go func(torem *Subscriber) {
 					torem.lk.Lock()
 					if !torem.cleanedUp {
···
 							},
 						}:
 						case <-time.After(time.Second * 5):
-							log.Warnw("failed to send error frame to backed up consumer", "ident", torem.ident)
+							em.log.Warn("failed to send error frame to backed up consumer", "ident", torem.ident)
 						}
 					}
 					torem.lk.Unlock()
···
 	// accept a uid. The lookup inside the persister is notably expensive (despite
 	// being an lru cache?)
 	if err := em.persister.Persist(ctx, evt); err != nil {
-		log.Errorf("failed to persist outbound event: %s", err)
+		em.log.Error("failed to persist outbound event", "err", err)
 	}
 }
 
···
 	return obj.MarshalCBOR(cborWriter)
 }
 
+func (xevt *XRPCStreamEvent) Deserialize(r io.Reader) error {
+	var header EventHeader
+	if err := header.UnmarshalCBOR(r); err != nil {
+		return fmt.Errorf("reading header: %w", err)
+	}
+	switch header.Op {
+	case EvtKindMessage:
+		switch header.MsgType {
+		case "#commit":
+			var evt comatproto.SyncSubscribeRepos_Commit
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return fmt.Errorf("reading repoCommit event: %w", err)
+			}
+			xevt.RepoCommit = &evt
+		case "#handle":
+			var evt comatproto.SyncSubscribeRepos_Handle
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoHandle = &evt
+		case "#identity":
+			var evt comatproto.SyncSubscribeRepos_Identity
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoIdentity = &evt
+		case "#account":
+			var evt comatproto.SyncSubscribeRepos_Account
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoAccount = &evt
+		case "#info":
+			// TODO: this might also be a LabelInfo (as opposed to RepoInfo)
+			var evt comatproto.SyncSubscribeRepos_Info
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoInfo = &evt
+		case "#migrate":
+			var evt comatproto.SyncSubscribeRepos_Migrate
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoMigrate = &evt
+		case "#tombstone":
+			var evt comatproto.SyncSubscribeRepos_Tombstone
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return err
+			}
+			xevt.RepoTombstone = &evt
+		case "#labels":
+			var evt comatproto.LabelSubscribeLabels_Labels
+			if err := evt.UnmarshalCBOR(r); err != nil {
+				return fmt.Errorf("reading Labels event: %w", err)
+			}
+			xevt.LabelLabels = &evt
+		}
+	case EvtKindErrorFrame:
+		var errframe ErrorFrame
+		if err := errframe.UnmarshalCBOR(r); err != nil {
+			return err
+		}
+		xevt.Error = &errframe
+	default:
+		return fmt.Errorf("unrecognized event stream type: %d", header.Op)
+	}
+	return nil
+}
+
+var ErrNoSeq = errors.New("event has no sequence number")
+
 // serialize content into Preserialized cache
 func (evt *XRPCStreamEvent) Preserialize() error {
 	if evt.Preserialized != nil {
···
 			case <-done:
 				return ErrPlaybackShutdown
 			case out <- e:
-				seq := sequenceForEvent(e)
+				seq := SequenceForEvent(e)
 				if seq > 0 {
 					lastSeq = seq
 				}
···
 			}
 		}); err != nil {
 			if errors.Is(err, ErrPlaybackShutdown) {
-				log.Warnf("events playback: %s", err)
+				em.log.Warn("events playback", "err", err)
 			} else {
-				log.Errorf("events playback: %s", err)
+				em.log.Error("events playback", "err", err)
 			}
 
 			// TODO: send an error frame or something?
···
 
 		// run playback again to get us to the events that have started buffering
 		if err := em.persister.Playback(ctx, lastSeq, func(e *XRPCStreamEvent) error {
-			seq := sequenceForEvent(e)
-			if seq > sequenceForEvent(first) {
+			seq := SequenceForEvent(e)
+			if seq > SequenceForEvent(first) {
 				return ErrCaughtUp
 			}
 
···
 			}
 		}); err != nil {
 			if !errors.Is(err, ErrCaughtUp) {
-				log.Errorf("events playback: %s", err)
+				em.log.Error("events playback", "err", err)
 
 				// TODO: send an error frame or something?
 				close(out)
···
 	return out, sub.cleanup, nil
 }
 
-func sequenceForEvent(evt *XRPCStreamEvent) int64 {
+func SequenceForEvent(evt *XRPCStreamEvent) int64 {
+	return evt.Sequence()
+}
+
+func (evt *XRPCStreamEvent) Sequence() int64 {
 	switch {
 	case evt == nil:
 		return -1
···
 		return evt.RepoTombstone.Seq
 	case evt.RepoIdentity != nil:
 		return evt.RepoIdentity.Seq
+	case evt.RepoAccount != nil:
+		return evt.RepoAccount.Seq
 	case evt.RepoInfo != nil:
 		return -1
 	case evt.Error != nil:

events/pebblepersist.go

···
+package events
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/bluesky-social/indigo/models"
+	"github.com/cockroachdb/pebble"
+)
+
+type PebblePersist struct {
+	broadcast func(*XRPCStreamEvent)
+	db        *pebble.DB
+
+	prevSeq      int64
+	prevSeqExtra uint32
+
+	cancel func()
+
+	options PebblePersistOptions
+}
+
+type PebblePersistOptions struct {
+	// path where pebble will create a directory full of files
+	DbPath string
+
+	// Throw away posts older than some time ago
+	PersistDuration time.Duration
+
+	// Throw away old posts every so often
+	GCPeriod time.Duration
+
+	// MaxBytes is what we _try_ to keep disk usage under
+	MaxBytes uint64
+}
+
+var DefaultPebblePersistOptions = PebblePersistOptions{
+	PersistDuration: time.Minute * 20,
+	GCPeriod:        time.Minute * 5,
+	MaxBytes:        1024 * 1024 * 1024, // 1 GiB
+}
+
+// Create a new EventPersistence which stores data in pebbledb
+// nil opts is ok
+func NewPebblePersistance(opts *PebblePersistOptions) (*PebblePersist, error) {
+	if opts == nil {
+		opts = &DefaultPebblePersistOptions
+	}
+	db, err := pebble.Open(opts.DbPath, &pebble.Options{})
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", opts.DbPath, err)
+	}
+	pp := new(PebblePersist)
+	pp.options = *opts
+	pp.db = db
+	return pp, nil
+}
+
+func setKeySeqMillis(key []byte, seq, millis int64) {
+	binary.BigEndian.PutUint64(key[:8], uint64(seq))
+	binary.BigEndian.PutUint64(key[8:16], uint64(millis))
+}
+
+func (pp *PebblePersist) Persist(ctx context.Context, e *XRPCStreamEvent) error {
+	err := e.Preserialize()
+	if err != nil {
+		return err
+	}
+	blob := e.Preserialized
+
+	seq := e.Sequence()
+	nowMillis := time.Now().UnixMilli()
+
+	if seq < 0 {
+		// persist with longer key {prev 8 byte key}{time}{int32 extra counter}
+		pp.prevSeqExtra++
+		var key [20]byte
+		setKeySeqMillis(key[:], seq, nowMillis)
+		binary.BigEndian.PutUint32(key[16:], pp.prevSeqExtra)
+
+		err = pp.db.Set(key[:], blob, pebble.Sync)
+	} else {
+		pp.prevSeq = seq
+		pp.prevSeqExtra = 0
+		var key [16]byte
+		setKeySeqMillis(key[:], seq, nowMillis)
+
+		err = pp.db.Set(key[:], blob, pebble.Sync)
+	}
+
+	if err != nil {
+		return err
+	}
+	pp.broadcast(e)
+
+	return err
+}
+
+func eventFromPebbleIter(iter *pebble.Iterator) (*XRPCStreamEvent, error) {
+	blob, err := iter.ValueAndErr()
+	if err != nil {
+		return nil, err
+	}
+	br := bytes.NewReader(blob)
+	evt := new(XRPCStreamEvent)
+	err = evt.Deserialize(br)
+	if err != nil {
+		return nil, err
+	}
+	evt.Preserialized = bytes.Clone(blob)
+	return evt, nil
+}
+
+func (pp *PebblePersist) Playback(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error) error {
+	var key [8]byte
+	binary.BigEndian.PutUint64(key[:], uint64(since))
+
+	iter, err := pp.db.NewIterWithContext(ctx, &pebble.IterOptions{LowerBound: key[:]})
+	if err != nil {
+		return err
+	}
+	defer iter.Close()
+
+	for iter.First(); iter.Valid(); iter.Next() {
+		evt, err := eventFromPebbleIter(iter)
+		if err != nil {
+			return err
+		}
+
+		err = cb(evt)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+func (pp *PebblePersist) TakeDownRepo(ctx context.Context, usr models.Uid) error {
+	// TODO: implement filter on playback to ignore taken-down-repos?
+	return nil
+}
+func (pp *PebblePersist) Flush(context.Context) error {
+	return pp.db.Flush()
+}
+func (pp *PebblePersist) Shutdown(context.Context) error {
+	if pp.cancel != nil {
+		pp.cancel()
+	}
+	err := pp.db.Close()
+	pp.db = nil
+	return err
+}
+
+func (pp *PebblePersist) SetEventBroadcaster(broadcast func(*XRPCStreamEvent)) {
+	pp.broadcast = broadcast
+}
+
+var ErrNoLast = errors.New("no last event")
+
+func (pp *PebblePersist) GetLast(ctx context.Context) (seq, millis int64, evt *XRPCStreamEvent, err error) {
+	iter, err := pp.db.NewIterWithContext(ctx, &pebble.IterOptions{})
+	if err != nil {
+		return 0, 0, nil, err
+	}
+	ok := iter.Last()
+	if !ok {
+		return 0, 0, nil, ErrNoLast
+	}
+	evt, err = eventFromPebbleIter(iter)
+	keyblob := iter.Key()
+	seq = int64(binary.BigEndian.Uint64(keyblob[:8]))
+	millis = int64(binary.BigEndian.Uint64(keyblob[8:16]))
+	return seq, millis, evt, nil
+}
+
+// example;
+// ```
+// pp := NewPebblePersistance("/tmp/foo.pebble")
+// go pp.GCThread(context.Background(), 48 * time.Hour, 5 * time.Minute)
+// ```
+func (pp *PebblePersist) GCThread(ctx context.Context) {
+	ctx, cancel := context.WithCancel(ctx)
+	pp.cancel = cancel
+	ticker := time.NewTicker(pp.options.GCPeriod)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			err := pp.GarbageCollect(ctx)
+			if err != nil {
+				log.Error("GC err", "err", err)
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+var zeroKey [16]byte
+var ffffKey [16]byte
+
+func init() {
+	setKeySeqMillis(zeroKey[:], 0, 0)
+	for i := range ffffKey {
+		ffffKey[i] = 0xff
+	}
+}
+
+func (pp *PebblePersist) GarbageCollect(ctx context.Context) error {
+	nowMillis := time.Now().UnixMilli()
+	expired := nowMillis - pp.options.PersistDuration.Milliseconds()
+	iter, err := pp.db.NewIterWithContext(ctx, &pebble.IterOptions{})
+	if err != nil {
+		return err
+	}
+	defer iter.Close()
+	// scan keys to find last expired, then delete range
+	var seq int64 = int64(-1)
+	var lastKeyTime int64
+	for iter.First(); iter.Valid(); iter.Next() {
+		keyblob := iter.Key()
+
+		keyTime := int64(binary.BigEndian.Uint64(keyblob[8:16]))
+		if keyTime <= expired {
+			lastKeyTime = keyTime
+			seq = int64(binary.BigEndian.Uint64(keyblob[:8]))
+		} else {
+			break
+		}
+	}
+
+	// TODO: use pp.options.MaxBytes
+
+	sizeBefore, _ := pp.db.EstimateDiskUsage(zeroKey[:], ffffKey[:])
+	if seq == -1 {
+		// nothing to delete
+		log.Info("pebble gc nop", "size", sizeBefore)
+		return nil
+	}
+	var key [16]byte
+	setKeySeqMillis(key[:], seq, lastKeyTime)
+	log.Info("pebble gc start", "to", hex.EncodeToString(key[:]))
+	err = pp.db.DeleteRange(zeroKey[:], key[:], pebble.Sync)
+	if err != nil {
+		return err
+	}
+	sizeAfter, _ := pp.db.EstimateDiskUsage(zeroKey[:], ffffKey[:])
+	log.Info("pebble gc", "before", sizeBefore, "after", sizeAfter)
+	start := time.Now()
+	err = pp.db.Compact(zeroKey[:], key[:], true)
+	if err != nil {
+		log.Warn("pebble gc compact", "err", err)
+	}
+	dt := time.Since(start)
+	log.Info("pebble gc compact ok", "dt", dt)
+	return nil
+}

events/pebblepersist_test.go

···
+package events
+
+import (
+	"gorm.io/gorm"
+	"path/filepath"
+	"testing"
+)
+
+func TestPebblePersist(t *testing.T) {
+	factory := func(tempPath string, db *gorm.DB) (EventPersistence, error) {
+		opts := DefaultPebblePersistOptions
+		opts.DbPath = filepath.Join(tempPath, "pebble.db")
+		return NewPebblePersistance(&opts)
+	}
+	testPersister(t, factory)
+}

events/schedulers/autoscaling/autoscaling.go

···
 
 import (
 	"context"
+	"log/slog"
 	"sync"
 	"time"
 
 	"github.com/bluesky-social/indigo/events"
 	"github.com/bluesky-social/indigo/events/schedulers"
-	logging "github.com/ipfs/go-log"
 	"github.com/prometheus/client_golang/prometheus"
 )
-
-var log = logging.Logger("autoscaling-scheduler")
 
 // Scheduler is a scheduler that will scale up and down the number of workers based on the throughput of the workers.
 type Scheduler struct {
···
 	autoscaleFrequency time.Duration
 	autoscalerIn       chan struct{}
 	autoscalerOut      chan struct{}
+
+	log *slog.Logger
 }
 
 type AutoscaleSettings struct {
···
 		autoscaleFrequency: autoscaleSettings.AutoscaleFrequency,
 		autoscalerIn:       make(chan struct{}),
 		autoscalerOut:      make(chan struct{}),
+
+		log: slog.Default().With("system", "autoscaling-scheduler"),
 	}
 
 	for i := 0; i < p.concurrency; i++ {
···
 }
 
 func (p *Scheduler) Shutdown() {
-	log.Debugf("shutting down autoscaling scheduler for %s", p.ident)
+	p.log.Debug("shutting down autoscaling scheduler", "ident", p.ident)
 
 	// stop autoscaling
 	p.autoscalerIn <- struct{}{}
 	close(p.autoscalerIn)
 	<-p.autoscalerOut
 
-	log.Debug("stopping autoscaling scheduler workers")
+	p.log.Debug("stopping autoscaling scheduler workers")
 	// stop workers
 	for i := 0; i < p.concurrency; i++ {
 		p.feeder <- &consumerTask{signal: "stop"}
 	}
 	close(p.feeder)
 
-	log.Debug("waiting for autoscaling scheduler workers to stop")
+	p.log.Debug("waiting for autoscaling scheduler workers to stop")
 
 	p.workerGroup.Wait()
 
-	log.Debug("stopping autoscaling scheduler throughput manager")
+	p.log.Debug("stopping autoscaling scheduler throughput manager")
 	p.throughputManager.Stop()
 
-	log.Debug("autoscaling scheduler shutdown complete")
+	p.log.Debug("autoscaling scheduler shutdown complete")
 }
 
 // Add autoscaling function
···
 }
 
 func (p *Scheduler) worker() {
-	log.Debugf("starting autoscaling worker for %s", p.ident)
+	p.log.Debug("starting autoscaling worker", "ident", p.ident)
 	p.workersActive.Inc()
 	p.workerGroup.Add(1)
 	defer p.workerGroup.Done()
···
 		for work != nil {
 			// Check if the work item contains a signal to stop the worker.
 			if work.signal == "stop" {
-				log.Debugf("stopping autoscaling worker for %s", p.ident)
+				p.log.Debug("stopping autoscaling worker", "ident", p.ident)
 				p.workersActive.Dec()
 				return
 			}
 
 			p.itemsActive.Inc()
 			if err := p.do(context.TODO(), work.val); err != nil {
-				log.Errorf("event handler failed: %s", err)
+				p.log.Error("event handler failed", "err", err)
 			}
 			p.itemsProcessed.Inc()
 
 			p.lk.Lock()
 			rem, ok := p.active[work.repo]
 			if !ok {
-				log.Errorf("should always have an 'active' entry if a worker is processing a job")
+				p.log.Error("should always have an 'active' entry if a worker is processing a job")
 			}
 
 			if len(rem) == 0 {

events/schedulers/parallel/parallel.go

···
 
 import (
 	"context"
+	"log/slog"
 	"sync"
 
 	"github.com/bluesky-social/indigo/events"
 	"github.com/bluesky-social/indigo/events/schedulers"
-	logging "github.com/ipfs/go-log"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
-
-var log = logging.Logger("parallel-scheduler")
 
 // Scheduler is a parallel scheduler that will run work on a fixed number of workers
 type Scheduler struct {
···
 	itemsProcessed prometheus.Counter
 	itemsActive    prometheus.Counter
 	workesActive   prometheus.Gauge
+
+	log *slog.Logger
 }
 
 func NewScheduler(maxC, maxQ int, ident string, do func(context.Context, *events.XRPCStreamEvent) error) *Scheduler {
···
 		itemsProcessed: schedulers.WorkItemsProcessed.WithLabelValues(ident, "parallel"),
 		itemsActive:    schedulers.WorkItemsActive.WithLabelValues(ident, "parallel"),
 		workesActive:   schedulers.WorkersActive.WithLabelValues(ident, "parallel"),
+
+		log: slog.Default().With("system", "parallel-scheduler"),
 	}
 
 	for i := 0; i < maxC; i++ {
···
 }
 
 func (p *Scheduler) Shutdown() {
-	log.Infof("shutting down parallel scheduler for %s", p.ident)
+	p.log.Info("shutting down parallel scheduler", "ident", p.ident)
 
 	for i := 0; i < p.maxConcurrency; i++ {
 		p.feeder <- &consumerTask{
···
 		<-p.out
 	}
 
-	log.Info("parallel scheduler shutdown complete")
+	p.log.Info("parallel scheduler shutdown complete")
 }
 
 type consumerTask struct {
···
 
 			p.itemsActive.Inc()
 			if err := p.do(context.TODO(), work.val); err != nil {
-				log.Errorf("event handler failed: %s", err)
+				p.log.Error("event handler failed", "err", err)
 			}
 			p.itemsProcessed.Inc()
 
 			p.lk.Lock()
 			rem, ok := p.active[work.repo]
 			if !ok {
-				log.Errorf("should always have an 'active' entry if a worker is processing a job")
+				p.log.Error("should always have an 'active' entry if a worker is processing a job")
 			}
 
 			if len(rem) == 0 {

events/schedulers/sequential/sequential.go

···
 
 import (
 	"context"
-
 	"github.com/bluesky-social/indigo/events"
 	"github.com/bluesky-social/indigo/events/schedulers"
-	logging "github.com/ipfs/go-log"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-var log = logging.Logger("sequential-scheduler")
+// var log = slog.Default().With("system", "sequential-scheduler")
 
 // Scheduler is a sequential scheduler that will run work on a single worker
 type Scheduler struct {

fakedata/accounts.go

···
 
 func (ac *AccountCatalog) Combined() []AccountContext {
 	var combined []AccountContext
-	for _, c := range ac.Celebs {
-		combined = append(combined, c)
-	}
-	for _, r := range ac.Regulars {
-		combined = append(combined, r)
-	}
+	combined = append(combined, ac.Celebs...)
+	combined = append(combined, ac.Regulars...)
 	return combined
 }
 
···
 			return nil, fmt.Errorf("account index didn't match: %d != %d (%s)", i, u.Index, u.AccountType)
 		}
 	}
-	log.Infof("loaded account catalog: regular=%d celebrity=%d", len(catalog.Regulars), len(catalog.Celebs))
+	log.Info("loaded account catalog", "regular", len(catalog.Regulars), "celebrity", len(catalog.Celebs))
 	return catalog, nil
 }
 

fakedata/generators.go

···
 	"bytes"
 	"context"
 	"fmt"
+	"log/slog"
 	"math/rand"
 	"time"
 
···
 	"github.com/bluesky-social/indigo/xrpc"
 
 	"github.com/brianvoe/gofakeit/v6"
-	logging "github.com/ipfs/go-log"
 )
 
-var log = logging.Logger("fakedata")
+var log = slog.Default().With("system", "fakedata")
+
+func SetLogger(logger *slog.Logger) {
+	log = logger
+}
 
 func MeasureIterations(name string) func(int) {
 	start := time.Now()
···
 			return
 		}
 		total := time.Since(start)
-		log.Infof("%s wall runtime: count=%d total=%s mean=%s", name, count, total, total/time.Duration(count))
+		log.Info("wall runtime", "name", name, "count", count, "total", total, "rate", total/time.Duration(count))
 	}
 }
 
···
 func BrowseAccount(xrpcc *xrpc.Client, acc *AccountContext) error {
 	// fetch notifications
 	maxNotif := 50
-	resp, err := appbsky.NotificationListNotifications(context.TODO(), xrpcc, "", int64(maxNotif), false, "")
+	resp, err := appbsky.NotificationListNotifications(context.TODO(), xrpcc, "", int64(maxNotif), false, nil, "")
 	if err != nil {
 		return err
 	}

go.mod

···
 module github.com/bluesky-social/indigo
 
-go 1.22
+go 1.23
 
 require (
 	contrib.go.opencensus.io/exporter/prometheus v0.4.2
···
 	github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b
 	github.com/adrg/xdg v0.5.0
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
+	github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874
 	github.com/brianvoe/gofakeit/v6 v6.25.0
 	github.com/carlmjohnson/versioninfo v0.22.5
+	github.com/cockroachdb/pebble v1.1.2
 	github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2
 	github.com/flosch/pongo2/v6 v6.0.0
 	github.com/go-redis/cache/v9 v9.0.0
 	github.com/goccy/go-json v0.10.2
-	github.com/gocql/gocql v1.7.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/gorilla/websocket v1.5.1
 	github.com/hashicorp/go-retryablehttp v0.7.5
···
 	github.com/ipfs/go-ipld-cbor v0.1.0
 	github.com/ipfs/go-ipld-format v0.6.0
 	github.com/ipfs/go-libipfs v0.7.0
-	github.com/ipfs/go-log v1.0.5
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4
 	github.com/ipld/go-car/v2 v2.13.1
···
 	github.com/urfave/cli/v2 v2.25.7
 	github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e
 	github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6
+	github.com/xlab/treeprint v1.2.0
 	gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1
 	go.opentelemetry.io/otel v1.21.0
···
 	go.opentelemetry.io/otel/sdk v1.21.0
 	go.opentelemetry.io/otel/trace v1.21.0
 	go.uber.org/automaxprocs v1.5.3
+	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.21.0
-	golang.org/x/sync v0.5.0
+	golang.org/x/sync v0.7.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.3.0
 	golang.org/x/tools v0.15.0
···
 )
 
 require (
+	github.com/DataDog/zstd v1.4.5 // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/go-redis/redis v6.15.9+incompatible // indirect
-	github.com/golang/snappy v0.0.3 // indirect
-	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/klauspost/compress v1.17.3 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/labstack/gommon v0.4.1 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/vmihailenco/go-tinylfu v0.2.2 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
-	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
 )
 
 require (
···
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
 	google.golang.org/grpc v1.59.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect

go.sum

···
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
+github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/PuerkitoBio/purell v1.2.1 h1:QsZ4TjvwiMpat6gBCBxEQI0rcS9ehtkKtSpiUnd9N28=
 github.com/PuerkitoBio/purell v1.2.1/go.mod h1:ZwHcC/82TOaovDi//J/804umJFFmbOHPngi8iYYv/Eo=
 github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU=
···
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
-github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
-github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
-github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous=
+github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c=
 github.com/brianvoe/gofakeit/v6 v6.25.0 h1:ZpFjktOpLZUeF8q223o0rUuXtA+m5qW5srjvVi+JkXk=
 github.com/brianvoe/gofakeit/v6 v6.25.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
···
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
+github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
+github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
+github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
+github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
+github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
+github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
+github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
+github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
+github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
+github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
+github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE=
 github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
···
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
+github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
···
 github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus=
-github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
···
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
···
 github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y=
-github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
-github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
···
 github.com/orandin/slog-gorm v1.3.2/go.mod h1:MoZ51+b7xE9lwGNPYEhxcUtRNrYzjdcKvA8QXQQGEPA=
 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
···
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
···
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 h1:yJ9/LwIGIk/c0CdoavpC9RNSGSruIspSZtxG3Nnldic=
 github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6/go.mod h1:39U9RRVr4CKbXpXYopWn+FSH5s+vWu6+RmguSPWAq5s=
+github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
+github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
···
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
···
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
···
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
···
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
-gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

indexer/crawler.go

···
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"sync"
+	"time"
 
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/models"
···
 	doRepoCrawl func(context.Context, *crawlWork) error
 
 	concurrency int
+
+	log *slog.Logger
+
+	done chan struct{}
 }
 
-func NewCrawlDispatcher(repoFn func(context.Context, *crawlWork) error, concurrency int) (*CrawlDispatcher, error) {
+func NewCrawlDispatcher(repoFn func(context.Context, *crawlWork) error, concurrency int, log *slog.Logger) (*CrawlDispatcher, error) {
 	if concurrency < 1 {
 		return nil, fmt.Errorf("must specify a non-zero positive integer for crawl dispatcher concurrency")
 	}
 
-	return &CrawlDispatcher{
+	out := &CrawlDispatcher{
 		ingest:      make(chan *models.ActorInfo),
 		repoSync:    make(chan *crawlWork),
 		complete:    make(chan models.Uid),
···
 		concurrency: concurrency,
 		todo:        make(map[models.Uid]*crawlWork),
 		inProgress:  make(map[models.Uid]*crawlWork),
-	}, nil
+		log:         log,
+		done:        make(chan struct{}),
+	}
+	go out.CatchupRepoGaugePoller()
+
+	return out, nil
 }
 
 func (c *CrawlDispatcher) Run() {
···
 	for i := 0; i < c.concurrency; i++ {
 		go c.fetchWorker()
 	}
+}
+
+func (c *CrawlDispatcher) Shutdown() {
+	close(c.done)
 }
 
 type catchupJob struct {
···
 }
 
 func (c *CrawlDispatcher) addToCatchupQueue(catchup *catchupJob) *crawlWork {
-	catchupEventsEnqueued.Inc()
 	c.maplk.Lock()
 	defer c.maplk.Unlock()
 
 	// If the actor crawl is enqueued, we can append to the catchup queue which gets emptied during the crawl
 	job, ok := c.todo[catchup.user.Uid]
 	if ok {
+		catchupEventsEnqueued.WithLabelValues("todo").Inc()
 		job.catchup = append(job.catchup, catchup)
 		return nil
 	}
···
 	// If the actor crawl is in progress, we can append to the nextr queue which gets emptied after the crawl
 	job, ok = c.inProgress[catchup.user.Uid]
 	if ok {
+		catchupEventsEnqueued.WithLabelValues("prog").Inc()
 		job.next = append(job.next, catchup)
 		return nil
 	}
 
+	catchupEventsEnqueued.WithLabelValues("new").Inc()
 	// Otherwise, we need to create a new crawl job for this actor and enqueue it
 	cw := &crawlWork{
 		act:     catchup.user,
···
 		select {
 		case job := <-c.repoSync:
 			if err := c.doRepoCrawl(context.TODO(), job); err != nil {
-				log.Errorf("failed to perform repo crawl of %q: %s", job.act.Did, err)
+				c.log.Error("failed to perform repo crawl", "did", job.act.Did, "err", err)
 			}
 
 			// TODO: do we still just do this if it errors?
···
 
 	return false
 }
+
+func (c *CrawlDispatcher) countReposInSlowPath() int {
+	c.maplk.Lock()
+	defer c.maplk.Unlock()
+	return len(c.inProgress) + len(c.todo)
+}
+
+func (c *CrawlDispatcher) CatchupRepoGaugePoller() {
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-c.done:
+		case <-ticker.C:
+			catchupReposGauge.Set(float64(c.countReposInSlowPath()))
+		}
+	}
+}

indexer/indexer.go

···
 	"database/sql"
 	"errors"
 	"fmt"
+	"log/slog"
 	"time"
 
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
···
 	"github.com/bluesky-social/indigo/xrpc"
 
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
 	"go.opentelemetry.io/otel"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
 )
 
-var log = logging.Logger("indexer")
-
 const MaxEventSliceLength = 1000000
 const MaxOpsSliceLength = 200
 
···
 	SendRemoteFollow       func(context.Context, string, uint) error
 	CreateExternalUser     func(context.Context, string) (*models.ActorInfo, error)
 	ApplyPDSClientSettings func(*xrpc.Client)
+
+	log *slog.Logger
 }
 
 func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events.EventManager, didr did.Resolver, fetcher *RepoFetcher, crawl, aggregate, spider bool) (*Indexer, error) {
···
 			return nil
 		},
 		ApplyPDSClientSettings: func(*xrpc.Client) {},
+		log:                    slog.Default().With("system", "indexer"),
 	}
 
 	if crawl {
-		c, err := NewCrawlDispatcher(fetcher.FetchAndIndexRepo, fetcher.MaxConcurrency)
+		c, err := NewCrawlDispatcher(fetcher.FetchAndIndexRepo, fetcher.MaxConcurrency, ix.log)
 		if err != nil {
 			return nil, err
 		}
···
 	return ix, nil
 }
 
+func (ix *Indexer) Shutdown() {
+	if ix.Crawler != nil {
+		ix.Crawler.Shutdown()
+	}
+}
+
 func (ix *Indexer) HandleRepoEvent(ctx context.Context, evt *repomgr.RepoEvent) error {
 	ctx, span := otel.Tracer("indexer").Start(ctx, "HandleRepoEvent")
 	defer span.End()
 
-	log.Debugw("Handling Repo Event!", "uid", evt.User)
+	ix.log.Debug("Handling Repo Event!", "uid", evt.User)
 
 	outops := make([]*comatproto.SyncSubscribeRepos_RepoOp, 0, len(evt.Ops))
 	for _, op := range evt.Ops {
···
 		})
 
 		if err := ix.handleRepoOp(ctx, evt, &op); err != nil {
-			log.Errorw("failed to handle repo op", "err", err)
+			ix.log.Error("failed to handle repo op", "err", err)
 		}
 	}
 
···
 		toobig = true
 	}
 
-	log.Debugw("Sending event", "did", did)
+	ix.log.Debug("Sending event", "did", did)
 	if err := ix.events.AddEvent(ctx, &events.XRPCStreamEvent{
 		RepoCommit: &comatproto.SyncSubscribeRepos_Commit{
 			Repo:   did,
···
 			if e.Type == "mention" {
 				_, err := ix.GetUserOrMissing(ctx, e.Value)
 				if err != nil {
-					log.Infow("failed to parse user mention", "ref", e.Value, "err", err)
+					ix.log.Info("failed to parse user mention", "ref", e.Value, "err", err)
 				}
 			}
 		}
···
 		if rec.Reply != nil {
 			if rec.Reply.Parent != nil {
 				if err := ix.crawlAtUriRef(ctx, rec.Reply.Parent.Uri); err != nil {
-					log.Infow("failed to crawl reply parent", "cid", op.RecCid, "replyuri", rec.Reply.Parent.Uri, "err", err)
+					ix.log.Info("failed to crawl reply parent", "cid", op.RecCid, "replyuri", rec.Reply.Parent.Uri, "err", err)
 				}
 			}
 
 			if rec.Reply.Root != nil {
 				if err := ix.crawlAtUriRef(ctx, rec.Reply.Root.Uri); err != nil {
-					log.Infow("failed to crawl reply root", "cid", op.RecCid, "rooturi", rec.Reply.Root.Uri, "err", err)
+					ix.log.Info("failed to crawl reply root", "cid", op.RecCid, "rooturi", rec.Reply.Root.Uri, "err", err)
 				}
 			}
 		}
···
 	case *bsky.FeedRepost:
 		if rec.Subject != nil {
 			if err := ix.crawlAtUriRef(ctx, rec.Subject.Uri); err != nil {
-				log.Infow("failed to crawl repost subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err)
+				ix.log.Info("failed to crawl repost subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err)
 			}
 		}
 		return nil
 	case *bsky.FeedLike:
 		if rec.Subject != nil {
 			if err := ix.crawlAtUriRef(ctx, rec.Subject.Uri); err != nil {
-				log.Infow("failed to crawl like subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err)
+				ix.log.Info("failed to crawl like subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err)
 			}
 		}
 		return nil
 	case *bsky.GraphFollow:
 		_, err := ix.GetUserOrMissing(ctx, rec.Subject)
 		if err != nil {
-			log.Infow("failed to crawl follow subject", "cid", op.RecCid, "subjectdid", rec.Subject, "err", err)
+			ix.log.Info("failed to crawl follow subject", "cid", op.RecCid, "subjectdid", rec.Subject, "err", err)
 		}
 		return nil
 	case *bsky.GraphBlock:
 		_, err := ix.GetUserOrMissing(ctx, rec.Subject)
 		if err != nil {
-			log.Infow("failed to crawl follow subject", "cid", op.RecCid, "subjectdid", rec.Subject, "err", err)
+			ix.log.Info("failed to crawl follow subject", "cid", op.RecCid, "subjectdid", rec.Subject, "err", err)
 		}
 		return nil
 	case *bsky.ActorProfile:
···
 	case *bsky.FeedGenerator:
 		return nil
 	default:
-		log.Warnw("unrecognized record type (crawling references)", "record", op.Record, "collection", op.Collection)
+		ix.log.Warn("unrecognized record type (crawling references)", "record", op.Record, "collection", op.Collection)
 		return nil
 	}
 }
···
 }
 
 func (ix *Indexer) addUserToCrawler(ctx context.Context, ai *models.ActorInfo) error {
-	log.Debugw("Sending user to crawler: ", "did", ai.Did)
+	ix.log.Debug("Sending user to crawler: ", "did", ai.Did)
 	if ix.Crawler == nil {
 		return nil
 	}
···
 }
 
 func (ix *Indexer) handleRecordDelete(ctx context.Context, evt *repomgr.RepoEvent, op *repomgr.RepoOp, local bool) error {
-	log.Debugw("record delete event", "collection", op.Collection)
+	ix.log.Debug("record delete event", "collection", op.Collection)
 
 	switch op.Collection {
 	case "app.bsky.feed.post":
···
 		fp, err := ix.GetPost(ctx, uri)
 		if err != nil {
 			if errors.Is(err, gorm.ErrRecordNotFound) {
-				log.Warnw("deleting post weve never seen before. Weird.", "user", evt.User, "rkey", op.Rkey)
+				ix.log.Warn("deleting post weve never seen before. Weird.", "user", evt.User, "rkey", op.Rkey)
 				return nil
 			}
 			return err
···
 			return err
 		}
 
-		log.Warn("TODO: remove notifications on delete")
+		ix.log.Warn("TODO: remove notifications on delete")
 		/*
 		   if err := ix.notifman.RemoveRepost(ctx, fp.Author, rr.ID, evt.User); err != nil {
 		           return nil, err
···
 		return err
 	}
 
-	log.Warnf("need to delete vote notification")
+	ix.log.Warn("need to delete vote notification")
 	return nil
 }
 
···
 	}
 
 	if q.RowsAffected == 0 {
-		log.Warnw("attempted to delete follow we did not have a record for", "user", evt.User, "rkey", op.Rkey)
+		ix.log.Warn("attempted to delete follow we did not have a record for", "user", evt.User, "rkey", op.Rkey)
 		return nil
 	}
 
···
 }
 
 func (ix *Indexer) handleRecordCreate(ctx context.Context, evt *repomgr.RepoEvent, op *repomgr.RepoOp, local bool) ([]uint, error) {
-	log.Debugw("record create event", "collection", op.Collection)
+	ix.log.Debug("record create event", "collection", op.Collection)
 
 	var out []uint
 	switch rec := op.Record.(type) {
···
 	case *bsky.FeedGenerator:
 		return out, nil
 	case *bsky.ActorProfile:
-		log.Debugf("TODO: got actor profile record creation, need to do something with this")
+		ix.log.Debug("TODO: got actor profile record creation, need to do something with this")
 	default:
+		ix.log.Warn("unrecognized record", "record", op.Record, "collection", op.Collection)
 		return nil, fmt.Errorf("unrecognized record type (creation): %s", op.Collection)
 	}
 
···
 }
 
 func (ix *Indexer) handleRecordUpdate(ctx context.Context, evt *repomgr.RepoEvent, op *repomgr.RepoOp, local bool) error {
-	log.Debugw("record update event", "collection", op.Collection)
+	ix.log.Debug("record update event", "collection", op.Collection)
 
 	switch rec := op.Record.(type) {
 	case *bsky.FeedPost:
···
 
 		if oldReply != newReply {
 			// the 'replyness' of the post was changed... that's weird
-			log.Errorf("need to properly handle case where reply-ness of posts is changed")
+			ix.log.Error("need to properly handle case where reply-ness of posts is changed")
 			return nil
 		}
 
···
 			}
 
 			if replyto.ID != fp.ReplyTo {
-				log.Errorf("post was changed to be a reply to a different post")
+				ix.log.Error("post was changed to be a reply to a different post")
 				return nil
 			}
 		}
···
 
 		return ix.handleRecordCreateGraphFollow(ctx, rec, evt, op)
 	case *bsky.ActorProfile:
-		log.Debugf("TODO: got actor profile record update, need to do something with this")
+		ix.log.Debug("TODO: got actor profile record update, need to do something with this")
 	default:
 		return fmt.Errorf("unrecognized record type (update): %s", op.Collection)
 	}
···
 		// we're likely filling in a missing reference
 		if !maybe.Missing {
 			// TODO: we've already processed this record creation
-			log.Warnw("potentially erroneous event, duplicate create", "rkey", rkey, "user", user)
+			ix.log.Warn("potentially erroneous event, duplicate create", "rkey", rkey, "user", user)
 		}
 
 		if err := ix.db.Clauses(clause.OnConflict{
···
 }
 
 func (ix *Indexer) createMissingPostRecord(ctx context.Context, puri *util.ParsedUri) (*models.FeedPost, error) {
-	log.Warn("creating missing post record")
+	ix.log.Warn("creating missing post record")
 	ai, err := ix.GetUserOrMissing(ctx, puri.Did)
 	if err != nil {
 		return nil, err
···
 	if post.Reply != nil {
 		replyto, err := ix.GetPost(ctx, post.Reply.Parent.Uri)
 		if err != nil {
-			log.Error("probably shouldn't error when processing a reply to a not-found post")
+			ix.log.Error("probably shouldn't error when processing a reply to a not-found post")
 			return err
 		}
 

indexer/keymgr.go

···
 import (
 	"context"
 	"fmt"
+	"log/slog"
 
 	did "github.com/whyrusleeping/go-did"
 	"go.opentelemetry.io/otel"
···
 	didr DidResolver
 
 	signingKey *did.PrivKey
+
+	log *slog.Logger
 }
 
 type DidResolver interface {
···
 	return &KeyManager{
 		didr:       didr,
 		signingKey: k,
+		log:        slog.Default().With("system", "indexer"),
 	}
 }
 
···
 
 	err = k.Verify(msg, sig)
 	if err != nil {
-		log.Warnw("signature failed to verify", "err", err, "did", did, "pubKey", k, "sigBytes", sig, "msgBytes", msg)
+		km.log.Warn("signature failed to verify", "err", err, "did", did, "pubKey", k, "sigBytes", sig, "msgBytes", msg)
 	}
 	return err
 }

indexer/metrics.go

···
 	Help: "Number of repos fetched",
 }, []string{"status"})
 
-var catchupEventsEnqueued = promauto.NewCounter(prometheus.CounterOpts{
+var catchupEventsEnqueued = promauto.NewCounterVec(prometheus.CounterOpts{
 	Name: "indexer_catchup_events_enqueued",
 	Help: "Number of catchup events enqueued",
-})
+}, []string{"how"})
 
 var catchupEventsProcessed = promauto.NewCounter(prometheus.CounterOpts{
 	Name: "indexer_catchup_events_processed",
 	Help: "Number of catchup events processed",
 })
+
+var catchupEventsFailed = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indexer_catchup_events_failed",
+	Help: "Number of catchup events processed",
+}, []string{"err"})
+
+var catchupReposGauge = promauto.NewGauge(prometheus.GaugeOpts{
+	Name: "indexer_catchup_repos",
+	Help: "Number of repos waiting on catchup",
+})

indexer/posts_test.go

···
 		t.Fatal(err)
 	}
 
-	cs, err := carstore.NewCarStore(cardb, cspath)
+	cs, err := carstore.NewCarStore(cardb, []string{cspath})
 	if err != nil {
 		t.Fatal(err)
 	}
···
 	if ix.dir != "" {
 		_ = os.RemoveAll(ix.dir)
 	}
+	ix.ix.Shutdown()
 }
 
 // TODO: dedupe this out into some testing utility package

indexer/repofetch.go

···
 	"fmt"
 	"io"
 	"io/fs"
+	"log/slog"
 	"sync"
 
 	"github.com/bluesky-social/indigo/api/atproto"
···
 		Limiters:               make(map[uint]*rate.Limiter),
 		ApplyPDSClientSettings: func(*xrpc.Client) {},
 		MaxConcurrency:         maxConcurrency,
+		log:                    slog.Default().With("system", "indexer"),
 	}
 }
 
···
 	MaxConcurrency int
 
 	ApplyPDSClientSettings func(*xrpc.Client)
+
+	log *slog.Logger
 }
 
 func (rf *RepoFetcher) GetLimiter(pdsID uint) *rate.Limiter {
···
 }
 
 func (rf *RepoFetcher) GetOrCreateLimiter(pdsID uint, pdsrate float64) *rate.Limiter {
-	rf.LimitMux.RLock()
-	defer rf.LimitMux.RUnlock()
+	rf.LimitMux.Lock()
+	defer rf.LimitMux.Unlock()
 
 	lim, ok := rf.Limiters[pdsID]
 	if !ok {
···
 	// Wait to prevent DOSing the PDS when connecting to a new stream with lots of active repos
 	limiter.Wait(ctx)
 
-	log.Debugw("SyncGetRepo", "did", did, "since", rev)
+	rf.log.Debug("SyncGetRepo", "did", did, "since", rev)
 	// TODO: max size on these? A malicious PDS could just send us a petabyte sized repo here and kill us
 	repo, err := atproto.SyncGetRepo(ctx, c, did, rev)
 	if err != nil {
···
 
 	var pds models.PDS
 	if err := rf.db.First(&pds, "id = ?", ai.PDS).Error; err != nil {
+		catchupEventsFailed.WithLabelValues("nopds").Inc()
 		return fmt.Errorf("expected to find pds record (%d) in db for crawling one of their users: %w", ai.PDS, err)
 	}
 
 	rev, err := rf.repoman.GetRepoRev(ctx, ai.Uid)
 	if err != nil && !isNotFound(err) {
+		catchupEventsFailed.WithLabelValues("noroot").Inc()
 		return fmt.Errorf("failed to get repo root: %w", err)
 	}
 
···
 			for i, j := range job.catchup {
 				catchupEventsProcessed.Inc()
 				if err := rf.repoman.HandleExternalUserEvent(ctx, pds.ID, ai.Uid, ai.Did, j.evt.Since, j.evt.Rev, j.evt.Blocks, j.evt.Ops); err != nil {
-					log.Errorw("buffered event catchup failed", "error", err, "did", ai.Did, "i", i, "jobCount", len(job.catchup), "seq", j.evt.Seq)
+					rf.log.Error("buffered event catchup failed", "error", err, "did", ai.Did, "i", i, "jobCount", len(job.catchup), "seq", j.evt.Seq)
 					resync = true // fall back to a repo sync
 					break
 				}
···
 		span.RecordError(err)
 
 		if ipld.IsNotFound(err) || errors.Is(err, io.EOF) || errors.Is(err, fs.ErrNotExist) {
-			log.Errorw("partial repo fetch was missing data", "did", ai.Did, "pds", pds.Host, "rev", rev)
+			rf.log.Error("partial repo fetch was missing data", "did", ai.Did, "pds", pds.Host, "rev", rev)
 			repo, err := rf.fetchRepo(ctx, c, &pds, ai.Did, "")
 			if err != nil {
 				return err

lex/type_schema.go

···
 			}
 			returndef = fmt.Sprintf("(*%s.%s, error)", impname, outname)
 		case "application/cbor", "application/vnd.ipld.car", "*/*":
-			returndef = fmt.Sprintf("(io.Reader, error)")
+			returndef = "(io.Reader, error)"
 		default:
 			return fmt.Errorf("unrecognized output encoding (handler stub): %q", s.Output.Encoding)
 		}

mst/cbor_gen.go

···
 var _ = math.E
 var _ = sort.Sort
 
-func (t *nodeData) MarshalCBOR(w io.Writer) error {
+func (t *NodeData) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
···
 		return err
 	}
 
-	// t.Entries ([]mst.treeEntry) (slice)
+	// t.Entries ([]mst.TreeEntry) (slice)
 	if len("e") > 1000000 {
 		return xerrors.Errorf("Value in field \"e\" was too long")
 	}
···
 	return nil
 }
 
-func (t *nodeData) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = nodeData{}
+func (t *NodeData) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = NodeData{}
 
 	cr := cbg.NewCborReader(r)
 
···
 	}
 
 	if extra > cbg.MaxLength {
-		return fmt.Errorf("nodeData: map struct too large (%d)", extra)
+		return fmt.Errorf("NodeData: map struct too large (%d)", extra)
 	}
 
 	n := extra
···
 		}
 
 		switch string(nameBuf[:nameLen]) {
-		// t.Entries ([]mst.treeEntry) (slice)
+		// t.Entries ([]mst.TreeEntry) (slice)
 		case "e":
 
 			maj, extra, err = cr.ReadHeader()
···
 			}
 
 			if extra > 0 {
-				t.Entries = make([]treeEntry, extra)
+				t.Entries = make([]TreeEntry, extra)
 			}
 
 			for i := 0; i < int(extra); i++ {
···
 
 	return nil
 }
-func (t *treeEntry) MarshalCBOR(w io.Writer) error {
+func (t *TreeEntry) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
···
 	return nil
 }
 
-func (t *treeEntry) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = treeEntry{}
+func (t *TreeEntry) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = TreeEntry{}
 
 	cr := cbg.NewCborReader(r)
 
···
 	}
 
 	if extra > cbg.MaxLength {
-		return fmt.Errorf("treeEntry: map struct too large (%d)", extra)
+		return fmt.Errorf("TreeEntry: map struct too large (%d)", extra)
 	}
 
 	n := extra

mst/mst.go

···
 // the CBOR codec.
 func CBORTypes() []reflect.Type {
 	return []reflect.Type{
-		reflect.TypeOf(nodeData{}),
-		reflect.TypeOf(treeEntry{}),
+		reflect.TypeOf(NodeData{}),
+		reflect.TypeOf(TreeEntry{}),
 	}
 }
 
 // MST tree node as gets serialized to CBOR. Note that the CBOR fields are all
 // single-character.
-type nodeData struct {
+type NodeData struct {
 	Left    *cid.Cid    `cborgen:"l"` // [nullable] pointer to lower-level subtree to the "left" of this path/key
-	Entries []treeEntry `cborgen:"e"` // ordered list of entries at this node
+	Entries []TreeEntry `cborgen:"e"` // ordered list of entries at this node
 }
 
-// treeEntry are elements of nodeData's Entries.
-type treeEntry struct {
+// TreeEntry are elements of NodeData's Entries.
+type TreeEntry struct {
 	PrefixLen int64    `cborgen:"p"` // count of characters shared with previous path/key in tree
 	KeySuffix []byte   `cborgen:"k"` // remaining part of path/key (appended to "previous key")
 	Val       cid.Cid  `cborgen:"v"` // CID pointer at this path/key
···
 	// otherwise this is a virtual/pointer struct and we need to hydrate from
 	// blockstore before returning entries
 	if mst.pointer != cid.Undef {
-		var nd nodeData
+		var nd NodeData
 		if err := mst.cst.Get(ctx, mst.pointer, &nd); err != nil {
 			return nil, err
 		}
···
 }
 
 // golang-specific helper that calls in to deserializeNodeData
-func entriesFromNodeData(ctx context.Context, nd *nodeData, cst cbor.IpldStore) ([]nodeEntry, error) {
+func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore) ([]nodeEntry, error) {
 	layer := -1
 	if len(nd.Entries) > 0 {
 		// NOTE(bnewbold): can compute the layer on the first KeySuffix, because for the first entry that field is a complete key

mst/mst_interop_test.go

···
 		t.Fatal(err)
 	}
 
-	simple_nd := nodeData{
+	simple_nd := NodeData{
 		Left: nil,
-		Entries: []treeEntry{
+		Entries: []TreeEntry{
 			{
 				PrefixLen: 0,
 				KeySuffix: []byte("com.example.record/3jqfcqzm3fo2j"),

mst/mst_util.go

···
 }
 
 // Typescript: deserializeNodeData(storage, data, layer)
-func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *nodeData, layer int) ([]nodeEntry, error) {
+func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *NodeData, layer int) ([]nodeEntry, error) {
 	entries := []nodeEntry{}
 	if nd.Left != nil {
 		// Note: like Typescript, this is actually a lazy load
···
 }
 
 // Typescript: serializeNodeData(entries) -> NodeData
-func serializeNodeData(entries []nodeEntry) (*nodeData, error) {
-	var data nodeData
+func serializeNodeData(entries []nodeEntry) (*NodeData, error) {
+	var data NodeData
 
 	i := 0
 	if len(entries) > 0 && entries[0].isTree() {
···
 		}
 
 		prefixLen := countPrefixLen(lastKey, leaf.Key)
-		data.Entries = append(data.Entries, treeEntry{
+		data.Entries = append(data.Entries, TreeEntry{
 			PrefixLen: int64(prefixLen),
 			KeySuffix: []byte(leaf.Key)[prefixLen:],
 			Val:       leaf.Val,

pds/data/types.go

···
+package data
+
+import (
+	"github.com/bluesky-social/indigo/models"
+	"gorm.io/gorm"
+	"time"
+)
+
+type User struct {
+	ID          models.Uid `gorm:"primarykey"`
+	CreatedAt   time.Time
+	UpdatedAt   time.Time
+	DeletedAt   gorm.DeletedAt `gorm:"index"`
+	Handle      string         `gorm:"uniqueIndex"`
+	Password    string
+	RecoveryKey string
+	Email       string
+	Did         string `gorm:"uniqueIndex"`
+	PDS         uint
+}
+
+type Peering struct {
+	gorm.Model
+	Host     string
+	Did      string
+	Approved bool
+}

pds/feedgen.go

···
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"sort"
 	"strings"
 	"time"
···
 	ix *indexer.Indexer
 
 	readRecord ReadRecordFunc
+
+	log *slog.Logger
 }
 
-func NewFeedGenerator(db *gorm.DB, ix *indexer.Indexer, readRecord ReadRecordFunc) (*FeedGenerator, error) {
+func NewFeedGenerator(db *gorm.DB, ix *indexer.Indexer, readRecord ReadRecordFunc, log *slog.Logger) (*FeedGenerator, error) {
 	return &FeedGenerator{
 		db:         db,
 		ix:         ix,
 		readRecord: readRecord,
+		log:        log,
 	}, nil
 }
 
···
 
 func (fg *FeedGenerator) GetVotes(ctx context.Context, uri string, pcid cid.Cid, limit int, before string) ([]*HydratedVote, error) {
 	if before != "" {
-		log.Warn("not respecting 'before' yet")
+		fg.log.Warn("not respecting 'before' yet")
 	}
 
 	p, err := fg.ix.GetPost(ctx, uri)

pds/handlers_test.go

···
 		t.Fatal(err)
 	}
 
-	cs, err := carstore.NewCarStore(db, sharddir)
+	cs, err := carstore.NewCarStore(db, []string{sharddir})
 	if err != nil {
 		t.Fatal(err)
 	}

pds/server.go

···
 	"database/sql"
 	"errors"
 	"fmt"
+	"log/slog"
 	"net"
 	"net/http"
 	"net/mail"
···
 
 	"github.com/bluesky-social/indigo/api/atproto"
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
-	bsky "github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/carstore"
 	"github.com/bluesky-social/indigo/events"
 	"github.com/bluesky-social/indigo/indexer"
 	lexutil "github.com/bluesky-social/indigo/lex/util"
 	"github.com/bluesky-social/indigo/models"
 	"github.com/bluesky-social/indigo/notifs"
+	pdsdata "github.com/bluesky-social/indigo/pds/data"
 	"github.com/bluesky-social/indigo/plc"
 	"github.com/bluesky-social/indigo/repomgr"
 	"github.com/bluesky-social/indigo/util"
···
 	gojwt "github.com/golang-jwt/jwt"
 	"github.com/gorilla/websocket"
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	"github.com/lestrrat-go/jwx/v2/jwt"
···
 	"gorm.io/gorm"
 )
 
-var log = logging.Logger("pds")
-
 type Server struct {
 	db             *gorm.DB
 	cs             carstore.CarStore
···
 	serviceUrl   string
 
 	plc plc.PLCClient
+
+	log *slog.Logger
 }
 
 // serverListenerBootTimeout is how long to wait for the requested server socket
···
 		serviceUrl:     serviceUrl,
 		jwtSigningKey:  jwtkey,
 		enforcePeering: false,
+
+		log: slog.Default().With("system", "pds"),
 	}
 
 	repoman.SetEventHandler(func(ctx context.Context, evt *repomgr.RepoEvent) {
 		if err := ix.HandleRepoEvent(ctx, evt); err != nil {
-			log.Errorw("handle repo event failed", "user", evt.User, "err", err)
+			s.log.Error("handle repo event failed", "user", evt.User, "err", err)
 		}
 	}, true)
 
 	//ix.SendRemoteFollow = s.sendRemoteFollow
 	ix.CreateExternalUser = s.createExternalUser
 
-	feedgen, err := NewFeedGenerator(db, ix, s.readRecordFunc)
+	feedgen, err := NewFeedGenerator(db, ix, s.readRecordFunc, s.log)
 	if err != nil {
 		return nil, err
 	}
···
 		handle = hurl.Host
 	}
 
-	profile, err := bsky.ActorGetProfile(ctx, c, did)
-	if err != nil {
-		return nil, err
-	}
-
-	if handle != profile.Handle {
-		return nil, fmt.Errorf("mismatch in handle between did document and pds profile (%s != %s)", handle, profile.Handle)
-	}
-
 	// TODO: request this users info from their server to fill out our data...
 	u := User{
 		Handle: handle,
···
 	subj := &models.ActorInfo{
 		Uid:         u.ID,
 		Handle:      sql.NullString{String: handle, Valid: true},
-		DisplayName: *profile.DisplayName,
+		DisplayName: "missing display name",
 		Did:         did,
 		Type:        "",
 		PDS:         peering.ID,
···
 
 func (s *Server) HandleHealthCheck(c echo.Context) error {
 	if err := s.db.Exec("SELECT 1").Error; err != nil {
-		log.Errorf("healthcheck can't connect to database: %v", err)
+		s.log.Error("healthcheck can't connect to database", "err", err)
 		return c.JSON(500, HealthStatus{Status: "error", Message: "can't connect to database"})
 	} else {
 		return c.JSON(200, HealthStatus{Status: "ok"})
···
 	return c.String(200, u.Did)
 }
 
-type User struct {
-	ID          models.Uid `gorm:"primarykey"`
-	CreatedAt   time.Time
-	UpdatedAt   time.Time
-	DeletedAt   gorm.DeletedAt `gorm:"index"`
-	Handle      string         `gorm:"uniqueIndex"`
-	Password    string
-	RecoveryKey string
-	Email       string
-	Did         string `gorm:"uniqueIndex"`
-	PDS         uint
-}
+type User = pdsdata.User
 
 type RefreshToken struct {
 	gorm.Model
···
 	panic("nyi")
 }
 
-type Peering struct {
-	gorm.Model
-	Host     string
-	Did      string
-	Approved bool
-}
+type Peering = pdsdata.Peering
 
 func (s *Server) EventsHandler(c echo.Context) error {
 	conn, err := websocket.Upgrade(c.Response().Writer, c.Request(), c.Response().Header(), 1<<10, 1<<10)
···
 func (s *Server) UpdateUserHandle(ctx context.Context, u *User, handle string) error {
 	if u.Handle == handle {
 		// no change? move on
-		log.Warnw("attempted to change handle to current handle", "did", u.Did, "handle", handle)
+		s.log.Warn("attempted to change handle to current handle", "did", u.Did, "handle", handle)
 		return nil
 	}
 

plc/memcached.go

···
+package plc
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/bradfitz/gomemcache/memcache"
+	"go.opentelemetry.io/otel/attribute"
+	"time"
+
+	"github.com/bluesky-social/indigo/did"
+	"go.opentelemetry.io/otel"
+)
+
+type MemcachedDidResolver struct {
+	mcd    *memcache.Client
+	res    did.Resolver
+	maxAge int32
+}
+
+func NewMemcachedDidResolver(res did.Resolver, maxAge time.Duration, servers []string) *MemcachedDidResolver {
+	expiry := int32(0)
+	if maxAge.Seconds() > (30 * 24 * 60 * 60) {
+		// clamp expiry at 30 days minus a minute for memcached
+		expiry = (30 * 24 * 60 * 60) - 60
+	} else {
+		expiry = int32(maxAge.Seconds())
+	}
+	client := memcache.New(servers...)
+	return &MemcachedDidResolver{
+		mcd:    client,
+		res:    res,
+		maxAge: expiry,
+	}
+}
+
+func (r *MemcachedDidResolver) FlushCacheFor(didstr string) {
+	r.mcd.Delete(didstr)
+	r.res.FlushCacheFor(didstr)
+}
+
+func (r *MemcachedDidResolver) tryCache(didstr string) (*did.Document, bool) {
+	ob, err := r.mcd.Get(didstr)
+	if (ob == nil) || (err != nil) {
+		return nil, false
+	}
+	var doc did.Document
+	err = json.Unmarshal(ob.Value, &doc)
+	if err != nil {
+		// TODO: log error?
+		return nil, false
+	}
+
+	return &doc, true
+}
+
+func (r *MemcachedDidResolver) putCache(did string, doc *did.Document) {
+	blob, err := json.Marshal(doc)
+	if err != nil {
+		// TODO: log error
+		return
+	}
+	item := memcache.Item{
+		Key:        did,
+		Value:      blob,
+		Expiration: int32(r.maxAge),
+	}
+	r.mcd.Set(&item)
+}
+
+func (r *MemcachedDidResolver) GetDocument(ctx context.Context, didstr string) (*did.Document, error) {
+	ctx, span := otel.Tracer("cacheResolver").Start(ctx, "getDocument")
+	defer span.End()
+
+	doc, ok := r.tryCache(didstr)
+	if ok {
+		span.SetAttributes(attribute.Bool("cache", true))
+		memcacheHitsTotal.Inc()
+		return doc, nil
+	}
+	memcacheMissesTotal.Inc()
+	span.SetAttributes(attribute.Bool("cache", false))
+
+	doc, err := r.res.GetDocument(ctx, didstr)
+	if err != nil {
+		return nil, err
+	}
+
+	r.putCache(didstr, doc)
+	return doc, nil
+}

plc/metrics.go

···
 	Name: "plc_cache_misses_total",
 	Help: "Total number of cache misses",
 })
+
+var memcacheHitsTotal = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "plc_memcache_hits_total",
+	Help: "Total number of cache hits",
+})
+
+var memcacheMissesTotal = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "plc_memcache_misses_total",
+	Help: "Total number of cache misses",
+})

repomgr/bench_test.go

···
 		b.Fatal(err)
 	}
 
-	cs, err := carstore.NewCarStore(cardb, cspath)
+	// TODO: constructor for 'either type'
+	/*
+		cs, err := carstore.NewCarStore(cardb, []string{cspath})
+		if err != nil {
+			b.Fatal(err)
+		}
+	*/
+	cs, err := carstore.NewNonArchivalCarstore(cardb)
 	if err != nil {
 		b.Fatal(err)
 	}
 
 	repoman := NewRepoManager(cs, &util.FakeKeyManager{})
+	repoman.noArchive = true
 
 	ctx := context.TODO()
 	if err := repoman.InitNewActor(ctx, 1, "hello.world", "did:foo:bar", "catdog", "", ""); err != nil {

repomgr/ingest_test.go

···
 		t.Fatal(err)
 	}
 
-	cs, err := carstore.NewCarStore(cardb, cspath)
+	cs, err := carstore.NewCarStore(cardb, []string{cspath})
 	if err != nil {
 		t.Fatal(err)
 	}
···
 	}
 }
 
-func testCarstore(t *testing.T, dir string) carstore.CarStore {
+func testCarstore(t *testing.T, dir string, archive bool) carstore.CarStore {
 	cardb, err := gorm.Open(sqlite.Open(filepath.Join(dir, "car.sqlite")))
 	if err != nil {
 		t.Fatal(err)
···
 		t.Fatal(err)
 	}
 
-	cs, err := carstore.NewCarStore(cardb, cspath)
-	if err != nil {
-		t.Fatal(err)
-	}
+	if archive {
+		cs, err := carstore.NewCarStore(cardb, []string{cspath})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return cs
+	} else {
+		cs, err := carstore.NewNonArchivalCarstore(cardb)
+		if err != nil {
+			t.Fatal(err)
+		}
 
-	return cs
+		return cs
+	}
 }
 
 func TestIngestWithGap(t *testing.T) {
···
 		Uid: 1,
 	})
 
-	cs := testCarstore(t, dir)
+	cs := testCarstore(t, dir, true)
 
 	repoman := NewRepoManager(cs, &util.FakeKeyManager{})
 
···
 	if err != nil {
 		t.Fatal(err)
 	}
-	cs2 := testCarstore(t, dir2)
+	cs2 := testCarstore(t, dir2, true)
 
 	var since *string
 	ctx := context.TODO()
···
 		Uid: 1,
 	})
 
-	cs := testCarstore(t, dir)
+	cs := testCarstore(t, dir, true)
 
 	repoman := NewRepoManager(cs, &util.FakeKeyManager{})
 
···
 	}
 
 	p1, _, err := repoman.CreateRecord(ctx, 1, "app.bsky.feed.post", &bsky.FeedPost{
-		Text: fmt.Sprintf("hello friend"),
+		Text: "hello friend",
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	p2, _, err := repoman.CreateRecord(ctx, 1, "app.bsky.feed.post", &bsky.FeedPost{
-		Text: fmt.Sprintf("hello friend"),
+		Text: "hello friend",
 	})
 	if err != nil {
 		t.Fatal(err)

repomgr/metrics.go

···
 	Name: "repomgr_repo_ops_imported",
 	Help: "Number of repo ops imported",
 })
+
+var openAndSigCheckDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "repomgr_open_and_sig_check_duration",
+	Help:    "Duration of opening and signature check",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+})
+
+var calcDiffDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "repomgr_calc_diff_duration",
+	Help:    "Duration of calculating diff",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+})
+
+var writeCarSliceDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "repomgr_write_car_slice_duration",
+	Help:    "Duration of writing car slice",
+	Buckets: prometheus.ExponentialBuckets(0.001, 2, 15),
+})

repomgr/repomgr.go

···
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"strings"
 	"sync"
+	"time"
 
 	atproto "github.com/bluesky-social/indigo/api/atproto"
 	bsky "github.com/bluesky-social/indigo/api/bsky"
···
 	"github.com/ipfs/go-datastore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	ipld "github.com/ipfs/go-ipld-format"
-	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipld/go-car"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"go.opentelemetry.io/otel"
···
 	"gorm.io/gorm"
 )
 
-var log = logging.Logger("repomgr")
+func NewRepoManager(cs carstore.CarStore, kmgr KeyManager) *RepoManager {
 
-func NewRepoManager(cs carstore.CarStore, kmgr KeyManager) *RepoManager {
+	var noArchive bool
+	if _, ok := cs.(*carstore.NonArchivalCarstore); ok {
+		noArchive = true
+	}
 
 	return &RepoManager{
 		cs:        cs,
 		userLocks: make(map[models.Uid]*userLock),
 		kmgr:      kmgr,
+		log:       slog.Default().With("system", "repomgr"),
+		noArchive: noArchive,
 	}
 }
 
···
 
 	events         func(context.Context, *RepoEvent)
 	hydrateRecords bool
+
+	log       *slog.Logger
+	noArchive bool
 }
 
 type ActorInfo struct {
···
 		return cid.Undef, nil, err
 	}
 
-	_, _, err = r.GetRecord(ctx, collection+"/"+rkey)
+	_, _, err = r.GetRecordBytes(ctx, collection+"/"+rkey)
 	if err != nil {
 		return cid.Undef, nil, err
 	}
···
 }
 
 func (rm *RepoManager) HandleExternalUserEvent(ctx context.Context, pdsid uint, uid models.Uid, did string, since *string, nrev string, carslice []byte, ops []*atproto.SyncSubscribeRepos_RepoOp) error {
+	if rm.noArchive {
+		return rm.handleExternalUserEventNoArchive(ctx, pdsid, uid, did, since, nrev, carslice, ops)
+	} else {
+		return rm.handleExternalUserEventArchive(ctx, pdsid, uid, did, since, nrev, carslice, ops)
+	}
+}
+
+func (rm *RepoManager) handleExternalUserEventNoArchive(ctx context.Context, pdsid uint, uid models.Uid, did string, since *string, nrev string, carslice []byte, ops []*atproto.SyncSubscribeRepos_RepoOp) error {
 	ctx, span := otel.Tracer("repoman").Start(ctx, "HandleExternalUserEvent")
 	defer span.End()
 
 	span.SetAttributes(attribute.Int64("uid", int64(uid)))
 
-	log.Debugw("HandleExternalUserEvent", "pds", pdsid, "uid", uid, "since", since, "nrev", nrev)
+	rm.log.Debug("HandleExternalUserEvent", "pds", pdsid, "uid", uid, "since", since, "nrev", nrev)
+
+	unlock := rm.lockUser(ctx, uid)
+	defer unlock()
+
+	start := time.Now()
+	root, ds, err := rm.cs.ImportSlice(ctx, uid, since, carslice)
+	if err != nil {
+		return fmt.Errorf("importing external carslice: %w", err)
+	}
+
+	r, err := repo.OpenRepo(ctx, ds, root)
+	if err != nil {
+		return fmt.Errorf("opening external user repo (%d, root=%s): %w", uid, root, err)
+	}
+
+	if err := rm.CheckRepoSig(ctx, r, did); err != nil {
+		return fmt.Errorf("check repo sig: %w", err)
+	}
+	openAndSigCheckDuration.Observe(time.Since(start).Seconds())
+
+	evtops := make([]RepoOp, 0, len(ops))
+	for _, op := range ops {
+		parts := strings.SplitN(op.Path, "/", 2)
+		if len(parts) != 2 {
+			return fmt.Errorf("invalid rpath in mst diff, must have collection and rkey")
+		}
+
+		switch EventKind(op.Action) {
+		case EvtKindCreateRecord:
+			rop := RepoOp{
+				Kind:       EvtKindCreateRecord,
+				Collection: parts[0],
+				Rkey:       parts[1],
+				RecCid:     (*cid.Cid)(op.Cid),
+			}
+
+			if rm.hydrateRecords {
+				_, rec, err := r.GetRecord(ctx, op.Path)
+				if err != nil {
+					return fmt.Errorf("reading changed record from car slice: %w", err)
+				}
+				rop.Record = rec
+			}
+
+			evtops = append(evtops, rop)
+		case EvtKindUpdateRecord:
+			rop := RepoOp{
+				Kind:       EvtKindUpdateRecord,
+				Collection: parts[0],
+				Rkey:       parts[1],
+				RecCid:     (*cid.Cid)(op.Cid),
+			}
+
+			if rm.hydrateRecords {
+				_, rec, err := r.GetRecord(ctx, op.Path)
+				if err != nil {
+					return fmt.Errorf("reading changed record from car slice: %w", err)
+				}
+
+				rop.Record = rec
+			}
+
+			evtops = append(evtops, rop)
+		case EvtKindDeleteRecord:
+			evtops = append(evtops, RepoOp{
+				Kind:       EvtKindDeleteRecord,
+				Collection: parts[0],
+				Rkey:       parts[1],
+			})
+		default:
+			return fmt.Errorf("unrecognized external user event kind: %q", op.Action)
+		}
+	}
+
+	if rm.events != nil {
+		rm.events(ctx, &RepoEvent{
+			User: uid,
+			//OldRoot:   prev,
+			NewRoot:   root,
+			Rev:       nrev,
+			Since:     since,
+			Ops:       evtops,
+			RepoSlice: carslice,
+			PDS:       pdsid,
+		})
+	}
+
+	return nil
+}
+
+func (rm *RepoManager) handleExternalUserEventArchive(ctx context.Context, pdsid uint, uid models.Uid, did string, since *string, nrev string, carslice []byte, ops []*atproto.SyncSubscribeRepos_RepoOp) error {
+	ctx, span := otel.Tracer("repoman").Start(ctx, "HandleExternalUserEvent")
+	defer span.End()
+
+	span.SetAttributes(attribute.Int64("uid", int64(uid)))
+
+	rm.log.Debug("HandleExternalUserEvent", "pds", pdsid, "uid", uid, "since", since, "nrev", nrev)
 
 	unlock := rm.lockUser(ctx, uid)
 	defer unlock()
 
+	start := time.Now()
 	root, ds, err := rm.cs.ImportSlice(ctx, uid, since, carslice)
 	if err != nil {
 		return fmt.Errorf("importing external carslice: %w", err)
···
 	if err := rm.CheckRepoSig(ctx, r, did); err != nil {
 		return err
 	}
+	openAndSigCheckDuration.Observe(time.Since(start).Seconds())
 
 	var skipcids map[cid.Cid]bool
 	if ds.BaseCid().Defined() {
···
 		}
 	}
 
+	start = time.Now()
 	if err := ds.CalcDiff(ctx, skipcids); err != nil {
 		return fmt.Errorf("failed while calculating mst diff (since=%v): %w", since, err)
-
 	}
+	calcDiffDuration.Observe(time.Since(start).Seconds())
 
 	evtops := make([]RepoOp, 0, len(ops))
 
···
 		}
 	}
 
+	start = time.Now()
 	rslice, err := ds.CloseWithRoot(ctx, root, nrev)
 	if err != nil {
 		return fmt.Errorf("close with root: %w", err)
 	}
+	writeCarSliceDuration.Observe(time.Since(start).Seconds())
 
 	if rm.events != nil {
 		rm.events(ctx, &RepoEvent{
···
 		ops := make([]RepoOp, 0, len(diffops))
 		for _, op := range diffops {
 			repoOpsImported.Inc()
-			out, err := processOp(ctx, bs, op, rm.hydrateRecords)
+			out, err := rm.processOp(ctx, bs, op, rm.hydrateRecords)
 			if err != nil {
-				log.Errorw("failed to process repo op", "err", err, "path", op.Rpath, "repo", repoDid)
+				rm.log.Error("failed to process repo op", "err", err, "path", op.Rpath, "repo", repoDid)
 			}
 
 			if out != nil {
···
 	return nil
 }
 
-func processOp(ctx context.Context, bs blockstore.Blockstore, op *mst.DiffOp, hydrateRecords bool) (*RepoOp, error) {
+func (rm *RepoManager) processOp(ctx context.Context, bs blockstore.Blockstore, op *mst.DiffOp, hydrateRecords bool) (*RepoOp, error) {
 	parts := strings.SplitN(op.Rpath, "/", 2)
 	if len(parts) != 2 {
 		return nil, fmt.Errorf("repo mst had invalid rpath: %q", op.Rpath)
···
 					return nil, err
 				}
 
-				log.Warnf("failed processing repo diff: %s", err)
+				rm.log.Warn("failed processing repo diff", "err", err)
 			} else {
 				outop.Record = rec
 			}
···
 	// the repos lifecycle, this will end up erroneously not including
 	// them. We should compute the set of blocks needed to read any repo
 	// ops that happened in the commit and use that for our 'output' blocks
-	cids, err := walkTree(ctx, seen, root, membs, true)
+	cids, err := rm.walkTree(ctx, seen, root, membs, true)
 	if err != nil {
 		return fmt.Errorf("walkTree: %w", err)
 	}
···
 
 // walkTree returns all cids linked recursively by the root, skipping any cids
 // in the 'skip' map, and not erroring on 'not found' if prevMissing is set
-func walkTree(ctx context.Context, skip map[cid.Cid]bool, root cid.Cid, bs blockstore.Blockstore, prevMissing bool) ([]cid.Cid, error) {
+func (rm *RepoManager) walkTree(ctx context.Context, skip map[cid.Cid]bool, root cid.Cid, bs blockstore.Blockstore, prevMissing bool) ([]cid.Cid, error) {
 	// TODO: what if someone puts non-cbor links in their repo?
 	if root.Prefix().Codec != cid.DagCBOR {
 		return nil, fmt.Errorf("can only handle dag-cbor objects in repos (%s is %d)", root, root.Prefix().Codec)
···
 	var links []cid.Cid
 	if err := cbg.ScanForLinks(bytes.NewReader(blk.RawData()), func(c cid.Cid) {
 		if c.Prefix().Codec == cid.Raw {
-			log.Debugw("skipping 'raw' CID in record", "recordCid", root, "rawCid", c)
+			rm.log.Debug("skipping 'raw' CID in record", "recordCid", root, "rawCid", c)
 			return
 		}
 		if skip[c] {
···
 
 	// TODO: should do this non-recursive since i expect these may get deep
 	for _, c := range links {
-		sub, err := walkTree(ctx, skip, c, bs, prevMissing)
+		sub, err := rm.walkTree(ctx, skip, c, bs, prevMissing)
 		if err != nil {
 			if prevMissing && !ipld.IsNotFound(err) {
 				return nil, err

search/firehose.go

···
 			idx.relayhost,
 			rsc.EventHandler,
 		),
+		idx.logger,
 	)
 }
 

search/handlers.go

···
 	if offset > 10000 {
 		return 0, 0, &echo.HTTPError{
 			Code:    400,
-			Message: fmt.Sprintf("invalid value for 'cursor' (can't paginate so deep)"),
+			Message: "invalid value for 'cursor' (can't paginate so deep)",
 		}
 	}
 

search/indexing.go

···
 		opts.SyncRequestsPerSecond = 8
 	}
 
-	opts.CheckoutPath = fmt.Sprintf("%s/xrpc/com.atproto.sync.getRepo", relayHTTP)
+	opts.RelayHost = relayHTTP
 	if config.IndexMaxConcurrency > 0 {
 		opts.ParallelRecordCreates = config.IndexMaxConcurrency
 	} else {
···
 		idx.handleDelete,
 		opts,
 	)
+	// reuse identity directory (for efficient caching)
+	bf.Directory = dir
 
 	idx.bfs = bfstore
 	idx.bf = bf

search/transform.go

···
 			}
 		}
 	}
-	for _, t := range p.Tags {
-		ret = append(ret, t)
-	}
+	ret = append(ret, p.Tags...)
 	if len(ret) == 0 {
 		return nil
 	}

splitter/metrics.go

···
+package splitter
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var eventsSentCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "spl_events_sent_counter",
+	Help: "The total number of events sent to consumers",
+}, []string{"remote_addr", "user_agent"})
+
+var activeClientGauge = promauto.NewGauge(prometheus.GaugeOpts{
+	Name: "spl_active_clients",
+	Help: "Current number of active clients",
+})

splitter/ringbuf.go

···
+package splitter
+
+import (
+	"context"
+	"sync"
+
+	events "github.com/bluesky-social/indigo/events"
+	"github.com/bluesky-social/indigo/models"
+)
+
+func NewEventRingBuffer(chunkSize, nchunks int) *EventRingBuffer {
+	return &EventRingBuffer{
+		chunkSize:     chunkSize,
+		maxChunkCount: nchunks,
+	}
+}
+
+type EventRingBuffer struct {
+	lk            sync.Mutex
+	chunks        []*ringChunk
+	chunkSize     int
+	maxChunkCount int
+
+	broadcast func(*events.XRPCStreamEvent)
+}
+
+type ringChunk struct {
+	lk  sync.Mutex
+	buf []*events.XRPCStreamEvent
+}
+
+func (rc *ringChunk) append(evt *events.XRPCStreamEvent) {
+	rc.lk.Lock()
+	defer rc.lk.Unlock()
+	rc.buf = append(rc.buf, evt)
+}
+
+func (rc *ringChunk) events() []*events.XRPCStreamEvent {
+	rc.lk.Lock()
+	defer rc.lk.Unlock()
+	return rc.buf
+}
+
+func (er *EventRingBuffer) Persist(ctx context.Context, evt *events.XRPCStreamEvent) error {
+	er.lk.Lock()
+	defer er.lk.Unlock()
+
+	if len(er.chunks) == 0 {
+		er.chunks = []*ringChunk{new(ringChunk)}
+	}
+
+	last := er.chunks[len(er.chunks)-1]
+	if len(last.buf) >= er.chunkSize {
+		last = new(ringChunk)
+		er.chunks = append(er.chunks, last)
+		if len(er.chunks) > er.maxChunkCount {
+			er.chunks = er.chunks[1:]
+		}
+	}
+
+	last.append(evt)
+
+	er.broadcast(evt)
+	return nil
+}
+
+func (er *EventRingBuffer) Flush(context.Context) error {
+	return nil
+}
+
+func (er *EventRingBuffer) Playback(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error) error {
+	// run playback a few times to get as close to 'live' as possible before returning
+	for i := 0; i < 10; i++ {
+		n, err := er.playbackRound(ctx, since, cb)
+		if err != nil {
+			return err
+		}
+
+		// playback had no new events
+		if n-since == 0 {
+			return nil
+		}
+		since = n
+	}
+
+	return nil
+}
+
+func (er *EventRingBuffer) playbackRound(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error) (int64, error) {
+	// grab a snapshot of the current chunks
+	er.lk.Lock()
+	chunks := er.chunks
+	er.lk.Unlock()
+
+	i := len(chunks) - 1
+	for ; i >= 0; i-- {
+		c := chunks[i]
+		evts := c.events()
+		if since > events.SequenceForEvent(evts[len(evts)-1]) {
+			i++
+			break
+		}
+	}
+	if i < 0 {
+		i = 0
+	}
+
+	var lastSeq int64 = since
+	for _, c := range chunks[i:] {
+		var nread int
+		evts := c.events()
+		for nread < len(evts) {
+			for _, e := range evts[nread:] {
+				nread++
+				seq := events.SequenceForEvent(e)
+				if seq <= since {
+					continue
+				}
+
+				if err := cb(e); err != nil {
+					return 0, err
+				}
+				lastSeq = seq
+			}
+
+			// recheck evts buffer to see if more were added while we were here
+			evts = c.events()
+		}
+	}
+
+	return lastSeq, nil
+}
+
+func (er *EventRingBuffer) SetEventBroadcaster(brc func(*events.XRPCStreamEvent)) {
+	er.broadcast = brc
+}
+
+func (er *EventRingBuffer) Shutdown(context.Context) error {
+	return nil
+}
+
+func (er *EventRingBuffer) TakeDownRepo(context.Context, models.Uid) error {
+	return nil
+}

splitter/splitter.go

···
+package splitter
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"go.opentelemetry.io/otel"
+	"io"
+	"log/slog"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/bluesky-social/indigo/api/atproto"
+	comatproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/bgs"
+	events "github.com/bluesky-social/indigo/events"
+	"github.com/bluesky-social/indigo/events/schedulers/sequential"
+	"github.com/bluesky-social/indigo/util"
+	"github.com/bluesky-social/indigo/xrpc"
+	"github.com/gorilla/websocket"
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+	promclient "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	dto "github.com/prometheus/client_model/go"
+)
+
+type Splitter struct {
+	erb    *EventRingBuffer
+	pp     *events.PebblePersist
+	events *events.EventManager
+
+	// Management of Socket Consumers
+	consumersLk    sync.RWMutex
+	nextConsumerID uint64
+	consumers      map[uint64]*SocketConsumer
+
+	conf SplitterConfig
+
+	log *slog.Logger
+
+	httpC        *http.Client
+	nextCrawlers []*url.URL
+}
+
+type SplitterConfig struct {
+	UpstreamHost  string
+	CursorFile    string
+	PebbleOptions *events.PebblePersistOptions
+}
+
+func (sc *SplitterConfig) XrpcRootUrl() string {
+	if strings.HasPrefix(sc.UpstreamHost, "http://") {
+		return sc.UpstreamHost
+	}
+	if strings.HasPrefix(sc.UpstreamHost, "https://") {
+		return sc.UpstreamHost
+	}
+	if strings.HasPrefix(sc.UpstreamHost, "ws://") {
+		return "http://" + sc.UpstreamHost[5:]
+	}
+	if strings.HasPrefix(sc.UpstreamHost, "wss://") {
+		return "https://" + sc.UpstreamHost[6:]
+	}
+	return "https://" + sc.UpstreamHost
+}
+
+func NewSplitter(conf SplitterConfig, nextCrawlers []string) (*Splitter, error) {
+	var nextCrawlerURLs []*url.URL
+	log := slog.Default().With("system", "splitter")
+	if len(nextCrawlers) > 0 {
+		nextCrawlerURLs = make([]*url.URL, len(nextCrawlers))
+		for i, tu := range nextCrawlers {
+			var err error
+			nextCrawlerURLs[i], err = url.Parse(tu)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse next-crawler url: %w", err)
+			}
+			log.Info("configuring relay for requestCrawl", "host", nextCrawlerURLs[i])
+		}
+	}
+
+	s := &Splitter{
+		conf:         conf,
+		consumers:    make(map[uint64]*SocketConsumer),
+		log:          log,
+		httpC:        util.RobustHTTPClient(),
+		nextCrawlers: nextCrawlerURLs,
+	}
+
+	if conf.PebbleOptions == nil {
+		// mem splitter
+		erb := NewEventRingBuffer(20_000, 10_000)
+		s.erb = erb
+		s.events = events.NewEventManager(erb)
+	} else {
+		pp, err := events.NewPebblePersistance(conf.PebbleOptions)
+		if err != nil {
+			return nil, err
+		}
+		go pp.GCThread(context.Background())
+		s.pp = pp
+		s.events = events.NewEventManager(pp)
+	}
+
+	return s, nil
+}
+func NewDiskSplitter(host, path string, persistHours float64, maxBytes int64) (*Splitter, error) {
+	ppopts := events.PebblePersistOptions{
+		DbPath:          path,
+		PersistDuration: time.Duration(float64(time.Hour) * persistHours),
+		GCPeriod:        5 * time.Minute,
+		MaxBytes:        uint64(maxBytes),
+	}
+	conf := SplitterConfig{
+		UpstreamHost:  host,
+		CursorFile:    "cursor-file",
+		PebbleOptions: &ppopts,
+	}
+	pp, err := events.NewPebblePersistance(&ppopts)
+	if err != nil {
+		return nil, err
+	}
+
+	go pp.GCThread(context.Background())
+	em := events.NewEventManager(pp)
+	return &Splitter{
+		conf:      conf,
+		pp:        pp,
+		events:    em,
+		consumers: make(map[uint64]*SocketConsumer),
+		log:       slog.Default().With("system", "splitter"),
+	}, nil
+}
+
+func (s *Splitter) Start(addr string) error {
+	var lc net.ListenConfig
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	curs, err := s.getLastCursor()
+	if err != nil {
+		return fmt.Errorf("loading cursor failed: %w", err)
+	}
+
+	go s.subscribeWithRedialer(context.Background(), s.conf.UpstreamHost, curs)
+
+	li, err := lc.Listen(ctx, "tcp", addr)
+	if err != nil {
+		return err
+	}
+	return s.StartWithListener(li)
+}
+
+func (s *Splitter) StartMetrics(listen string) error {
+	http.Handle("/metrics", promhttp.Handler())
+	return http.ListenAndServe(listen, nil)
+}
+
+func (s *Splitter) Shutdown() error {
+	return nil
+}
+
+func (s *Splitter) StartWithListener(listen net.Listener) error {
+	e := echo.New()
+	e.HideBanner = true
+
+	e.Use(middleware.CORSWithConfig(middleware.CORSConfig{
+		AllowOrigins: []string{"*"},
+		AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderContentType, echo.HeaderAccept, echo.HeaderAuthorization},
+	}))
+
+	/*
+		if !s.ssl {
+			e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
+				Format: "method=${method}, uri=${uri}, status=${status} latency=${latency_human}\n",
+			}))
+		} else {
+			e.Use(middleware.LoggerWithConfig(middleware.DefaultLoggerConfig))
+		}
+	*/
+
+	e.Use(bgs.MetricsMiddleware)
+
+	e.HTTPErrorHandler = func(err error, ctx echo.Context) {
+		switch err := err.(type) {
+		case *echo.HTTPError:
+			if err2 := ctx.JSON(err.Code, map[string]any{
+				"error": err.Message,
+			}); err2 != nil {
+				s.log.Error("Failed to write http error", "err", err2)
+			}
+		default:
+			sendHeader := true
+			if ctx.Path() == "/xrpc/com.atproto.sync.subscribeRepos" {
+				sendHeader = false
+			}
+
+			s.log.Warn("HANDLER ERROR", "path", ctx.Path(), "err", err)
+
+			if strings.HasPrefix(ctx.Path(), "/admin/") {
+				ctx.JSON(500, map[string]any{
+					"error": err.Error(),
+				})
+				return
+			}
+
+			if sendHeader {
+				ctx.Response().WriteHeader(500)
+			}
+		}
+	}
+
+	// TODO: this API is temporary until we formalize what we want here
+
+	e.POST("/xrpc/com.atproto.sync.requestCrawl", s.RequestCrawlHandler)
+	e.GET("/xrpc/com.atproto.sync.subscribeRepos", s.EventsHandler)
+	e.GET("/xrpc/com.atproto.sync.listRepos", s.HandleComAtprotoSyncListRepos)
+
+	e.GET("/xrpc/_health", s.HandleHealthCheck)
+	e.GET("/_health", s.HandleHealthCheck)
+	e.GET("/", s.HandleHomeMessage)
+
+	// In order to support booting on random ports in tests, we need to tell the
+	// Echo instance it's already got a port, and then use its StartServer
+	// method to re-use that listener.
+	e.Listener = listen
+	srv := &http.Server{}
+	return e.StartServer(srv)
+}
+
+type HealthStatus struct {
+	Status  string `json:"status"`
+	Message string `json:"msg,omitempty"`
+}
+
+func (s *Splitter) HandleHealthCheck(c echo.Context) error {
+	return c.JSON(200, HealthStatus{Status: "ok"})
+}
+
+var homeMessage string = `
+          _      _
+ _ _ __ _(_)_ _ | |__  _____ __ __
+| '_/ _' | | ' \| '_ \/ _ \ V  V /
+|_| \__,_|_|_||_|_.__/\___/\_/\_/
+
+This is an atproto [https://atproto.com] firehose fanout service, running the 'rainbow' codebase [https://github.com/bluesky-social/indigo]
+
+The firehose WebSocket path is at:  /xrpc/com.atproto.sync.subscribeRepos
+`
+
+func (s *Splitter) HandleHomeMessage(c echo.Context) error {
+	return c.String(http.StatusOK, homeMessage)
+}
+
+type XRPCError struct {
+	Message string `json:"message"`
+}
+
+func (s *Splitter) RequestCrawlHandler(c echo.Context) error {
+	ctx := c.Request().Context()
+	var body comatproto.SyncRequestCrawl_Input
+	if err := c.Bind(&body); err != nil {
+		return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid body: %s", err)})
+	}
+
+	host := body.Hostname
+	if host == "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname")
+	}
+
+	if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
+		host = "https://" + host
+	}
+
+	u, err := url.Parse(host)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, "failed to parse hostname")
+	}
+
+	if u.Scheme == "http" {
+		return echo.NewHTTPError(http.StatusBadRequest, "this server requires https")
+	}
+	if u.Path != "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without path")
+	}
+
+	if u.Query().Encode() != "" {
+		return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without query")
+	}
+
+	host = u.Host // potentially hostname:port
+
+	clientHost := fmt.Sprintf("%s://%s", u.Scheme, host)
+
+	xrpcC := &xrpc.Client{
+		Host:   clientHost,
+		Client: http.DefaultClient, // not using the client that auto-retries
+	}
+
+	desc, err := atproto.ServerDescribeServer(ctx, xrpcC)
+	if err != nil {
+		errMsg := fmt.Sprintf("requested host (%s) failed to respond to describe request", clientHost)
+		return echo.NewHTTPError(http.StatusBadRequest, errMsg)
+	}
+
+	// Maybe we could do something with this response later
+	_ = desc
+
+	if len(s.nextCrawlers) != 0 {
+		blob, err := json.Marshal(body)
+		if err != nil {
+			s.log.Warn("could not forward requestCrawl, json err", "err", err)
+		} else {
+			go func(bodyBlob []byte) {
+				for _, remote := range s.nextCrawlers {
+					if remote == nil {
+						continue
+					}
+
+					pu := remote.JoinPath("/xrpc/com.atproto.sync.requestCrawl")
+					response, err := s.httpC.Post(pu.String(), "application/json", bytes.NewReader(bodyBlob))
+					if response != nil && response.Body != nil {
+						response.Body.Close()
+					}
+					if err != nil || response == nil {
+						s.log.Warn("requestCrawl forward failed", "host", remote, "err", err)
+					} else if response.StatusCode != http.StatusOK {
+						s.log.Warn("requestCrawl forward failed", "host", remote, "status", response.Status)
+					} else {
+						s.log.Info("requestCrawl forward successful", "host", remote)
+					}
+				}
+			}(blob)
+		}
+	}
+
+	return c.JSON(200, HealthStatus{Status: "ok"})
+}
+
+func (s *Splitter) HandleComAtprotoSyncListRepos(c echo.Context) error {
+	ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncListRepos")
+	defer span.End()
+
+	cursorQuery := c.QueryParam("cursor")
+	limitQuery := c.QueryParam("limit")
+
+	var err error
+
+	limit := int64(500)
+	if limitQuery != "" {
+		limit, err = strconv.ParseInt(limitQuery, 10, 64)
+		if err != nil || limit < 1 || limit > 1000 {
+			return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid limit: %s", limitQuery)})
+		}
+	}
+
+	client := xrpc.Client{
+		Client: s.httpC,
+		Host:   s.conf.XrpcRootUrl(),
+	}
+
+	out, handleErr := atproto.SyncListRepos(ctx, &client, cursorQuery, limit)
+	if handleErr != nil {
+		return handleErr
+	}
+	return c.JSON(200, out)
+}
+
+func (s *Splitter) EventsHandler(c echo.Context) error {
+	var since *int64
+	if sinceVal := c.QueryParam("cursor"); sinceVal != "" {
+		sval, err := strconv.ParseInt(sinceVal, 10, 64)
+		if err != nil {
+			return err
+		}
+		since = &sval
+	}
+
+	ctx, cancel := context.WithCancel(c.Request().Context())
+	defer cancel()
+
+	// TODO: authhhh
+	conn, err := websocket.Upgrade(c.Response(), c.Request(), c.Response().Header(), 10<<10, 10<<10)
+	if err != nil {
+		return fmt.Errorf("upgrading websocket: %w", err)
+	}
+
+	lastWriteLk := sync.Mutex{}
+	lastWrite := time.Now()
+
+	// Start a goroutine to ping the client every 30 seconds to check if it's
+	// still alive. If the client doesn't respond to a ping within 5 seconds,
+	// we'll close the connection and teardown the consumer.
+	go func() {
+		ticker := time.NewTicker(30 * time.Second)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				lastWriteLk.Lock()
+				lw := lastWrite
+				lastWriteLk.Unlock()
+
+				if time.Since(lw) < 30*time.Second {
+					continue
+				}
+
+				if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(5*time.Second)); err != nil {
+					s.log.Error("failed to ping client", "err", err)
+					cancel()
+					return
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	conn.SetPingHandler(func(message string) error {
+		err := conn.WriteControl(websocket.PongMessage, []byte(message), time.Now().Add(time.Second*60))
+		if err == websocket.ErrCloseSent {
+			return nil
+		} else if e, ok := err.(net.Error); ok && e.Temporary() {
+			return nil
+		}
+		return err
+	})
+
+	// Start a goroutine to read messages from the client and discard them.
+	go func() {
+		for {
+			_, _, err := conn.ReadMessage()
+			if err != nil {
+				s.log.Error("failed to read message from client", "err", err)
+				cancel()
+				return
+			}
+		}
+	}()
+
+	ident := c.RealIP() + "-" + c.Request().UserAgent()
+
+	evts, cleanup, err := s.events.Subscribe(ctx, ident, func(evt *events.XRPCStreamEvent) bool { return true }, since)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	// Keep track of the consumer for metrics and admin endpoints
+	consumer := SocketConsumer{
+		RemoteAddr:  c.RealIP(),
+		UserAgent:   c.Request().UserAgent(),
+		ConnectedAt: time.Now(),
+	}
+	sentCounter := eventsSentCounter.WithLabelValues(consumer.RemoteAddr, consumer.UserAgent)
+	consumer.EventsSent = sentCounter
+
+	consumerID := s.registerConsumer(&consumer)
+	defer s.cleanupConsumer(consumerID)
+
+	s.log.Info("new consumer",
+		"remote_addr", consumer.RemoteAddr,
+		"user_agent", consumer.UserAgent,
+		"cursor", since,
+		"consumer_id", consumerID,
+	)
+	activeClientGauge.Inc()
+	defer activeClientGauge.Dec()
+
+	for {
+		select {
+		case evt, ok := <-evts:
+			if !ok {
+				s.log.Error("event stream closed unexpectedly")
+				return nil
+			}
+
+			wc, err := conn.NextWriter(websocket.BinaryMessage)
+			if err != nil {
+				s.log.Error("failed to get next writer", "err", err)
+				return err
+			}
+
+			if evt.Preserialized != nil {
+				_, err = wc.Write(evt.Preserialized)
+			} else {
+				err = evt.Serialize(wc)
+			}
+			if err != nil {
+				return fmt.Errorf("failed to write event: %w", err)
+			}
+
+			if err := wc.Close(); err != nil {
+				s.log.Warn("failed to flush-close our event write", "err", err)
+				return nil
+			}
+
+			lastWriteLk.Lock()
+			lastWrite = time.Now()
+			lastWriteLk.Unlock()
+			sentCounter.Inc()
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
+type SocketConsumer struct {
+	UserAgent   string
+	RemoteAddr  string
+	ConnectedAt time.Time
+	EventsSent  promclient.Counter
+}
+
+func (s *Splitter) registerConsumer(c *SocketConsumer) uint64 {
+	s.consumersLk.Lock()
+	defer s.consumersLk.Unlock()
+
+	id := s.nextConsumerID
+	s.nextConsumerID++
+
+	s.consumers[id] = c
+
+	return id
+}
+
+func (s *Splitter) cleanupConsumer(id uint64) {
+	s.consumersLk.Lock()
+	defer s.consumersLk.Unlock()
+
+	c := s.consumers[id]
+
+	var m = &dto.Metric{}
+	if err := c.EventsSent.Write(m); err != nil {
+		s.log.Error("failed to get sent counter", "err", err)
+	}
+
+	s.log.Info("consumer disconnected",
+		"consumer_id", id,
+		"remote_addr", c.RemoteAddr,
+		"user_agent", c.UserAgent,
+		"events_sent", m.Counter.GetValue())
+
+	delete(s.consumers, id)
+}
+
+func sleepForBackoff(b int) time.Duration {
+	if b == 0 {
+		return 0
+	}
+
+	if b < 50 {
+		return time.Millisecond * time.Duration(rand.Intn(100)+(5*b))
+	}
+
+	return time.Second * 5
+}
+
+func (s *Splitter) subscribeWithRedialer(ctx context.Context, host string, cursor int64) {
+	d := websocket.Dialer{}
+
+	protocol := "wss"
+
+	var backoff int
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		header := http.Header{
+			"User-Agent": []string{"bgs-rainbow-v0"},
+		}
+
+		var url string
+		if cursor < 0 {
+			url = fmt.Sprintf("%s://%s/xrpc/com.atproto.sync.subscribeRepos", protocol, host)
+		} else {
+			url = fmt.Sprintf("%s://%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", protocol, host, cursor)
+		}
+		con, res, err := d.DialContext(ctx, url, header)
+		if err != nil {
+			s.log.Warn("dialing failed", "host", host, "err", err, "backoff", backoff)
+			time.Sleep(sleepForBackoff(backoff))
+			backoff++
+
+			continue
+		}
+
+		s.log.Info("event subscription response", "code", res.StatusCode)
+
+		if err := s.handleConnection(ctx, host, con, &cursor); err != nil {
+			s.log.Warn("connection failed", "host", host, "err", err)
+		}
+	}
+}
+
+func (s *Splitter) handleConnection(ctx context.Context, host string, con *websocket.Conn, lastCursor *int64) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	sched := sequential.NewScheduler("splitter", func(ctx context.Context, evt *events.XRPCStreamEvent) error {
+		seq := events.SequenceForEvent(evt)
+		if seq < 0 {
+			// ignore info events and other unsupported types
+			return nil
+		}
+
+		if err := s.events.AddEvent(ctx, evt); err != nil {
+			return err
+		}
+
+		if seq%5000 == 0 {
+			// TODO: don't need this after we move to getting seq from pebble
+			if err := s.writeCursor(seq); err != nil {
+				s.log.Error("write cursor failed", "err", err)
+			}
+		}
+
+		*lastCursor = seq
+		return nil
+	})
+
+	return events.HandleRepoStream(ctx, con, sched, nil)
+}
+
+func (s *Splitter) getLastCursor() (int64, error) {
+	if s.pp != nil {
+		seq, millis, _, err := s.pp.GetLast(context.Background())
+		if err == nil {
+			s.log.Debug("got last cursor from pebble", "seq", seq, "millis", millis)
+			return seq, nil
+		} else if errors.Is(err, events.ErrNoLast) {
+			s.log.Info("pebble no last")
+		} else {
+			s.log.Error("pebble seq fail", "err", err)
+		}
+	}
+
+	fi, err := os.Open(s.conf.CursorFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return -1, nil
+		}
+		return -1, err
+	}
+
+	b, err := io.ReadAll(fi)
+	if err != nil {
+		return -1, err
+	}
+
+	v, err := strconv.ParseInt(string(b), 10, 64)
+	if err != nil {
+		return -1, err
+	}
+
+	return v, nil
+}
+
+func (s *Splitter) writeCursor(curs int64) error {
+	return os.WriteFile(s.conf.CursorFile, []byte(fmt.Sprint(curs)), 0664)
+}

testing/integ_test.go

···
 	"github.com/bluesky-social/indigo/repo"
 	"github.com/bluesky-social/indigo/xrpc"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-log/v2"
 	car "github.com/ipld/go-car"
 	"github.com/stretchr/testify/assert"
 )
 
-func init() {
-	log.SetAllLoggers(log.LevelInfo)
+func TestRelayBasic(t *testing.T) {
+	t.Helper()
+	testRelayBasic(t, true)
+}
+
+func TestRelayBasicNonArchive(t *testing.T) {
+	t.Helper()
+	testRelayBasic(t, false)
 }
 
-func TestRelayBasic(t *testing.T) {
+func testRelayBasic(t *testing.T, archive bool) {
 	if testing.Short() {
 		t.Skip("skipping Relay test in 'short' test mode")
 	}
···
 	p1 := MustSetupPDS(t, ".tpds", didr)
 	p1.Run(t)
 
-	b1 := MustSetupRelay(t, didr)
+	b1 := MustSetupRelay(t, didr, archive)
 	b1.Run(t)
 
 	b1.tr.TrialHosts = []string{p1.RawHost()}
···
 }
 
 func TestRelayMultiPDS(t *testing.T) {
+	t.Helper()
+	testRelayMultiPDS(t, true)
+}
+
+func TestRelayMultiPDSNonArchive(t *testing.T) {
+	t.Helper()
+	testRelayMultiPDS(t, false)
+}
+
+func testRelayMultiPDS(t *testing.T, archive bool) {
 	if testing.Short() {
 		t.Skip("skipping Relay test in 'short' test mode")
 	}
···
 	p2 := MustSetupPDS(t, ".pdsdos", didr)
 	p2.Run(t)
 
-	b1 := MustSetupRelay(t, didr)
+	b1 := MustSetupRelay(t, didr, archive)
 	b1.Run(t)
 
 	b1.tr.TrialHosts = []string{p1.RawHost(), p2.RawHost()}
···
 	p2 := MustSetupPDS(t, ".pdsdos", didr)
 	p2.Run(t)
 
-	b1 := MustSetupRelay(t, didr)
+	b1 := MustSetupRelay(t, didr, true)
 	b1.Run(t)
 
 	b1.tr.TrialHosts = []string{p1.RawHost(), p2.RawHost()}
···
 	p1 := MustSetupPDS(t, ".pdsuno", didr)
 	p1.Run(t)
 
-	b1 := MustSetupRelay(t, didr)
+	b1 := MustSetupRelay(t, didr, true)
 	b1.Run(t)
 
 	b1.tr.TrialHosts = []string{p1.RawHost()}
···
 	p1 := MustSetupPDS(t, ".pdsuno", didr)
 	p1.Run(t)
 
-	b1 := MustSetupRelay(t, didr)
+	b1 := MustSetupRelay(t, didr, true)
 	b1.Run(t)
 
 	b1.tr.TrialHosts = []string{p1.RawHost()}
···
 }
 
 func TestRelayTakedown(t *testing.T) {
+	testRelayTakedown(t, true)
+}
+
+func TestRelayTakedownNonArchive(t *testing.T) {
+	testRelayTakedown(t, false)
+}
+
+func testRelayTakedown(t *testing.T, archive bool) {
 	if testing.Short() {
 		t.Skip("skipping Relay test in 'short' test mode")
 	}
···
 	p1 := MustSetupPDS(t, ".tpds", didr)
 	p1.Run(t)
 
-	b1 := MustSetupRelay(t, didr)
+	b1 := MustSetupRelay(t, didr, true)
 	b1.Run(t)
 
 	b1.tr.TrialHosts = []string{p1.RawHost()}
···
 	}
 	didr := TestPLC(t)
 
-	b1 := MustSetupRelay(t, didr)
+	b1 := MustSetupRelay(t, didr, true)
 	b1.Run(t)
 
 	b1.BanDomain(t, "foo.com")
···
 	p1 := MustSetupPDS(t, ".tpds", didr)
 	p1.Run(t)
 
-	b1 := MustSetupRelay(t, didr)
+	b1 := MustSetupRelay(t, didr, true)
 	b1.Run(t)
 
 	b1.tr.TrialHosts = []string{p1.RawHost()}
···
 	e1 := evts.Next()
 	assert.NotNil(e1.RepoCommit)
 	assert.Equal(e1.RepoCommit.Repo, bob.DID())
+	fmt.Println(e1.RepoCommit.Ops[0])
 
 	ctx := context.TODO()
 	rm := p1.server.Repoman()
···
 	}
 
 	e2 := evts.Next()
+	//fmt.Println(e2.RepoCommit.Ops[0])
 	assert.Equal(len(e2.RepoCommit.Ops), 0)
 	assert.Equal(e2.RepoCommit.Repo, bob.DID())
 }

testing/utils.go

···
 		return nil, err
 	}
 
-	cs, err := carstore.NewCarStore(cardb, cspath)
+	cs, err := carstore.NewCarStore(cardb, []string{cspath})
 	if err != nil {
 		return nil, err
 	}
···
 	t.Helper()
 
 	ctx := context.TODO()
-	resp, err := bsky.NotificationListNotifications(ctx, u.client, "", 100, false, "")
+	resp, err := bsky.NotificationListNotifications(ctx, u.client, "", 100, false, nil, "")
 	if err != nil {
 		t.Fatal(err)
 	}
···
 	return t.listener.Addr().String()
 }
 
-func MustSetupRelay(t *testing.T, didr plc.PLCClient) *TestRelay {
+func MustSetupRelay(t *testing.T, didr plc.PLCClient, archive bool) *TestRelay {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	tbgs, err := SetupRelay(ctx, didr)
+	tbgs, err := SetupRelay(ctx, didr, archive)
 	if err != nil {
 		t.Fatal(err)
 	}
···
 	return tbgs
 }
 
-func SetupRelay(ctx context.Context, didr plc.PLCClient) (*TestRelay, error) {
+func SetupRelay(ctx context.Context, didr plc.PLCClient, archive bool) (*TestRelay, error) {
 	dir, err := os.MkdirTemp("", "integtest")
 	if err != nil {
 		return nil, err
···
 		return nil, err
 	}
 
-	cs, err := carstore.NewCarStore(cardb, cspath)
-	if err != nil {
-		return nil, err
+	var cs carstore.CarStore
+	if archive {
+		arccs, err := carstore.NewCarStore(cardb, []string{cspath})
+		if err != nil {
+			return nil, err
+		}
+		cs = arccs
+	} else {
+		nacs, err := carstore.NewNonArchivalCarstore(cardb)
+		if err != nil {
+			return nil, err
+		}
+		cs = nacs
 	}
 
 	//kmgr := indexer.NewKeyManager(didr, nil)
···
 			},
 		}
 		seqScheduler := sequential.NewScheduler("test", rsc.EventHandler)
-		if err := events.HandleRepoStream(ctx, con, seqScheduler); err != nil {
+		if err := events.HandleRepoStream(ctx, con, seqScheduler, nil); err != nil {
 			fmt.Println(err)
 		}
 	}()

util/cliutil/ipfslog.go

···
+package cliutil
+
+import (
+	"io"
+
+	ipfslog "github.com/ipfs/go-log/v2"
+	"go.uber.org/zap/zapcore"
+)
+
+func SetIpfsWriter(out io.Writer, format string, level string) {
+	var ze zapcore.Encoder
+	switch format {
+	case "json":
+		ze = zapcore.NewJSONEncoder(zapcore.EncoderConfig{})
+	case "text":
+		ze = zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
+	default:
+		ze = zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
+	}
+	var zl zapcore.LevelEnabler
+	switch level {
+	case "debug":
+		zl = zapcore.DebugLevel
+	case "info":
+		zl = zapcore.InfoLevel
+	case "warn":
+		zl = zapcore.WarnLevel
+	case "error":
+		zl = zapcore.ErrorLevel
+	default:
+		zl = zapcore.InfoLevel
+	}
+	nc := zapcore.NewCore(ze, zapcore.AddSync(out), zl)
+	ipfslog.SetPrimaryCore(nc)
+}

util/cliutil/util.go

···
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"io/fs"
+	"log/slog"
 	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
···
 
 	return db, nil
 }
+
+type LogOptions struct {
+	// e.g. 1_000_000_000
+	LogRotateBytes int64
+
+	// path to write to, if rotating, %T gets UnixMilli at file open time
+	// NOTE: substitution is simple replace("%T", "")
+	LogPath string
+
+	// text|json
+	LogFormat string
+
+	// info|debug|warn|error
+	LogLevel string
+
+	// Keep N old logs (not including current); <0 disables removal, 0==remove all old log files immediately
+	KeepOld int
+}
+
+func firstenv(env_var_names ...string) string {
+	for _, env_var_name := range env_var_names {
+		val := os.Getenv(env_var_name)
+		if val != "" {
+			return val
+		}
+	}
+	return ""
+}
+
+// SetupSlog integrates passed in options and env vars.
+//
+// passing default cliutil.LogOptions{} is ok.
+//
+// BSKYLOG_LOG_LEVEL=info|debug|warn|error
+//
+// BSKYLOG_LOG_FMT=text|json
+//
+// BSKYLOG_FILE=path (or "-" or "" for stdout), %T gets UnixMilli; if a path with '/', {prefix}/current becomes a link to active log file
+//
+// BSKYLOG_ROTATE_BYTES=int maximum size of log chunk before rotating
+//
+// BSKYLOG_ROTATE_KEEP=int keep N olg logs (not including current)
+//
+// The env vars were derived from ipfs logging library, and also respond to some GOLOG_ vars from that library,
+// but BSKYLOG_ variables are preferred because imported code still using the ipfs log library may misbehave
+// if some GOLOG values are set, especially GOLOG_FILE.
+func SetupSlog(options LogOptions) (*slog.Logger, error) {
+	fmt.Fprintf(os.Stderr, "SetupSlog\n")
+	var hopts slog.HandlerOptions
+	hopts.Level = slog.LevelInfo
+	hopts.AddSource = true
+	if options.LogLevel == "" {
+		options.LogLevel = firstenv("BSKYLOG_LOG_LEVEL", "GOLOG_LOG_LEVEL")
+	}
+	if options.LogLevel == "" {
+		hopts.Level = slog.LevelInfo
+		options.LogLevel = "info"
+	} else {
+		level := strings.ToLower(options.LogLevel)
+		switch level {
+		case "debug":
+			hopts.Level = slog.LevelDebug
+		case "info":
+			hopts.Level = slog.LevelInfo
+		case "warn":
+			hopts.Level = slog.LevelWarn
+		case "error":
+			hopts.Level = slog.LevelError
+		default:
+			return nil, fmt.Errorf("unknown log level: %#v", options.LogLevel)
+		}
+	}
+	if options.LogFormat == "" {
+		options.LogFormat = firstenv("BSKYLOG_LOG_FMT", "GOLOG_LOG_FMT")
+	}
+	if options.LogFormat == "" {
+		options.LogFormat = "text"
+	} else {
+		format := strings.ToLower(options.LogFormat)
+		if format == "json" || format == "text" {
+			// ok
+		} else {
+			return nil, fmt.Errorf("invalid log format: %#v", options.LogFormat)
+		}
+		options.LogFormat = format
+	}
+
+	if options.LogPath == "" {
+		options.LogPath = firstenv("BSKYLOG_FILE", "GOLOG_FILE")
+	}
+	if options.LogRotateBytes == 0 {
+		rotateBytesStr := os.Getenv("BSKYLOG_ROTATE_BYTES") // no GOLOG equivalent
+		if rotateBytesStr != "" {
+			rotateBytes, err := strconv.ParseInt(rotateBytesStr, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("invalid BSKYLOG_ROTATE_BYTES value: %w", err)
+			}
+			options.LogRotateBytes = rotateBytes
+		}
+	}
+	if options.KeepOld == 0 {
+		keepOldUnset := true
+		keepOldStr := os.Getenv("BSKYLOG_ROTATE_KEEP") // no GOLOG equivalent
+		if keepOldStr != "" {
+			keepOld, err := strconv.ParseInt(keepOldStr, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("invalid BSKYLOG_ROTATE_KEEP value: %w", err)
+			}
+			keepOldUnset = false
+			options.KeepOld = int(keepOld)
+		}
+		if keepOldUnset {
+			options.KeepOld = 2
+		}
+	}
+	logaround := make(chan string, 100)
+	go logbouncer(logaround)
+	var out io.Writer
+	if (options.LogPath == "") || (options.LogPath == "-") {
+		out = os.Stdout
+	} else if options.LogRotateBytes != 0 {
+		out = &logRotateWriter{
+			rotateBytes:     options.LogRotateBytes,
+			outPathTemplate: options.LogPath,
+			keep:            options.KeepOld,
+			logaround:       logaround,
+		}
+	} else {
+		var err error
+		out, err = os.Create(options.LogPath)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", options.LogPath, err)
+		}
+		fmt.Fprintf(os.Stderr, "SetupSlog create %#v\n", options.LogPath)
+	}
+	var handler slog.Handler
+	switch options.LogFormat {
+	case "text":
+		handler = slog.NewTextHandler(out, &hopts)
+	case "json":
+		handler = slog.NewJSONHandler(out, &hopts)
+	default:
+		return nil, fmt.Errorf("unknown log format: %#v", options.LogFormat)
+	}
+	logger := slog.New(handler)
+	slog.SetDefault(logger)
+	templateDirPart, _ := filepath.Split(options.LogPath)
+	ents, _ := os.ReadDir(templateDirPart)
+	for _, ent := range ents {
+		fmt.Fprintf(os.Stdout, "%s\n", filepath.Join(templateDirPart, ent.Name()))
+	}
+	SetIpfsWriter(out, options.LogFormat, options.LogLevel)
+	return logger, nil
+}
+
+type logRotateWriter struct {
+	currentWriter io.WriteCloser
+
+	// how much has been written to current log file
+	currentBytes int64
+
+	// e.g. path/to/logs/foo%T
+	currentPath string
+
+	// e.g. path/to/logs/current
+	currentPathCurrent string
+
+	rotateBytes int64
+
+	outPathTemplate string
+
+	// keep the most recent N log files (not including current)
+	keep int
+
+	// write strings to this from inside the log system, a task outside the log system hands them to slog.Info()
+	logaround chan<- string
+}
+
+func logbouncer(out <-chan string) {
+	var logger *slog.Logger
+	for line := range out {
+		fmt.Fprintf(os.Stderr, "ll %s\n", line)
+		if logger == nil {
+			// lazy to make sure it crops up after slog Default has been set
+			logger = slog.Default().With("system", "logging")
+		}
+		logger.Info(line)
+	}
+}
+
+var currentMatcher = regexp.MustCompile("current_\\d+")
+
+func (w *logRotateWriter) cleanOldLogs() {
+	if w.keep < 0 {
+		// old log removal is disabled
+		return
+	}
+	// w.currentPath was recently set as the new log
+	dirpart, _ := filepath.Split(w.currentPath)
+	// find old logs
+	templateDirPart, templateNamePart := filepath.Split(w.outPathTemplate)
+	if dirpart != templateDirPart {
+		w.logaround <- fmt.Sprintf("current dir part %#v != template dir part %#v\n", w.currentPath, w.outPathTemplate)
+		return
+	}
+	// build a regexp that is string literal parts with \d+ replacing the UnixMilli part
+	templateNameParts := strings.Split(templateNamePart, "%T")
+	var sb strings.Builder
+	first := true
+	for _, part := range templateNameParts {
+		if first {
+			first = false
+		} else {
+			sb.WriteString("\\d+")
+		}
+		sb.WriteString(regexp.QuoteMeta(part))
+	}
+	tmre, err := regexp.Compile(sb.String())
+	if err != nil {
+		w.logaround <- fmt.Sprintf("failed to compile old log template regexp: %#v\n", err)
+		return
+	}
+	dir, err := os.ReadDir(dirpart)
+	if err != nil {
+		w.logaround <- fmt.Sprintf("failed to read old log template dir: %#v\n", err)
+		return
+	}
+	var found []fs.FileInfo
+	for _, ent := range dir {
+		name := ent.Name()
+		if tmre.MatchString(name) || currentMatcher.MatchString(name) {
+			fi, err := ent.Info()
+			if err != nil {
+				continue
+			}
+			found = append(found, fi)
+		}
+	}
+	if len(found) <= w.keep {
+		// not too many, nothing to do
+		return
+	}
+	foundMtimeLess := func(i, j int) bool {
+		return found[i].ModTime().Before(found[j].ModTime())
+	}
+	sort.Slice(found, foundMtimeLess)
+	drops := found[:len(found)-w.keep]
+	for _, fi := range drops {
+		fullpath := filepath.Join(dirpart, fi.Name())
+		err = os.Remove(fullpath)
+		if err != nil {
+			w.logaround <- fmt.Sprintf("failed to rm old log: %#v\n", err)
+			// but keep going
+		}
+		// maybe it would be safe to debug-log old log removal from within the logging infrastructure?
+	}
+}
+
+func (w *logRotateWriter) closeOldLog() []error {
+	if w.currentWriter == nil {
+		return nil
+	}
+	var earlyWeakErrors []error
+	err := w.currentWriter.Close()
+	if err != nil {
+		earlyWeakErrors = append(earlyWeakErrors, err)
+	}
+	w.currentWriter = nil
+	w.currentBytes = 0
+	w.currentPath = ""
+	if w.currentPathCurrent != "" {
+		err = os.Remove(w.currentPathCurrent) // not really an error until something else goes wrong
+		if err != nil {
+			earlyWeakErrors = append(earlyWeakErrors, err)
+		}
+		w.currentPathCurrent = ""
+	}
+	return earlyWeakErrors
+}
+
+func (w *logRotateWriter) openNewLog(earlyWeakErrors []error) (badErr error, weakErrors []error) {
+	nowMillis := time.Now().UnixMilli()
+	nows := strconv.FormatInt(nowMillis, 10)
+	w.currentPath = strings.Replace(w.outPathTemplate, "%T", nows, -1)
+	var err error
+	w.currentWriter, err = os.Create(w.currentPath)
+	if err != nil {
+		earlyWeakErrors = append(earlyWeakErrors, err)
+		return errors.Join(earlyWeakErrors...), nil
+	}
+	w.logaround <- fmt.Sprintf("new log file %#v", w.currentPath)
+	w.cleanOldLogs()
+	dirpart, _ := filepath.Split(w.currentPath)
+	if dirpart != "" {
+		w.currentPathCurrent = filepath.Join(dirpart, "current")
+		fi, err := os.Stat(w.currentPathCurrent)
+		if err == nil && fi.Mode().IsRegular() {
+			// move aside unknown "current" from a previous run
+			// see also currentMatcher regexp current_\d+
+			err = os.Rename(w.currentPathCurrent, w.currentPathCurrent+"_"+nows)
+			if err != nil {
+				// not crucial if we can't move aside "current"
+				// TODO: log warning ... but not from inside log writer?
+				earlyWeakErrors = append(earlyWeakErrors, err)
+			}
+		}
+		err = os.Link(w.currentPath, w.currentPathCurrent)
+		if err != nil {
+			// not crucial if we can't make "current" link
+			// TODO: log warning ... but not from inside log writer?
+			earlyWeakErrors = append(earlyWeakErrors, err)
+		}
+	}
+	return nil, earlyWeakErrors
+}
+
+func (w *logRotateWriter) Write(p []byte) (n int, err error) {
+	var earlyWeakErrors []error
+	if int64(len(p))+w.currentBytes > w.rotateBytes {
+		// next write would be over the limit
+		earlyWeakErrors = w.closeOldLog()
+	}
+	if w.currentWriter == nil {
+		// start new log file
+		var err error
+		err, earlyWeakErrors = w.openNewLog(earlyWeakErrors)
+		if err != nil {
+			return 0, err
+		}
+	}
+	var wrote int
+	wrote, err = w.currentWriter.Write(p)
+	w.currentBytes += int64(wrote)
+	if err != nil {
+		earlyWeakErrors = append(earlyWeakErrors, err)
+		return wrote, errors.Join(earlyWeakErrors...)
+	}
+	if earlyWeakErrors != nil {
+		w.logaround <- fmt.Sprintf("ok, but: %s", errors.Join(earlyWeakErrors...).Error())
+	}
+	return wrote, nil
+}