add support for jetstream backend and multiple sync backends

README.md

···
 
 It will take a minute but it should pull all records from that user.
 
+## Upstream Firehose Configuration
+
+Konbini supports both standard firehose endpoints as well as jetstream. If
+bandwidth and CPU usage is a concern, and you trust the jetstream endpoint,
+then it may be worth trying that out.
+
+The configuration file is formatted as follows:
+
+```json
+{
+  "backends": [
+    {
+      "type": "jetstream",
+      "host": "jetstream1.us-west.bsky.network"
+    }
+  ]
+}
+```
+
+The default (implicit) configuration file looks like this:
+
+```json
+{
+  "backends": [
+    {
+      "type": "firehose",
+      "host": "bsky.network"
+    }
+  ]
+}
+```
+
+Note that this is an array of backends, you can specify multiple upstreams, and
+konbini will read from all of them. The main intended purpose of this is to be
+able to subscribe directly to PDSs. PDSs currently only support the full
+firehose endpoint, not jetstream, so be sure to specify a type of "firehose"
+for individual PDS endpoints.
+
 ## License
 
 MIT (whyrusleeping)

backend/events.go

···
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"log/slog"
 	"strings"
···
 	"github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/atproto/syntax"
+	lexutil "github.com/bluesky-social/indigo/lex/util"
 	"github.com/bluesky-social/indigo/repo"
+	jsmodels "github.com/bluesky-social/jetstream/pkg/models"
 	"github.com/ipfs/go-cid"
 	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/prometheus/client_golang/prometheus"
···
 			return fmt.Errorf("failed to update rev: %w", err)
 		}
 	*/
+
+	return nil
+}
+
+func cborBytesFromEvent(evt *jsmodels.Event) ([]byte, error) {
+	val, err := lexutil.NewFromType(evt.Commit.Collection)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load event record type: %w", err)
+	}
+
+	if err := json.Unmarshal(evt.Commit.Record, val); err != nil {
+		return nil, err
+	}
+
+	cval, ok := val.(lexutil.CBOR)
+	if !ok {
+		return nil, fmt.Errorf("decoded type was not cbor marshalable")
+	}
+
+	buf := new(bytes.Buffer)
+	if err := cval.MarshalCBOR(buf); err != nil {
+		return nil, fmt.Errorf("failed to marshal event to cbor: %w", err)
+	}
+
+	rec := buf.Bytes()
+	return rec, nil
+}
+
+func (b *PostgresBackend) HandleEventJetstream(ctx context.Context, evt *jsmodels.Event) error {
+
+	path := evt.Commit.Collection + "/" + evt.Commit.RKey
+	switch evt.Commit.Operation {
+	case jsmodels.CommitOperationCreate:
+		rec, err := cborBytesFromEvent(evt)
+		if err != nil {
+			return err
+		}
+
+		c, err := cid.Decode(evt.Commit.CID)
+		if err != nil {
+			return err
+		}
+
+		if err := b.HandleCreate(ctx, evt.Did, evt.Commit.Rev, path, &rec, &c); err != nil {
+			return fmt.Errorf("create record failed: %w", err)
+		}
+	case jsmodels.CommitOperationUpdate:
+		rec, err := cborBytesFromEvent(evt)
+		if err != nil {
+			return err
+		}
+
+		c, err := cid.Decode(evt.Commit.CID)
+		if err != nil {
+			return err
+		}
+
+		if err := b.HandleUpdate(ctx, evt.Did, evt.Commit.Rev, path, &rec, &c); err != nil {
+			return fmt.Errorf("update record failed: %w", err)
+		}
+	case jsmodels.CommitOperationDelete:
+		if err := b.HandleDelete(ctx, evt.Did, evt.Commit.Rev, path); err != nil {
+			return fmt.Errorf("delete record failed: %w", err)
+		}
+	}
 
 	return nil
 }

go.mod

···
 go 1.25.1
 
 require (
-	github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f
+	github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe
+	github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1
 	github.com/gorilla/websocket v1.5.1
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/ipfs/go-cid v0.4.1
···
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipfs/go-verifcid v0.0.3 // indirect
-	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 // indirect
+	github.com/ipld/go-car v0.6.2 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
···
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
-	github.com/klauspost/compress v1.17.3 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.1 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
···
 	github.com/orandin/slog-gorm v1.3.2 // indirect
 	github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.48.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/prometheus/common v0.54.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/redis/go-redis/v9 v9.3.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect

go.sum

···
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f h1:FugOoTzh0nCMTWGqNGsjttFWVPcwxaaGD3p/nE9V8qY=
-github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f/go.mod h1:n6QE1NDPFoi7PRbMUZmc2y7FibCqiVU4ePpsvhHUBR8=
+github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe h1:VBhaqE5ewQgXbY5SfSWFZC/AwHFo7cHxZKFYi2ce9Yo=
+github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe/go.mod h1:RuQVrCGm42QNsgumKaR6se+XkFKfCPNwdCiTvqKRUck=
+github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1 h1:ovcRKN1iXZnY5WApVg+0Hw2RkwMH0ziA7lSAA8vellU=
+github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1/go.mod h1:5PtGi4r/PjEVBBl+0xWuQn4mBEjr9h6xsfDBADS6cHs=
 github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous=
 github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
···
 github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
 github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs=
 github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw=
-github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 h1:oFo19cBmcP0Cmg3XXbrr0V/c+xU9U1huEZp8+OgBzdI=
-github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4/go.mod h1:6nkFF8OmR5wLKBzRKi7/YFJpyYR7+oEn1DX+mMWnlLA=
+github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc=
+github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
 github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4=
 github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
···
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
-github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
 github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8=
···
 github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
-github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
-github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
-github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8=
+github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA=
 github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0=
 github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=

hydration/post.go

···
 	if dbPost.NotFound || len(dbPost.Raw) == 0 {
 		if autoFetch {
 			h.AddMissingRecord(uri, true)
-			if err := h.db.Raw(`SELECT * FROM posts WHERE author = ? AND rkey = ? `, r.ID, extractRkeyFromURI(uri)).Scan(&dbPost).Error; err != nil {
+			if err := h.db.Raw(`SELECT * FROM posts WHERE author = ? AND rkey = ?`, r.ID, extractRkeyFromURI(uri)).Scan(&dbPost).Error; err != nil {
 				return nil, fmt.Errorf("failed to query post: %w", err)
 			}
 			if dbPost.NotFound || len(dbPost.Raw) == 0 {

main.go

···
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"log/slog"
···
 	"github.com/bluesky-social/indigo/atproto/identity"
 	"github.com/bluesky-social/indigo/atproto/identity/redisdir"
 	"github.com/bluesky-social/indigo/atproto/syntax"
-	"github.com/bluesky-social/indigo/cmd/relay/stream"
-	"github.com/bluesky-social/indigo/cmd/relay/stream/schedulers/parallel"
 	"github.com/bluesky-social/indigo/repo"
 	"github.com/bluesky-social/indigo/util/cliutil"
 	xrpclib "github.com/bluesky-social/indigo/xrpc"
-	"github.com/gorilla/websocket"
 	"github.com/ipfs/go-cid"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/prometheus/client_golang/prometheus"
···
 		},
 		&cli.StringFlag{
 			Name: "redis-url",
+		},
+		&cli.StringFlag{
+			Name: "sync-config",
 		},
 	}
 	app.Action = func(cctx *cli.Context) error {
···
 
 			db: db,
 		}
-		fmt.Println("MY DID: ", s.mydid)
 
 		pgb, err := backend.NewPostgresBackend(mydid, db, pool, cc, dir)
 		if err != nil {
···
 			http.ListenAndServe(":4445", nil)
 		}()
 
-		seqno, err := loadLastSeq(db, "firehose_seq")
-		if err != nil {
-			fmt.Println("failed to load sequence number, starting over", err)
+		sc := SyncConfig{
+			Backends: []SyncBackend{
+				{
+					Type: "firehose",
+					Host: "bsky.network",
+				},
+			},
+		}
+
+		if scfn := cctx.String("sync-config"); scfn != "" {
+			{
+				scfi, err := os.Open(scfn)
+				if err != nil {
+					return err
+				}
+				defer scfi.Close()
+
+				var lsc SyncConfig
+				if err := json.NewDecoder(scfi).Decode(&lsc); err != nil {
+					return err
+				}
+				sc = lsc
+			}
 		}
 
-		return s.startLiveTail(ctx, int(seqno), 10, 20)
+		/*
+			sc.Backends[0] = SyncBackend{
+				Type: "jetstream",
+				Host: "jetstream1.us-west.bsky.network",
+			}
+		*/
+
+		return s.StartSyncEngine(ctx, &sc)
+
 	}
 
 	app.RunAndExitOnError()
···
 func (s *Server) getXrpcClient() (*xrpclib.Client, error) {
 	// TODO: handle refreshing the token periodically
 	return s.client, nil
-}
-
-func (s *Server) startLiveTail(ctx context.Context, curs int, parWorkers, maxQ int) error {
-	slog.Info("starting live tail")
-
-	// Connect to the Relay websocket
-	urlStr := fmt.Sprintf("wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", curs)
-
-	d := websocket.DefaultDialer
-	con, _, err := d.Dial(urlStr, http.Header{
-		"User-Agent": []string{"market/0.0.1"},
-	})
-	if err != nil {
-		return fmt.Errorf("failed to connect to relay: %w", err)
-	}
-
-	var lelk sync.Mutex
-	lastEvent := time.Now()
-
-	go func() {
-		for range time.Tick(time.Second) {
-			lelk.Lock()
-			let := lastEvent
-			lelk.Unlock()
-
-			if time.Since(let) > time.Second*30 {
-				slog.Error("firehose connection timed out")
-				con.Close()
-				return
-			}
-
-		}
-
-	}()
-
-	var cclk sync.Mutex
-	var completeCursor int64
-
-	rsc := &stream.RepoStreamCallbacks{
-		RepoCommit: func(evt *atproto.SyncSubscribeRepos_Commit) error {
-			ctx := context.Background()
-
-			firehoseCursorGauge.WithLabelValues("ingest").Set(float64(evt.Seq))
-
-			s.seqLk.Lock()
-			if evt.Seq > s.lastSeq {
-				curs = int(evt.Seq)
-				s.lastSeq = evt.Seq
-
-				if evt.Seq%1000 == 0 {
-					if err := storeLastSeq(s.db, "firehose_seq", evt.Seq); err != nil {
-						fmt.Println("failed to store seqno: ", err)
-					}
-				}
-			}
-			s.seqLk.Unlock()
-
-			lelk.Lock()
-			lastEvent = time.Now()
-			lelk.Unlock()
-
-			if err := s.backend.HandleEvent(ctx, evt); err != nil {
-				return fmt.Errorf("handle event (%s,%d): %w", evt.Repo, evt.Seq, err)
-			}
-
-			cclk.Lock()
-			if evt.Seq > completeCursor {
-				completeCursor = evt.Seq
-				firehoseCursorGauge.WithLabelValues("complete").Set(float64(evt.Seq))
-			}
-			cclk.Unlock()
-
-			return nil
-		},
-		RepoInfo: func(info *atproto.SyncSubscribeRepos_Info) error {
-			return nil
-		},
-		// TODO: all the other event types
-		Error: func(errf *stream.ErrorFrame) error {
-			return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message)
-		},
-	}
-
-	sched := parallel.NewScheduler(parWorkers, maxQ, con.RemoteAddr().String(), rsc.EventHandler)
-
-	//s.eventScheduler = sched
-	//s.streamFinished = make(chan struct{})
-
-	return stream.HandleRepoStream(ctx, con, sched, slog.Default())
 }
 
 func (s *Server) resolveAccountIdent(ctx context.Context, acc string) (string, error) {

sync-config-jetstream.json

···
+{
+	"backends": [
+		{
+			"type": "jetstream",
+			"host": "jetstream1.us-west.bsky.network"
+		}
+	]
+}

sync.go

···
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/cmd/relay/stream"
+	"github.com/bluesky-social/indigo/cmd/relay/stream/schedulers/parallel"
+	jsclient "github.com/bluesky-social/jetstream/pkg/client"
+	jsparallel "github.com/bluesky-social/jetstream/pkg/client/schedulers/parallel"
+	"github.com/bluesky-social/jetstream/pkg/models"
+	"github.com/gorilla/websocket"
+)
+
+type SyncConfig struct {
+	Backends []SyncBackend `json:"backends"`
+}
+
+type SyncBackend struct {
+	Type       string `json:"type"`
+	Host       string `json:"host"`
+	MaxWorkers int    `json:"max_workers,omitempty"`
+}
+
+func (s *Server) StartSyncEngine(ctx context.Context, sc *SyncConfig) error {
+	for _, be := range sc.Backends {
+		switch be.Type {
+		case "firehose":
+			go s.runSyncFirehose(ctx, be)
+		case "jetstream":
+			go s.runSyncJetstream(ctx, be)
+		default:
+			return fmt.Errorf("unrecognized sync backend type: %q", be.Type)
+		}
+	}
+
+	<-ctx.Done()
+	return fmt.Errorf("exiting sync routine")
+}
+
+const failureTimeInterval = time.Second * 5
+
+func (s *Server) runSyncFirehose(ctx context.Context, be SyncBackend) {
+	var failures int
+	for {
+		seqno, err := loadLastSeq(s.db, be.Host)
+		if err != nil {
+			fmt.Println("failed to load sequence number, starting over", err)
+		}
+
+		maxWorkers := 10
+		if be.MaxWorkers != 0 {
+			maxWorkers = be.MaxWorkers
+		}
+
+		start := time.Now()
+		if err := s.startLiveTail(ctx, be.Host, int(seqno), maxWorkers, 20); err != nil {
+			slog.Error("firehose connection lost", "host", be.Host, "error", err)
+		}
+
+		elapsed := time.Since(start)
+
+		if elapsed > failureTimeInterval {
+			failures = 0
+			continue
+		}
+		failures++
+
+		delay := delayForFailureCount(failures)
+		slog.Warn("retrying connection after delay", "host", be.Host, "delay", delay)
+	}
+}
+
+func (s *Server) runSyncJetstream(ctx context.Context, be SyncBackend) {
+	var failures int
+	for {
+		// Load last cursor (stored as sequence number in same table)
+		cursor, err := loadLastSeq(s.db, be.Host)
+		if err != nil {
+			slog.Warn("failed to load jetstream cursor, starting from live", "error", err)
+			cursor = 0
+		}
+
+		maxWorkers := 10
+		if be.MaxWorkers != 0 {
+			maxWorkers = be.MaxWorkers
+		}
+
+		start := time.Now()
+		if err := s.startJetstreamTail(ctx, be.Host, cursor, maxWorkers); err != nil {
+			slog.Error("jetstream connection lost", "host", be.Host, "error", err)
+		}
+
+		elapsed := time.Since(start)
+
+		if elapsed > failureTimeInterval {
+			failures = 0
+			continue
+		}
+		failures++
+
+		delay := delayForFailureCount(failures)
+		slog.Warn("retrying jetstream connection after delay", "host", be.Host, "delay", delay)
+		time.Sleep(delay)
+	}
+}
+
+func delayForFailureCount(n int) time.Duration {
+	if n < 5 {
+		return (time.Second * 5) + (time.Second * 2 * time.Duration(n))
+	}
+
+	return time.Second * 30
+}
+
+func (s *Server) startLiveTail(ctx context.Context, host string, curs int, parWorkers, maxQ int) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	slog.Info("starting live tail")
+
+	// Connect to the Relay websocket
+	urlStr := fmt.Sprintf("wss://%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", host, curs)
+
+	d := websocket.DefaultDialer
+	con, _, err := d.Dial(urlStr, http.Header{
+		"User-Agent": []string{"konbini/0.0.1"},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to connect to relay: %w", err)
+	}
+
+	var lelk sync.Mutex
+	lastEvent := time.Now()
+
+	go func() {
+		tick := time.NewTicker(time.Second)
+		defer tick.Stop()
+		for {
+			select {
+			case <-tick.C:
+				lelk.Lock()
+				let := lastEvent
+				lelk.Unlock()
+
+				if time.Since(let) > time.Second*30 {
+					slog.Error("firehose connection timed out")
+					con.Close()
+					return
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	var cclk sync.Mutex
+	var completeCursor int64
+
+	rsc := &stream.RepoStreamCallbacks{
+		RepoCommit: func(evt *atproto.SyncSubscribeRepos_Commit) error {
+			ctx := context.Background()
+
+			firehoseCursorGauge.WithLabelValues("ingest").Set(float64(evt.Seq))
+
+			s.seqLk.Lock()
+			if evt.Seq > s.lastSeq {
+				curs = int(evt.Seq)
+				s.lastSeq = evt.Seq
+
+				if evt.Seq%1000 == 0 {
+					if err := storeLastSeq(s.db, host, evt.Seq); err != nil {
+						fmt.Println("failed to store seqno: ", err)
+					}
+				}
+			}
+			s.seqLk.Unlock()
+
+			lelk.Lock()
+			lastEvent = time.Now()
+			lelk.Unlock()
+
+			if err := s.backend.HandleEvent(ctx, evt); err != nil {
+				return fmt.Errorf("handle event (%s,%d): %w", evt.Repo, evt.Seq, err)
+			}
+
+			cclk.Lock()
+			if evt.Seq > completeCursor {
+				completeCursor = evt.Seq
+				firehoseCursorGauge.WithLabelValues("complete").Set(float64(evt.Seq))
+			}
+			cclk.Unlock()
+
+			return nil
+		},
+		RepoInfo: func(info *atproto.SyncSubscribeRepos_Info) error {
+			return nil
+		},
+		// TODO: all the other event types
+		Error: func(errf *stream.ErrorFrame) error {
+			return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message)
+		},
+	}
+
+	sched := parallel.NewScheduler(parWorkers, maxQ, con.RemoteAddr().String(), rsc.EventHandler)
+
+	return stream.HandleRepoStream(ctx, con, sched, slog.Default())
+}
+
+func (s *Server) startJetstreamTail(ctx context.Context, host string, cursor int64, parWorkers int) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	slog.Info("starting jetstream tail", "host", host, "cursor", cursor)
+
+	// Create a scheduler for parallel processing
+	lastStored := int64(0)
+	sched := jsparallel.NewScheduler(
+		parWorkers,
+		host,
+		slog.Default(),
+		func(ctx context.Context, event *models.Event) error {
+			// Update cursor tracking
+			s.seqLk.Lock()
+			if event.TimeUS > s.lastSeq {
+				s.lastSeq = event.TimeUS
+				if event.TimeUS-lastStored > 1_000_000 {
+					// Store checkpoint periodically
+					if err := storeLastSeq(s.db, host, event.TimeUS); err != nil {
+						slog.Error("failed to store jetstream cursor", "error", err)
+					}
+					lastStored = event.TimeUS
+				}
+			}
+			s.seqLk.Unlock()
+
+			// Update metrics
+			firehoseCursorGauge.WithLabelValues("ingest").Set(float64(event.TimeUS))
+
+			// Convert Jetstream event to ATProto event format
+			if event.Commit != nil {
+
+				if err := s.backend.HandleEventJetstream(ctx, event); err != nil {
+					return fmt.Errorf("handle event (%s,%d): %w", event.Did, event.TimeUS, err)
+				}
+
+				firehoseCursorGauge.WithLabelValues("complete").Set(float64(event.TimeUS))
+			}
+
+			return nil
+		},
+	)
+
+	// Configure Jetstream client
+	config := jsclient.DefaultClientConfig()
+	config.WebsocketURL = fmt.Sprintf("wss://%s/subscribe", host)
+
+	// Prepare cursor pointer
+	var cursorPtr *int64
+	if cursor > 0 {
+		cursorPtr = &cursor
+	}
+
+	// Create and connect client
+	client, err := jsclient.NewClient(
+		config,
+		slog.Default(),
+		sched,
+	)
+	if err != nil {
+		return fmt.Errorf("create jetstream client: %w", err)
+	}
+
+	// Start reading from Jetstream
+	return client.ConnectAndRead(ctx, cursorPtr)
+}