refactor persisters in to sub-packages

events/dbpersist.go events/dbpersist/dbpersist.go

···
-package events
+package dbpersist
 
 import (
 	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"sync"
 	"time"
 
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/carstore"
+	"github.com/bluesky-social/indigo/events"
 	lexutil "github.com/bluesky-social/indigo/lex/util"
 	"github.com/bluesky-social/indigo/models"
 	"github.com/bluesky-social/indigo/util"
···
 	"gorm.io/gorm"
 )
 
+var log = slog.Default().With("system", "dbpersist")
+
 type PersistenceBatchItem struct {
 	Record *RepoEventRecord
-	Event  *XRPCStreamEvent
+	Event  *events.XRPCStreamEvent
 }
 
 type Options struct {
···
 
 	lk sync.Mutex
 
-	broadcast func(*XRPCStreamEvent)
+	broadcast func(*events.XRPCStreamEvent)
 
 	batch        []*PersistenceBatchItem
 	batchOptions Options
···
 	}
 }
 
-func (p *DbPersistence) SetEventBroadcaster(brc func(*XRPCStreamEvent)) {
+func (p *DbPersistence) SetEventBroadcaster(brc func(*events.XRPCStreamEvent)) {
 	p.broadcast = brc
 }
 
···
 	return nil
 }
 
-func (p *DbPersistence) AddItemToBatch(ctx context.Context, rec *RepoEventRecord, evt *XRPCStreamEvent) error {
+func (p *DbPersistence) AddItemToBatch(ctx context.Context, rec *RepoEventRecord, evt *events.XRPCStreamEvent) error {
 	p.lk.Lock()
 	defer p.lk.Unlock()
 	p.batch = append(p.batch, &PersistenceBatchItem{
···
 	return nil
 }
 
-func (p *DbPersistence) Persist(ctx context.Context, e *XRPCStreamEvent) error {
+func (p *DbPersistence) Persist(ctx context.Context, e *events.XRPCStreamEvent) error {
 	var rer *RepoEventRecord
 	var err error
 
···
 	return &rer, nil
 }
 
-func (p *DbPersistence) Playback(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error) error {
+func (p *DbPersistence) Playback(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error) error {
 	pageSize := 1000
 
 	for {
···
 	return nil
 }
 
-func (p *DbPersistence) hydrateBatch(ctx context.Context, batch []*RepoEventRecord, cb func(*XRPCStreamEvent) error) error {
-	events := make([]*XRPCStreamEvent, len(batch))
+func (p *DbPersistence) hydrateBatch(ctx context.Context, batch []*RepoEventRecord, cb func(*events.XRPCStreamEvent) error) error {
+	evts := make([]*events.XRPCStreamEvent, len(batch))
 
 	type Result struct {
-		Event *XRPCStreamEvent
+		Event *events.XRPCStreamEvent
 		Index int
 		Err   error
 	}
···
 			// release the semaphore at the end of the goroutine
 			defer func() { <-sem }()
 
-			var streamEvent *XRPCStreamEvent
+			var streamEvent *events.XRPCStreamEvent
 			var err error
 
 			switch {
···
 			return result.Err
 		}
 
-		events[result.Index] = result.Event
+		evts[result.Index] = result.Event
 
-		for ; cur < len(events) && events[cur] != nil; cur++ {
-			if err := cb(events[cur]); err != nil {
+		for ; cur < len(evts) && evts[cur] != nil; cur++ {
+			if err := cb(evts[cur]); err != nil {
 				return err
 			}
 		}
···
 	return u.Did, nil
 }
 
-func (p *DbPersistence) hydrateHandleChange(ctx context.Context, rer *RepoEventRecord) (*XRPCStreamEvent, error) {
+func (p *DbPersistence) hydrateHandleChange(ctx context.Context, rer *RepoEventRecord) (*events.XRPCStreamEvent, error) {
 	if rer.NewHandle == nil {
 		return nil, fmt.Errorf("NewHandle is nil")
 	}
···
 		return nil, err
 	}
 
-	return &XRPCStreamEvent{
+	return &events.XRPCStreamEvent{
 		RepoHandle: &comatproto.SyncSubscribeRepos_Handle{
 			Did:    did,
 			Handle: *rer.NewHandle,
···
 	}, nil
 }
 
-func (p *DbPersistence) hydrateIdentityEvent(ctx context.Context, rer *RepoEventRecord) (*XRPCStreamEvent, error) {
+func (p *DbPersistence) hydrateIdentityEvent(ctx context.Context, rer *RepoEventRecord) (*events.XRPCStreamEvent, error) {
 	did, err := p.didForUid(ctx, rer.Repo)
 	if err != nil {
 		return nil, err
 	}
 
-	return &XRPCStreamEvent{
+	return &events.XRPCStreamEvent{
 		RepoIdentity: &comatproto.SyncSubscribeRepos_Identity{
 			Did:  did,
 			Time: rer.Time.Format(util.ISO8601),
···
 	}, nil
 }
 
-func (p *DbPersistence) hydrateAccountEvent(ctx context.Context, rer *RepoEventRecord) (*XRPCStreamEvent, error) {
+func (p *DbPersistence) hydrateAccountEvent(ctx context.Context, rer *RepoEventRecord) (*events.XRPCStreamEvent, error) {
 	did, err := p.didForUid(ctx, rer.Repo)
 	if err != nil {
 		return nil, err
 	}
 
-	return &XRPCStreamEvent{
+	return &events.XRPCStreamEvent{
 		RepoAccount: &comatproto.SyncSubscribeRepos_Account{
 			Did:    did,
 			Time:   rer.Time.Format(util.ISO8601),
···
 	}, nil
 }
 
-func (p *DbPersistence) hydrateTombstone(ctx context.Context, rer *RepoEventRecord) (*XRPCStreamEvent, error) {
+func (p *DbPersistence) hydrateTombstone(ctx context.Context, rer *RepoEventRecord) (*events.XRPCStreamEvent, error) {
 	did, err := p.didForUid(ctx, rer.Repo)
 	if err != nil {
 		return nil, err
 	}
 
-	return &XRPCStreamEvent{
+	return &events.XRPCStreamEvent{
 		RepoTombstone: &comatproto.SyncSubscribeRepos_Tombstone{
 			Did:  did,
 			Time: rer.Time.Format(util.ISO8601),
···
 	}, nil
 }
 
-func (p *DbPersistence) hydrateCommit(ctx context.Context, rer *RepoEventRecord) (*XRPCStreamEvent, error) {
+func (p *DbPersistence) hydrateCommit(ctx context.Context, rer *RepoEventRecord) (*events.XRPCStreamEvent, error) {
 	if rer.Commit == nil {
 		return nil, fmt.Errorf("commit is nil")
 	}
···
 		out.Blocks = cs
 	}
 
-	return &XRPCStreamEvent{RepoCommit: out}, nil
+	return &events.XRPCStreamEvent{RepoCommit: out}, nil
 }
 
 func (p *DbPersistence) readCarSlice(ctx context.Context, rer *RepoEventRecord) ([]byte, error) {

events/dbpersist_test.go events/dbpersist/dbpersist_test.go

···
-package events
+package dbpersist
 
 import (
 	"context"
···
 	atproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/carstore"
+	"github.com/bluesky-social/indigo/events"
 	lexutil "github.com/bluesky-social/indigo/lex/util"
 	"github.com/bluesky-social/indigo/models"
 	pds "github.com/bluesky-social/indigo/pds/data"
···
 	}
 
 	// Create a bunch of events
-	evtman := NewEventManager(dbp)
+	evtman := events.NewEventManager(dbp)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	inEvts := make([]*XRPCStreamEvent, b.N)
+	inEvts := make([]*events.XRPCStreamEvent, b.N)
 	for i := 0; i < b.N; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &XRPCStreamEvent{
+		inEvts[i] = &events.XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 
 	b.StopTimer()
 
-	dbp.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
+	dbp.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
 		outEvtCount++
 		return nil
 	})
···
 	}
 
 	// Create a bunch of events
-	evtman := NewEventManager(dbp)
+	evtman := events.NewEventManager(dbp)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	inEvts := make([]*XRPCStreamEvent, n)
+	inEvts := make([]*events.XRPCStreamEvent, n)
 	for i := 0; i < n; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &XRPCStreamEvent{
+		inEvts[i] = &events.XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 
 	b.ResetTimer()
 
-	dbp.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
+	dbp.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
 		outEvtCount++
 		return nil
 	})

events/diskpersist.go events/diskpersist/diskpersist.go

···
-package events
+package diskpersist
 
 import (
 	"bufio"
···
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
 	"github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/events"
 	"github.com/bluesky-social/indigo/models"
 	arc "github.com/hashicorp/golang-lru/arc/v2"
 	"github.com/prometheus/client_golang/prometheus"
···
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"gorm.io/gorm"
 )
+
+var log = slog.Default().With("system", "diskpersist")
 
 type DiskPersistence struct {
 	primaryDir      string
···
 
 	meta *gorm.DB
 
-	broadcast func(*XRPCStreamEvent)
+	broadcast func(*events.XRPCStreamEvent)
 
 	logfi *os.File
 
···
 
 type persistJob struct {
 	Bytes  []byte
-	Evt    *XRPCStreamEvent
+	Evt    *events.XRPCStreamEvent
 	Buffer *bytes.Buffer // so we can put it back in the pool when we're done
 }
 
···
 	EvtFlagRebased
 )
 
-var _ (EventPersistence) = (*DiskPersistence)(nil)
+var _ (events.EventPersistence) = (*DiskPersistence)(nil)
 
 type DiskPersistOptions struct {
 	UIDCacheSize    int
···
 	return nil
 }
 
-func (dp *DiskPersistence) Persist(ctx context.Context, e *XRPCStreamEvent) error {
+func (dp *DiskPersistence) Persist(ctx context.Context, e *events.XRPCStreamEvent) error {
 	buffer := dp.buffers.Get().(*bytes.Buffer)
 	cw := dp.writers.Get().(*cbg.CborWriter)
 	cw.SetWriter(buffer)
···
 	return u.Uid, nil
 }
 
-func (dp *DiskPersistence) Playback(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error) error {
+func (dp *DiskPersistence) Playback(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error) error {
 	base := since - (since % dp.eventsPerFile)
 	var logs []LogFileRef
 	if err := dp.meta.Debug().Order("seq_start asc").Find(&logs, "seq_start >= ?", base).Error; err != nil {
···
 	return nil
 }
 
-func (dp *DiskPersistence) PlaybackLogfiles(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error, logFiles []LogFileRef) (*int64, error) {
+func (dp *DiskPersistence) PlaybackLogfiles(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error, logFiles []LogFileRef) (*int64, error) {
 	for i, lf := range logFiles {
 		lastSeq, err := dp.readEventsFrom(ctx, since, filepath.Join(dp.primaryDir, lf.Path), cb)
 		if err != nil {
···
 	return false
 }
 
-func (dp *DiskPersistence) readEventsFrom(ctx context.Context, since int64, fn string, cb func(*XRPCStreamEvent) error) (*int64, error) {
+func (dp *DiskPersistence) readEventsFrom(ctx context.Context, since int64, fn string, cb func(*events.XRPCStreamEvent) error) (*int64, error) {
 	fi, err := os.OpenFile(fn, os.O_RDONLY, 0)
 	if err != nil {
 		return nil, err
···
 				return nil, err
 			}
 			evt.Seq = h.Seq
-			if err := cb(&XRPCStreamEvent{RepoCommit: &evt}); err != nil {
+			if err := cb(&events.XRPCStreamEvent{RepoCommit: &evt}); err != nil {
 				return nil, err
 			}
 		case evtKindHandle:
···
 				return nil, err
 			}
 			evt.Seq = h.Seq
-			if err := cb(&XRPCStreamEvent{RepoHandle: &evt}); err != nil {
+			if err := cb(&events.XRPCStreamEvent{RepoHandle: &evt}); err != nil {
 				return nil, err
 			}
 		case evtKindIdentity:
···
 				return nil, err
 			}
 			evt.Seq = h.Seq
-			if err := cb(&XRPCStreamEvent{RepoIdentity: &evt}); err != nil {
+			if err := cb(&events.XRPCStreamEvent{RepoIdentity: &evt}); err != nil {
 				return nil, err
 			}
 		case evtKindAccount:
···
 				return nil, err
 			}
 			evt.Seq = h.Seq
-			if err := cb(&XRPCStreamEvent{RepoAccount: &evt}); err != nil {
+			if err := cb(&events.XRPCStreamEvent{RepoAccount: &evt}); err != nil {
 				return nil, err
 			}
 		case evtKindTombstone:
···
 				return nil, err
 			}
 			evt.Seq = h.Seq
-			if err := cb(&XRPCStreamEvent{RepoTombstone: &evt}); err != nil {
+			if err := cb(&events.XRPCStreamEvent{RepoTombstone: &evt}); err != nil {
 				return nil, err
 			}
 		default:
···
 	return nil
 }
 
-func (dp *DiskPersistence) SetEventBroadcaster(f func(*XRPCStreamEvent)) {
+func (dp *DiskPersistence) SetEventBroadcaster(f func(*events.XRPCStreamEvent)) {
 	dp.broadcast = f
 }

events/diskpersist_test.go events/diskpersist/diskpersist_test.go

···
-package events
+package diskpersist
 
 import (
 	"context"
···
 	atproto "github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/carstore"
+	"github.com/bluesky-social/indigo/events"
 	lexutil "github.com/bluesky-social/indigo/lex/util"
 	"github.com/bluesky-social/indigo/models"
 	pds "github.com/bluesky-social/indigo/pds/data"
 	"github.com/bluesky-social/indigo/repomgr"
 	"github.com/bluesky-social/indigo/util"
+	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )
 
-func testPersister(t *testing.T, perisistenceFactory func(path string, db *gorm.DB) (EventPersistence, error)) {
+func testPersister(t *testing.T, perisistenceFactory func(path string, db *gorm.DB) (events.EventPersistence, error)) {
 	ctx := context.Background()
 
 	db, _, cs, tempPath, err := setupDBs(t)
···
 	}
 
 	// Create a bunch of events
-	evtman := NewEventManager(dp)
+	evtman := events.NewEventManager(dp)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
···
 	}
 
 	n := 100
-	inEvts := make([]*XRPCStreamEvent, n)
+	inEvts := make([]*events.XRPCStreamEvent, n)
 	for i := 0; i < n; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &XRPCStreamEvent{
+		inEvts[i] = &events.XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 	outEvtCount := 0
 	expectedEvtCount := n
 
-	dp.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
+	dp.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
 		outEvtCount++
 		return nil
 	})
···
 		t.Fatal(err)
 	}
 
-	evtman2 := NewEventManager(dp2)
+	evtman2 := events.NewEventManager(dp2)
 
-	inEvts = make([]*XRPCStreamEvent, n)
+	inEvts = make([]*events.XRPCStreamEvent, n)
 	for i := 0; i < n; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &XRPCStreamEvent{
+		inEvts[i] = &events.XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 	}
 }
 func TestDiskPersist(t *testing.T) {
-	factory := func(tempPath string, db *gorm.DB) (EventPersistence, error) {
+	factory := func(tempPath string, db *gorm.DB) (events.EventPersistence, error) {
 		return NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &DiskPersistOptions{
 			EventsPerFile: 10,
 			UIDCacheSize:  100000,
···
 
 }
 
-func runPersisterBenchmark(b *testing.B, cs carstore.CarStore, db *gorm.DB, p EventPersistence) {
+func runPersisterBenchmark(b *testing.B, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) {
 	ctx := context.Background()
 
 	db.AutoMigrate(&pds.User{})
···
 	}
 
 	// Create a bunch of events
-	evtman := NewEventManager(p)
+	evtman := events.NewEventManager(p)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	inEvts := make([]*XRPCStreamEvent, b.N)
+	inEvts := make([]*events.XRPCStreamEvent, b.N)
 	for i := 0; i < b.N; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &XRPCStreamEvent{
+		inEvts[i] = &events.XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 	runEventManagerTest(t, cs, db, dp)
 }
 
-func runEventManagerTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p EventPersistence) {
+func runEventManagerTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) {
 	ctx := context.Background()
 
 	db.AutoMigrate(&pds.User{})
···
 		t.Fatal(err)
 	}
 
-	evtman := NewEventManager(p)
+	evtman := events.NewEventManager(p)
 
 	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
 	if err != nil {
···
 	}
 
 	testSize := 100 // you can adjust this number as needed
-	inEvts := make([]*XRPCStreamEvent, testSize)
+	inEvts := make([]*events.XRPCStreamEvent, testSize)
 	for i := 0; i < testSize; i++ {
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &XRPCStreamEvent{
+		inEvts[i] = &events.XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   "did:example:123",
 				Commit: headLink,
···
 	}
 
 	outEvtCount := 0
-	p.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
+	p.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
 		// Check that the contents of the output events match the input events
 		// Clear cache, don't care if one has it and not the other
 		inEvts[outEvtCount].Preserialized = nil
···
 	runTakedownTest(t, cs, db, dp)
 }
 
-func runTakedownTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p EventPersistence) {
+func runTakedownTest(t *testing.T, cs carstore.CarStore, db *gorm.DB, p events.EventPersistence) {
 	ctx := context.TODO()
 
 	db.AutoMigrate(&pds.User{})
···
 		}
 	}
 
-	evtman := NewEventManager(p)
+	evtman := events.NewEventManager(p)
 
 	testSize := 100 // you can adjust this number as needed
-	inEvts := make([]*XRPCStreamEvent, testSize*userCount)
+	inEvts := make([]*events.XRPCStreamEvent, testSize*userCount)
 	for i := 0; i < testSize*userCount; i++ {
 		user := users[i%userCount]
 		_, cid, err := mgr.CreateRecord(ctx, user.Uid, "app.bsky.feed.post", &bsky.FeedPost{
···
 
 		cidLink := lexutil.LexLink(cid)
 		headLink := lexutil.LexLink(userRepoHead)
-		inEvts[i] = &XRPCStreamEvent{
+		inEvts[i] = &events.XRPCStreamEvent{
 			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
 				Repo:   user.Did,
 				Commit: headLink,
···
 
 	// Verify that the events of the user have been removed from the event stream
 	var evtsCount int
-	if err := p.Playback(ctx, 0, func(evt *XRPCStreamEvent) error {
+	if err := p.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
 		evtsCount++
 		if evt.RepoCommit.Repo == takeDownUser.Did {
 			t.Fatalf("found event for user %d after takedown", takeDownUser.Uid)
···
 		t.Fatalf("wrong number of events out: %d != %d", evtsCount, exp)
 	}
 }
+
+func setupDBs(t testing.TB) (*gorm.DB, *gorm.DB, carstore.CarStore, string, error) {
+	dir, err := os.MkdirTemp("", "integtest")
+	if err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	maindb, err := gorm.Open(sqlite.Open(filepath.Join(dir, "test.sqlite?cache=shared&mode=rwc")))
+	if err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	tx := maindb.Exec("PRAGMA journal_mode=WAL;")
+	if tx.Error != nil {
+		return nil, nil, nil, "", tx.Error
+	}
+
+	tx.Commit()
+
+	cardb, err := gorm.Open(sqlite.Open(filepath.Join(dir, "car.sqlite?cache=shared&mode=rwc")))
+	if err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	tx = cardb.Exec("PRAGMA journal_mode=WAL;")
+	if tx.Error != nil {
+		return nil, nil, nil, "", tx.Error
+	}
+
+	cspath := filepath.Join(dir, "carstore")
+	if err := os.Mkdir(cspath, 0775); err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	cs, err := carstore.NewCarStore(cardb, []string{cspath})
+	if err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	return maindb, cardb, cs, dir, nil
+}

events/pebblepersist.go events/pebblepersist/pebblepersist.go

···
-package events
+package pebblepersist
 
 import (
 	"bytes"
···
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"log/slog"
 	"time"
 
+	"github.com/bluesky-social/indigo/events"
 	"github.com/bluesky-social/indigo/models"
 	"github.com/cockroachdb/pebble"
 )
 
+var log = slog.Default().With("system", "pebblepersist")
+
 type PebblePersist struct {
-	broadcast func(*XRPCStreamEvent)
+	broadcast func(*events.XRPCStreamEvent)
 	db        *pebble.DB
 
 	prevSeq      int64
···
 	binary.BigEndian.PutUint64(key[8:16], uint64(millis))
 }
 
-func (pp *PebblePersist) Persist(ctx context.Context, e *XRPCStreamEvent) error {
+func (pp *PebblePersist) Persist(ctx context.Context, e *events.XRPCStreamEvent) error {
 	err := e.Preserialize()
 	if err != nil {
 		return err
···
 	return err
 }
 
-func eventFromPebbleIter(iter *pebble.Iterator) (*XRPCStreamEvent, error) {
+func eventFromPebbleIter(iter *pebble.Iterator) (*events.XRPCStreamEvent, error) {
 	blob, err := iter.ValueAndErr()
 	if err != nil {
 		return nil, err
 	}
 	br := bytes.NewReader(blob)
-	evt := new(XRPCStreamEvent)
+	evt := new(events.XRPCStreamEvent)
 	err = evt.Deserialize(br)
 	if err != nil {
 		return nil, err
···
 	return evt, nil
 }
 
-func (pp *PebblePersist) Playback(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error) error {
+func (pp *PebblePersist) Playback(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error) error {
 	var key [8]byte
 	binary.BigEndian.PutUint64(key[:], uint64(since))
 
···
 	return err
 }
 
-func (pp *PebblePersist) SetEventBroadcaster(broadcast func(*XRPCStreamEvent)) {
+func (pp *PebblePersist) SetEventBroadcaster(broadcast func(*events.XRPCStreamEvent)) {
 	pp.broadcast = broadcast
 }
 
 var ErrNoLast = errors.New("no last event")
 
-func (pp *PebblePersist) GetLast(ctx context.Context) (seq, millis int64, evt *XRPCStreamEvent, err error) {
+func (pp *PebblePersist) GetLast(ctx context.Context) (seq, millis int64, evt *events.XRPCStreamEvent, err error) {
 	iter, err := pp.db.NewIterWithContext(ctx, &pebble.IterOptions{})
 	if err != nil {
 		return 0, 0, nil, err

events/pebblepersist/pebblepersist_test.go

···
+package pebblepersist
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	atproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/api/bsky"
+	"github.com/bluesky-social/indigo/carstore"
+	"github.com/bluesky-social/indigo/events"
+	"github.com/bluesky-social/indigo/events/diskpersist"
+	lexutil "github.com/bluesky-social/indigo/lex/util"
+	"github.com/bluesky-social/indigo/models"
+	pds "github.com/bluesky-social/indigo/pds/data"
+	"github.com/bluesky-social/indigo/repomgr"
+	"github.com/bluesky-social/indigo/util"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+func TestPebblePersist(t *testing.T) {
+	factory := func(tempPath string, db *gorm.DB) (events.EventPersistence, error) {
+		opts := DefaultPebblePersistOptions
+		opts.DbPath = filepath.Join(tempPath, "pebble.db")
+		return NewPebblePersistance(&opts)
+	}
+	testPersister(t, factory)
+}
+
+func testPersister(t *testing.T, perisistenceFactory func(path string, db *gorm.DB) (events.EventPersistence, error)) {
+	ctx := context.Background()
+
+	db, _, cs, tempPath, err := setupDBs(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db.AutoMigrate(&pds.User{})
+	db.AutoMigrate(&pds.Peering{})
+	db.AutoMigrate(&models.ActorInfo{})
+
+	db.Create(&models.ActorInfo{
+		Uid: 1,
+		Did: "did:example:123",
+	})
+
+	mgr := repomgr.NewRepoManager(cs, &util.FakeKeyManager{})
+
+	err = mgr.InitNewActor(ctx, 1, "alice", "did:example:123", "Alice", "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, cid, err := mgr.CreateRecord(ctx, 1, "app.bsky.feed.post", &bsky.FeedPost{
+		Text:      "hello world",
+		CreatedAt: time.Now().Format(util.ISO8601),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer os.RemoveAll(tempPath)
+
+	// Initialize a persister
+	dp, err := perisistenceFactory(tempPath, db)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a bunch of events
+	evtman := events.NewEventManager(dp)
+
+	userRepoHead, err := mgr.GetRepoRoot(ctx, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n := 100
+	inEvts := make([]*events.XRPCStreamEvent, n)
+	for i := 0; i < n; i++ {
+		cidLink := lexutil.LexLink(cid)
+		headLink := lexutil.LexLink(userRepoHead)
+		inEvts[i] = &events.XRPCStreamEvent{
+			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
+				Repo:   "did:example:123",
+				Commit: headLink,
+				Ops: []*atproto.SyncSubscribeRepos_RepoOp{
+					{
+						Action: "add",
+						Cid:    &cidLink,
+						Path:   "path1",
+					},
+				},
+				Time: time.Now().Format(util.ISO8601),
+				Seq:  int64(i),
+			},
+		}
+	}
+
+	// Add events in parallel
+	for i := 0; i < n; i++ {
+		err = evtman.AddEvent(ctx, inEvts[i])
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if err := dp.Flush(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	outEvtCount := 0
+	expectedEvtCount := n
+
+	dp.Playback(ctx, 0, func(evt *events.XRPCStreamEvent) error {
+		outEvtCount++
+		return nil
+	})
+
+	if outEvtCount != expectedEvtCount {
+		t.Fatalf("expected %d events, got %d", expectedEvtCount, outEvtCount)
+	}
+
+	dp.Shutdown(ctx)
+
+	time.Sleep(time.Millisecond * 100)
+
+	dp2, err := diskpersist.NewDiskPersistence(filepath.Join(tempPath, "diskPrimary"), filepath.Join(tempPath, "diskArchive"), db, &diskpersist.DiskPersistOptions{
+		EventsPerFile: 10,
+		UIDCacheSize:  100000,
+		DIDCacheSize:  100000,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	evtman2 := events.NewEventManager(dp2)
+
+	inEvts = make([]*events.XRPCStreamEvent, n)
+	for i := 0; i < n; i++ {
+		cidLink := lexutil.LexLink(cid)
+		headLink := lexutil.LexLink(userRepoHead)
+		inEvts[i] = &events.XRPCStreamEvent{
+			RepoCommit: &atproto.SyncSubscribeRepos_Commit{
+				Repo:   "did:example:123",
+				Commit: headLink,
+				Ops: []*atproto.SyncSubscribeRepos_RepoOp{
+					{
+						Action: "add",
+						Cid:    &cidLink,
+						Path:   "path1",
+					},
+				},
+				Time: time.Now().Format(util.ISO8601),
+			},
+		}
+	}
+
+	for i := 0; i < n; i++ {
+		err = evtman2.AddEvent(ctx, inEvts[i])
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func setupDBs(t testing.TB) (*gorm.DB, *gorm.DB, carstore.CarStore, string, error) {
+	dir, err := os.MkdirTemp("", "integtest")
+	if err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	maindb, err := gorm.Open(sqlite.Open(filepath.Join(dir, "test.sqlite?cache=shared&mode=rwc")))
+	if err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	tx := maindb.Exec("PRAGMA journal_mode=WAL;")
+	if tx.Error != nil {
+		return nil, nil, nil, "", tx.Error
+	}
+
+	tx.Commit()
+
+	cardb, err := gorm.Open(sqlite.Open(filepath.Join(dir, "car.sqlite?cache=shared&mode=rwc")))
+	if err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	tx = cardb.Exec("PRAGMA journal_mode=WAL;")
+	if tx.Error != nil {
+		return nil, nil, nil, "", tx.Error
+	}
+
+	cspath := filepath.Join(dir, "carstore")
+	if err := os.Mkdir(cspath, 0775); err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	cs, err := carstore.NewCarStore(cardb, []string{cspath})
+	if err != nil {
+		return nil, nil, nil, "", err
+	}
+
+	return maindb, cardb, cs, dir, nil
+}

events/pebblepersist_test.go

···
-package events
-
-import (
-	"gorm.io/gorm"
-	"path/filepath"
-	"testing"
-)
-
-func TestPebblePersist(t *testing.T) {
-	factory := func(tempPath string, db *gorm.DB) (EventPersistence, error) {
-		opts := DefaultPebblePersistOptions
-		opts.DbPath = filepath.Join(tempPath, "pebble.db")
-		return NewPebblePersistance(&opts)
-	}
-	testPersister(t, factory)
-}

events/yolopersist.go events/yolopersist/yolopersist.go

···
-package events
+package yolopersist
 
 import (
 	"context"
 	"fmt"
 	"sync"
 
+	"github.com/bluesky-social/indigo/events"
 	"github.com/bluesky-social/indigo/models"
 )
 
···
 	lk  sync.Mutex
 	seq int64
 
-	broadcast func(*XRPCStreamEvent)
+	broadcast func(*events.XRPCStreamEvent)
 }
 
 func NewYoloPersister() *YoloPersister {
 	return &YoloPersister{}
 }
 
-func (yp *YoloPersister) Persist(ctx context.Context, e *XRPCStreamEvent) error {
+func (yp *YoloPersister) Persist(ctx context.Context, e *events.XRPCStreamEvent) error {
 	yp.lk.Lock()
 	defer yp.lk.Unlock()
 	yp.seq++
···
 	return nil
 }
 
-func (mp *YoloPersister) Playback(ctx context.Context, since int64, cb func(*XRPCStreamEvent) error) error {
+func (mp *YoloPersister) Playback(ctx context.Context, since int64, cb func(*events.XRPCStreamEvent) error) error {
 	return fmt.Errorf("playback not supported by yolo persister, test usage only")
 }
 
···
 	return fmt.Errorf("repo takedowns not currently supported by memory persister, test usage only")
 }
 
-func (yp *YoloPersister) SetEventBroadcaster(brc func(*XRPCStreamEvent)) {
+func (yp *YoloPersister) SetEventBroadcaster(brc func(*events.XRPCStreamEvent)) {
 	yp.broadcast = brc
 }