diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 55ca7e4..22e2331 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -26,6 +26,9 @@ const ( // defaultPurgeInterval is how often posts older than their feed's max_age are removed. const defaultPurgeInterval = 10 * time.Minute +// defaultStatusInterval is how often Run logs its cursor and lag. +const defaultStatusInterval = time.Minute + // Ingester applies firehose events to the store. All of its state is owned by // the goroutine calling Run (or ProcessBatch and Purge), so it needs no locking. type Ingester struct { @@ -40,8 +43,16 @@ type Ingester struct { // purgeInterval is how often Run purges expired posts. purgeInterval time.Duration lastPurge time.Time - now func() time.Time - log *slog.Logger + + // statusInterval is how often Run logs its cursor and lag. + statusInterval time.Duration + lastStatus time.Time + // cursor and timeUS are the cursor and time_us of the last committed event. + cursor uint64 + timeUS int64 + + now func() time.Time + log *slog.Logger } // New builds an Ingester for feeds and loads the tracked post set from st. @@ -51,13 +62,14 @@ func New(ctx context.Context, st *store.Store, feeds []config.Feed, log *slog.Lo return nil, err } i := &Ingester{ - store: st, - matcher: m, - feeds: make(map[string]config.Feed, len(feeds)), - gravityByFeed: make(map[string]float64), - purgeInterval: defaultPurgeInterval, - now: time.Now, - log: log, + store: st, + matcher: m, + feeds: make(map[string]config.Feed, len(feeds)), + gravityByFeed: make(map[string]float64), + purgeInterval: defaultPurgeInterval, + statusInterval: defaultStatusInterval, + now: time.Now, + log: log, } for _, feed := range feeds { i.feeds[feed.Name] = feed @@ -114,6 +126,9 @@ func (i *Ingester) Run(ctx context.Context, host string) error { return err } } + if i.cursor != 0 && i.now().Sub(i.lastStatus) >= i.statusInterval { + i.logStatus() + } } if err := ctx.Err(); err != nil { return err @@ -132,7 +147,7 @@ func (i *Ingester) collections() []string { // ProcessBatch applies events and saves cursor in a single transaction. // A cursor of 0 (an empty batch) is not saved. func (i *Ingester) ProcessBatch(ctx context.Context, events []jetstream.Event, cursor uint64) error { - return i.store.WithTx(ctx, func(tx *store.Tx) error { + err := i.store.WithTx(ctx, func(tx *store.Tx) error { for _, event := range events { if err := i.processEvent(ctx, tx, event); err != nil { return err @@ -143,6 +158,20 @@ func (i *Ingester) ProcessBatch(ctx context.Context, events []jetstream.Event, c } return tx.SaveCursor(ctx, cursor) }) + if err != nil || cursor == 0 || len(events) == 0 { + return err + } + i.cursor = cursor + i.timeUS = events[len(events)-1].TimeUS + return nil +} + +// logStatus logs the last committed cursor and how far its time_us trails now. +func (i *Ingester) logStatus() { + now := i.now() + lag := now.Sub(time.UnixMicro(i.timeUS)).Round(time.Second) + i.log.Info("ingest status", "cursor", i.cursor, "time_us", i.timeUS, "lag", lag) + i.lastStatus = now } func (i *Ingester) processEvent(ctx context.Context, tx *store.Tx, event jetstream.Event) error { diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 4d4d79d..a58d74d 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -6,6 +6,7 @@ import ( "log/slog" "path/filepath" "slices" + "strings" "testing" "time" @@ -196,6 +197,26 @@ func TestPurgeUsesPerFeedMaxAge(t *testing.T) { } } +func TestLogStatusReportsCursorAndLag(t *testing.T) { + i, _ := newTestIngester(t, testFeeds) + var out strings.Builder + i.log = slog.New(slog.NewTextHandler(&out, nil)) + now := time.UnixMicro(10 * time.Hour.Microseconds()) + i.now = func() time.Time { return now } + + process(t, i, 5, postCreate("did:plc:a", "p1", "hello", now.Add(-90*time.Second).UnixMicro())) + i.logStatus() + + for _, want := range []string{"cursor=5", "lag=1m30s"} { + if !strings.Contains(out.String(), want) { + t.Errorf("status log %q missing %q", out.String(), want) + } + } + if i.lastStatus != now { + t.Errorf("lastStatus = %v, want %v", i.lastStatus, now) + } +} + func TestNewLoadsTrackedSetFromStore(t *testing.T) { i, st := newTestIngester(t, testFeeds) process(t, i, 1, postCreate("did:plc:a", "1", "golang rust", 100), postCreate("did:plc:a", "2", "rust", 100))