package spindle import ( "context" "database/sql" "encoding/json" "errors" "fmt" "slices" "sync" "time" "github.com/bluesky-social/indigo/atproto/syntax" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "tangled.org/core/api/org_tangled" "tangled.org/core/jetstream" "tangled.org/core/spindle/db" "tangled.org/core/spindle/knotclient" "tangled.org/core/spindle/observability" "tangled.org/core/tapc" ) type Ingester func(ctx context.Context, e *jetstream.Event) error func (s *Spindle) ingest() Ingester { var shards *ingestShards var start sync.Once wipes := make(chan struct{}, accountJobSlots) return func(ctx context.Context, e *jetstream.Event) error { if e.Kind != jetstream.EventKindAccount && e.Kind != jetstream.EventKindCommit { return nil } ticket := jetstream.Hold(ctx) if ticket == nil { err := s.ingestEvent(ctx, e) // ingestEvent already logged it and nothing retries out here if err != nil && e.Kind == jetstream.EventKindCommit && ctx.Err() == nil { return nil } return err } // wipes do io, they skip the shards and hold the ticket until done if e.Kind == jetstream.EventKindAccount { return s.startAccountJob(ctx, e, ticket, wipes) } start.Do(func() { shards = newIngestShards(ctx, func(ctx context.Context, e *jetstream.Event) error { return s.ingestEvent(withMarkOnly(ctx), e) }) }) return shards.dispatch(ctx, e, ticket) } } const accountJobSlots = 4 // wipes that run past this budget are left to the repo jobs var accountWipeBudget = 10 * time.Minute func (s *Spindle) startAccountJob(ctx context.Context, e *jetstream.Event, ticket ingestTicket, slots chan struct{}) error { select { case slots <- struct{}{}: case <-ctx.Done(): ticket.Drop() return ctx.Err() } go func() { defer func() { <-slots }() s.ingestAccountJob(ctx, e, ticket) }() return nil } func (s *Spindle) ingestAccountJob(ctx context.Context, e *jetstream.Event, ticket ingestTicket) { jobCtx, cancel := context.WithTimeout(ctx, accountWipeBudget) defer cancel() err := s.ingestEvent(jobCtx, e) // a failed wipe already left a wipes row for the repo jobs, and the ticket finishes anyway if err != nil && ctx.Err() == nil { s.l.Error("account event wipe failed, its wipes are retried", "did", e.Did, "err", err) } finishTicket(ctx, ticket, err) } func (s *Spindle) ingestEvent(ctx context.Context, e *jetstream.Event) error { if e.Kind == jetstream.EventKindAccount { return s.ingestAccount(ctx, e) } if e.Kind != jetstream.EventKindCommit { return nil } ctx, span := observability.Tracer().Start(ctx, "jetstream.ingest") defer span.End() if span.IsRecording() { var attrs []attribute.KeyValue if e.Did != "" { attrs = append(attrs, attribute.String(observability.UserDIDKey, e.Did)) } if e.Commit != nil { if e.Commit.Collection != "" { attrs = append(attrs, attribute.String(observability.CollectionKey, e.Commit.Collection)) } if e.Commit.RKey != "" { attrs = append(attrs, attribute.String(observability.RKeyKey, e.Commit.RKey)) } } span.SetAttributes(attrs...) } var err error if evt, ok := jetstreamToTapEvent(e); ok { err = s.processEvent(ctx, evt) } // a cancelled event stays unconsumed, or the cursor moves past it if err != nil && ctx.Err() != nil { span.SetStatus(codes.Error, "cancelled while processing a jetstream event") return err } if err != nil { s.l.WarnContext(ctx, "failed to process message", "nsid", e.Commit.Collection, "did", e.Did, "rkey", e.Commit.RKey, "err", err) s.metrics.RecordEventIngestion("jetstream", "error") span.SetStatus(codes.Error, "failed to process jetstream event") return err } s.metrics.RecordEventIngestion("jetstream", "success") span.SetStatus(codes.Ok, "success") return nil } // the firehose sees every repo, so each handler drops what spindle doesn't track func (s *Spindle) processEvent(ctx context.Context, evt tapc.Event) error { if evt.Type != tapc.EvtRecord || evt.Record == nil { return nil } rec := evt.Record var handle func(context.Context, *tapc.RecordEventData) error switch rec.Collection.String() { case org_tangled.RepoManifestNSID: handle = s.onManifest case org_tangled.GraphCollaboratorNSID: handle = s.onCollaborator case org_tangled.RepoDeclarationNSID: handle = s.onDeclaration case org_tangled.TrackTicketNSID: handle = s.onTicket case org_tangled.ReviewPatchNSID: handle = s.onPatch default: return nil } source := "backfill" if rec.Live { source = "live" } if err := handle(ctx, rec); err != nil { s.metrics.RecordRepoRecord(source, "failed") return err } s.metrics.RecordRepoRecord(source, "applied") return nil } func (s *Spindle) onManifest(ctx context.Context, rec *tapc.RecordEventData) error { if rec.Rkey != knotclient.ManifestRkey { return nil } known, err := s.knowsRepo(rec.Did) if err != nil { return err } if !known { if rec.Action == tapc.RecordDeleteAction { return nil } var manifest org_tangled.RepoManifest if err := json.Unmarshal(rec.Record, &manifest); err != nil { s.l.Warn("skipping invalid manifest record", "repo", rec.Did, "err", err) return nil } if manifest.Spindle == nil || *manifest.Spindle != s.cfg.Server.Did().String() { return nil } // only enrolled owners get knot and pds reads, since anyone can point a manifest at this spindle if manifest.Declaration == nil { return nil } owner, err := syntax.ParseDID(manifest.Declaration.Owner) if err != nil { return nil } if enrolled, err := db.Enrolls(s.db, &s.cfg.Server, owner); err != nil || !enrolled { return err } } return s.refreshForEvent(ctx, rec, rec.Did) } func (s *Spindle) onCollaborator(ctx context.Context, rec *tapc.RecordEventData) error { known, err := s.knowsRepo(rec.Did) if err != nil || !known { return err } if markOnly(ctx) { if err := s.markRecord(rec); err != nil { return err } } return s.refreshForEvent(ctx, rec, rec.Did) } // delete events carry no record, so the repo has to be found by owner and rkey func (s *Spindle) onDeclaration(ctx context.Context, rec *tapc.RecordEventData) error { if rec.Action == tapc.RecordDeleteAction { repo, err := s.db.GetRepoByOwnerRkey(rec.Did, rec.Rkey) if errors.Is(err, sql.ErrNoRows) { return nil } if err != nil { return err } // old rows can have an empty repo did, so there's nothing to refresh if repo.RepoDid == "" { return nil } return s.refreshForEvent(ctx, rec, repo.RepoDid) } var declaration org_tangled.RepoDeclaration if err := json.Unmarshal(rec.Record, &declaration); err != nil { s.l.Warn("skipping invalid declaration record", "owner", rec.Did, "err", err) return nil } repoDid, err := syntax.ParseDID(declaration.Repo) if err != nil { s.l.Warn("skipping declaration with a malformed repo did", "owner", rec.Did, "repo", declaration.Repo) return nil } rows, err := s.db.ReposByDid(repoDid) if err != nil { return err } if len(rows) > 0 { // anyone can write a declaration naming a served repo, and every refresh costs round trips if !slices.ContainsFunc(rows, func(r db.Repo) bool { return r.Owner == rec.Did }) { return nil } } else if enrolled, err := db.Enrolls(s.db, &s.cfg.Server, rec.Did); err != nil || !enrolled { return err } return s.refreshForEvent(ctx, rec, repoDid) } func jetstreamToTapEvent(e *jetstream.Event) (tapc.Event, bool) { if e.Commit == nil { return tapc.Event{}, false } did, err := syntax.ParseDID(e.Did) if err != nil { return tapc.Event{}, false } var action tapc.RecordAction switch e.Commit.Operation { case jetstream.CommitOperationCreate: action = tapc.RecordCreateAction case jetstream.CommitOperationUpdate: action = tapc.RecordUpdateAction case jetstream.CommitOperationDelete: action = tapc.RecordDeleteAction default: return tapc.Event{}, false } return tapc.Event{ Type: tapc.EvtRecord, Record: &tapc.RecordEventData{ Did: did, Rkey: syntax.RecordKey(e.Commit.RKey), Collection: syntax.NSID(e.Commit.Collection), Rev: e.Commit.Rev, Action: action, Record: e.Commit.Record, // jetstream is only used for live Live: true, }, }, true } func (s *Spindle) ingestAccount(ctx context.Context, e *jetstream.Event) error { if e.Account == nil || e.Account.Active { return nil } status := "unknown" if e.Account.Status != nil { status = *e.Account.Status } did, err := syntax.ParseDID(e.Account.Did) if err != nil { return fmt.Errorf("parsing account did: %w", err) } s.l.Warn("account went inactive, purging its repos", "did", did, "status", status) return s.WipeOwner(ctx, did, "account "+status) }