clean up missing post logic

backend/backend.go

···
 
 	"github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/api/bsky"
+	"github.com/bluesky-social/indigo/atproto/identity"
 	"github.com/bluesky-social/indigo/atproto/syntax"
 	"github.com/bluesky-social/indigo/util"
 	"github.com/bluesky-social/indigo/xrpc"
···
 
 // PostgresBackend handles database operations
 type PostgresBackend struct {
-	db      *gorm.DB
-	pgx     *pgxpool.Pool
-	tracker RecordTracker
+	db  *gorm.DB
+	pgx *pgxpool.Pool
+
+	dir identity.Directory
 
 	client *xrpc.Client
 
···
 	didByIDCache *lru.TwoQueueCache[uint, string]
 
 	postInfoCache *lru.TwoQueueCache[string, cachedPostInfo]
+
+	missingRecords chan MissingRecord
 }
 
 type cachedPostInfo struct {
···
 }
 
 // NewPostgresBackend creates a new PostgresBackend
-func NewPostgresBackend(mydid string, db *gorm.DB, pgx *pgxpool.Pool, client *xrpc.Client, tracker RecordTracker) (*PostgresBackend, error) {
+func NewPostgresBackend(mydid string, db *gorm.DB, pgx *pgxpool.Pool, client *xrpc.Client, dir identity.Directory) (*PostgresBackend, error) {
 	rc, _ := lru.New2Q[string, *Repo](1_000_000)
 	pc, _ := lru.New2Q[string, cachedPostInfo](1_000_000)
 	revc, _ := lru.New2Q[uint, string](1_000_000)
···
 		mydid:         mydid,
 		db:            db,
 		pgx:           pgx,
-		tracker:       tracker,
 		relevantDids:  make(map[string]bool),
 		repoCache:     rc,
 		postInfoCache: pc,
 		revCache:      revc,
 		didByIDCache:  dbic,
+		dir:           dir,
+
+		missingRecords: make(chan MissingRecord, 1000),
 	}
 
 	r, err := b.GetOrCreateRepo(context.TODO(), mydid)
···
 	}
 
 	b.myrepo = r
+
+	go b.missingRecordFetcher()
 	return b, nil
 }
 
 // TrackMissingRecord implements the RecordTracker interface
 func (b *PostgresBackend) TrackMissingRecord(identifier string, wait bool) {
-	if b.tracker != nil {
-		b.tracker.TrackMissingRecord(identifier, wait)
+	mr := MissingRecord{
+		Type:       mrTypeFromIdent(identifier),
+		Identifier: identifier,
+		Wait:       wait,
+	}
+
+	b.addMissingRecord(context.TODO(), mr)
+}
+
+func mrTypeFromIdent(ident string) MissingRecordType {
+	if strings.HasPrefix(ident, "did:") {
+		return MissingRecordTypeProfile
+	}
+
+	puri, _ := syntax.ParseATURI(ident)
+	switch puri.Collection().String() {
+	case "app.bsky.feed.post":
+		return MissingRecordTypePost
+	case "app.bsky.feed.generator":
+		return MissingRecordTypeFeedGenerator
+	default:
+		return MissingRecordTypeUnknown
 	}
+
 }
 
 // DidToID converts a DID to a database ID

handlers.go

···
 	}
 
 	if profile.Raw == nil || len(profile.Raw) == 0 {
-		s.addMissingProfile(ctx, accdid)
+		s.backend.TrackMissingRecord(accdid, false)
 		return e.JSON(404, map[string]any{
 			"error": "missing profile info for user",
 		})
···
 	}
 
 	if profile.Raw == nil || len(profile.Raw) == 0 {
-		s.addMissingProfile(ctx, r.Did)
+		s.backend.TrackMissingRecord(r.Did, false)
 		return &authorInfo{
 			Handle: resp.Handle.String(),
 			Did:    r.Did,
···
 
 			uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", r.Did, p.Rkey)
 			if len(p.Raw) == 0 || p.NotFound {
-				s.addMissingPost(ctx, uri)
+				s.backend.TrackMissingRecord(uri, false)
 				posts[ix] = postResponse{
 					Uri:     uri,
 					Missing: true,
···
 	quotedPost, err := s.backend.GetPostByUri(ctx, quotedURI, "*")
 	if err != nil {
 		slog.Warn("failed to get quoted post", "uri", quotedURI, "error", err)
-		s.addMissingPost(ctx, quotedURI)
+		s.backend.TrackMissingRecord(quotedURI, false)
 		return s.buildQuoteFallback(quotedURI, quotedCid)
 	}
 
 	if quotedPost == nil || quotedPost.Raw == nil || len(quotedPost.Raw) == 0 || quotedPost.NotFound {
-		s.addMissingPost(ctx, quotedURI)
+		s.backend.TrackMissingRecord(quotedURI, false)
 		return s.buildQuoteFallback(quotedURI, quotedCid)
 	}
 
···
 				prof = &p
 			}
 		} else {
-			s.addMissingProfile(ctx, r.Did)
+			s.backend.TrackMissingRecord(r.Did, false)
 		}
 
 		users = append(users, engagementUser{
···
 				prof = &p
 			}
 		} else {
-			s.addMissingProfile(ctx, r.Did)
+			s.backend.TrackMissingRecord(r.Did, false)
 		}
 
 		users = append(users, engagementUser{
···
 				prof = &p
 			}
 		} else {
-			s.addMissingProfile(ctx, r.Did)
+			s.backend.TrackMissingRecord(r.Did, false)
 		}
 
 		users = append(users, engagementUser{

hydration/post.go

···
 
 // hydrateEmbeddedRecord hydrates an embedded record (for quote posts, etc.)
 func (h *Hydrator) hydrateEmbeddedRecord(ctx context.Context, uri string, viewerDID string) *bsky.EmbedRecord_View_Record {
+	ctx, span := tracer.Start(ctx, "hydrateEmbeddedRecord")
+	defer span.End()
+
 	// Check if it's a post URI
 	if !isPostURI(uri) {
 		// Could be a feed generator, list, labeler, or starter pack

main.go

···
 			client: cc,
 			dir:    dir,
 
-			missingRecords: make(chan MissingRecord, 1024),
-			db:             db,
+			db: db,
 		}
 		fmt.Println("MY DID: ", s.mydid)
 
-		pgb, err := backend.NewPostgresBackend(mydid, db, pool, cc, nil)
+		pgb, err := backend.NewPostgresBackend(mydid, db, pool, cc, dir)
 		if err != nil {
 			return err
 		}
···
 			http.ListenAndServe(":4445", nil)
 		}()
 
-		go s.missingRecordFetcher()
-
 		seqno, err := loadLastSeq(db, "firehose_seq")
 		if err != nil {
 			fmt.Println("failed to load sequence number, starting over", err)
···
 	seqLk   sync.Mutex
 	lastSeq int64
 
-	mpLk           sync.Mutex
-	missingRecords chan MissingRecord
+	mpLk sync.Mutex
 
 	db *gorm.DB
 }

missing.go backend/missing.go

···
-package main
+package backend
 
 import (
 	"bytes"
···
 	MissingRecordTypeProfile       MissingRecordType = "profile"
 	MissingRecordTypePost          MissingRecordType = "post"
 	MissingRecordTypeFeedGenerator MissingRecordType = "feedgenerator"
+	MissingRecordTypeUnknown       MissingRecordType = "unknown"
 )
 
 type MissingRecord struct {
···
 	waitch chan struct{}
 }
 
-func (s *Server) addMissingRecord(ctx context.Context, rec MissingRecord) {
+func (b *PostgresBackend) addMissingRecord(ctx context.Context, rec MissingRecord) {
 	if rec.Wait {
 		rec.waitch = make(chan struct{})
 	}
 
 	select {
-	case s.missingRecords <- rec:
+	case b.missingRecords <- rec:
 	case <-ctx.Done():
 	}
 
···
 	}
 }
 
-// Legacy methods for backward compatibility
-func (s *Server) addMissingProfile(ctx context.Context, did string) {
-	s.addMissingRecord(ctx, MissingRecord{
-		Type:       MissingRecordTypeProfile,
-		Identifier: did,
-	})
-}
-
-func (s *Server) addMissingPost(ctx context.Context, uri string) {
-	slog.Info("adding missing post to fetch queue", "uri", uri)
-	s.addMissingRecord(ctx, MissingRecord{
-		Type:       MissingRecordTypePost,
-		Identifier: uri,
-	})
-}
-
-func (s *Server) addMissingFeedGenerator(ctx context.Context, uri string) {
-	slog.Info("adding missing feed generator to fetch queue", "uri", uri)
-	s.addMissingRecord(ctx, MissingRecord{
-		Type:       MissingRecordTypeFeedGenerator,
-		Identifier: uri,
-	})
-}
-
-func (s *Server) missingRecordFetcher() {
-	for rec := range s.missingRecords {
+func (b *PostgresBackend) missingRecordFetcher() {
+	for rec := range b.missingRecords {
 		var err error
 		switch rec.Type {
 		case MissingRecordTypeProfile:
-			err = s.fetchMissingProfile(context.TODO(), rec.Identifier)
+			err = b.fetchMissingProfile(context.TODO(), rec.Identifier)
 		case MissingRecordTypePost:
-			err = s.fetchMissingPost(context.TODO(), rec.Identifier)
+			err = b.fetchMissingPost(context.TODO(), rec.Identifier)
 		case MissingRecordTypeFeedGenerator:
-			err = s.fetchMissingFeedGenerator(context.TODO(), rec.Identifier)
+			err = b.fetchMissingFeedGenerator(context.TODO(), rec.Identifier)
 		default:
 			slog.Error("unknown missing record type", "type", rec.Type)
 			continue
···
 	}
 }
 
-func (s *Server) fetchMissingProfile(ctx context.Context, did string) error {
-	s.backend.AddRelevantDid(did)
+func (b *PostgresBackend) fetchMissingProfile(ctx context.Context, did string) error {
+	b.AddRelevantDid(did)
 
-	repo, err := s.backend.GetOrCreateRepo(ctx, did)
+	repo, err := b.GetOrCreateRepo(ctx, did)
 	if err != nil {
 		return err
 	}
 
-	resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
+	resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
 	if err != nil {
 		return err
 	}
···
 		return err
 	}
 
-	return s.backend.HandleUpdateProfile(ctx, repo, "self", "", buf.Bytes(), cc)
+	return b.HandleUpdateProfile(ctx, repo, "self", "", buf.Bytes(), cc)
 }
 
-func (s *Server) fetchMissingPost(ctx context.Context, uri string) error {
+func (b *PostgresBackend) fetchMissingPost(ctx context.Context, uri string) error {
 	puri, err := syntax.ParseATURI(uri)
 	if err != nil {
 		return fmt.Errorf("invalid AT URI: %s", uri)
···
 	collection := puri.Collection().String()
 	rkey := puri.RecordKey().String()
 
-	s.backend.AddRelevantDid(did)
+	b.AddRelevantDid(did)
 
-	repo, err := s.backend.GetOrCreateRepo(ctx, did)
+	repo, err := b.GetOrCreateRepo(ctx, did)
 	if err != nil {
 		return err
 	}
 
-	resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
+	resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
 	if err != nil {
 		return err
 	}
···
 		return err
 	}
 
-	return s.backend.HandleCreatePost(ctx, repo, rkey, buf.Bytes(), cc)
+	return b.HandleCreatePost(ctx, repo, rkey, buf.Bytes(), cc)
 }
 
-func (s *Server) fetchMissingFeedGenerator(ctx context.Context, uri string) error {
+func (b *PostgresBackend) fetchMissingFeedGenerator(ctx context.Context, uri string) error {
 	puri, err := syntax.ParseATURI(uri)
 	if err != nil {
 		return fmt.Errorf("invalid AT URI: %s", uri)
···
 	did := puri.Authority().String()
 	collection := puri.Collection().String()
 	rkey := puri.RecordKey().String()
-	s.backend.AddRelevantDid(did)
+	b.AddRelevantDid(did)
 
-	repo, err := s.backend.GetOrCreateRepo(ctx, did)
+	repo, err := b.GetOrCreateRepo(ctx, did)
 	if err != nil {
 		return err
 	}
 
-	resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
+	resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
 	if err != nil {
 		return err
 	}
···
 		return err
 	}
 
-	return s.backend.HandleCreateFeedGenerator(ctx, repo, rkey, buf.Bytes(), cc)
+	return b.HandleCreateFeedGenerator(ctx, repo, rkey, buf.Bytes(), cc)
 }

xrpc/notification/listNotifications.go

···
 		cursorPtr = &cursor
 	}
 
+	var lastSeen time.Time
+	if err := db.Raw("SELECT seen_at FROM notification_seens WHERE repo = (select id from repos where did = ?)", viewer).Scan(&lastSeen).Error; err != nil {
+		return err
+	}
+
+	var lastSeenStr *string
+	if !lastSeen.IsZero() {
+		s := lastSeen.Format(time.RFC3339)
+		lastSeenStr = &s
+	}
+
 	output := &bsky.NotificationListNotifications_Output{
 		Notifications: notifications,
 		Cursor:        cursorPtr,
+		SeenAt:        lastSeenStr,
 	}
 
 	return c.JSON(http.StatusOK, output)