did doc hosting and some other tweaks

hydration/hydrator.go

···
 	db  *gorm.DB
 	dir identity.Directory
 
-	missingActorCallback         func(string)
-	missingPostCallback          func(string)
-	missingFeedGeneratorCallback func(string)
+	missingRecordCallback func(string, bool)
 }
 
 // NewHydrator creates a new Hydrator
···
 	}
 }
 
-func (h *Hydrator) SetMissingActorCallback(fn func(string)) {
-	h.missingActorCallback = fn
+// SetMissingRecordCallback sets the callback for when a record is missing
+// The callback receives an identifier which can be:
+// - A DID (e.g., "did:plc:...") for actors/profiles
+// - An AT-URI (e.g., "at://did:plc:.../app.bsky.feed.post/...") for posts
+// - An AT-URI (e.g., "at://did:plc:.../app.bsky.feed.generator/...") for feed generators
+func (h *Hydrator) SetMissingRecordCallback(fn func(string, bool)) {
+	h.missingRecordCallback = fn
 }
 
-func (h *Hydrator) addMissingActor(did string) {
-	if h.missingActorCallback != nil {
-		h.missingActorCallback(did)
+// AddMissingRecord reports a missing record that needs to be fetched
+func (h *Hydrator) AddMissingRecord(identifier string, wait bool) {
+	if h.missingRecordCallback != nil {
+		h.missingRecordCallback(identifier, wait)
 	}
 }
 
-func (h *Hydrator) SetMissingFeedGeneratorCallback(fn func(string)) {
-	h.missingFeedGeneratorCallback = fn
-}
-
-func (h *Hydrator) AddMissingFeedGenerator(uri string) {
-	if h.missingFeedGeneratorCallback != nil {
-		h.missingFeedGeneratorCallback(uri)
-	}
+// addMissingActor is a convenience method for adding missing actors
+func (h *Hydrator) addMissingActor(did string) {
+	h.AddMissingRecord(did, false)
 }
 
 // HydrateCtx contains context for hydration operations

missing.go

···
 type MissingRecord struct {
 	Type       MissingRecordType
 	Identifier string // DID for profiles, AT-URI for posts/feedgens
+	Wait       bool
+
+	waitch chan struct{}
 }
 
 func (s *Server) addMissingRecord(ctx context.Context, rec MissingRecord) {
+	if rec.Wait {
+		rec.waitch = make(chan struct{})
+	}
+
 	select {
 	case s.missingRecords <- rec:
 	case <-ctx.Done():
 	}
+
+	if rec.Wait {
+		select {
+		case <-rec.waitch:
+		case <-ctx.Done():
+		}
+	}
 }
 
 // Legacy methods for backward compatibility
···
 
 		if err != nil {
 			slog.Warn("failed to fetch missing record", "type", rec.Type, "identifier", rec.Identifier, "error", err)
+		}
+
+		if rec.Wait {
+			close(rec.waitch)
 		}
 	}
 }
 
 func (s *Server) fetchMissingProfile(ctx context.Context, did string) error {
+	s.backend.addRelevantDid(did)
+
 	repo, err := s.backend.getOrCreateRepo(ctx, did)
 	if err != nil {
 		return err
···
 	collection := puri.Collection().String()
 	rkey := puri.RecordKey().String()
 
+	s.backend.addRelevantDid(did)
+
 	repo, err := s.backend.getOrCreateRepo(ctx, did)
 	if err != nil {
 		return err
···
 	did := puri.Authority().String()
 	collection := puri.Collection().String()
 	rkey := puri.RecordKey().String()
+	s.backend.addRelevantDid(did)
 
 	repo, err := s.backend.getOrCreateRepo(ctx, did)
 	if err != nil {

pgbackend.go

···
 	return &r, nil
 }
 
-func (b *PostgresBackend) TrackMissingActor(did string) {
-	b.s.addMissingProfile(context.TODO(), did)
+func (b *PostgresBackend) TrackMissingRecord(identifier string, wait bool) {
+	b.s.addMissingRecord(context.TODO(), MissingRecord{
+		Type:       inferRecordType(identifier),
+		Identifier: identifier,
+		Wait:       wait,
+	})
 }
 
-func (b *PostgresBackend) TrackMissingFeedGenerator(uri string) {
-	b.s.addMissingFeedGenerator(context.TODO(), uri)
+// inferRecordType determines the record type based on the identifier format
+func inferRecordType(identifier string) MissingRecordType {
+	if strings.HasPrefix(identifier, "did:") {
+		return MissingRecordTypeProfile
+	}
+
+	if strings.HasPrefix(identifier, "at://") {
+		if strings.Contains(identifier, "/app.bsky.feed.post/") {
+			return MissingRecordTypePost
+		}
+		if strings.Contains(identifier, "/app.bsky.feed.generator/") {
+			return MissingRecordTypeFeedGenerator
+		}
+	}
+
+	// Default to post if we can't determine
+	return MissingRecordTypePost
 }

xrpc/feed/getFeed.go

···
 	"log/slog"
 	"net/http"
 	"strconv"
+	"strings"
+	"sync"
 
 	"github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/atproto/identity"
···
 	}
 
 	if feedGen.ID == 0 {
-		hydrator.AddMissingFeedGenerator(feedURI)
+		hydrator.AddMissingRecord(feedURI, true)
 		return c.JSON(http.StatusNotFound, map[string]any{
 			"error":   "NotFound",
 			"message": "feed generator not found",
···
 
 	// Hydrate the posts from the skeleton
 	posts := make([]*bsky.FeedDefs_FeedViewPost, 0, len(skeleton.Feed))
-	for _, skeletonPost := range skeleton.Feed {
-		postURI, err := syntax.ParseATURI(skeletonPost.Post)
-		if err != nil {
-			slog.Warn("invalid post URI in skeleton", "uri", skeletonPost.Post, "error", err)
-			continue
-		}
+	var wg sync.WaitGroup
+	for i := range skeleton.Feed {
+		wg.Add(1)
+		go func(ix int) {
+			defer wg.Done()
+			skeletonPost := skeleton.Feed[ix]
+			postURI, err := syntax.ParseATURI(skeletonPost.Post)
+			if err != nil {
+				slog.Warn("invalid post URI in skeleton", "uri", skeletonPost.Post, "error", err)
+				return
+			}
 
-		postInfo, err := hydrator.HydratePost(ctx, string(postURI), viewer)
-		if err != nil {
-			slog.Warn("failed to hydrate post", "uri", postURI, "error", err)
-			continue
-		}
+			postInfo, err := hydrator.HydratePost(ctx, postURI.String(), viewer)
+			if err != nil {
+				if strings.Contains(err.Error(), "post not found") {
+					hydrator.AddMissingRecord(postURI.String(), true)
+					postInfo, err = hydrator.HydratePost(ctx, postURI.String(), viewer)
+					if err != nil {
+						slog.Error("failed to hydrate post after fetch missing", "uri", postURI, "error", err)
+						return
+					}
+				} else {
+					slog.Warn("failed to hydrate post", "uri", postURI, "error", err)
+					return
+				}
+			}
 
-		authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
-		if err != nil {
-			slog.Warn("failed to hydrate author", "did", postInfo.Author, "error", err)
-			continue
-		}
+			authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
+			if err != nil {
+				slog.Warn("failed to hydrate author", "did", postInfo.Author, "error", err)
+				return
+			}
 
-		posts = append(posts, views.FeedViewPost(postInfo, authorInfo))
+			posts[ix] = views.FeedViewPost(postInfo, authorInfo)
+		}(i)
 	}
+	wg.Wait()
 
 	output := &bsky.FeedGetFeed_Output{
 		Feed:   posts,

xrpc/feed/getFeedGenerator.go

···
 
 	if err != nil || feedGen.ID == 0 {
 		// Track this missing feed generator for fetching
-		hydrator.AddMissingFeedGenerator(feedURI)
+		hydrator.AddMissingRecord(feedURI, true)
 
 		return c.JSON(http.StatusNotFound, map[string]any{
 			"error":   "NotFound",

xrpc/server.go

···
 type Backend interface {
 	// Add methods as needed for data access
 
-	TrackMissingActor(did string)
-	TrackMissingFeedGenerator(uri string)
+	TrackMissingRecord(identifier string, wait bool)
 }
 
 // NewServer creates a new XRPC server
···
 		hydrator: hydration.NewHydrator(db, dir),
 	}
 
-	s.hydrator.SetMissingActorCallback(backend.TrackMissingActor)
-	s.hydrator.SetMissingFeedGeneratorCallback(backend.TrackMissingFeedGenerator)
+	s.hydrator.SetMissingRecordCallback(backend.TrackMissingRecord)
 
 	// Register XRPC endpoints
 	s.registerEndpoints()
···
 // registerEndpoints registers all XRPC endpoints
 func (s *Server) registerEndpoints() {
 	// XRPC endpoints follow the pattern: /xrpc/<namespace>.<method>
+
+	s.e.GET("/.well-known/did.json", func(c echo.Context) error {
+		return c.File("did.json")
+	})
+
 	xrpcGroup := s.e.Group("/xrpc")
 
 	// com.atproto.identity.*