add instuctions for running social app against konbini

README.md

···
 
 The frontend will be available at http://localhost:3000 and will connect to the API at http://localhost:4444.
 
+## Running the Bluesky App against Konbini
+
+Konbini implements a large portion of the app.bsky.\* appview endpoints that
+are required for pointing the main app to it and having it work reasonably
+well.
+
+To accomplish this you will need a few things:
+
+### Service DID
+
+You will need a DID, preferably a did:web for your appview that points at a
+public endpoint where your appview is accessible via https.
+I'll get into the https proxy next, but for the did, I've just pointed a domain
+I own (in my case appview1.bluesky.day) to a VPS, and used caddy to host a file
+at `/.well-known/did.json`.
+That file should look like this:
+
+```json
+{
+  "@context": [
+    "https://www.w3.org/ns/did/v1",
+    "https://w3id.org/security/multikey/v1"
+  ],
+  "id": "did:web:appview1.bluesky.day",
+  "verificationMethod": [
+    {
+      "id": "did:web:api.bsky.app#atproto",
+      "type": "Multikey",
+      "controller": "did:web:api.bsky.app",
+      "publicKeyMultibase": "zQ3shpRzb2NDriwCSSsce6EqGxG23kVktHZc57C3NEcuNy1jg"
+    }
+  ],
+  "service": [
+    {
+      "id": "#bsky_notif",
+      "type": "BskyNotificationService",
+      "serviceEndpoint": "YOUR APPVIEW HTTPS URL"
+    },
+    {
+      "id": "#bsky_appview",
+      "type": "BskyAppView",
+      "serviceEndpoint": "YOUR APPVIEW HTTPS URL"
+    }
+  ]
+}
+```
+
+The verificationMethod isn't used but i'm not sure if _something_ is required
+there or not, so i'm just leaving that there, it works on my machine.
+
+### HTTPS Endpoint
+
+I've been using ngrok to proxy traffic from a publicly accessible https url to my appview.
+You can simply run `ngrok http 4446` and it will give you an https url that you
+can then put in your DID doc above.
+
+### The Social App
+
+Now, clone and build the social app:
+
+```
+git clone https://github.com/bluesky-social/social-app
+cd social-app
+yarn
+```
+
+And then set this environment variable that tells it to use your appview:
+
+```
+export EXPO_PUBLIC_BLUESKY_PROXY_DID=did:web:YOURDIDWEB
+```
+
+And finally run the app:
+
+```
+yarn web
+```
+
+This takes a while on first load since its building everything.
+After that, load the localhost url it gives you and it _should_ work.
+
 ## License
 
 MIT (whyrusleeping)
+
+```
+
+```

main.go

···
 package main
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"log"
···
 	"github.com/bluesky-social/indigo/atproto/syntax"
 	"github.com/bluesky-social/indigo/cmd/relay/stream"
 	"github.com/bluesky-social/indigo/cmd/relay/stream/schedulers/parallel"
+	"github.com/bluesky-social/indigo/repo"
 	"github.com/bluesky-social/indigo/util/cliutil"
 	xrpclib "github.com/bluesky-social/indigo/xrpc"
 	"github.com/gorilla/websocket"
···
 			client: cc,
 			dir:    dir,
 
-			missingProfiles:       make(chan string, 1024),
-			missingPosts:          make(chan string, 1024),
-			missingFeedGenerators: make(chan string, 1024),
+			missingRecords: make(chan MissingRecord, 1024),
 		}
 		fmt.Println("MY DID: ", s.mydid)
 
···
 			http.ListenAndServe(":4445", nil)
 		}()
 
-		go s.missingProfileFetcher()
-		go s.missingPostFetcher()
-		go s.missingFeedGeneratorFetcher()
+		go s.missingRecordFetcher()
 
 		seqno, err := loadLastSeq(db, "firehose_seq")
 		if err != nil {
···
 	seqLk   sync.Mutex
 	lastSeq int64
 
-	mpLk                  sync.Mutex
-	missingProfiles       chan string
-	missingPosts          chan string
-	missingFeedGenerators chan string
+	mpLk           sync.Mutex
+	missingRecords chan MissingRecord
 }
 
 func (s *Server) getXrpcClient() (*xrpclib.Client, error) {
···
 		return err
 	}
 
-	_ = resp
-	return nil
+	c := &xrpclib.Client{
+		Host: resp.PDSEndpoint(),
+	}
+
+	repob, err := atproto.SyncGetRepo(ctx, c, did, "")
+	if err != nil {
+		return err
+	}
+
+	rep, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(repob))
+	if err != nil {
+		return err
+	}
+
+	return rep.ForEach(ctx, "", func(k string, v cid.Cid) error {
+		blk, err := rep.Blockstore().Get(ctx, v)
+		if err != nil {
+			slog.Error("record missing in repo", "path", k, "cid", v, "error", err)
+			return nil
+		}
+
+		d := blk.RawData()
+		if err := s.backend.HandleCreate(ctx, did, "", k, &d, &v); err != nil {
+			slog.Error("failed to index record", "path", k, "cid", v, "error", err)
+		}
+		return nil
+	})
 
 }

missing.go

···
 	"context"
 	"fmt"
 	"log/slog"
-	"strings"
 
 	"github.com/bluesky-social/indigo/api/atproto"
 	"github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/atproto/syntax"
 	xrpclib "github.com/bluesky-social/indigo/xrpc"
 	"github.com/ipfs/go-cid"
-	"github.com/labstack/gommon/log"
 )
 
-func (s *Server) addMissingProfile(ctx context.Context, did string) {
+type MissingRecordType string
+
+const (
+	MissingRecordTypeProfile       MissingRecordType = "profile"
+	MissingRecordTypePost          MissingRecordType = "post"
+	MissingRecordTypeFeedGenerator MissingRecordType = "feedgenerator"
+)
+
+type MissingRecord struct {
+	Type       MissingRecordType
+	Identifier string // DID for profiles, AT-URI for posts/feedgens
+}
+
+func (s *Server) addMissingRecord(ctx context.Context, rec MissingRecord) {
 	select {
-	case s.missingProfiles <- did:
+	case s.missingRecords <- rec:
 	case <-ctx.Done():
 	}
 }
 
-func (s *Server) missingProfileFetcher() {
-	for did := range s.missingProfiles {
-		if err := s.fetchMissingProfile(context.TODO(), did); err != nil {
-			log.Warn("failed to fetch missing profile", "did", did, "error", err)
+// Legacy methods for backward compatibility
+func (s *Server) addMissingProfile(ctx context.Context, did string) {
+	s.addMissingRecord(ctx, MissingRecord{
+		Type:       MissingRecordTypeProfile,
+		Identifier: did,
+	})
+}
+
+func (s *Server) addMissingPost(ctx context.Context, uri string) {
+	slog.Info("adding missing post to fetch queue", "uri", uri)
+	s.addMissingRecord(ctx, MissingRecord{
+		Type:       MissingRecordTypePost,
+		Identifier: uri,
+	})
+}
+
+func (s *Server) addMissingFeedGenerator(ctx context.Context, uri string) {
+	slog.Info("adding missing feed generator to fetch queue", "uri", uri)
+	s.addMissingRecord(ctx, MissingRecord{
+		Type:       MissingRecordTypeFeedGenerator,
+		Identifier: uri,
+	})
+}
+
+func (s *Server) missingRecordFetcher() {
+	for rec := range s.missingRecords {
+		var err error
+		switch rec.Type {
+		case MissingRecordTypeProfile:
+			err = s.fetchMissingProfile(context.TODO(), rec.Identifier)
+		case MissingRecordTypePost:
+			err = s.fetchMissingPost(context.TODO(), rec.Identifier)
+		case MissingRecordTypeFeedGenerator:
+			err = s.fetchMissingFeedGenerator(context.TODO(), rec.Identifier)
+		default:
+			slog.Error("unknown missing record type", "type", rec.Type)
+			continue
+		}
+
+		if err != nil {
+			slog.Warn("failed to fetch missing record", "type", rec.Type, "identifier", rec.Identifier, "error", err)
 		}
 	}
 }
···
 	return s.backend.HandleUpdateProfile(ctx, repo, "self", "", buf.Bytes(), cc)
 }
 
-func (s *Server) addMissingPost(ctx context.Context, uri string) {
-	slog.Info("adding missing post to fetch queue", "uri", uri)
-	select {
-	case s.missingPosts <- uri:
-	case <-ctx.Done():
-	}
-}
-
-func (s *Server) missingPostFetcher() {
-	for uri := range s.missingPosts {
-		if err := s.fetchMissingPost(context.TODO(), uri); err != nil {
-			log.Warn("failed to fetch missing post", "uri", uri, "error", err)
-		}
-	}
-}
-
 func (s *Server) fetchMissingPost(ctx context.Context, uri string) error {
-	// Parse AT URI: at://did:plc:xxx/app.bsky.feed.post/rkey
-	parts := strings.Split(uri, "/")
-	if len(parts) < 5 || !strings.HasPrefix(parts[2], "did:") {
+	puri, err := syntax.ParseATURI(uri)
+	if err != nil {
 		return fmt.Errorf("invalid AT URI: %s", uri)
 	}
 
-	did := parts[2]
-	collection := parts[3]
-	rkey := parts[4]
+	did := puri.Authority().String()
+	collection := puri.Collection().String()
+	rkey := puri.RecordKey().String()
 
 	repo, err := s.backend.getOrCreateRepo(ctx, did)
 	if err != nil {
···
 	return s.backend.HandleCreatePost(ctx, repo, rkey, buf.Bytes(), cc)
 }
 
-func (s *Server) addMissingFeedGenerator(ctx context.Context, uri string) {
-	slog.Info("adding missing feed generator to fetch queue", "uri", uri)
-	select {
-	case s.missingFeedGenerators <- uri:
-	case <-ctx.Done():
-	}
-}
-
-func (s *Server) missingFeedGeneratorFetcher() {
-	for uri := range s.missingFeedGenerators {
-		if err := s.fetchMissingFeedGenerator(context.TODO(), uri); err != nil {
-			log.Warn("failed to fetch missing feed generator", "uri", uri, "error", err)
-		}
-	}
-}
-
 func (s *Server) fetchMissingFeedGenerator(ctx context.Context, uri string) error {
-	// Parse AT URI: at://did:plc:xxx/app.bsky.feed.generator/rkey
 	puri, err := syntax.ParseATURI(uri)
 	if err != nil {
 		return fmt.Errorf("invalid AT URI: %s", uri)

xrpc/feed/getFeed.go

···
 	// Parse parameters
 	feedURI := c.QueryParam("feed")
 	if feedURI == "" {
-		return c.JSON(http.StatusBadRequest, map[string]interface{}{
+		return c.JSON(http.StatusBadRequest, map[string]any{
 			"error":   "InvalidRequest",
 			"message": "feed parameter is required",
 		})
···
 	rkey := extractRkeyFromURI(feedURI)
 
 	if did == "" || rkey == "" {
-		return c.JSON(http.StatusBadRequest, map[string]interface{}{
+		return c.JSON(http.StatusBadRequest, map[string]any{
 			"error":   "InvalidRequest",
 			"message": "invalid feed URI format",
 		})
···
 
 	if feedGen.ID == 0 {
 		hydrator.AddMissingFeedGenerator(feedURI)
-		return c.JSON(http.StatusNotFound, map[string]interface{}{
+		return c.JSON(http.StatusNotFound, map[string]any{
 			"error":   "NotFound",
 			"message": "feed generator not found",
 		})
···
 	var feedGenRecord bsky.FeedGenerator
 	if err := feedGenRecord.UnmarshalCBOR(bytes.NewReader(feedGen.Raw)); err != nil {
 		slog.Error("failed to decode feed generator record", "error", err)
-		return c.JSON(http.StatusInternalServerError, map[string]interface{}{
+		return c.JSON(http.StatusInternalServerError, map[string]any{
 			"error":   "InternalError",
 			"message": "failed to decode feed generator record",
 		})
···
 	serviceDID, err := syntax.ParseDID(feedGenRecord.Did)
 	if err != nil {
 		slog.Error("invalid service DID in feed generator", "error", err, "did", feedGenRecord.Did)
-		return c.JSON(http.StatusInternalServerError, map[string]interface{}{
+		return c.JSON(http.StatusInternalServerError, map[string]any{
 			"error":   "InternalError",
 			"message": "invalid service DID",
 		})
···
 	serviceIdent, err := dir.LookupDID(ctx, serviceDID)
 	if err != nil {
 		slog.Error("failed to resolve service DID", "error", err, "did", serviceDID)
-		return c.JSON(http.StatusInternalServerError, map[string]interface{}{
+		return c.JSON(http.StatusInternalServerError, map[string]any{
 			"error":   "InternalError",
 			"message": "failed to resolve service endpoint",
 		})
···
 	serviceEndpoint := serviceIdent.GetServiceEndpoint("bsky_fg")
 	if serviceEndpoint == "" {
 		slog.Error("service has no bsky_fg endpoint", "did", serviceDID)
-		return c.JSON(http.StatusInternalServerError, map[string]interface{}{
+		return c.JSON(http.StatusInternalServerError, map[string]any{
 			"error":   "InternalError",
 			"message": "service has no endpoint",
 		})
 	}
 
 	// Create XRPC client for the feed generator service
+	// Pass through headers from the original request so feed generators can
+	// customize feeds based on the viewer
+	headers := make(map[string]string)
+
+	// Set User-Agent to identify konbini
+	headers["User-Agent"] = "konbini/0.0.1"
+
+	// Pass through Authorization header if present (for authenticated feed requests)
+	if authHeader := c.Request().Header.Get("Authorization"); authHeader != "" {
+		headers["Authorization"] = authHeader
+	}
+
+	// Pass through Accept-Language header if present
+	if langHeader := c.Request().Header.Get("Accept-Language"); langHeader != "" {
+		headers["Accept-Language"] = langHeader
+	}
+
+	// Pass through X-Bsky-Topics header if present
+	if topicsHeader := c.Request().Header.Get("X-Bsky-Topics"); topicsHeader != "" {
+		headers["X-Bsky-Topics"] = topicsHeader
+	}
+
 	client := &xrpc.Client{
-		Host: serviceEndpoint,
+		Host:    serviceEndpoint,
+		Headers: headers,
 	}
 
 	// Call getFeedSkeleton on the service