aturilist i forgot what i was doing

.gitignore

···
+cmd/aturilist/badger_data

cmd/appview/main.go

···
 
 	did "github.com/whyrusleeping/go-did"
 	"tangled.org/whey.party/red-dwarf-server/auth"
+	aturilist "tangled.org/whey.party/red-dwarf-server/cmd/aturilist/client"
 	"tangled.org/whey.party/red-dwarf-server/microcosm/constellation"
 	"tangled.org/whey.party/red-dwarf-server/microcosm/slingshot"
 	appbskyactordefs "tangled.org/whey.party/red-dwarf-server/shims/lex/app/bsky/actor/defs"
···
 	SPACEDUST_URL     string
 	SLINGSHOT_URL     string
 	CONSTELLATION_URL string
+	ATURILIST_URL     string
 )
 
 func initURLs(prod bool) {
···
 		SPACEDUST_URL = "wss://spacedust.whey.party/subscribe"
 		SLINGSHOT_URL = "https://slingshot.whey.party"
 		CONSTELLATION_URL = "https://constellation.whey.party"
+		ATURILIST_URL = "http://localhost:7155"
 	} else {
 		JETSTREAM_URL = "ws://localhost:6008/subscribe"
 		SPACEDUST_URL = "ws://localhost:9998/subscribe"
 		SLINGSHOT_URL = "http://localhost:7729"
 		CONSTELLATION_URL = "http://localhost:7728"
+		ATURILIST_URL = "http://localhost:7155"
 	}
 }
 
···
 )
 
 func main() {
-	log.Println("red-dwarf-server started")
+	log.Println("red-dwarf-server AppView Service started")
 	prod := flag.Bool("prod", false, "use production URLs instead of localhost")
 	flag.Parse()
 
···
 	mailbox := sticket.New()
 	sl := slingshot.NewSlingshot(SLINGSHOT_URL)
 	cs := constellation.NewConstellation(CONSTELLATION_URL)
+	al := aturilist.NewClient(ATURILIST_URL)
 	// spacedust is type definitions only
 	// jetstream types is probably available from jetstream/pkg/models
 
···
 	router_raw := gin.New()
 	router_raw.Use(gin.Logger())
 	router_raw.Use(gin.Recovery())
-	router_raw.Use(cors.Default())
+	//router_raw.Use(cors.Default())
+	router_raw.Use(cors.New(cors.Config{
+		AllowAllOrigins: true,
+		AllowMethods:    []string{"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"},
+		// You must explicitly allow the custom ATProto headers here
+		AllowHeaders: []string{
+			"Origin",
+			"Content-Length",
+			"Content-Type",
+			"Authorization",
+			"Accept",
+			"Accept-Language",
+			"atproto-accept-labelers", // <--- The specific fix for your error
+			"atproto-proxy",           // Good to have for future compatibility
+		},
+		ExposeHeaders:    []string{"Content-Length", "Link"},
+		AllowCredentials: true,
+		MaxAge:           12 * time.Hour,
+	}))
 
 	router_raw.GET("/.well-known/did.json", GetWellKnownDID)
 
···
 			// if err == nil && kvkey != "" {
 			// 	kv.Set(kvkey, bytes, 1*time.Minute)
 			// }
+		})
+
+	router.GET("/xrpc/app.bsky.feed.getAuthorFeed",
+		func(c *gin.Context) {
+
+			rawdid := c.GetString("user_did")
+			log.Println("getFeed router_unsafe user_did: " + rawdid)
+			var viewer *utils.DID
+			didval, errdid := utils.NewDID(rawdid)
+			if errdid != nil {
+				viewer = nil
+			} else {
+				viewer = &didval
+			}
+
+			actorDidParam := c.Query("actor")
+			if actorDidParam == "" {
+				c.JSON(http.StatusBadRequest, gin.H{"error": "Missing actor param"})
+				return
+			}
+			cursorRawParam := c.Query("cursor")
+
+			listResp, err := al.ListRecords(ctx, actorDidParam, "app.bsky.feed.post", cursorRawParam, true)
+			if err != nil {
+				log.Fatalf("Failed to list: %v", err)
+			}
+
+			concurrentResults := MapConcurrent(
+				ctx,
+				listResp.Aturis,
+				20,
+				func(ctx context.Context, raw string, idx int) (*appbsky.FeedDefs_FeedViewPost, error) {
+					post, _, err := appbskyfeeddefs.PostView(ctx, raw, sl, cs, BSKYIMAGECDN_URL, viewer, 2)
+					if err != nil {
+						return nil, err
+					}
+					if post == nil {
+						return nil, fmt.Errorf("post not found")
+					}
+
+					return &appbsky.FeedDefs_FeedViewPost{
+						// FeedContext *string                       `json:"feedContext,omitempty" cborgen:"feedContext,omitempty"`
+						// Post        *FeedDefs_PostView            `json:"post" cborgen:"post"`
+						Post: post,
+						// Reason      *FeedDefs_FeedViewPost_Reason `json:"reason,omitempty" cborgen:"reason,omitempty"`
+						// Reason: &appbsky.FeedDefs_FeedViewPost_Reason{
+						// 	// FeedDefs_ReasonRepost *FeedDefs_ReasonRepost
+						// 	FeedDefs_ReasonRepost: &appbsky.FeedDefs_ReasonRepost{
+						// 		// LexiconTypeID string                      `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#reasonRepost"`
+						// 		LexiconTypeID: "app.bsky.feed.defs#reasonRepost",
+						// 		// By            *ActorDefs_ProfileViewBasic `json:"by" cborgen:"by"`
+						// 		// Cid           *string                     `json:"cid,omitempty" cborgen:"cid,omitempty"`
+						// 		// IndexedAt     string                      `json:"indexedAt" cborgen:"indexedAt"`
+						// 		// Uri           *string                     `json:"uri,omitempty" cborgen:"uri,omitempty"`
+						// 		Uri: &raw.Reason.FeedDefs_SkeletonReasonRepost.Repost,
+						// 	},
+						// 	// FeedDefs_ReasonPin    *FeedDefs_ReasonPin
+						// 	FeedDefs_ReasonPin: &appbsky.FeedDefs_ReasonPin{
+						// 		// LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#reasonPin"`
+						// 		LexiconTypeID: "app.bsky.feed.defs#reasonPin",
+						// 	},
+						// },
+						// Reply       *FeedDefs_ReplyRef            `json:"reply,omitempty" cborgen:"reply,omitempty"`
+						// // reqId: Unique identifier per request that may be passed back alongside interactions.
+						// ReqId *string `json:"reqId,omitempty" cborgen:"reqId,omitempty"`
+					}, nil
+				},
+			)
+
+			// build final slice
+			out := make([]*appbsky.FeedDefs_FeedViewPost, 0, len(concurrentResults))
+			for _, r := range concurrentResults {
+				if r.Err == nil && r.Value != nil && r.Value.Post != nil {
+					out = append(out, r.Value)
+				}
+			}
+
+			c.JSON(http.StatusOK, &appbsky.FeedGetAuthorFeed_Output{
+				Cursor: &listResp.Cursor,
+				Feed:   out,
+			})
 		})
 
 	// weird stuff

cmd/aturilist/client/client.go

···
+package aturilist
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// Constants for the XRPC methods
+const (
+	MethodListRecords     = "app.reddwarf.aturilist.listRecords"
+	MethodCountRecords    = "app.reddwarf.aturilist.countRecords"
+	MethodIndexRecord     = "app.reddwarf.aturilist.indexRecord"
+	MethodValidateRecord  = "app.reddwarf.aturilist.validateRecord"
+	DefaultProductionHost = "https://aturilist.reddwarf.app"
+)
+
+// Client is the API client for the Red Dwarf AtURI List Service.
+type Client struct {
+	Host       string
+	HTTPClient *http.Client
+	// AuthToken is the JWT used for the Authorization header
+	AuthToken string
+}
+
+// NewClient creates a new client. If host is empty, it defaults to production.
+func NewClient(host string) *Client {
+	if host == "" {
+		host = DefaultProductionHost
+	}
+	return &Client{
+		Host: host,
+		HTTPClient: &http.Client{
+			Timeout: 10 * time.Second,
+		},
+	}
+}
+
+// --- Response Models ---
+
+type ListRecordsResponse struct {
+	Aturis []string `json:"aturis"`
+	Count  int      `json:"count"`
+	Cursor string   `json:"cursor,omitempty"`
+}
+
+type CountRecordsResponse struct {
+	Repo       string `json:"repo"`
+	Collection string `json:"collection"`
+	Count      int    `json:"count"`
+}
+
+type ErrorResponse struct {
+	Error string `json:"error"`
+}
+
+// --- Request Models ---
+
+type RecordRequest struct {
+	Repo       string `json:"repo"`
+	Collection string `json:"collection"`
+	RKey       string `json:"rkey"`
+}
+
+// --- Methods ---
+
+// ListRecords retrieves a list of AT URIs.
+// Set reverse=true to get newest records first.
+func (c *Client) ListRecords(ctx context.Context, repo, collection, cursor string, reverse bool) (*ListRecordsResponse, error) {
+	params := url.Values{}
+	params.Set("repo", repo)
+	params.Set("collection", collection)
+
+	if cursor != "" {
+		params.Set("cursor", cursor)
+	}
+
+	if reverse {
+		params.Set("reverse", "true")
+	}
+
+	var resp ListRecordsResponse
+	if err := c.doRequest(ctx, http.MethodGet, MethodListRecords, params, nil, &resp); err != nil {
+		return nil, err
+	}
+
+	return &resp, nil
+}
+
+// CountRecords returns the total number of records indexed for a collection.
+func (c *Client) CountRecords(ctx context.Context, repo, collection string) (*CountRecordsResponse, error) {
+	params := url.Values{}
+	params.Set("repo", repo)
+	params.Set("collection", collection)
+
+	var resp CountRecordsResponse
+	if err := c.doRequest(ctx, http.MethodGet, MethodCountRecords, params, nil, &resp); err != nil {
+		return nil, err
+	}
+
+	return &resp, nil
+}
+
+// IndexRecord triggers a manual index of a specific record.
+// This endpoint is rate-limited on the server.
+func (c *Client) IndexRecord(ctx context.Context, repo, collection, rkey string) error {
+	reqBody := RecordRequest{
+		Repo:       repo,
+		Collection: collection,
+		RKey:       rkey,
+	}
+
+	// Server returns 200 OK on success, body is empty or status only.
+	return c.doRequest(ctx, http.MethodPost, MethodIndexRecord, nil, reqBody, nil)
+}
+
+// ValidateRecord checks if a specific record exists in the local DB.
+// Returns true if exists, false if 404, error otherwise.
+func (c *Client) ValidateRecord(ctx context.Context, repo, collection, rkey string) (bool, error) {
+	reqBody := RecordRequest{
+		Repo:       repo,
+		Collection: collection,
+		RKey:       rkey,
+	}
+
+	err := c.doRequest(ctx, http.MethodPost, MethodValidateRecord, nil, reqBody, nil)
+	if err != nil {
+		// Parse standard error to see if it was a 404
+		if clientErr, ok := err.(*ClientError); ok && clientErr.StatusCode == 404 {
+			return false, nil
+		}
+		return false, err
+	}
+
+	return true, nil
+}
+
+// --- Internal Helpers ---
+
+type ClientError struct {
+	StatusCode int
+	Message    string
+}
+
+func (e *ClientError) Error() string {
+	return fmt.Sprintf("api error (status %d): %s", e.StatusCode, e.Message)
+}
+
+func (c *Client) doRequest(ctx context.Context, method, xrpcMethod string, params url.Values, body interface{}, dest interface{}) error {
+	u, err := url.Parse(fmt.Sprintf("%s/xrpc/%s", c.Host, xrpcMethod))
+	if err != nil {
+		return fmt.Errorf("invalid url: %w", err)
+	}
+
+	if len(params) > 0 {
+		u.RawQuery = params.Encode()
+	}
+
+	var bodyReader io.Reader
+	if body != nil {
+		jsonBytes, err := json.Marshal(body)
+		if err != nil {
+			return fmt.Errorf("failed to marshal body: %w", err)
+		}
+		bodyReader = bytes.NewBuffer(jsonBytes)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, u.String(), bodyReader)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	if c.AuthToken != "" {
+		req.Header.Set("Authorization", "Bearer "+c.AuthToken)
+	}
+
+	resp, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Handle non-200 responses
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		var errResp ErrorResponse
+		// Try to decode server error message
+		if decodeErr := json.NewDecoder(resp.Body).Decode(&errResp); decodeErr == nil && errResp.Error != "" {
+			return &ClientError{StatusCode: resp.StatusCode, Message: errResp.Error}
+		}
+		// Fallback if JSON decode fails or empty
+		return &ClientError{StatusCode: resp.StatusCode, Message: resp.Status}
+	}
+
+	// Decode response if destination provided
+	if dest != nil {
+		if err := json.NewDecoder(resp.Body).Decode(dest); err != nil {
+			return fmt.Errorf("failed to decode response: %w", err)
+		}
+	}
+
+	return nil
+}

cmd/aturilist/main.go

···
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"log/slog"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/bluesky-social/indigo/api/agnostic"
+	"github.com/bluesky-social/indigo/atproto/syntax"
+	"github.com/bluesky-social/jetstream/pkg/client"
+	"github.com/bluesky-social/jetstream/pkg/client/schedulers/sequential"
+	"github.com/bluesky-social/jetstream/pkg/models"
+	"github.com/dgraph-io/badger/v4"
+	"github.com/gin-gonic/gin"
+
+	// Restored your specific imports
+	"tangled.org/whey.party/red-dwarf-server/auth"
+	"tangled.org/whey.party/red-dwarf-server/microcosm"
+	"tangled.org/whey.party/red-dwarf-server/microcosm/slingshot"
+)
+
+type Server struct {
+	db     *badger.DB
+	logger *slog.Logger
+
+	// Locks for specific operations if needed, though Badger is thread-safe
+	backfillTracker map[string]*sync.WaitGroup
+	backfillMutex   sync.Mutex
+}
+
+var (
+	JETSTREAM_URL     string
+	SPACEDUST_URL     string
+	SLINGSHOT_URL     string
+	CONSTELLATION_URL string
+)
+
+func initURLs(prod bool) {
+	if !prod {
+		JETSTREAM_URL = "wss://jetstream.whey.party/subscribe"
+		SPACEDUST_URL = "wss://spacedust.whey.party/subscribe"
+		SLINGSHOT_URL = "https://slingshot.whey.party"
+		CONSTELLATION_URL = "https://constellation.whey.party"
+	} else {
+		JETSTREAM_URL = "ws://localhost:6008/subscribe"
+		SPACEDUST_URL = "ws://localhost:9998/subscribe"
+		SLINGSHOT_URL = "http://localhost:7729"
+		CONSTELLATION_URL = "http://localhost:7728"
+	}
+}
+
+const (
+	BSKYIMAGECDN_URL = "https://cdn.bsky.app"
+	BSKYVIDEOCDN_URL = "https://video.bsky.app"
+	serviceWebDID    = "did:web:aturilist.reddwarf.app"
+	serviceWebHost   = "https://aturilist.reddwarf.app"
+)
+
+func main() {
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	log.Println("red-dwarf-server AtURI List Service started")
+
+	prod := flag.Bool("prod", false, "use production URLs instead of localhost")
+	dbPath := flag.String("db", "./badger_data", "path to badger db")
+	flag.Parse()
+
+	initURLs(*prod)
+
+	// 1. Initialize DB
+	db, err := badger.Open(badger.DefaultOptions(*dbPath))
+	if err != nil {
+		logger.Error("Failed to open BadgerDB", "error", err)
+		os.Exit(1)
+	}
+	defer db.Close()
+
+	srv := &Server{
+		db:     db,
+		logger: logger,
+	}
+
+	// 2. Initialize Auth
+	auther, err := auth.NewAuth(
+		100_000,
+		time.Hour*12,
+		5,
+		serviceWebDID, //+"#bsky_appview",
+	)
+	if err != nil {
+		log.Fatalf("Failed to create Auth: %v", err)
+	}
+
+	// 3. Initialize Clients
+	ctx := context.Background()
+	sl := slingshot.NewSlingshot(SLINGSHOT_URL)
+
+	// 4. Initialize Jetstream
+	config := client.DefaultClientConfig()
+	config.WebsocketURL = JETSTREAM_URL
+	config.Compress = true
+
+	handler := &JetstreamHandler{srv: srv}
+	scheduler := sequential.NewScheduler("my_app", logger, handler.HandleEvent)
+
+	c, err := client.NewClient(config, logger, scheduler)
+	if err != nil {
+		logger.Error("failed to create client", "error", err)
+		return
+	}
+
+	// Connect with cursor (5 minutes ago)
+	cursor := time.Now().Add(-5 * time.Minute).UnixMicro()
+
+	go func() {
+		logger.Info("Connecting to Jetstream...")
+		/*
+			If you resume a jetstream firehose from a cursor, everything works fine until you catch up to real time.
+			At that point, the connection drops. If you connect without a cursor (going straight to realtime), it keeps working.
+		*/
+		for {
+			if err := c.ConnectAndRead(ctx, &cursor); err != nil {
+				logger.Error("jetstream connection disconnected", "error", err)
+			}
+
+			select {
+			case <-ctx.Done():
+				return // Context cancelled, exit loop
+			default:
+				logger.Info("Reconnecting to Jetstream in 5 seconds...", "cursor", cursor)
+				time.Sleep(5 * time.Second)
+			}
+		}
+	}()
+
+	// 5. Initialize Router
+	router := gin.New()
+	router.Use(auther.AuthenticateGinRequestViaJWT)
+
+	router.GET("/xrpc/app.reddwarf.aturilist.listRecords", srv.handleListRecords)
+
+	router.GET("/xrpc/app.reddwarf.aturilist.countRecords", srv.handleCountRecords)
+
+	// heavily rate limited because can be used for spam.
+	router.POST("/xrpc/app.reddwarf.aturilist.indexRecord", func(c *gin.Context) {
+		srv.handleIndexRecord(c, sl)
+	})
+
+	router.POST("/xrpc/app.reddwarf.aturilist.validateRecord", srv.handleValidateRecord)
+
+	// router.GET("/xrpc/app.reddwarf.aturilist.requestBackfill", )
+
+	router.Run(":7155")
+}
+
+// --- Jetstream Handler ---
+
+type JetstreamHandler struct {
+	srv *Server
+}
+
+func (h *JetstreamHandler) HandleEvent(ctx context.Context, event *models.Event) error {
+	if event != nil {
+		if event.Commit != nil {
+			// Identify Delete operation
+			isDelete := event.Commit.Operation == models.CommitOperationDelete
+
+			// Process
+			h.srv.processRecord(event.Did, event.Commit.Collection, event.Commit.RKey, isDelete)
+
+		}
+	}
+	return nil
+}
+
+// --- DB Helpers ---
+
+func makeKey(repo, collection, rkey string) []byte {
+	return []byte(fmt.Sprintf("%s|%s|%s", repo, collection, rkey))
+}
+
+func parseKey(key []byte) (repo, collection, rkey string, err error) {
+	parts := strings.Split(string(key), "|")
+	if len(parts) != 3 {
+		return "", "", "", errors.New("invalid key format")
+	}
+	return parts[0], parts[1], parts[2], nil
+}
+
+// processRecord handles the DB write/delete.
+// isDelete=true removes the key. isDelete=false sets the key.
+func (s *Server) processRecord(repo, collection, rkey string, isDelete bool) {
+	key := makeKey(repo, collection, rkey)
+
+	err := s.db.Update(func(txn *badger.Txn) error {
+		if isDelete {
+			return txn.Delete(key)
+		}
+		// On create/update, store current timestamp.
+		// You can store more data (Cid, etc) here if needed later.
+		return txn.Set(key, []byte(time.Now().Format(time.RFC3339)))
+	})
+
+	if err != nil {
+		s.logger.Error("Failed to update DB", "repo", repo, "rkey", rkey, "err", err)
+	}
+}
+
+// --- HTTP Handlers ---
+
+func (s *Server) handleListRecords(c *gin.Context) {
+	repo := c.Query("repo")
+	collection := c.Query("collection")
+	cursor := c.Query("cursor")
+	reverse := c.Query("reverse") == "true" // 1. Check param
+	limit := 50
+
+	if repo == "" || collection == "" {
+		c.JSON(400, gin.H{"error": "repo and collection required"})
+		return
+	}
+
+	// Base prefix: "repo|collection|"
+	prefixStr := fmt.Sprintf("%s|%s|", repo, collection)
+	prefix := []byte(prefixStr)
+
+	var aturis []string
+	var lastRkey string
+
+	err := s.db.View(func(txn *badger.Txn) error {
+		// 2. Configure Iterator Options
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchValues = false
+		opts.Reverse = reverse // Set reverse mode
+
+		it := txn.NewIterator(opts)
+		defer it.Close()
+
+		// 3. Determine Start Key
+		var startKey []byte
+		if cursor != "" {
+			// If cursor exists, we seek to it regardless of direction
+			startKey = makeKey(repo, collection, cursor)
+		} else {
+			if reverse {
+				// REVERSE START: "repo|collection|" + 0xFF
+				// This seeks to the theoretical end of this prefix range
+				startKey = append([]byte(prefixStr), 0xFF)
+			} else {
+				// FORWARD START: "repo|collection|"
+				startKey = prefix
+			}
+		}
+
+		// 4. Seek and Iterate
+		it.Seek(startKey)
+
+		// Handle Cursor Pagination Skip
+		// If we provided a cursor, we likely landed exactly ON that cursor.
+		// We want the record *after* (or *before* in reverse) the cursor.
+		if cursor != "" && it.Valid() {
+			// Badger's Seek moves to key >= seek_key (even in reverse mode logic varies slightly,
+			// but practically we check if we landed on the exact cursor).
+			if string(it.Item().Key()) == string(startKey) {
+				it.Next() // Skip the cursor itself
+			}
+		}
+
+		// Iterate as long as the key still starts with our prefix
+		for ; it.ValidForPrefix(prefix); it.Next() {
+			if len(aturis) >= limit {
+				break
+			}
+			item := it.Item()
+			k := item.Key()
+			_, _, rkey, err := parseKey(k)
+			if err == nil {
+				aturis = append(aturis, fmt.Sprintf("at://%s/%s/%s", repo, collection, rkey))
+				lastRkey = rkey
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		c.JSON(500, gin.H{"error": err.Error()})
+		return
+	}
+
+	resp := gin.H{
+		"aturis": aturis,
+		"count":  len(aturis),
+	}
+
+	// Only return cursor if we hit the limit, allowing the client to request the next page
+	if lastRkey != "" && len(aturis) == limit {
+		resp["cursor"] = lastRkey
+	}
+
+	c.JSON(200, resp)
+}
+
+func (s *Server) handleCountRecords(c *gin.Context) {
+	repo := c.Query("repo")
+	collection := c.Query("collection")
+
+	if repo == "" || collection == "" {
+		c.JSON(400, gin.H{"error": "repo and collection required"})
+		return
+	}
+
+	prefix := []byte(fmt.Sprintf("%s|%s|", repo, collection))
+	count := 0
+
+	err := s.db.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchValues = false
+		it := txn.NewIterator(opts)
+		defer it.Close()
+
+		for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
+			count++
+		}
+		return nil
+	})
+
+	if err != nil {
+		c.JSON(500, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(200, gin.H{
+		"repo":       repo,
+		"collection": collection,
+		"count":      count,
+	})
+}
+
+// handleIndexRecord now takes the Slingshot client specifically
+func (s *Server) handleIndexRecord(c *gin.Context, sl *microcosm.MicrocosmClient) {
+	//authedUserDid := c.GetString("user_did")
+	// Support JSON body preferentially, fallback to Query/Form
+	var req struct {
+		Collection string `json:"collection"`
+		Repo       string `json:"repo"`
+		RKey       string `json:"rkey"`
+	}
+
+	if err := c.BindJSON(&req); err != nil {
+		req.Collection = c.PostForm("collection")
+		req.Repo = c.PostForm("repo")
+		req.RKey = c.PostForm("rkey")
+	}
+
+	if req.Collection == "" || req.Repo == "" || req.RKey == "" {
+		c.JSON(400, gin.H{"error": "invalid parameters"})
+		return
+	}
+
+	// Verify existence using Slingshot/Agnostic
+	recordResponse, err := agnostic.RepoGetRecord(c.Request.Context(), sl, "", req.Collection, req.Repo, req.RKey)
+	if err != nil {
+		// Does not exist remotely -> Delete locally
+		s.processRecord(req.Repo, req.Collection, req.RKey, true)
+
+		// You might want to return 200 even if deleted, to confirm "indexing done"
+		c.Status(200)
+		return
+	}
+
+	// Exists remotely -> Parse and Insert locally
+	uri := recordResponse.Uri
+	aturi, err := syntax.ParseATURI(uri)
+	if err != nil {
+		c.JSON(400, gin.H{"error": "failed to parse aturi from remote"})
+		return
+	}
+
+	s.processRecord(aturi.Authority().String(), string(aturi.Collection()), string(aturi.RecordKey()), false)
+	c.Status(200)
+}
+
+func (s *Server) handleValidateRecord(c *gin.Context) {
+	var req struct {
+		Collection string `json:"collection"`
+		Repo       string `json:"repo"`
+		RKey       string `json:"rkey"`
+	}
+	if err := c.BindJSON(&req); err != nil {
+		c.JSON(400, gin.H{"error": "invalid json"})
+		return
+	}
+
+	key := makeKey(req.Repo, req.Collection, req.RKey)
+	exists := false
+
+	err := s.db.View(func(txn *badger.Txn) error {
+		_, err := txn.Get(key)
+		if err == nil {
+			exists = true
+		} else if err == badger.ErrKeyNotFound {
+			exists = false
+			return nil
+		}
+		return err
+	})
+
+	if err != nil {
+		c.JSON(500, gin.H{"error": err.Error()})
+		return
+	}
+
+	if exists {
+		c.Status(200)
+	} else {
+		c.Status(404)
+	}
+}

go.mod

···
 go 1.25.4
 
 require (
-	github.com/bluesky-social/indigo v0.0.0-20251202051123-81f317e322bc
+	github.com/bluesky-social/indigo v0.0.0-20251206005924-d49b45419635
 	github.com/ericvolp12/jwt-go-secp256k1 v0.0.2
 	github.com/gin-contrib/cors v1.7.6
 	github.com/gin-gonic/gin v1.11.0
···
 
 require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
+	github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/flatbuffers v25.2.10+incompatible // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-block-format v0.2.0 // indirect
···
 	github.com/ipfs/go-merkledag v0.11.0 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/ipfs/go-verifcid v0.0.3 // indirect
-	github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 // indirect
+	github.com/ipld/go-car v0.6.2 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
···
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1
 	github.com/bytedance/sonic v1.14.0 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
+	github.com/dgraph-io/badger/v4 v4.8.0
 	github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.9 // indirect

go.sum

···
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bluesky-social/indigo v0.0.0-20251202051123-81f317e322bc h1:2t+uAvfzJiCsTMwn5fW85t/IGa0+2I7BXS2ORastK4o=
 github.com/bluesky-social/indigo v0.0.0-20251202051123-81f317e322bc/go.mod h1:Pm2I1+iDXn/hLbF7XCg/DsZi6uDCiOo7hZGWprSM7k0=
+github.com/bluesky-social/indigo v0.0.0-20251206005924-d49b45419635 h1:kNeRrgGJH2g5OvjLqtaQ744YXqduliZYpFkJ/ld47c0=
+github.com/bluesky-social/indigo v0.0.0-20251206005924-d49b45419635/go.mod h1:Pm2I1+iDXn/hLbF7XCg/DsZi6uDCiOo7hZGWprSM7k0=
+github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1 h1:ovcRKN1iXZnY5WApVg+0Hw2RkwMH0ziA7lSAA8vellU=
+github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1/go.mod h1:5PtGi4r/PjEVBBl+0xWuQn4mBEjr9h6xsfDBADS6cHs=
 github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ=
 github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA=
 github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
···
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
+github.com/dgraph-io/badger/v4 v4.8.0 h1:JYph1ChBijCw8SLeybvPINizbDKWZ5n/GYbz2yhN/bs=
+github.com/dgraph-io/badger/v4 v4.8.0/go.mod h1:U6on6e8k/RTbUWxqKR0MvugJuVmkxSNc79ap4917h4w=
+github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM=
+github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg=
 github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw=
 github.com/ericvolp12/jwt-go-secp256k1 v0.0.2 h1:puGwrNTY2vCt8eakkSEq2yeNxUD3zb2kPhv1OsF1hPs=
···
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
+github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
···
 github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw=
 github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 h1:oFo19cBmcP0Cmg3XXbrr0V/c+xU9U1huEZp8+OgBzdI=
 github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4/go.mod h1:6nkFF8OmR5wLKBzRKi7/YFJpyYR7+oEn1DX+mMWnlLA=
+github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc=
+github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
 github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=

readme.md

···
 ### `/cmd/backstream`
 experimental backfiller that kinda (but not really) conforms to the jetstream event shape. designed to be ingested by consumers expecting jetstream
 
+### `/cmd/aturilist`
+experimental listRecords replacement. is not backfilled. uses the official jetstream go client, which means it suffers from this [bug](https://github.com/bluesky-social/jetstream/pull/45)
+
 ## Packages
 
 ### `/auth`