embedding_firehose_client / batch.go

package main

import "fmt"

// Batch is the columnar wire format for the embedding firehose.
// Each field is a parallel array — index i across all fields describes one text.
//
// Embeddings are EmbeddingGemma-300m (Matryoshka) float32 vectors. Two per text:
//   - C (cluster): for clustering/similarity analysis
//   - R (retrieval): for semantic search
//
// Open clients receive 128d (L2-normalized Matryoshka truncation).
// Token-authenticated clients receive full 768d.
type Batch struct {
	DID  []string    `json:"did"`  // AT Protocol DID (e.g. "did:plc:abc123")
	Col  []string    `json:"col"`  // Collection NSID (e.g. "app.bsky.feed.post")
	Rkey []string    `json:"rkey"` // Record key
	Lang []string    `json:"lang"` // Detected language ("en", "de")
	C    [][]float32 `json:"c"`    // Cluster embeddings (128d open, 768d token)
	R    [][]float32 `json:"r"`    // Retrieval embeddings (128d open, 768d token)
}

// Len returns the number of items in the batch.
func (b *Batch) Len() int { return len(b.DID) }

// ATURI returns the AT URI for item i.
func (b *Batch) ATURI(i int) string {
	return fmt.Sprintf("at://%s/%s/%s", b.DID[i], b.Col[i], b.Rkey[i])
}