A lil service that creates embeddings of posts, profiles, and avatars to store them in Qdrant
at main 1.3 kB view raw
1from pydantic_settings import BaseSettings, SettingsConfigDict 2 3 4class Config(BaseSettings): 5 qdrant_url: str = "http://localhost:6333" 6 qdrant_profile_collection_name: str = "bluesky_profile" 7 qdrant_avatar_collection_name: str = "bluesky_avatar" 8 qdrant_post_collection_name: str = "bluesky_post" 9 10 embedding_model: str = "all-MiniLM-L6-v2" 11 embedding_device: str = "cuda" 12 batch_size: int = 256 13 14 kafka_bootstrap_servers: str = "localhost:9092" 15 kafka_topic: str = "firehose_events" 16 kafka_group_id: str = "skyembed" 17 kafka_auto_offset_reset: str = "latest" 18 19 retina_host: str = "http://localhost:7075" 20 21 metrics_port: int = 8500 22 metrics_host: str = "0.0.0.0" 23 24 model_config = SettingsConfigDict( 25 env_file=".env", 26 env_file_encoding="utf-8", 27 case_sensitive=False, 28 ) 29 30 @property 31 def embedding_size(self) -> int: 32 if "mpnet" in self.embedding_model.lower(): 33 return 768 34 elif "minilm" in self.embedding_model.lower(): 35 return 384 36 elif "gte-large" in self.embedding_model.lower(): 37 return 1024 38 else: 39 # this might not always be correct depending on model used 40 return 384 41 42 43CONFIG = Config()