add prom metrics

database.py

···
-from dataclasses import dataclass
-from datetime import datetime, timezone
 import logging
 import sys
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from time import time
 from typing import List, Optional
-import uuid
 
+from qdrant_client import QdrantClient
 from qdrant_client.grpc import OptimizersConfigDiff
 from qdrant_client.http.models import BinaryQuantizationConfig
-
-from config import CONFIG
-from qdrant_client import QdrantClient
 from qdrant_client.models import (
     BinaryQuantization,
     Distance,
···
     ScalarType,
     VectorParams,
 )
+
+from config import CONFIG
+from metrics import prom_metrics
 
 logger = logging.getLogger(__name__)
 
···
             logger.info("Collection created successfully")
 
     def upsert_profile(self, did: str, description: str, vector: List[float]):
+        status = "error"
+        start_time = time()
+
         try:
             payload = {
                 "did": did,
···
                 points=[point],
             )
 
+            status = "ok"
+
             return True
         except Exception as e:
             logger.error(f"Error upserting profile: {e}")
             return False
+        finally:
+            prom_metrics.upserts.labels(kind="profile", status=status).inc()
+            prom_metrics.upsert_duration.labels(kind="profile", status=status).observe(
+                time() - start_time
+            )
 
     def upsert_avatar(self, did: str, cid: str, vector: List[float]):
+        status = "error"
+        start_time = time()
+
         try:
             payload = {
                 "did": did,
···
                 collection_name=self.avatar_collection_name,
                 points=[point],
             )
+
+            status = "ok"
 
             return True
         except Exception as e:
             logger.error(f"Error upserting avatar: {e}")
             return False
+        finally:
+            prom_metrics.upserts.labels(kind="avatar", status=status).inc()
+            prom_metrics.upsert_duration.labels(kind="avatar", status=status).observe(
+                time() - start_time
+            )
 
     def upsert_post(self, did: str, uri: str, text: str, vector: List[float]):
+        status = "error"
+        start_time = time()
+
         word_ct = len(text.split())
 
         try:
···
                 points=[point],
             )
 
+            status = "ok"
+
             return True
         except Exception as e:
             logger.error(f"Error upserting post: {e}")
             return False
+        finally:
+            prom_metrics.upserts.labels(kind="avatar", status=status).inc()
+            prom_metrics.upsert_duration.labels(kind="avatar", status=status).observe(
+                time() - start_time
+            )
 
     def search_similar(
         self,

embedder.py

···
 import logging
 from typing import List
+from time import time
 
 from sentence_transformers import SentenceTransformer
 import torch
 
 from config import CONFIG
+from metrics import prom_metrics
 
 
 logger = logging.getLogger(__name__)
···
         if not text or not text.strip():
             return [0.0] * CONFIG.embedding_size
 
-        vector = self.model.encode(text, convert_to_numpy=True)
-        return vector.tolist()
+        status = "error"
+        start_time = time()
+        try:
+            vector = self.model.encode(text, convert_to_numpy=True)
+            status = "ok"
+            return vector.tolist()
+        except Exception as e:
+            logger.error(f"Error getting embedding: {e}")
+            raise e
+        finally:
+            prom_metrics.embedding_performed.labels(status=status).inc()
+            prom_metrics.embedding_duration.labels(status=status).observe(
+                time() - start_time
+            )
 
 
 EMBEDDING_SERVICE = EmbeddingService()

indexer.py

···
 from embedder import EMBEDDING_SERVICE
 from retina import RETINA_CLIENT, binary_to_float_vector, hex_to_binary
 from models import AtKafkaEvent
+from metrics import prom_metrics
 
 
 logger = logging.getLogger(__name__)
···
         record = event.operation.record
 
         description = record.get("description")
-        if not isinstance(description, str):
-            return
+        if isinstance(description, str) and description and description.strip():
+            status = "error"
 
-        if not description or not description.strip():
-            return
+            try:
+                vector = EMBEDDING_SERVICE.encode(description)
 
-        vector = EMBEDDING_SERVICE.encode(description)
+                QDRANT_SERVICE.upsert_profile(
+                    did=event.did, description=description, vector=vector
+                )
 
-        QDRANT_SERVICE.upsert_profile(
-            did=event.did, description=description, vector=vector
-        )
+                status = "ok"
+            except Exception as e:
+                logger.error(f"Error handling profile: {e}")
+            finally:
+                prom_metrics.events_handled.labels(kind="profile", status=status).inc()
 
-        try:
-            avatar = record.get("avatar")
+        avatar = record.get("avatar")
+        if isinstance(avatar, dict):
+            status = "error"
+            try:
+                ref = avatar.get("ref")
 
-            if not isinstance(avatar, dict):
-                return
+                if not isinstance(ref, dict):
+                    return
 
-            ref = avatar.get("ref")
+                link = ref.get("$link")
+                if not isinstance(link, str):
+                    return
 
-            if not isinstance(ref, dict):
-                return
+                resp = RETINA_CLIENT.get_image_hash(event.did, link)
 
-            link = ref.get("$link")
-            if not isinstance(link, str):
-                return
+                if resp.quality_too_low:
+                    logger.info("avatar quality was too low")
+                    return
 
-            resp = RETINA_CLIENT.get_image_hash(event.did, link)
-        except Exception as e:
-            logger.error(f"Failed to get avatar hash: {e}")
-            return
+                if not resp.hash:
+                    logger.error("no hash in response")
+                    return
 
-        if resp.quality_too_low:
-            logger.info("avatar quality was too low")
-            return
+                bin = hex_to_binary(resp.hash)
+                vector = binary_to_float_vector(bin)
 
-        if not resp.hash:
-            logger.error("no hash in response")
-            return
+                QDRANT_SERVICE.upsert_avatar(
+                    did=event.did,
+                    cid=link,
+                    vector=vector,
+                )
 
-        bin = hex_to_binary(resp.hash)
-        vector = binary_to_float_vector(bin)
-
-        QDRANT_SERVICE.upsert_avatar(
-            did=event.did,
-            cid=link,
-            vector=vector,
-        )
+                status = "ok"
+            except Exception as e:
+                logger.error(f"Failed to get avatar hash: {e}")
+            finally:
+                prom_metrics.events_handled.labels(kind="avatar", status=status).inc()
 
     def _handle_post(self, event: AtKafkaEvent):
         text = event.operation.record.get("text")
···
         if not text or not text.strip():
             return
 
-        vector = EMBEDDING_SERVICE.encode(text)
+        status = "error"
+
+        try:
+            vector = EMBEDDING_SERVICE.encode(text)
+
+            QDRANT_SERVICE.upsert_post(
+                did=event.did,
+                uri=f"at://{event.did}/app.bsky.feed.post/{event.operation.rkey}",
+                text=text,
+                vector=vector,
+            )
 
-        QDRANT_SERVICE.upsert_post(
-            did=event.did,
-            uri=f"at://{event.did}/app.bsky.feed.post/{event.operation.rkey}",
-            text=text,
-            vector=vector,
-        )
+            status = "ok"
+        except Exception as e:
+            logger.error(f"Error handling post: {e}")
+        finally:
+            prom_metrics.events_handled.labels(kind="post", status=status).inc()

main.py

···
-import logging
 import asyncio
-import sys
-import signal
 import json
+import logging
+import signal
+import sys
 from typing import Optional
 
 import click
···
 from database import QDRANT_SERVICE
 from embedder import EMBEDDING_SERVICE
 from indexer import Indexer
+from metrics import prom_metrics
 from models import AtKafkaEvent
 
 shutdown_requested = False
···
 ):
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
+
+    prom_metrics.start_http(8500, "0.0.0.0")
 
     EMBEDDING_SERVICE.initialize()
 

metrics.py

···
+import logging
+from os import name
+
+from prometheus_client import Counter, Histogram, start_http_server
+
+NAMESPACE = "skyembed"
+
+logger = logging.getLogger(__name__)
+
+
+class PromMetrics:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+
+        self.embedding_performed = Counter(
+            name="requests",
+            namespace=NAMESPACE,
+            documentation="Number of embeddings performed",
+            labelnames=["status"],
+        )
+
+        self.embedding_duration = Histogram(
+            name="embedding_duration_seconds",
+            namespace=NAMESPACE,
+            buckets=(
+                0.001,
+                0.005,
+                0.01,
+                0.025,
+                0.05,
+                0.1,
+                0.25,
+                0.5,
+                1.0,
+                2.5,
+                5.0,
+                10.0,
+            ),
+            labelnames=["status"],
+            documentation="Time taken to create an embedding",
+        )
+
+        self.events_handled = Counter(
+            name="events_handled",
+            namespace=NAMESPACE,
+            documentation="Number of events handled",
+            labelnames=["kind", "status"],
+        )
+
+        self.upserts = Counter(
+            name="upserts",
+            namespace=NAMESPACE,
+            documentation="Number of database upserts",
+            labelnames=["kind", "status"],
+        )
+
+        self.upsert_duration = Histogram(
+            name="upsert_duration_seconds",
+            namespace=NAMESPACE,
+            buckets=(
+                0.001,
+                0.005,
+                0.01,
+                0.025,
+                0.05,
+                0.1,
+                0.25,
+                0.5,
+                1.0,
+                2.5,
+                5.0,
+                10.0,
+            ),
+            labelnames=["kind", "status"],
+            documentation="Time taken to perform an upsert",
+        )
+
+        self._initialized = True
+
+    def start_http(self, port: int, addr: str = "0.0.0.0"):
+        logger.info(f"Starting Prometheus client on {addr}:{port}")
+        start_http_server(port=port, addr=addr)
+        logger.info(f"Prometheus client running on {addr}:{port}")
+
+
+prom_metrics = PromMetrics()

pyproject.toml

···
     "aiokafka>=0.12.0",
     "click>=8.3.1",
     "confluent-kafka>=2.12.2",
+    "prometheus-client>=0.23.1",
     "pydantic>=2.12.5",
     "pydantic-settings>=2.12.0",
     "python-snappy>=0.7.3",

uv.lock

···
 ]
 
 [[package]]
+name = "prometheus-client"
+version = "0.23.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" },
+]
+
+[[package]]
 name = "protobuf"
 version = "6.33.2"
 source = { registry = "https://pypi.org/simple" }
···
     { name = "aiokafka" },
     { name = "click" },
     { name = "confluent-kafka" },
+    { name = "prometheus-client" },
     { name = "pydantic" },
     { name = "pydantic-settings" },
     { name = "python-snappy" },
···
     { name = "aiokafka", specifier = ">=0.12.0" },
     { name = "click", specifier = ">=8.3.1" },
     { name = "confluent-kafka", specifier = ">=2.12.2" },
+    { name = "prometheus-client", specifier = ">=0.23.1" },
     { name = "pydantic", specifier = ">=2.12.5" },
     { name = "pydantic-settings", specifier = ">=2.12.0" },
     { name = "python-snappy", specifier = ">=0.7.3" },