observability stuff

Cargo.lock

···
  "tiny-skia",
  "tokio",
  "tower",
+ "tower-http",
  "tracing",
  "tracing-subscriber",
  "tracing-wasm",
···
 name = "weaver-common"
 version = "0.1.0"
 dependencies = [
+ "axum",
  "blake3",
  "chrono",
  "futures-util",

crates/weaver-app/Cargo.toml

···
 web = ["dioxus/web", "dioxus-primitives/web"]
 desktop = ["dioxus/desktop"]
 mobile = ["dioxus/mobile"]
-server = [ "dioxus/server", "dep:jacquard-axum", "dep:axum", "dep:axum-extra", "dep:tower", "dep:resvg", "dep:usvg", "dep:tiny-skia", "dep:textwrap", "dep:askama", "dep:fontdb", "dep:lightningcss"]
+server = [ "dioxus/server", "dep:jacquard-axum", "dep:axum", "dep:axum-extra", "dep:tower", "dep:tower-http", "dep:resvg", "dep:usvg", "dep:tiny-skia", "dep:textwrap", "dep:askama", "dep:fontdb", "dep:lightningcss", "weaver-common/telemetry"]
 collab-worker = ["weaver-common/iroh"]
 
 
···
 axum = { version = "0.8.6", optional = true }
 axum-extra = { version = "0.10", optional = true, features = ["typed-header"] }
 tower = { version = "0.5", optional = true }
+tower-http = { version = "0.6", optional = true, features = ["trace"] }
 mime-sniffer = {version = "^0.1"}
 chrono = { version = "0.4" }
 serde = { version = "1.0" }

crates/weaver-app/src/components/app_link.rs

···
 #[component]
 pub fn AppLink(props: AppLinkProps) -> Element {
     let link_mode = use_context::<LinkMode>();
-    tracing::info!(?link_mode, "AppLink: reading LinkMode context");
     let class = props.class.clone().unwrap_or_default();
 
     match link_mode {

crates/weaver-app/src/main.rs

···
         // Filter out noisy crates
         // Use weaver_app=trace for detailed editor debugging
         let filter = EnvFilter::new(
-            "debug,weaver_app=trace,loro_internal=warn,jacquard_identity=info,jacquard_common=info,iroh=info",
+            "debug,weaver_app=trace,loro_internal=warn,jacquard_identity=info,jacquard_common=info,iroh=info,reqwest=warn",
         );
 
         let reg = Registry::default()
···
         let _ = set_global_default(reg);
     }
 
+    // Initialize telemetry (metrics + tracing) before server starts.
+    // Loki task is spawned inside dioxus::serve where tokio runtime exists.
+    // Wrapped in Arc<Mutex> so the FnMut closure can clone and take() on first call.
+    #[cfg(feature = "server")]
+    let loki_task = {
+        use weaver_common::telemetry::{self, TelemetryConfig};
+        let config = TelemetryConfig::from_env("weaver-app");
+        std::sync::Arc::new(std::sync::Mutex::new(telemetry::init_sync(config)))
+    };
+
     #[cfg(feature = "server")]
     std::panic::set_hook(Box::new(|panic_info| {
         tracing::error!("PANIC: {:?}", panic_info);
···
 
     // Run `serve()` on the server only
     #[cfg(feature = "server")]
-    dioxus::serve(|| async move {
-        #[cfg(feature = "fullstack-server")]
-        use axum::middleware;
-        use axum::middleware::Next;
-        use axum::{Router, body::Body, extract::Request, response::Response, routing::get};
-        use axum_extra::extract::Host;
-        use jacquard::oauth::{client::OAuthClient, session::ClientData};
-        use std::convert::Infallible;
-        use weaver_app::auth::AuthStore;
-        use weaver_app::blobcache::BlobCache;
+    dioxus::serve({
+        let loki_task = loki_task.clone();
+        move || {
+            let loki_task = loki_task.clone();
+            async move {
+                #[cfg(feature = "fullstack-server")]
+                use axum::middleware;
+                use axum::middleware::Next;
+                use axum::{
+                    Router, body::Body, extract::Request, response::Response, routing::get,
+                };
+                use axum_extra::extract::Host;
+                use jacquard::oauth::{client::OAuthClient, session::ClientData};
+                use std::convert::Infallible;
+                use weaver_app::auth::AuthStore;
+                use weaver_app::blobcache::BlobCache;
+                use weaver_common::telemetry;
 
-        #[cfg(not(feature = "fullstack-server"))]
-        let router = { Router::new().merge(dioxus::server::router(App)) };
+                // Spawn the Loki background task now that we're in tokio runtime
+                if let Some(task) = loki_task.lock().unwrap().take() {
+                    telemetry::spawn_loki_task(task);
+                }
 
-        #[cfg(feature = "fullstack-server")]
-        let router = {
-            let fetcher = Arc::new(fetch::Fetcher::new(OAuthClient::new(
-                AuthStore::new(),
-                ClientData::new_public(CONFIG.oauth.clone()),
-            )));
+                #[cfg(not(feature = "fullstack-server"))]
+                let router = {
+                    Router::new()
+                        .merge(dioxus::server::router(App))
+                        .layer(middleware::from_fn(telemetry::http_metrics))
+                        .layer(tower_http::trace::TraceLayer::new_for_http())
+                };
 
-            let blob_cache = Arc::new(BlobCache::new(fetcher.clone()));
-            axum::Router::new()
-                .route("/favicon.ico", get(weaver_app::favicon))
-                .serve_dioxus_application(ServeConfig::builder(), App)
-                // Host context resolution.
-                .layer(middleware::from_fn({
-                    let fetcher = fetcher.clone();
-                    move |req: Request, next: Next| {
-                        let fetcher = fetcher.clone();
-                        async move {
-                            weaver_app::middleware::host_context_middleware(req, next, fetcher)
-                                .await
-                        }
-                    }
-                }))
-                // Insert fetcher and blob cache into extensions.
-                .layer(middleware::from_fn({
-                    let blob_cache = blob_cache.clone();
-                    let fetcher = fetcher.clone();
-                    move |mut req: Request, next: Next| {
-                        let blob_cache = blob_cache.clone();
-                        let fetcher = fetcher.clone();
-                        async move {
-                            req.extensions_mut().insert(blob_cache);
-                            req.extensions_mut().insert(fetcher);
-                            Ok::<_, Infallible>(next.run(req).await)
-                        }
-                    }
-                }))
-        };
-        Ok(router)
+                #[cfg(feature = "fullstack-server")]
+                let router = {
+                    let fetcher = Arc::new(fetch::Fetcher::new(OAuthClient::new(
+                        AuthStore::new(),
+                        ClientData::new_public(CONFIG.oauth.clone()),
+                    )));
+
+                    let blob_cache = Arc::new(BlobCache::new(fetcher.clone()));
+                    axum::Router::new()
+                        .route("/favicon.ico", get(weaver_app::favicon))
+                        .route("/metrics", get(|| async { telemetry::render() }))
+                        .serve_dioxus_application(ServeConfig::builder(), App)
+                        // Host context resolution.
+                        .layer(middleware::from_fn({
+                            let fetcher = fetcher.clone();
+                            move |req: Request, next: Next| {
+                                let fetcher = fetcher.clone();
+                                async move {
+                                    weaver_app::middleware::host_context_middleware(
+                                        req, next, fetcher,
+                                    )
+                                    .await
+                                }
+                            }
+                        }))
+                        // Insert fetcher and blob cache into extensions.
+                        .layer(middleware::from_fn({
+                            let blob_cache = blob_cache.clone();
+                            let fetcher = fetcher.clone();
+                            move |mut req: Request, next: Next| {
+                                let blob_cache = blob_cache.clone();
+                                let fetcher = fetcher.clone();
+                                async move {
+                                    req.extensions_mut().insert(blob_cache);
+                                    req.extensions_mut().insert(fetcher);
+                                    Ok::<_, Infallible>(next.run(req).await)
+                                }
+                            }
+                        }))
+                        // HTTP metrics (request count, duration)
+                        .layer(middleware::from_fn(telemetry::http_metrics))
+                        .layer(tower_http::trace::TraceLayer::new_for_http())
+                };
+                Ok(router)
+            }
+        }
     });
 
     #[cfg(not(feature = "server"))]

crates/weaver-app/src/views/subdomain_navbar.rs

···
                         }
                     }
                 }
-                // Author profile link
+                // Author profile link - temporarily disabled to debug SSR hang
                 nav { class: "nav-tools",
-                    AuthorProfileLink { ident: ctx.owner.clone() }
+                    // AuthorProfileLink { ident: ctx.owner.clone() }
+                    "DEBUG: navbar without author link"
                 }
 
                 // Auth button

crates/weaver-common/Cargo.toml

···
 native = ["jacquard/dns"]
 use-index = []
 iroh = ["dep:iroh", "dep:iroh-gossip", "dep:iroh-tickets"]
-telemetry = ["dep:metrics", "dep:metrics-exporter-prometheus", "dep:tracing-subscriber", "dep:tracing-loki"]
+telemetry = ["dep:metrics", "dep:metrics-exporter-prometheus", "dep:tracing-subscriber", "dep:tracing-loki", "dep:axum"]
 cache = ["dep:mini-moka-wasm"]
 perf = []
 
···
 metrics-exporter-prometheus = { version = "0.17.2", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"], optional = true }
 tracing-loki = { version = "0.2", optional = true }
+axum = { version = "0.8", optional = true }
 
 getrandom = { version = "0.3", features = [] }
 ring = { version = "0.17", default-features = false }

crates/weaver-common/src/telemetry.rs

···
     }
 }
 
+/// Opaque handle for the Loki background task.
+pub struct LokiTask(tracing_loki::BackgroundTask);
+
 /// Initialize telemetry (metrics + tracing).
 ///
 /// Call once at application startup. If `LOKI_URL` is set, spawns a background
···
     // Initialize prometheus metrics
     init_metrics();
 
-    // Initialize tracing
-    init_tracing(config).await;
+    // Initialize tracing subscriber
+    if let Some(task) = init_tracing(config) {
+        // Spawn the loki background task
+        tokio::spawn(task.0);
+    }
+}
+
+/// Initialize telemetry without spawning the Loki task.
+///
+/// Use this when you need to set up tracing before a tokio runtime is available.
+/// Returns the Loki task if configured - caller must spawn it later with `spawn_loki_task`.
+pub fn init_sync(config: TelemetryConfig) -> Option<LokiTask> {
+    init_metrics();
+    init_tracing(config)
+}
+
+/// Spawn the Loki background task.
+///
+/// Call this inside a tokio runtime after `init_sync`.
+pub fn spawn_loki_task(task: LokiTask) {
+    tokio::spawn(task.0);
 }
 
 /// Initialize just the prometheus metrics recorder.
 pub fn init_metrics() -> &'static PrometheusHandle {
     PROMETHEUS_HANDLE.get_or_init(|| {
+        // HTTP request duration buckets (in seconds)
+        let http_buckets = vec![
+            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
+        ];
+
         PrometheusBuilder::new()
+            .set_buckets_for_metric(
+                metrics_exporter_prometheus::Matcher::Prefix("http_request_duration".to_string()),
+                &http_buckets,
+            )
+            .expect("failed to set histogram buckets")
             .install_recorder()
             .expect("failed to install prometheus recorder")
     })
 }
 
 /// Initialize tracing with console + optional Loki layers.
-async fn init_tracing(config: TelemetryConfig) {
+///
+/// Returns the Loki background task if Loki is configured.
+fn init_tracing(config: TelemetryConfig) -> Option<LokiTask> {
     let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| {
-        EnvFilter::new(format!(
-            "{}",
-            config.console_level.as_str().to_lowercase()
-        ))
+        EnvFilter::new(format!("{}", config.console_level.as_str().to_lowercase()))
     });
 
     // Pretty console layer for human-readable stdout
···
                     .with(loki_layer)
                     .init();
 
-                // Spawn the background task that pushes to Loki
-                tokio::spawn(loki_task);
-
                 tracing::info!(
                     service = %config.service_name,
                     loki_url = %loki_url,
                     "telemetry initialized with loki"
                 );
+
+                Some(LokiTask(loki_task))
             }
             Err(e) => {
                 // Invalid URL - fall back to console only
···
                     loki_url = %loki_url,
                     "invalid LOKI_URL, falling back to console only"
                 );
+                None
             }
         }
     } else {
···
             service = %config.service_name,
             "telemetry initialized (console only, set LOKI_URL to enable loki)"
         );
+        None
     }
 }
 
···
 
 // Re-export the metrics crate for convenience
 pub use metrics::{counter, gauge, histogram};
+
+/// HTTP metrics middleware for axum.
+///
+/// Records `http_requests_total` counter and `http_request_duration_seconds` histogram.
+/// Use with `axum::middleware::from_fn`.
+///
+/// # Example
+/// ```ignore
+/// use axum::middleware;
+/// use weaver_common::telemetry::http_metrics;
+///
+/// let app = Router::new()
+///     .route("/", get(handler))
+///     .layer(middleware::from_fn(http_metrics));
+/// ```
+#[cfg(feature = "telemetry")]
+pub async fn http_metrics(
+    req: axum::extract::Request,
+    next: axum::middleware::Next,
+) -> axum::response::Response {
+    let start = std::time::Instant::now();
+    let method = req.method().to_string();
+    let path = req.uri().path().to_string();
+
+    let response = next.run(req).await;
+
+    let duration = start.elapsed().as_secs_f64();
+    let status = response.status().as_u16().to_string();
+
+    metrics::counter!(
+        "http_requests_total",
+        "method" => method.clone(),
+        "path" => normalize_path(&path),
+        "status" => status
+    )
+    .increment(1);
+
+    metrics::histogram!(
+        "http_request_duration_seconds",
+        "method" => method,
+        "path" => normalize_path(&path)
+    )
+    .record(duration);
+
+    response
+}
+
+/// Normalize path for metrics labels.
+/// Keeps first 3 segments, collapses rest to reduce cardinality.
+#[cfg(feature = "telemetry")]
+fn normalize_path(path: &str) -> String {
+    let parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+    match parts.len() {
+        0 => "/".to_string(),
+        1 => format!("/{}", parts[0]),
+        2 => format!("/{}/{}", parts[0], parts[1]),
+        3 => format!("/{}/{}/{}", parts[0], parts[1], parts[2]),
+        _ => format!("/{}/{}/{}/*", parts[0], parts[1], parts[2]),
+    }
+}

crates/weaver-index/src/server.rs

···
 use std::sync::Arc;
 
 use axum::{
-    Json, Router,
+    Json, Router, middleware,
     extract::State,
     http::{StatusCode, header},
     response::{Html, IntoResponse},
···
         .merge(GetEditHistoryRequest::into_router(edit::get_edit_history))
         .merge(GetContributorsRequest::into_router(edit::get_contributors))
         .merge(ListDraftsRequest::into_router(edit::list_drafts))
+        .layer(middleware::from_fn(telemetry::http_metrics))
         .layer(TraceLayer::new_for_http())
         .layer(CorsLayer::permissive().max_age(std::time::Duration::from_secs(86400)))
         .with_state(state)

docker-compose.yml

···
     ports:
       - "80:80"
       - "443:443"
+      - "2019:2019" # Admin API with metrics
     volumes:
       - ./infra/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
       - caddy_data:/data
       - caddy_config:/config
     environment:
       CLOUDFLARE_API_TOKEN: ${CLOUDFLARE_API_TOKEN}
-      ACME_EMAIL: ${ACME_EMAIL:[e-mail]}
+      ACME_EMAIL: ${ACME_EMAIL:[e-mail]}
     depends_on:
       - weaver-app
       - index
     restart: unless-stopped
 
-  # ClickHouse - analytics database (internal only, no host ports exposed)
+  # ClickHouse - analytics database
   clickhouse:
     image: clickhouse/clickhouse-server:25.11
     container_name: weaver-clickhouse
-    # No ports exposed to host - only accessible via docker network
     ports:
       - "8123:8123"
       - "9000:9000"
+      - "9363:9363" # Prometheus metrics
     volumes:
       - ~/data/clickhouse:/var/lib/clickhouse
       - ~/data/clickhouse-logs:/var/log/clickhouse-server
       - ~/data/clickhouse-config:/etc/clickhouse-server/config.d
+      - ./infra/clickhouse/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
     environment:
       CLICKHOUSE_DB: ${CLICKHOUSE_DATABASE:-weaver}
       CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
···
     image: ghcr.io/bluesky-social/indigo/tap:latest
     ports:
       - "2480:2480"
+      - "2481:2481"
     volumes:
       - tap_data:/data/tap
     environment:
       TAP_DATABASE_URL: sqlite:///data/tap/tap.db
       TAP_BIND: ":2480"
+      TAP_METRICS_LISTEN: ":2481"
       TAP_DISABLE_ACKS: "false"
-      TAP_LOG_LEVEL: info
+      TAP_LOG_LEVEL: debug
       TAP_OUTBOX_PARALLELISM: 5
       #TAP_FULL_NETWORK: true
       #TAP_SIGNAL_COLLECTION: place.stream.chat.profile
···
       - index_data:/app/data
     environment:
       RUST_LOG: info,weaver_index=debug,hyper_util::client::legacy::pool=info
+      LOKI_URL: ${LOKI_URL:-}
       CLICKHOUSE_URL: http://clickhouse:8123
       CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-weaver}
       CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
···
     environment:
       PORT: 8080
       IP: 0.0.0.0
-      RUST_LOG: info
+      RUST_LOG: info,weaver-app=debug,weaver-common=debug,hyper=warn,hyper_util=warn,tower=warn,h2=warn,rustls=warn,reqwest=info,dioxus_core=warn,dioxus_signals=warn
+      LOKI_URL: ${LOKI_URL:-}
     healthcheck:
       test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/"]
       interval: 20s
       timeout: 5s
       retries: 3
+    restart: unless-stopped
+
+  # ============ OBSERVABILITY STACK ============
+
+  # Node exporter - host metrics (CPU, memory, disk, network)
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: weaver-node-exporter
+    ports:
+      - "9100:9100"
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    restart: unless-stopped
+
+  # cAdvisor - container metrics (per-container CPU, memory, network)
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    container_name: weaver-cadvisor
+    ports:
+      - "9080:8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    privileged: true
+    restart: unless-stopped
+
+  # Promtail - ship container logs to Loki
+  promtail:
+    image: grafana/promtail:latest
+    container_name: weaver-promtail
+    volumes:
+      - ./infra/promtail/config.yml:/etc/promtail/config.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    command: -config.file=/etc/promtail/config.yml -config.expand-env=true
+    environment:
+      LOKI_URL: ${LOKI_URL:-http://localhost:3100}
     restart: unless-stopped
 
 volumes:

infra/caddy/Caddyfile

···
     on_demand_tls {
         ask http://index:3000/internal/verify-domain
     }
+
+    admin 0.0.0.0:2019
+
+    servers {
+        metrics
+    }
 }
 
 # Index service

infra/clickhouse/prometheus.xml

···
+<clickhouse>
+    <prometheus>
+        <endpoint>/metrics</endpoint>
+        <port>9363</port>
+        <metrics>true</metrics>
+        <events>true</events>
+        <asynchronous_metrics>true</asynchronous_metrics>
+    </prometheus>
+</clickhouse>

infra/grafana/dashboards/weaver-clickhouse.json

···
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 100,
+      "title": "ClickHouse Overview",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "green", "value": 1 }
+            ]
+          }
+        }
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "up{service=\"weaver-clickhouse\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "Status",
+      "type": "stat"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 50 },
+              { "color": "red", "value": 100 }
+            ]
+          }
+        }
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "ClickHouseMetrics_Query{service=\"weaver-clickhouse\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "Active Queries",
+      "type": "stat"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "unit": "bytes",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 4294967296 },
+              { "color": "red", "value": 8589934592 }
+            ]
+          }
+        }
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "ClickHouseMetrics_MemoryTracking{service=\"weaver-clickhouse\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "Memory Used",
+      "type": "stat"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 50 },
+              { "color": "red", "value": 100 }
+            ]
+          }
+        }
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "ClickHouseMetrics_TCPConnection{service=\"weaver-clickhouse\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "TCP Connections",
+      "type": "stat"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 50 },
+              { "color": "red", "value": 100 }
+            ]
+          }
+        }
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "ClickHouseMetrics_HTTPConnection{service=\"weaver-clickhouse\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "HTTP Connections",
+      "type": "stat"
+    },
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+      "id": 101,
+      "title": "Query Performance",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "id": 6,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(ClickHouseProfileEvents_Query{service=\"weaver-clickhouse\"}[5m])",
+          "legendFormat": "Queries/s",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(ClickHouseProfileEvents_SelectQuery{service=\"weaver-clickhouse\"}[5m])",
+          "legendFormat": "Selects/s",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(ClickHouseProfileEvents_InsertQuery{service=\"weaver-clickhouse\"}[5m])",
+          "legendFormat": "Inserts/s",
+          "refId": "C"
+        }
+      ],
+      "title": "Query Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "id": 7,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(ClickHouseProfileEvents_FailedQuery{service=\"weaver-clickhouse\"}[5m])",
+          "legendFormat": "Failed",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(ClickHouseProfileEvents_FailedSelectQuery{service=\"weaver-clickhouse\"}[5m])",
+          "legendFormat": "Failed Selects",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(ClickHouseProfileEvents_FailedInsertQuery{service=\"weaver-clickhouse\"}[5m])",
+          "legendFormat": "Failed Inserts",
+          "refId": "C"
+        }
+      ],
+      "title": "Failed Queries",
+      "type": "timeseries"
+    },
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+      "id": 102,
+      "title": "Resources",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
+      "id": 8,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "ClickHouseMetrics_MemoryTracking{service=\"weaver-clickhouse\"}",
+          "legendFormat": "Memory Tracking",
+          "refId": "A"
+        },
+        {
+          "expr": "ClickHouseAsyncMetrics_MemoryResident{service=\"weaver-clickhouse\"}",
+          "legendFormat": "Resident",
+          "refId": "B"
+        }
+      ],
+      "title": "Memory Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "Bps"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
+      "id": 9,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{service=\"weaver-clickhouse\"}[5m])",
+          "legendFormat": "Read",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{service=\"weaver-clickhouse\"}[5m])",
+          "legendFormat": "Write",
+          "refId": "B"
+        }
+      ],
+      "title": "Disk I/O",
+      "type": "timeseries"
+    },
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
+      "id": 103,
+      "title": "Merges & Parts",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+      "id": 10,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "ClickHouseMetrics_Merge{service=\"weaver-clickhouse\"}",
+          "legendFormat": "Active Merges",
+          "refId": "A"
+        },
+        {
+          "expr": "ClickHouseMetrics_BackgroundMergesAndMutationsPoolTask{service=\"weaver-clickhouse\"}",
+          "legendFormat": "Pool Tasks",
+          "refId": "B"
+        }
+      ],
+      "title": "Merge Activity",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+      "id": 11,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "ClickHouseMetrics_PartsActive{service=\"weaver-clickhouse\"}",
+          "legendFormat": "Active Parts",
+          "refId": "A"
+        },
+        {
+          "expr": "ClickHouseMetrics_PartsOutdated{service=\"weaver-clickhouse\"}",
+          "legendFormat": "Outdated Parts",
+          "refId": "B"
+        }
+      ],
+      "title": "Parts",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["weaver", "clickhouse"],
+  "templating": {
+    "list": [
+      {
+        "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": { "from": "now-1h", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Weaver ClickHouse",
+  "uid": "weaver-clickhouse",
+  "version": 1
+}

infra/grafana/dashboards/weaver-infra.json

···
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 100,
+      "title": "Host Metrics (booskie-box)",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        }
+      },
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 1 },
+      "id": 1,
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "1 - avg(rate(node_cpu_seconds_total{service=\"weaver-node\", mode=\"idle\"}[5m]))",
+          "legendFormat": "CPU Usage",
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": { "h": 6, "w": 8, "x": 8, "y": 1 },
+      "id": 2,
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "node_memory_MemTotal_bytes{service=\"weaver-node\"} - node_memory_MemAvailable_bytes{service=\"weaver-node\"}",
+          "legendFormat": "Used",
+          "refId": "A"
+        },
+        {
+          "expr": "node_memory_MemAvailable_bytes{service=\"weaver-node\"}",
+          "legendFormat": "Available",
+          "refId": "B"
+        }
+      ],
+      "title": "Memory",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        }
+      },
+      "gridPos": { "h": 6, "w": 8, "x": 16, "y": 1 },
+      "id": 3,
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "1 - (node_filesystem_avail_bytes{service=\"weaver-node\", mountpoint=\"/\"} / node_filesystem_size_bytes{service=\"weaver-node\", mountpoint=\"/\"})",
+          "legendFormat": "/ usage",
+          "refId": "A"
+        }
+      ],
+      "title": "Disk Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "Bps"
+        }
+      },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 7 },
+      "id": 4,
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(node_network_receive_bytes_total{service=\"weaver-node\", device!~\"lo|veth.*|docker.*|br-.*\"}[5m])",
+          "legendFormat": "{{device}} rx",
+          "refId": "A"
+        },
+        {
+          "expr": "-rate(node_network_transmit_bytes_total{service=\"weaver-node\", device!~\"lo|veth.*|docker.*|br-.*\"}[5m])",
+          "legendFormat": "{{device}} tx",
+          "refId": "B"
+        }
+      ],
+      "title": "Network I/O",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "Bps"
+        }
+      },
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 7 },
+      "id": 5,
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(node_disk_read_bytes_total{service=\"weaver-node\"}[5m])",
+          "legendFormat": "{{device}} read",
+          "refId": "A"
+        },
+        {
+          "expr": "-rate(node_disk_written_bytes_total{service=\"weaver-node\"}[5m])",
+          "legendFormat": "{{device}} write",
+          "refId": "B"
+        }
+      ],
+      "title": "Disk I/O",
+      "type": "timeseries"
+    },
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 },
+      "id": 101,
+      "title": "Container Metrics",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "percentunit"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
+      "id": 6,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(container_cpu_usage_seconds_total{service=\"weaver-cadvisor\", name=~\"weaver-.*\"}[5m])",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Container CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
+      "id": 7,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "container_memory_usage_bytes{service=\"weaver-cadvisor\", name=~\"weaver-.*\"}",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Container Memory Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "Bps"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
+      "id": 8,
+      "options": {
+        "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(container_network_receive_bytes_total{service=\"weaver-cadvisor\", name=~\"weaver-.*\"}[5m])",
+          "legendFormat": "{{name}} rx",
+          "refId": "A"
+        },
+        {
+          "expr": "-rate(container_network_transmit_bytes_total{service=\"weaver-cadvisor\", name=~\"weaver-.*\"}[5m])",
+          "legendFormat": "{{name}} tx",
+          "refId": "B"
+        }
+      ],
+      "title": "Container Network I/O",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
+      "id": 9,
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "container_last_seen{service=\"weaver-cadvisor\", name=~\"weaver-.*\"} - time()",
+          "legendFormat": "{{name}}",
+          "refId": "A",
+          "hide": true
+        },
+        {
+          "expr": "count(container_last_seen{service=\"weaver-cadvisor\", name=~\"weaver-.*\"} > (time() - 60))",
+          "legendFormat": "Running containers",
+          "refId": "B"
+        }
+      ],
+      "title": "Running Containers",
+      "type": "stat"
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["weaver", "infrastructure"],
+  "templating": {
+    "list": [
+      {
+        "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": { "from": "now-1h", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Weaver Infrastructure",
+  "uid": "weaver-infra",
+  "version": 1
+}

infra/grafana/dashboards/weaver-overview.json

···
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": 0,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 100,
+      "panels": [],
+      "title": "Weaver Services",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "rate(http_requests_total{service=~\"weaver-.*\"}[5m])",
+          "legendFormat": "{{service}} {{method}} {{status}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Request Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "yellow",
+                "value": 0.5
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max", "p99"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{service=~\"weaver-.*\"}[5m]))",
+          "legendFormat": "{{service}} p99",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.50, rate(http_request_duration_seconds_bucket{service=~\"weaver-.*\"}[5m]))",
+          "legendFormat": "{{service}} p50",
+          "refId": "B"
+        }
+      ],
+      "title": "Request Latency",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": ["sum"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "rate(http_requests_total{service=~\"weaver-.*\", status=~\"5..\"}[5m])",
+          "legendFormat": "{{service}} {{status}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Error Rate (5xx)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "text": "DOWN"
+                },
+                "1": {
+                  "color": "green",
+                  "text": "UP"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 9
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "up{service=\"weaver-index\"}",
+          "legendFormat": "weaver-index",
+          "refId": "A"
+        }
+      ],
+      "title": "Index Status",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "text": "DOWN"
+                },
+                "1": {
+                  "color": "green",
+                  "text": "UP"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 9
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "up{service=\"weaver-app\"}",
+          "legendFormat": "weaver-app",
+          "refId": "A"
+        }
+      ],
+      "title": "App Status",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 17
+      },
+      "id": 101,
+      "panels": [],
+      "title": "Caddy Proxy",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "rate(caddy_admin_http_requests_total{service=\"weaver-caddy\"}[5m])",
+          "legendFormat": "{{handler}} {{code}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Caddy Request Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.99, rate(caddy_http_request_duration_seconds_bucket{service=\"weaver-caddy\"}[5m]))",
+          "legendFormat": "p99",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.50, rate(caddy_http_request_duration_seconds_bucket{service=\"weaver-caddy\"}[5m]))",
+          "legendFormat": "p50",
+          "refId": "B"
+        }
+      ],
+      "title": "Caddy Latency",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 26
+      },
+      "id": 102,
+      "panels": [],
+      "title": "Logs",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki}"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 27
+      },
+      "id": 8,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showControls": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "{service_name=\"weaver-index\"} |= ``",
+          "refId": "A"
+        }
+      ],
+      "title": "Index Logs",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki}"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 27
+      },
+      "id": 9,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showControls": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "{service=\"weaver-app\"} != `dioxus_core` or `hyper_util` or `dioxus_signals` or `reqwest` or `axum`",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "App Logs",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 37
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "expr": "sum by (service_name) (count_over_time({service_name=~\"weaver-.*\"} | level=~\"error|ERROR|err\" [1m]))",
+          "legendFormat": "{{service_name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Error Log Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki}"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 43
+      },
+      "id": 11,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showControls": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "pluginVersion": "12.3.1",
+      "targets": [
+        {
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "{container_name=~\"weaver-clickhouse|weaver-caddy|weaver-tap\"} |= ``",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Infrastructure Logs",
+      "type": "logs"
+    }
+  ],
+  "preload": false,
+  "schemaVersion": 42,
+  "tags": ["weaver"],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": "Prometheus",
+          "value": "PBFA97CFB590B2093"
+        },
+        "includeAll": false,
+        "label": "Metrics",
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      },
+      {
+        "current": {
+          "text": "Loki",
+          "value": "P8E80F9AEF21F6940"
+        },
+        "includeAll": false,
+        "label": "Logs",
+        "name": "loki",
+        "options": [],
+        "query": "loki",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Weaver Overview",
+  "uid": "weaver-overview",
+  "version": 9
+}

infra/grafana/dashboards/weaver-tap.json

···
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "id": 100,
+      "title": "Firehose",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 1 },
+      "id": 1,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(tap_firehose_events_received_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Received",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(tap_firehose_events_processed_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Processed",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(tap_firehose_events_skipped_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Skipped",
+          "refId": "C"
+        }
+      ],
+      "title": "Firehose Event Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "none"
+        }
+      },
+      "gridPos": { "h": 6, "w": 4, "x": 8, "y": 1 },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "tap_firehose_last_seq{service=\"weaver-tap\"}",
+          "legendFormat": "Last Seq",
+          "refId": "A"
+        }
+      ],
+      "title": "Last Sequence",
+      "type": "stat"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 10000 },
+              { "color": "red", "value": 50000 }
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 6, "w": 4, "x": 12, "y": 1 },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "tap_event_cache_size{service=\"weaver-tap\"}",
+          "legendFormat": "Cache Size",
+          "refId": "A"
+        }
+      ],
+      "title": "Event Cache Size",
+      "type": "stat"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 6, "w": 8, "x": 16, "y": 1 },
+      "id": 4,
+      "options": {
+        "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "increase(tap_firehose_events_received_total{service=\"weaver-tap\"}[1h])",
+          "legendFormat": "Received",
+          "refId": "A"
+        },
+        {
+          "expr": "increase(tap_firehose_events_processed_total{service=\"weaver-tap\"}[1h])",
+          "legendFormat": "Processed",
+          "refId": "B"
+        }
+      ],
+      "title": "Events (1h)",
+      "type": "timeseries"
+    },
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
+      "id": 101,
+      "title": "Resyncs",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "id": 5,
+      "options": {
+        "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(tap_resyncs_started_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Started",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(tap_resyncs_completed_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Completed",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(tap_resyncs_failed_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Failed",
+          "refId": "C"
+        }
+      ],
+      "title": "Resync Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "s"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "id": 6,
+      "options": {
+        "legend": { "calcs": ["mean", "max", "p99"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.99, rate(tap_resync_duration_seconds_bucket{service=\"weaver-tap\"}[5m]))",
+          "legendFormat": "p99",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.50, rate(tap_resync_duration_seconds_bucket{service=\"weaver-tap\"}[5m]))",
+          "legendFormat": "p50",
+          "refId": "B"
+        }
+      ],
+      "title": "Resync Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null }
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 16 },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "tap_resyncs_completed_total{service=\"weaver-tap\"}",
+          "legendFormat": "Completed",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Completed",
+      "type": "stat"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 1 }
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 16 },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
+      },
+      "targets": [
+        {
+          "expr": "tap_resyncs_failed_total{service=\"weaver-tap\"}",
+          "legendFormat": "Failed",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Failed",
+      "type": "stat"
+    },
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 },
+      "id": 102,
+      "title": "Event Delivery",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 },
+      "id": 9,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(tap_events_delivered_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Delivered",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(tap_events_acked_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Acked",
+          "refId": "B"
+        }
+      ],
+      "title": "Event Delivery Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 },
+      "id": 10,
+      "options": {
+        "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi" }
+      },
+      "targets": [
+        {
+          "expr": "rate(tap_crawler_repos_discovered_total{service=\"weaver-tap\"}[5m])",
+          "legendFormat": "Repos Discovered",
+          "refId": "A"
+        }
+      ],
+      "title": "Crawler Discovery Rate",
+      "type": "timeseries"
+    },
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 },
+      "id": 103,
+      "title": "Logs",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "loki", "uid": "${loki}" },
+      "gridPos": { "h": 10, "w": 24, "x": 0, "y": 30 },
+      "id": 11,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "expr": "{container_name=\"weaver-tap\"} |= ``",
+          "refId": "A"
+        }
+      ],
+      "title": "Tap Logs",
+      "type": "logs"
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["weaver", "tap", "atproto"],
+  "templating": {
+    "list": [
+      {
+        "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Metrics",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": { "selected": false, "text": "Loki", "value": "Loki" },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Logs",
+        "multi": false,
+        "name": "loki",
+        "options": [],
+        "query": "loki",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": { "from": "now-1h", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Weaver Tap",
+  "uid": "weaver-tap",
+  "version": 1
+}

infra/promtail/config.yml

···
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: ${LOKI_URL}/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    relabel_configs:
+      # Only scrape weaver containers
+      - source_labels: [__meta_docker_container_name]
+        regex: "/(weaver-.+)"
+        action: keep
+      # Set container_name label (matches dashboard queries)
+      - source_labels: [__meta_docker_container_name]
+        regex: "/(.+)"
+        target_label: container_name
+      # Add instance label
+      - target_label: instance
+        replacement: "booskie-box"
+    pipeline_stages:
+      # Parse JSON logs if present
+      - json:
+          expressions:
+            level: level
+            msg: msg
+            timestamp: timestamp
+      # Use extracted level if available
+      - labels:
+          level:
+      # Timestamp from log if available
+      - timestamp:
+          source: timestamp
+          format: RFC3339Nano
+          fallback_formats:
+            - RFC3339
+            - UnixMs