From 4705bfcc43e5b3530f7283962325ed5b4b643693 Mon Sep 17 00:00:00 2001 From: Orual Date: Sun, 11 Jan 2026 14:32:24 -0500 Subject: [PATCH] observability stuff --- Cargo.lock | 2 + crates/weaver-app/Cargo.toml | 3 +- crates/weaver-app/src/components/app_link.rs | 1 - crates/weaver-app/src/main.rs | 141 ++- .../weaver-app/src/views/subdomain_navbar.rs | 5 +- crates/weaver-common/Cargo.toml | 3 +- crates/weaver-common/src/telemetry.rs | 112 ++- crates/weaver-index/src/server.rs | 3 +- docker-compose.yml | 63 +- infra/caddy/Caddyfile | 6 + infra/clickhouse/prometheus.xml | 9 + .../grafana/dashboards/weaver-clickhouse.json | 414 ++++++++ infra/grafana/dashboards/weaver-infra.json | 294 ++++++ infra/grafana/dashboards/weaver-overview.json | 943 ++++++++++++++++++ infra/grafana/dashboards/weaver-tap.json | 412 ++++++++ infra/promtail/config.yml | 44 + 16 files changed, 2381 insertions(+), 74 deletions(-) create mode 100644 infra/clickhouse/prometheus.xml create mode 100644 infra/grafana/dashboards/weaver-clickhouse.json create mode 100644 infra/grafana/dashboards/weaver-infra.json create mode 100644 infra/grafana/dashboards/weaver-overview.json create mode 100644 infra/grafana/dashboards/weaver-tap.json create mode 100644 infra/promtail/config.yml diff --git a/Cargo.lock b/Cargo.lock index 0b5e0a6..4297903 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12049,6 +12049,7 @@ dependencies = [ "tiny-skia", "tokio", "tower", + "tower-http", "tracing", "tracing-subscriber", "tracing-wasm", @@ -12092,6 +12093,7 @@ dependencies = [ name = "weaver-common" version = "0.1.0" dependencies = [ + "axum", "blake3", "chrono", "futures-util", diff --git a/crates/weaver-app/Cargo.toml b/crates/weaver-app/Cargo.toml index c119446..a97694a 100644 --- a/crates/weaver-app/Cargo.toml +++ b/crates/weaver-app/Cargo.toml @@ -39,7 +39,7 @@ use-index = ["weaver-common/use-index"] web = ["dioxus/web", "dioxus-primitives/web"] desktop = ["dioxus/desktop"] mobile = ["dioxus/mobile"] -server = [ "dioxus/server", "dep:jacquard-axum", "dep:axum", "dep:axum-extra", "dep:tower", "dep:resvg", "dep:usvg", "dep:tiny-skia", "dep:textwrap", "dep:askama", "dep:fontdb", "dep:lightningcss"] +server = [ "dioxus/server", "dep:jacquard-axum", "dep:axum", "dep:axum-extra", "dep:tower", "dep:tower-http", "dep:resvg", "dep:usvg", "dep:tiny-skia", "dep:textwrap", "dep:askama", "dep:fontdb", "dep:lightningcss", "weaver-common/telemetry"] collab-worker = ["weaver-common/iroh"] @@ -63,6 +63,7 @@ dioxus-primitives = { git = "https://github.com/DioxusLabs/components", version axum = { version = "0.8.6", optional = true } axum-extra = { version = "0.10", optional = true, features = ["typed-header"] } tower = { version = "0.5", optional = true } +tower-http = { version = "0.6", optional = true, features = ["trace"] } mime-sniffer = {version = "^0.1"} chrono = { version = "0.4" } serde = { version = "1.0" } diff --git a/crates/weaver-app/src/components/app_link.rs b/crates/weaver-app/src/components/app_link.rs index c6228d0..cd6bdc4 100644 --- a/crates/weaver-app/src/components/app_link.rs +++ b/crates/weaver-app/src/components/app_link.rs @@ -75,7 +75,6 @@ pub struct AppLinkProps { #[component] pub fn AppLink(props: AppLinkProps) -> Element { let link_mode = use_context::(); - tracing::info!(?link_mode, "AppLink: reading LinkMode context"); let class = props.class.clone().unwrap_or_default(); match link_mode { diff --git a/crates/weaver-app/src/main.rs b/crates/weaver-app/src/main.rs index fb9e1e9..f34da47 100644 --- a/crates/weaver-app/src/main.rs +++ b/crates/weaver-app/src/main.rs @@ -45,7 +45,7 @@ fn main() { // Filter out noisy crates // Use weaver_app=trace for detailed editor debugging let filter = EnvFilter::new( - "debug,weaver_app=trace,loro_internal=warn,jacquard_identity=info,jacquard_common=info,iroh=info", + "debug,weaver_app=trace,loro_internal=warn,jacquard_identity=info,jacquard_common=info,iroh=info,reqwest=warn", ); let reg = Registry::default() @@ -56,6 +56,16 @@ fn main() { let _ = set_global_default(reg); } + // Initialize telemetry (metrics + tracing) before server starts. + // Loki task is spawned inside dioxus::serve where tokio runtime exists. + // Wrapped in Arc so the FnMut closure can clone and take() on first call. + #[cfg(feature = "server")] + let loki_task = { + use weaver_common::telemetry::{self, TelemetryConfig}; + let config = TelemetryConfig::from_env("weaver-app"); + std::sync::Arc::new(std::sync::Mutex::new(telemetry::init_sync(config))) + }; + #[cfg(feature = "server")] std::panic::set_hook(Box::new(|panic_info| { tracing::error!("PANIC: {:?}", panic_info); @@ -63,58 +73,83 @@ fn main() { // Run `serve()` on the server only #[cfg(feature = "server")] - dioxus::serve(|| async move { - #[cfg(feature = "fullstack-server")] - use axum::middleware; - use axum::middleware::Next; - use axum::{Router, body::Body, extract::Request, response::Response, routing::get}; - use axum_extra::extract::Host; - use jacquard::oauth::{client::OAuthClient, session::ClientData}; - use std::convert::Infallible; - use weaver_app::auth::AuthStore; - use weaver_app::blobcache::BlobCache; - - #[cfg(not(feature = "fullstack-server"))] - let router = { Router::new().merge(dioxus::server::router(App)) }; - - #[cfg(feature = "fullstack-server")] - let router = { - let fetcher = Arc::new(fetch::Fetcher::new(OAuthClient::new( - AuthStore::new(), - ClientData::new_public(CONFIG.oauth.clone()), - ))); - - let blob_cache = Arc::new(BlobCache::new(fetcher.clone())); - axum::Router::new() - .route("/favicon.ico", get(weaver_app::favicon)) - .serve_dioxus_application(ServeConfig::builder(), App) - // Host context resolution. - .layer(middleware::from_fn({ - let fetcher = fetcher.clone(); - move |req: Request, next: Next| { - let fetcher = fetcher.clone(); - async move { - weaver_app::middleware::host_context_middleware(req, next, fetcher) - .await - } - } - })) - // Insert fetcher and blob cache into extensions. - .layer(middleware::from_fn({ - let blob_cache = blob_cache.clone(); - let fetcher = fetcher.clone(); - move |mut req: Request, next: Next| { - let blob_cache = blob_cache.clone(); - let fetcher = fetcher.clone(); - async move { - req.extensions_mut().insert(blob_cache); - req.extensions_mut().insert(fetcher); - Ok::<_, Infallible>(next.run(req).await) - } - } - })) - }; - Ok(router) + dioxus::serve({ + let loki_task = loki_task.clone(); + move || { + let loki_task = loki_task.clone(); + async move { + #[cfg(feature = "fullstack-server")] + use axum::middleware; + use axum::middleware::Next; + use axum::{ + Router, body::Body, extract::Request, response::Response, routing::get, + }; + use axum_extra::extract::Host; + use jacquard::oauth::{client::OAuthClient, session::ClientData}; + use std::convert::Infallible; + use weaver_app::auth::AuthStore; + use weaver_app::blobcache::BlobCache; + use weaver_common::telemetry; + + // Spawn the Loki background task now that we're in tokio runtime + if let Some(task) = loki_task.lock().unwrap().take() { + telemetry::spawn_loki_task(task); + } + + #[cfg(not(feature = "fullstack-server"))] + let router = { + Router::new() + .merge(dioxus::server::router(App)) + .layer(middleware::from_fn(telemetry::http_metrics)) + .layer(tower_http::trace::TraceLayer::new_for_http()) + }; + + #[cfg(feature = "fullstack-server")] + let router = { + let fetcher = Arc::new(fetch::Fetcher::new(OAuthClient::new( + AuthStore::new(), + ClientData::new_public(CONFIG.oauth.clone()), + ))); + + let blob_cache = Arc::new(BlobCache::new(fetcher.clone())); + axum::Router::new() + .route("/favicon.ico", get(weaver_app::favicon)) + .route("/metrics", get(|| async { telemetry::render() })) + .serve_dioxus_application(ServeConfig::builder(), App) + // Host context resolution. + .layer(middleware::from_fn({ + let fetcher = fetcher.clone(); + move |req: Request, next: Next| { + let fetcher = fetcher.clone(); + async move { + weaver_app::middleware::host_context_middleware( + req, next, fetcher, + ) + .await + } + } + })) + // Insert fetcher and blob cache into extensions. + .layer(middleware::from_fn({ + let blob_cache = blob_cache.clone(); + let fetcher = fetcher.clone(); + move |mut req: Request, next: Next| { + let blob_cache = blob_cache.clone(); + let fetcher = fetcher.clone(); + async move { + req.extensions_mut().insert(blob_cache); + req.extensions_mut().insert(fetcher); + Ok::<_, Infallible>(next.run(req).await) + } + } + })) + // HTTP metrics (request count, duration) + .layer(middleware::from_fn(telemetry::http_metrics)) + .layer(tower_http::trace::TraceLayer::new_for_http()) + }; + Ok(router) + } + } }); #[cfg(not(feature = "server"))] diff --git a/crates/weaver-app/src/views/subdomain_navbar.rs b/crates/weaver-app/src/views/subdomain_navbar.rs index c1453f4..8c7aae0 100644 --- a/crates/weaver-app/src/views/subdomain_navbar.rs +++ b/crates/weaver-app/src/views/subdomain_navbar.rs @@ -98,9 +98,10 @@ pub fn SubdomainNavbar() -> Element { } } } - // Author profile link + // Author profile link - temporarily disabled to debug SSR hang nav { class: "nav-tools", - AuthorProfileLink { ident: ctx.owner.clone() } + // AuthorProfileLink { ident: ctx.owner.clone() } + "DEBUG: navbar without author link" } // Auth button diff --git a/crates/weaver-common/Cargo.toml b/crates/weaver-common/Cargo.toml index ea38bed..46cfc17 100644 --- a/crates/weaver-common/Cargo.toml +++ b/crates/weaver-common/Cargo.toml @@ -10,7 +10,7 @@ default = [ "use-index"] native = ["jacquard/dns"] use-index = [] iroh = ["dep:iroh", "dep:iroh-gossip", "dep:iroh-tickets"] -telemetry = ["dep:metrics", "dep:metrics-exporter-prometheus", "dep:tracing-subscriber", "dep:tracing-loki"] +telemetry = ["dep:metrics", "dep:metrics-exporter-prometheus", "dep:tracing-subscriber", "dep:tracing-loki", "dep:axum"] cache = ["dep:mini-moka-wasm"] perf = [] @@ -51,6 +51,7 @@ metrics = { version = "0.24.2", optional = true } metrics-exporter-prometheus = { version = "0.17.2", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"], optional = true } tracing-loki = { version = "0.2", optional = true } +axum = { version = "0.8", optional = true } getrandom = { version = "0.3", features = [] } ring = { version = "0.17", default-features = false } diff --git a/crates/weaver-common/src/telemetry.rs b/crates/weaver-common/src/telemetry.rs index f3ab4ae..67e6e55 100644 --- a/crates/weaver-common/src/telemetry.rs +++ b/crates/weaver-common/src/telemetry.rs @@ -67,6 +67,9 @@ impl TelemetryConfig { } } +/// Opaque handle for the Loki background task. +pub struct LokiTask(tracing_loki::BackgroundTask); + /// Initialize telemetry (metrics + tracing). /// /// Call once at application startup. If `LOKI_URL` is set, spawns a background @@ -75,26 +78,54 @@ pub async fn init(config: TelemetryConfig) { // Initialize prometheus metrics init_metrics(); - // Initialize tracing - init_tracing(config).await; + // Initialize tracing subscriber + if let Some(task) = init_tracing(config) { + // Spawn the loki background task + tokio::spawn(task.0); + } +} + +/// Initialize telemetry without spawning the Loki task. +/// +/// Use this when you need to set up tracing before a tokio runtime is available. +/// Returns the Loki task if configured - caller must spawn it later with `spawn_loki_task`. +pub fn init_sync(config: TelemetryConfig) -> Option { + init_metrics(); + init_tracing(config) +} + +/// Spawn the Loki background task. +/// +/// Call this inside a tokio runtime after `init_sync`. +pub fn spawn_loki_task(task: LokiTask) { + tokio::spawn(task.0); } /// Initialize just the prometheus metrics recorder. pub fn init_metrics() -> &'static PrometheusHandle { PROMETHEUS_HANDLE.get_or_init(|| { + // HTTP request duration buckets (in seconds) + let http_buckets = vec![ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + PrometheusBuilder::new() + .set_buckets_for_metric( + metrics_exporter_prometheus::Matcher::Prefix("http_request_duration".to_string()), + &http_buckets, + ) + .expect("failed to set histogram buckets") .install_recorder() .expect("failed to install prometheus recorder") }) } /// Initialize tracing with console + optional Loki layers. -async fn init_tracing(config: TelemetryConfig) { +/// +/// Returns the Loki background task if Loki is configured. +fn init_tracing(config: TelemetryConfig) -> Option { let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { - EnvFilter::new(format!( - "{}", - config.console_level.as_str().to_lowercase() - )) + EnvFilter::new(format!("{}", config.console_level.as_str().to_lowercase())) }); // Pretty console layer for human-readable stdout @@ -121,14 +152,13 @@ async fn init_tracing(config: TelemetryConfig) { .with(loki_layer) .init(); - // Spawn the background task that pushes to Loki - tokio::spawn(loki_task); - tracing::info!( service = %config.service_name, loki_url = %loki_url, "telemetry initialized with loki" ); + + Some(LokiTask(loki_task)) } Err(e) => { // Invalid URL - fall back to console only @@ -139,6 +169,7 @@ async fn init_tracing(config: TelemetryConfig) { loki_url = %loki_url, "invalid LOKI_URL, falling back to console only" ); + None } } } else { @@ -149,6 +180,7 @@ async fn init_tracing(config: TelemetryConfig) { service = %config.service_name, "telemetry initialized (console only, set LOKI_URL to enable loki)" ); + None } } @@ -168,3 +200,63 @@ pub fn render() -> String { // Re-export the metrics crate for convenience pub use metrics::{counter, gauge, histogram}; + +/// HTTP metrics middleware for axum. +/// +/// Records `http_requests_total` counter and `http_request_duration_seconds` histogram. +/// Use with `axum::middleware::from_fn`. +/// +/// # Example +/// ```ignore +/// use axum::middleware; +/// use weaver_common::telemetry::http_metrics; +/// +/// let app = Router::new() +/// .route("/", get(handler)) +/// .layer(middleware::from_fn(http_metrics)); +/// ``` +#[cfg(feature = "telemetry")] +pub async fn http_metrics( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + let start = std::time::Instant::now(); + let method = req.method().to_string(); + let path = req.uri().path().to_string(); + + let response = next.run(req).await; + + let duration = start.elapsed().as_secs_f64(); + let status = response.status().as_u16().to_string(); + + metrics::counter!( + "http_requests_total", + "method" => method.clone(), + "path" => normalize_path(&path), + "status" => status + ) + .increment(1); + + metrics::histogram!( + "http_request_duration_seconds", + "method" => method, + "path" => normalize_path(&path) + ) + .record(duration); + + response +} + +/// Normalize path for metrics labels. +/// Keeps first 3 segments, collapses rest to reduce cardinality. +#[cfg(feature = "telemetry")] +fn normalize_path(path: &str) -> String { + let parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + match parts.len() { + 0 => "/".to_string(), + 1 => format!("/{}", parts[0]), + 2 => format!("/{}/{}", parts[0], parts[1]), + 3 => format!("/{}/{}/{}", parts[0], parts[1], parts[2]), + _ => format!("/{}/{}/{}/*", parts[0], parts[1], parts[2]), + } +} diff --git a/crates/weaver-index/src/server.rs b/crates/weaver-index/src/server.rs index 00dc9ff..de3970c 100644 --- a/crates/weaver-index/src/server.rs +++ b/crates/weaver-index/src/server.rs @@ -2,7 +2,7 @@ use std::net::SocketAddr; use std::sync::Arc; use axum::{ - Json, Router, + Json, Router, middleware, extract::State, http::{StatusCode, header}, response::{Html, IntoResponse}, @@ -166,6 +166,7 @@ pub fn router(state: AppState, did_doc: DidDocument<'static>) -> Router { .merge(GetEditHistoryRequest::into_router(edit::get_edit_history)) .merge(GetContributorsRequest::into_router(edit::get_contributors)) .merge(ListDraftsRequest::into_router(edit::list_drafts)) + .layer(middleware::from_fn(telemetry::http_metrics)) .layer(TraceLayer::new_for_http()) .layer(CorsLayer::permissive().max_age(std::time::Duration::from_secs(86400))) .with_state(state) diff --git a/docker-compose.yml b/docker-compose.yml index 510700b..61f7797 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,30 +6,32 @@ services: ports: - "80:80" - "443:443" + - "2019:2019" # Admin API with metrics volumes: - ./infra/caddy/Caddyfile:/etc/caddy/Caddyfile:ro - caddy_data:/data - caddy_config:/config environment: CLOUDFLARE_API_TOKEN: ${CLOUDFLARE_API_TOKEN} - ACME_EMAIL: ${ACME_EMAIL:-admin@weaver.sh} + ACME_EMAIL: ${ACME_EMAIL:-contact@weaver.sh} depends_on: - weaver-app - index restart: unless-stopped - # ClickHouse - analytics database (internal only, no host ports exposed) + # ClickHouse - analytics database clickhouse: image: clickhouse/clickhouse-server:25.11 container_name: weaver-clickhouse - # No ports exposed to host - only accessible via docker network ports: - "8123:8123" - "9000:9000" + - "9363:9363" # Prometheus metrics volumes: - ~/data/clickhouse:/var/lib/clickhouse - ~/data/clickhouse-logs:/var/log/clickhouse-server - ~/data/clickhouse-config:/etc/clickhouse-server/config.d + - ./infra/clickhouse/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro environment: CLICKHOUSE_DB: ${CLICKHOUSE_DATABASE:-weaver} CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} @@ -62,13 +64,15 @@ services: image: ghcr.io/bluesky-social/indigo/tap:latest ports: - "2480:2480" + - "2481:2481" volumes: - tap_data:/data/tap environment: TAP_DATABASE_URL: sqlite:///data/tap/tap.db TAP_BIND: ":2480" + TAP_METRICS_LISTEN: ":2481" TAP_DISABLE_ACKS: "false" - TAP_LOG_LEVEL: info + TAP_LOG_LEVEL: debug TAP_OUTBOX_PARALLELISM: 5 #TAP_FULL_NETWORK: true #TAP_SIGNAL_COLLECTION: place.stream.chat.profile @@ -92,6 +96,7 @@ services: - index_data:/app/data environment: RUST_LOG: info,weaver_index=debug,hyper_util::client::legacy::pool=info + LOKI_URL: ${LOKI_URL:-} CLICKHOUSE_URL: http://clickhouse:8123 CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-weaver} CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} @@ -122,7 +127,8 @@ services: environment: PORT: 8080 IP: 0.0.0.0 - RUST_LOG: info + RUST_LOG: info,weaver-app=debug,weaver-common=debug,hyper=warn,hyper_util=warn,tower=warn,h2=warn,rustls=warn,reqwest=info,dioxus_core=warn,dioxus_signals=warn + LOKI_URL: ${LOKI_URL:-} healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/"] interval: 20s @@ -130,6 +136,53 @@ services: retries: 3 restart: unless-stopped + # ============ OBSERVABILITY STACK ============ + + # Node exporter - host metrics (CPU, memory, disk, network) + node-exporter: + image: prom/node-exporter:latest + container_name: weaver-node-exporter + ports: + - "9100:9100" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + restart: unless-stopped + + # cAdvisor - container metrics (per-container CPU, memory, network) + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: weaver-cadvisor + ports: + - "9080:8080" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + privileged: true + restart: unless-stopped + + # Promtail - ship container logs to Loki + promtail: + image: grafana/promtail:latest + container_name: weaver-promtail + volumes: + - ./infra/promtail/config.yml:/etc/promtail/config.yml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + command: -config.file=/etc/promtail/config.yml -config.expand-env=true + environment: + LOKI_URL: ${LOKI_URL:-http://localhost:3100} + restart: unless-stopped + volumes: registry_data: tap_data: diff --git a/infra/caddy/Caddyfile b/infra/caddy/Caddyfile index 996a1d0..9dd582b 100644 --- a/infra/caddy/Caddyfile +++ b/infra/caddy/Caddyfile @@ -4,6 +4,12 @@ on_demand_tls { ask http://index:3000/internal/verify-domain } + + admin 0.0.0.0:2019 + + servers { + metrics + } } # Index service diff --git a/infra/clickhouse/prometheus.xml b/infra/clickhouse/prometheus.xml new file mode 100644 index 0000000..1d66246 --- /dev/null +++ b/infra/clickhouse/prometheus.xml @@ -0,0 +1,9 @@ + + + /metrics + 9363 + true + true + true + + diff --git a/infra/grafana/dashboards/weaver-clickhouse.json b/infra/grafana/dashboards/weaver-clickhouse.json new file mode 100644 index 0000000..2cd6a15 --- /dev/null +++ b/infra/grafana/dashboards/weaver-clickhouse.json @@ -0,0 +1,414 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "ClickHouse Overview", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + } + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "up{service=\"weaver-clickhouse\"}", + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 50 }, + { "color": "red", "value": 100 } + ] + } + } + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "ClickHouseMetrics_Query{service=\"weaver-clickhouse\"}", + "refId": "A" + } + ], + "title": "Active Queries", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 4294967296 }, + { "color": "red", "value": 8589934592 } + ] + } + } + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "ClickHouseMetrics_MemoryTracking{service=\"weaver-clickhouse\"}", + "refId": "A" + } + ], + "title": "Memory Used", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 50 }, + { "color": "red", "value": 100 } + ] + } + } + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "ClickHouseMetrics_TCPConnection{service=\"weaver-clickhouse\"}", + "refId": "A" + } + ], + "title": "TCP Connections", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 50 }, + { "color": "red", "value": 100 } + ] + } + } + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "ClickHouseMetrics_HTTPConnection{service=\"weaver-clickhouse\"}", + "refId": "A" + } + ], + "title": "HTTP Connections", + "type": "stat" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 101, + "title": "Query Performance", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "id": 6, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(ClickHouseProfileEvents_Query{service=\"weaver-clickhouse\"}[5m])", + "legendFormat": "Queries/s", + "refId": "A" + }, + { + "expr": "rate(ClickHouseProfileEvents_SelectQuery{service=\"weaver-clickhouse\"}[5m])", + "legendFormat": "Selects/s", + "refId": "B" + }, + { + "expr": "rate(ClickHouseProfileEvents_InsertQuery{service=\"weaver-clickhouse\"}[5m])", + "legendFormat": "Inserts/s", + "refId": "C" + } + ], + "title": "Query Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 7, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(ClickHouseProfileEvents_FailedQuery{service=\"weaver-clickhouse\"}[5m])", + "legendFormat": "Failed", + "refId": "A" + }, + { + "expr": "rate(ClickHouseProfileEvents_FailedSelectQuery{service=\"weaver-clickhouse\"}[5m])", + "legendFormat": "Failed Selects", + "refId": "B" + }, + { + "expr": "rate(ClickHouseProfileEvents_FailedInsertQuery{service=\"weaver-clickhouse\"}[5m])", + "legendFormat": "Failed Inserts", + "refId": "C" + } + ], + "title": "Failed Queries", + "type": "timeseries" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "id": 102, + "title": "Resources", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "bytes" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, + "id": 8, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "ClickHouseMetrics_MemoryTracking{service=\"weaver-clickhouse\"}", + "legendFormat": "Memory Tracking", + "refId": "A" + }, + { + "expr": "ClickHouseAsyncMetrics_MemoryResident{service=\"weaver-clickhouse\"}", + "legendFormat": "Resident", + "refId": "B" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "Bps" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, + "id": 9, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{service=\"weaver-clickhouse\"}[5m])", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{service=\"weaver-clickhouse\"}[5m])", + "legendFormat": "Write", + "refId": "B" + } + ], + "title": "Disk I/O", + "type": "timeseries" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "id": 103, + "title": "Merges & Parts", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "id": 10, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "ClickHouseMetrics_Merge{service=\"weaver-clickhouse\"}", + "legendFormat": "Active Merges", + "refId": "A" + }, + { + "expr": "ClickHouseMetrics_BackgroundMergesAndMutationsPoolTask{service=\"weaver-clickhouse\"}", + "legendFormat": "Pool Tasks", + "refId": "B" + } + ], + "title": "Merge Activity", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "id": 11, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "ClickHouseMetrics_PartsActive{service=\"weaver-clickhouse\"}", + "legendFormat": "Active Parts", + "refId": "A" + }, + { + "expr": "ClickHouseMetrics_PartsOutdated{service=\"weaver-clickhouse\"}", + "legendFormat": "Outdated Parts", + "refId": "B" + } + ], + "title": "Parts", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["weaver", "clickhouse"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Weaver ClickHouse", + "uid": "weaver-clickhouse", + "version": 1 +} diff --git a/infra/grafana/dashboards/weaver-infra.json b/infra/grafana/dashboards/weaver-infra.json new file mode 100644 index 0000000..e3bfc25 --- /dev/null +++ b/infra/grafana/dashboards/weaver-infra.json @@ -0,0 +1,294 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "Host Metrics (booskie-box)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "percentunit", + "min": 0, + "max": 1 + } + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 1 }, + "id": 1, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "1 - avg(rate(node_cpu_seconds_total{service=\"weaver-node\", mode=\"idle\"}[5m]))", + "legendFormat": "CPU Usage", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "bytes" + } + }, + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 1 }, + "id": 2, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{service=\"weaver-node\"} - node_memory_MemAvailable_bytes{service=\"weaver-node\"}", + "legendFormat": "Used", + "refId": "A" + }, + { + "expr": "node_memory_MemAvailable_bytes{service=\"weaver-node\"}", + "legendFormat": "Available", + "refId": "B" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "percentunit", + "min": 0, + "max": 1 + } + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 1 }, + "id": 3, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "1 - (node_filesystem_avail_bytes{service=\"weaver-node\", mountpoint=\"/\"} / node_filesystem_size_bytes{service=\"weaver-node\", mountpoint=\"/\"})", + "legendFormat": "/ usage", + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "Bps" + } + }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 7 }, + "id": 4, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{service=\"weaver-node\", device!~\"lo|veth.*|docker.*|br-.*\"}[5m])", + "legendFormat": "{{device}} rx", + "refId": "A" + }, + { + "expr": "-rate(node_network_transmit_bytes_total{service=\"weaver-node\", device!~\"lo|veth.*|docker.*|br-.*\"}[5m])", + "legendFormat": "{{device}} tx", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "Bps" + } + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 7 }, + "id": 5, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{service=\"weaver-node\"}[5m])", + "legendFormat": "{{device}} read", + "refId": "A" + }, + { + "expr": "-rate(node_disk_written_bytes_total{service=\"weaver-node\"}[5m])", + "legendFormat": "{{device}} write", + "refId": "B" + } + ], + "title": "Disk I/O", + "type": "timeseries" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "id": 101, + "title": "Container Metrics", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "percentunit" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "id": 6, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(container_cpu_usage_seconds_total{service=\"weaver-cadvisor\", name=~\"weaver-.*\"}[5m])", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Container CPU Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "bytes" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "id": 7, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "container_memory_usage_bytes{service=\"weaver-cadvisor\", name=~\"weaver-.*\"}", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Container Memory Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "Bps" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, + "id": 8, + "options": { + "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(container_network_receive_bytes_total{service=\"weaver-cadvisor\", name=~\"weaver-.*\"}[5m])", + "legendFormat": "{{name}} rx", + "refId": "A" + }, + { + "expr": "-rate(container_network_transmit_bytes_total{service=\"weaver-cadvisor\", name=~\"weaver-.*\"}[5m])", + "legendFormat": "{{name}} tx", + "refId": "B" + } + ], + "title": "Container Network I/O", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, + "id": 9, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "container_last_seen{service=\"weaver-cadvisor\", name=~\"weaver-.*\"} - time()", + "legendFormat": "{{name}}", + "refId": "A", + "hide": true + }, + { + "expr": "count(container_last_seen{service=\"weaver-cadvisor\", name=~\"weaver-.*\"} > (time() - 60))", + "legendFormat": "Running containers", + "refId": "B" + } + ], + "title": "Running Containers", + "type": "stat" + } + ], + "schemaVersion": 39, + "tags": ["weaver", "infrastructure"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Weaver Infrastructure", + "uid": "weaver-infra", + "version": 1 +} diff --git a/infra/grafana/dashboards/weaver-overview.json b/infra/grafana/dashboards/weaver-overview.json new file mode 100644 index 0000000..56d8b5b --- /dev/null +++ b/infra/grafana/dashboards/weaver-overview.json @@ -0,0 +1,943 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Weaver Services", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "rate(http_requests_total{service=~\"weaver-.*\"}[5m])", + "legendFormat": "{{service}} {{method}} {{status}}", + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["mean", "max", "p99"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{service=~\"weaver-.*\"}[5m]))", + "legendFormat": "{{service}} p99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, rate(http_request_duration_seconds_bucket{service=~\"weaver-.*\"}[5m]))", + "legendFormat": "{{service}} p50", + "refId": "B" + } + ], + "title": "Request Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "rate(http_requests_total{service=~\"weaver-.*\", status=~\"5..\"}[5m])", + "legendFormat": "{{service}} {{status}}", + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "DOWN" + }, + "1": { + "color": "green", + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "up{service=\"weaver-index\"}", + "legendFormat": "weaver-index", + "refId": "A" + } + ], + "title": "Index Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "DOWN" + }, + "1": { + "color": "green", + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "up{service=\"weaver-app\"}", + "legendFormat": "weaver-app", + "refId": "A" + } + ], + "title": "App Status", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 101, + "panels": [], + "title": "Caddy Proxy", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "rate(caddy_admin_http_requests_total{service=\"weaver-caddy\"}[5m])", + "legendFormat": "{{handler}} {{code}}", + "refId": "A" + } + ], + "title": "Caddy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(caddy_http_request_duration_seconds_bucket{service=\"weaver-caddy\"}[5m]))", + "legendFormat": "p99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, rate(caddy_http_request_duration_seconds_bucket{service=\"weaver-caddy\"}[5m]))", + "legendFormat": "p50", + "refId": "B" + } + ], + "title": "Caddy Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 102, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 8, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showControls": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "{service_name=\"weaver-index\"} |= ``", + "refId": "A" + } + ], + "title": "Index Logs", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 9, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showControls": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{service=\"weaver-app\"} != `dioxus_core` or `hyper_util` or `dioxus_signals` or `reqwest` or `axum`", + "queryType": "range", + "refId": "A" + } + ], + "title": "App Logs", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "sum by (service_name) (count_over_time({service_name=~\"weaver-.*\"} | level=~\"error|ERROR|err\" [1m]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "title": "Error Log Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 11, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showControls": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "direction": "backward", + "editorMode": "code", + "expr": "{container_name=~\"weaver-clickhouse|weaver-caddy|weaver-tap\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Infrastructure Logs", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": ["weaver"], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "includeAll": false, + "label": "Metrics", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": "Loki", + "value": "P8E80F9AEF21F6940" + }, + "includeAll": false, + "label": "Logs", + "name": "loki", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Weaver Overview", + "uid": "weaver-overview", + "version": 9 +} diff --git a/infra/grafana/dashboards/weaver-tap.json b/infra/grafana/dashboards/weaver-tap.json new file mode 100644 index 0000000..215658b --- /dev/null +++ b/infra/grafana/dashboards/weaver-tap.json @@ -0,0 +1,412 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "Firehose", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 1 }, + "id": 1, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(tap_firehose_events_received_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Received", + "refId": "A" + }, + { + "expr": "rate(tap_firehose_events_processed_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Processed", + "refId": "B" + }, + { + "expr": "rate(tap_firehose_events_skipped_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Skipped", + "refId": "C" + } + ], + "title": "Firehose Event Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "none" + } + }, + "gridPos": { "h": 6, "w": 4, "x": 8, "y": 1 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "tap_firehose_last_seq{service=\"weaver-tap\"}", + "legendFormat": "Last Seq", + "refId": "A" + } + ], + "title": "Last Sequence", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 10000 }, + { "color": "red", "value": 50000 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 6, "w": 4, "x": 12, "y": 1 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "tap_event_cache_size{service=\"weaver-tap\"}", + "legendFormat": "Cache Size", + "refId": "A" + } + ], + "title": "Event Cache Size", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 1 }, + "id": 4, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "increase(tap_firehose_events_received_total{service=\"weaver-tap\"}[1h])", + "legendFormat": "Received", + "refId": "A" + }, + { + "expr": "increase(tap_firehose_events_processed_total{service=\"weaver-tap\"}[1h])", + "legendFormat": "Processed", + "refId": "B" + } + ], + "title": "Events (1h)", + "type": "timeseries" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, + "id": 101, + "title": "Resyncs", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 5, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(tap_resyncs_started_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Started", + "refId": "A" + }, + { + "expr": "rate(tap_resyncs_completed_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Completed", + "refId": "B" + }, + { + "expr": "rate(tap_resyncs_failed_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Failed", + "refId": "C" + } + ], + "title": "Resync Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "s" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 6, + "options": { + "legend": { "calcs": ["mean", "max", "p99"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(tap_resync_duration_seconds_bucket{service=\"weaver-tap\"}[5m]))", + "legendFormat": "p99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, rate(tap_resync_duration_seconds_bucket{service=\"weaver-tap\"}[5m]))", + "legendFormat": "p50", + "refId": "B" + } + ], + "title": "Resync Duration", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 16 }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "tap_resyncs_completed_total{service=\"weaver-tap\"}", + "legendFormat": "Completed", + "refId": "A" + } + ], + "title": "Total Completed", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 16 }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + }, + "targets": [ + { + "expr": "tap_resyncs_failed_total{service=\"weaver-tap\"}", + "legendFormat": "Failed", + "refId": "A" + } + ], + "title": "Total Failed", + "type": "stat" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, + "id": 102, + "title": "Event Delivery", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, + "id": 9, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(tap_events_delivered_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Delivered", + "refId": "A" + }, + { + "expr": "rate(tap_events_acked_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Acked", + "refId": "B" + } + ], + "title": "Event Delivery Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, + "id": 10, + "options": { + "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "expr": "rate(tap_crawler_repos_discovered_total{service=\"weaver-tap\"}[5m])", + "legendFormat": "Repos Discovered", + "refId": "A" + } + ], + "title": "Crawler Discovery Rate", + "type": "timeseries" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, + "id": 103, + "title": "Logs", + "type": "row" + }, + { + "datasource": { "type": "loki", "uid": "${loki}" }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 30 }, + "id": 11, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "expr": "{container_name=\"weaver-tap\"} |= ``", + "refId": "A" + } + ], + "title": "Tap Logs", + "type": "logs" + } + ], + "schemaVersion": 39, + "tags": ["weaver", "tap", "atproto"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, + "hide": 0, + "includeAll": false, + "label": "Metrics", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { "selected": false, "text": "Loki", "value": "Loki" }, + "hide": 0, + "includeAll": false, + "label": "Logs", + "multi": false, + "name": "loki", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Weaver Tap", + "uid": "weaver-tap", + "version": 1 +} diff --git a/infra/promtail/config.yml b/infra/promtail/config.yml new file mode 100644 index 0000000..dea3cc9 --- /dev/null +++ b/infra/promtail/config.yml @@ -0,0 +1,44 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: ${LOKI_URL}/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + # Only scrape weaver containers + - source_labels: [__meta_docker_container_name] + regex: "/(weaver-.+)" + action: keep + # Set container_name label (matches dashboard queries) + - source_labels: [__meta_docker_container_name] + regex: "/(.+)" + target_label: container_name + # Add instance label + - target_label: instance + replacement: "booskie-box" + pipeline_stages: + # Parse JSON logs if present + - json: + expressions: + level: level + msg: msg + timestamp: timestamp + # Use extracted level if available + - labels: + level: + # Timestamp from log if available + - timestamp: + source: timestamp + format: RFC3339Nano + fallback_formats: + - RFC3339 + - UnixMs -- knotmirror 0.0.1