{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Comprehensive monitoring dashboard for TAP (ATProtocol) server metrics", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "\ud83d\udcca Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 1, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "tap_firehose_last_seq{instance=~\"$instance\"}", "legendFormat": "Firehose Sequence", "refId": "A" } ], "title": "Firehose Sequence", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "tap_crawler_repos_discovered_total{instance=~\"$instance\"}", "legendFormat": "Repos Discovered", "refId": "A" } ], "title": "Repos Discovered", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "tap_event_cache_size{instance=~\"$instance\"}", "legendFormat": "Event Cache Size", "refId": "A" } ], "title": "Event Cache Size", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "indigo_scheduler_workers_active{instance=~\"$instance\"}", "legendFormat": "Active Workers", "refId": "A" } ], "title": "Active Workers", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 50 }, { "color": "green", "value": 90 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "id": 5, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "atproto_directory_identity_cache_hits{instance=~\"$instance\"} / (atproto_directory_identity_cache_hits{instance=~\"$instance\"} + atproto_directory_identity_cache_misses{instance=~\"$instance\"})", "legendFormat": "Identity Cache Hit Rate", "refId": "A" } ], "title": "Identity Cache Hit Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 5 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "id": 6, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "tap_resyncs_failed_total{instance=~\"$instance\"} / tap_resyncs_started_total{instance=~\"$instance\"}", "legendFormat": "Resync Failure Rate", "refId": "A" } ], "title": "Resync Failure Rate", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 101, "panels": [], "title": "\ud83d\udd25 Firehose Processing", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 10, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(tap_firehose_events_received_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Received", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(tap_firehose_events_processed_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Processed", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(tap_firehose_events_skipped_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Skipped", "refId": "C" } ], "title": "Firehose Event Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 11, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "increase(tap_firehose_events_processed_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Processed", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "increase(tap_firehose_events_skipped_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Skipped", "refId": "B" } ], "title": "Firehose Events (Stacked)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, "id": 12, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(indigo_repo_stream_bytes_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "{{remote_addr}}", "refId": "A" } ], "title": "Stream Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, "id": 13, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(indigo_repo_stream_events_received_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "{{remote_addr}}", "refId": "A" } ], "title": "Stream Events Rate", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 102, "panels": [], "title": "\ud83d\udce4 Event Delivery", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Delivered" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Acked" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, "id": 20, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(tap_events_delivered_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Delivered", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(tap_events_acked_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Acked", "refId": "B" } ], "title": "Event Delivery Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, "id": 21, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "tap_events_delivered_total{instance=~\"$instance\"} - tap_events_acked_total{instance=~\"$instance\"}", "legendFormat": "Unacked Events", "refId": "A" } ], "title": "Unacknowledged Events", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 103, "panels": [], "title": "\ud83d\udd04 Resync Operations", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Started" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Completed" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Failed" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "id": 30, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(tap_resyncs_started_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Started", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(tap_resyncs_completed_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Completed", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(tap_resyncs_failed_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Failed", "refId": "C" } ], "title": "Resync Operations Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "id": 31, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, rate(tap_resync_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "p50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.90, rate(tap_resync_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "p90", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, rate(tap_resync_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "p99", "refId": "C" } ], "title": "Resync Duration Percentiles", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1 }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 40 }, "id": 32, "options": { "bucketCount": 20, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(tap_resync_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])) by (le)", "format": "heatmap", "legendFormat": "{{le}}", "refId": "A" } ], "title": "Resync Duration Distribution", "type": "histogram" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 48 }, "id": 104, "panels": [], "title": "\ud83d\udd10 ATProto Identity Resolution", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "cached" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "coalesced" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "id": 40, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(atproto_identity_resolve_did{instance=~\"$instance\", status=\"cached\"}[$__rate_interval])", "legendFormat": "cached", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(atproto_identity_resolve_did{instance=~\"$instance\", status=\"success\"}[$__rate_interval])", "legendFormat": "success", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(atproto_identity_resolve_did{instance=~\"$instance\", status=\"error\"}[$__rate_interval])", "legendFormat": "error", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(atproto_identity_resolve_did{instance=~\"$instance\", status=\"coalesced\"}[$__rate_interval])", "legendFormat": "coalesced", "refId": "D" } ], "title": "DID Resolution Rate by Status", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "id": 41, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, rate(atproto_identity_resolve_did_duration_bucket{instance=~\"$instance\", status=\"success\"}[$__rate_interval]))", "legendFormat": "p50 (success)", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.90, rate(atproto_identity_resolve_did_duration_bucket{instance=~\"$instance\", status=\"success\"}[$__rate_interval]))", "legendFormat": "p90 (success)", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, rate(atproto_identity_resolve_did_duration_bucket{instance=~\"$instance\", status=\"success\"}[$__rate_interval]))", "legendFormat": "p99 (success)", "refId": "C" } ], "title": "DID Resolution Latency (Success)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Hits" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Misses" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 57 }, "id": 42, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(atproto_directory_identity_cache_hits{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Hits", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(atproto_directory_identity_cache_misses{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Misses", "refId": "B" } ], "title": "Identity Cache Performance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "percentage", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 50 }, { "color": "green", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 57 }, "id": 43, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(atproto_directory_identity_cache_hits{instance=~\"$instance\"}[$__rate_interval]) / (rate(atproto_directory_identity_cache_hits{instance=~\"$instance\"}[$__rate_interval]) + rate(atproto_directory_identity_cache_misses{instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "Identity Cache", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(atproto_directory_handle_cache_hits{instance=~\"$instance\"}[$__rate_interval]) / (rate(atproto_directory_handle_cache_hits{instance=~\"$instance\"}[$__rate_interval]) + rate(atproto_directory_handle_cache_misses{instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "Handle Cache", "refId": "B" } ], "title": "Cache Hit Rates", "type": "gauge" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 65 }, "id": 105, "panels": [], "title": "\u2699\ufe0f Scheduler & Workers", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 66 }, "id": 50, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "indigo_scheduler_workers_active{instance=~\"$instance\"}", "legendFormat": "{{pool}}", "refId": "A" } ], "title": "Active Workers", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 66 }, "id": 51, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(indigo_scheduler_work_items_added_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Added", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(indigo_scheduler_work_items_processed_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "Processed", "refId": "B" } ], "title": "Work Items Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 66 }, "id": 52, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "indigo_scheduler_work_items_added_total{instance=~\"$instance\"} - indigo_scheduler_work_items_processed_total{instance=~\"$instance\"}", "legendFormat": "Queue Depth", "refId": "A" } ], "title": "Work Queue Backlog", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 74 }, "id": 106, "panels": [], "title": "\ud83d\udcbb Process & Runtime", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 75 }, "id": 60, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", "legendFormat": "Resident", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "go_memstats_heap_alloc_bytes{instance=~\"$instance\"}", "legendFormat": "Heap Alloc", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "go_memstats_heap_inuse_bytes{instance=~\"$instance\"}", "legendFormat": "Heap In Use", "refId": "C" } ], "title": "Memory Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 75 }, "id": 61, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "CPU Usage", "refId": "A" } ], "title": "CPU Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 75 }, "id": 62, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "go_goroutines{instance=~\"$instance\"}", "legendFormat": "Goroutines", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "go_threads{instance=~\"$instance\"}", "legendFormat": "OS Threads", "refId": "B" } ], "title": "Goroutines & Threads", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 83 }, "id": 63, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "go_gc_duration_seconds{instance=~\"$instance\", quantile=\"0.5\"}", "legendFormat": "p50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "go_gc_duration_seconds{instance=~\"$instance\", quantile=\"0.75\"}", "legendFormat": "p75", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "go_gc_duration_seconds{instance=~\"$instance\", quantile=\"1\"}", "legendFormat": "max", "refId": "C" } ], "title": "GC Pause Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Max FDs" }, "properties": [ { "id": "custom.lineStyle", "value": { "dash": [ 10, 10 ], "fill": "dash" } }, { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 83 }, "id": 64, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "process_open_fds{instance=~\"$instance\"}", "legendFormat": "Open FDs", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "process_max_fds{instance=~\"$instance\"}", "legendFormat": "Max FDs", "refId": "B" } ], "title": "File Descriptors", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "tap", "atproto", "prometheus" ], "templating": { "list": [ { "current": { "selected": false, "text": "grafana_prometheus", "value": "grafana_prometheus" }, "description": "Prometheus datasource for TAP metrics", "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(tap_firehose_last_seq, instance)", "description": "Filter metrics by TAP server instance", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [], "query": { "qryType": 1, "query": "label_values(tap_firehose_last_seq, instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "TAP Server Metrics", "uid": "tap-server-metrics", "version": 1, "weekStart": "" }