documentation: updates and bug fixes

docs/telegraf-timescaledb-metrics-guide.md

···
 
 This setup creates a metrics pipeline that:
 - Collects StatsD metrics via Telegraf on UDP port 8125
-- Stores metrics in PostgreSQL with TimescaleDB extensions
+- Creates individual PostgreSQL tables for each metric type
+- Stores metric tags as JSONB for flexible querying
 - Automatically creates hypertables for time-series optimization
 - Provides a complete Docker Compose configuration for easy deployment
 
+## Important Note on Table Structure
+
+The Telegraf PostgreSQL output plugin with the configuration in this guide creates **individual tables for each metric name**. For example:
+- `quickdid.http.request.count` becomes table `"quickdid.http.request.count"`
+- `quickdid.resolver.rate_limit.available_permits` becomes table `"quickdid.resolver.rate_limit.available_permits"`
+
+Each table has the following structure:
+- `time` (timestamptz) - The timestamp of the metric
+- `tags` (jsonb) - All tags stored as a JSON object
+- Metric-specific columns for values (e.g., `value`, `mean`, `p99`, etc.)
+
 ## Prerequisites
 
 - Docker and Docker Compose installed
···
 ├── docker-compose.yml
 ├── telegraf/
 │   └── telegraf.conf
+├── test-scripts/
+│   ├── send-metrics.sh
+│   └── verify-queries.sql
 └── .env
 ```
 
···
 [[outputs.postgresql]]
   connection = "host=timescaledb user=${TELEGRAF_DB_USER} password=${TELEGRAF_DB_PASSWORD} dbname=${POSTGRES_DB} sslmode=disable"
   schema = "public"
+  
+  # Create individual tables for each metric with hypertable support
   create_templates = [
     '''CREATE TABLE IF NOT EXISTS {{.table}} ({{.columns}})''',
     '''SELECT create_hypertable({{.table|quoteLiteral}}, 'time', if_not_exists => TRUE)''',
   ]
+  
+  # Store all tags as JSONB for flexible querying
   tags_as_jsonb = true
-
+  
+  # Keep fields as separate columns for better performance on aggregations
+  fields_as_jsonb = false
 ```
 
 ### 3. Docker Compose Configuration (docker-compose.yml)
···
 
 services:
   timescaledb:
-    image: timescale/timescaledb-ha:pg17
+    image: timescale/timescaledb:latest-pg17
     container_name: timescaledb
     restart: unless-stopped
     environment:
···
     volumes:
       - timescale_data:/home/postgres/pgdata/data
       - ./init-scripts:/docker-entrypoint-initdb.d:ro
+    command:
+      - postgres
+      - -c
+      - shared_buffers=1GB
+      - -c
+      - effective_cache_size=3GB
+      - -c
+      - maintenance_work_mem=512MB
+      - -c
+      - work_mem=32MB
+      - -c
+      - timescaledb.max_background_workers=8
+      - -c
+      - max_parallel_workers_per_gather=2
+      - -c
+      - max_parallel_workers=8
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
       interval: 10s
···
       - metrics_network
 
   telegraf:
-    image: telegraf
+    image: telegraf:1.35
     container_name: telegraf
     restart: unless-stopped
     environment:
···
 
 ### 4. Database Initialization Script (optional)
 
-Create `init-scripts/01-init.sql` to set up the Telegraf user and schema:
+Create `init-scripts/01-init.sql` to set up the TimescaleDB extension:
 
 ```sql
 -- Enable TimescaleDB extension
 CREATE EXTENSION IF NOT EXISTS timescaledb;
+
+-- Enable additional useful extensions
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+```
+
+## Test Scripts
+
+### 1. Send Test Metrics Script (test-scripts/send-metrics.sh)
+
+Create a script to send various types of metrics:
+
+```bash
+#!/bin/bash
+
+# Send test metrics to StatsD/Telegraf
+
+echo "Sending test metrics to StatsD on localhost:8125..."
+
+# Counter metrics
+for i in {1..10}; do
+  echo "quickdid.http.request.count:1|c|#method:GET,path:/resolve,status:200" | nc -u -w0 localhost 8125
+  echo "quickdid.http.request.count:1|c|#method:POST,path:/api,status:201" | nc -u -w0 localhost 8125
+  echo "quickdid.http.request.count:1|c|#method:GET,path:/resolve,status:404" | nc -u -w0 localhost 8125
+done
+
+# Gauge metrics
+echo "quickdid.resolver.rate_limit.available_permits:10|g" | nc -u -w0 localhost 8125
+echo "quickdid.resolver.rate_limit.available_permits:8|g" | nc -u -w0 localhost 8125
+echo "quickdid.resolver.rate_limit.available_permits:5|g" | nc -u -w0 localhost 8125
+
+# Timing metrics (in milliseconds)
+for i in {1..20}; do
+  duration=$((RANDOM % 100 + 10))
+  echo "quickdid.http.request.duration_ms:${duration}|ms|#method:GET,path:/resolve,status:200" | nc -u -w0 localhost 8125
+done
+
+for i in {1..10}; do
+  duration=$((RANDOM % 200 + 50))
+  echo "quickdid.http.request.duration_ms:${duration}|ms|#method:POST,path:/api,status:201" | nc -u -w0 localhost 8125
+done
+
+# Histogram metrics
+for i in {1..15}; do
+  resolution_time=$((RANDOM % 500 + 50))
+  echo "quickdid.resolver.resolution_time:${resolution_time}|h|#resolver:redis" | nc -u -w0 localhost 8125
+  echo "quickdid.resolver.resolution_time:$((resolution_time * 2))|h|#resolver:base" | nc -u -w0 localhost 8125
+done
+
+# Cache metrics
+echo "quickdid.cache.hit.count:45|c|#cache_type:redis" | nc -u -w0 localhost 8125
+echo "quickdid.cache.miss.count:5|c|#cache_type:redis" | nc -u -w0 localhost 8125
+echo "quickdid.cache.size:1024|g|#cache_type:memory" | nc -u -w0 localhost 8125
+
+echo "Metrics sent! Wait 15 seconds for Telegraf to flush..."
+sleep 15
+echo "Done!"
+```
+
+### 2. Verify Queries Script (test-scripts/verify-queries.sql)
+
+Create a SQL script to verify all queries work correctly:
+
+```sql
+-- Test script to verify all metrics queries work correctly
+-- Run this after sending test metrics with send-metrics.sh
+
+\echo '===== CHECKING AVAILABLE TABLES ====='
+SELECT table_name 
+FROM information_schema.tables 
+WHERE table_schema = 'public' 
+  AND table_name LIKE 'quickdid%'
+ORDER BY table_name;
+
+\echo ''
+\echo '===== CHECKING TABLE STRUCTURES ====='
+\echo 'Structure of quickdid.http.request.count table:'
+\d "quickdid.http.request.count"
+
+\echo ''
+\echo 'Structure of quickdid.http.request.duration_ms table:'
+\d "quickdid.http.request.duration_ms"
+
+\echo ''
+\echo '===== QUERY 1: Recent HTTP Request Counts ====='
+SELECT 
+    time,
+    tags,
+    tags->>'method' as method,
+    tags->>'path' as path,
+    tags->>'status' as status,
+    value
+FROM "quickdid.http.request.count"
+WHERE time > NOW() - INTERVAL '1 hour'
+ORDER BY time DESC
+LIMIT 10;
+
+\echo ''
+\echo '===== QUERY 2: HTTP Request Duration Statistics by Endpoint ====='
+SELECT 
+    time_bucket('1 minute', time) AS minute,
+    tags->>'method' as method,
+    tags->>'path' as path,
+    tags->>'status' as status,
+    COUNT(*) as request_count,
+    AVG(mean) as avg_duration_ms,
+    MAX(p99) as p99_duration_ms,
+    MIN(mean) as min_duration_ms
+FROM "quickdid.http.request.duration_ms"
+WHERE time > NOW() - INTERVAL '1 hour'
+  AND tags IS NOT NULL
+GROUP BY minute, tags->>'method', tags->>'path', tags->>'status'
+ORDER BY minute DESC
+LIMIT 10;
+
+\echo ''
+\echo '===== QUERY 3: Rate Limiter Status Over Time ====='
+SELECT 
+    time,
+    value as available_permits
+FROM "quickdid.resolver.rate_limit.available_permits"
+WHERE time > NOW() - INTERVAL '1 hour'
+ORDER BY time DESC
+LIMIT 10;
+
+\echo ''
+\echo '===== QUERY 4: Resolver Performance Comparison ====='
+SELECT 
+    tags->>'resolver' as resolver_type,
+    COUNT(*) as sample_count,
+    AVG(mean) as avg_resolution_time_ms,
+    MAX(p99) as p99_resolution_time_ms,
+    MIN(mean) as min_resolution_time_ms
+FROM "quickdid.resolver.resolution_time"
+WHERE time > NOW() - INTERVAL '1 hour'
+  AND tags->>'resolver' IS NOT NULL
+GROUP BY tags->>'resolver'
+ORDER BY avg_resolution_time_ms;
+
+\echo ''
+\echo '===== QUERY 5: Cache Hit Rate Analysis ====='
+WITH cache_stats AS (
+    SELECT 
+        'hits' as metric_type,
+        SUM(value) as total_count
+    FROM "quickdid.cache.hit.count"
+    WHERE time > NOW() - INTERVAL '1 hour'
+    UNION ALL
+    SELECT 
+        'misses' as metric_type,
+        SUM(value) as total_count
+    FROM "quickdid.cache.miss.count"
+    WHERE time > NOW() - INTERVAL '1 hour'
+)
+SELECT 
+    SUM(CASE WHEN metric_type = 'hits' THEN total_count ELSE 0 END) as total_hits,
+    SUM(CASE WHEN metric_type = 'misses' THEN total_count ELSE 0 END) as total_misses,
+    CASE 
+        WHEN SUM(total_count) > 0 THEN 
+            ROUND(100.0 * SUM(CASE WHEN metric_type = 'hits' THEN total_count ELSE 0 END) / SUM(total_count), 2)
+        ELSE 0 
+    END as hit_rate_percentage
+FROM cache_stats;
+
+\echo ''
+\echo '===== QUERY 6: Hypertable Information ====='
+SELECT 
+    hypertable_schema,
+    hypertable_name,
+    owner,
+    num_dimensions,
+    num_chunks,
+    compression_enabled
+FROM timescaledb_information.hypertables
+WHERE hypertable_name LIKE 'quickdid%'
+ORDER BY hypertable_name;
+
+\echo ''
+\echo '===== QUERY 7: HTTP Error Rate by Endpoint ====='
+WITH status_counts AS (
+    SELECT 
+        time_bucket('5 minutes', time) as period,
+        tags->>'path' as path,
+        CASE 
+            WHEN (tags->>'status')::int >= 400 THEN 'error'
+            ELSE 'success'
+        END as status_category,
+        SUM(value) as request_count
+    FROM "quickdid.http.request.count"
+    WHERE time > NOW() - INTERVAL '1 hour'
+    GROUP BY period, path, status_category
+)
+SELECT 
+    period,
+    path,
+    SUM(CASE WHEN status_category = 'error' THEN request_count ELSE 0 END) as error_count,
+    SUM(CASE WHEN status_category = 'success' THEN request_count ELSE 0 END) as success_count,
+    CASE 
+        WHEN SUM(request_count) > 0 THEN 
+            ROUND(100.0 * SUM(CASE WHEN status_category = 'error' THEN request_count ELSE 0 END) / SUM(request_count), 2)
+        ELSE 0 
+    END as error_rate_percentage
+FROM status_counts
+GROUP BY period, path
+HAVING SUM(request_count) > 0
+ORDER BY period DESC, error_rate_percentage DESC;
+
+\echo ''
+\echo '===== TEST COMPLETED ====='
 ```
 
 ## Usage
···
 cd metrics-stack
 ```
 
-2. Start the services:
+2. Make the test scripts executable:
+```bash
+chmod +x test-scripts/send-metrics.sh
+```
+
+3. Start the services:
 ```bash
 docker-compose up -d
 ```
 
-3. Check the logs to ensure everything is running:
+4. Check the logs to ensure everything is running:
 ```bash
 docker-compose logs -f
 ```
 
-### Sending Metrics
+5. Wait for services to be fully ready (about 30 seconds)
 
-Send test metrics to Telegraf using StatsD protocol:
+### Running the Test Suite
 
+1. Send test metrics:
 ```bash
-# Send a counter metric
-echo "quickdid.http.request.count:1|c|#method:GET,path:/resolve,status:200" | nc -u -w0 localhost 8125
-
-# Send a gauge metric
-echo "quickdid.resolver.rate_limit.available_permits:10|g" | nc -u -w0 localhost 8125
+./test-scripts/send-metrics.sh
+```
 
-# Send a timing metric
-echo "quickdid.http.request.duration_ms:42|ms|#method:POST,path:/api,status:201" | nc -u -w0 localhost 8125
-
-# Send a histogram
-echo "quickdid.resolver.resolution_time:100|h|#resolver:redis" | nc -u -w0 localhost 8125
+2. Verify all queries work:
+```bash
+docker exec -i timescaledb psql -U postgres -d metrics < test-scripts/verify-queries.sql
 ```
 
-### Querying Metrics
+### Manual Querying
 
-Connect to TimescaleDB to query your metrics:
+Connect to TimescaleDB to run queries manually:
 
 ```bash
 # Connect to the database
 docker exec -it timescaledb psql -U postgres -d metrics
 
-# Query recent metrics
+# List all metric tables
+\dt "quickdid*"
+
+# Describe a specific table structure
+\d "quickdid.http.request.duration_ms"
+
+# Query with JSONB tag filtering
 SELECT 
     time,
-    tags,
-    value
-FROM "quickdid.http.request.count"
-WHERE time > NOW() - INTERVAL '1 hour'
-ORDER BY time DESC
-LIMIT 10;
-
-# Query specific metric with tags
-SELECT 
-    time_bucket('1 minute', time) AS minute,
     tags->>'method' as method,
-    tags->>'path' as path,
-    tags->>'status' as status,
-    AVG(mean::numeric) as avg_duration
+    mean as avg_ms,
+    '99_percentile' as p99_ms
 FROM "quickdid.http.request.duration_ms"
-WHERE 
-    time > NOW() - INTERVAL '1 hour'
-GROUP BY minute, method, path, status
-ORDER BY minute DESC;
+WHERE tags @> '{"method": "GET"}'::jsonb
+  AND time > NOW() - INTERVAL '1 hour'
+ORDER BY time DESC
+LIMIT 10;
 ```
 
 ## Advanced Configuration
 
-### Continuous Aggregates
+### Continuous Aggregates for Performance
 
-Create continuous aggregates for better query performance:
+Create continuous aggregates for frequently queried data:
 
 ```sql
--- Create a continuous aggregate for HTTP metrics
+-- Create hourly aggregates for HTTP metrics
 CREATE MATERIALIZED VIEW http_metrics_hourly
 WITH (timescaledb.continuous) AS
 SELECT
     time_bucket('1 hour', time) AS hour,
-    name,
     tags->>'method' as method,
     tags->>'path' as path,
     tags->>'status' as status,
     COUNT(*) as request_count,
-    AVG((fields->>'value')::numeric) as avg_value,
-    MAX((fields->>'value')::numeric) as max_value,
-    MIN((fields->>'value')::numeric) as min_value
-FROM telegraf.metrics
-WHERE name LIKE 'quickdid_http_%'
-GROUP BY hour, name, method, path, status
+    AVG(mean) as avg_duration_ms,
+    MAX('99_percentile') as p99_duration_ms,
+    MIN(mean) as min_duration_ms
+FROM "quickdid.http.request.duration_ms"
+WHERE tags IS NOT NULL
+GROUP BY hour, method, path, status
 WITH NO DATA;
 
 -- Add refresh policy
···
     start_offset => INTERVAL '3 hours',
     end_offset => INTERVAL '1 hour',
     schedule_interval => INTERVAL '1 hour');
+
+-- Manually refresh to populate initial data
+CALL refresh_continuous_aggregate('http_metrics_hourly', NULL, NULL);
+
+-- Query the aggregate
+SELECT * FROM http_metrics_hourly
+ORDER BY hour DESC, request_count DESC
+LIMIT 20;
 ```
 
-## Monitoring and Maintenance
+### Data Retention Policies
 
-### Health Checks
+Set up automatic data retention:
 
-Check the health of your TimescaleDB hypertables:
+```sql
+-- Add retention policy to drop data older than 30 days
+SELECT add_retention_policy('"quickdid.http.request.count"', INTERVAL '30 days');
+SELECT add_retention_policy('"quickdid.http.request.duration_ms"', INTERVAL '30 days');
+
+-- View retention policies
+SELECT js.* FROM timescaledb_information.job_stats js
+JOIN timescaledb_information.jobs j ON js.job_id = j.job_id
+WHERE j.proc_name LIKE '%retention%';
+```
+
+### Compression for Storage Optimization
+
+Enable compression for older data:
 
 ```sql
--- View hypertable information
-SELECT * FROM timescaledb_information.hypertables;
+-- Enable compression on a hypertable
+ALTER TABLE "quickdid.http.request.duration_ms" SET (
+    timescaledb.compress,
+    timescaledb.compress_segmentby = 'tags'
+);
 
--- Check chunk information
+-- Add compression policy (compress chunks older than 7 days)
+SELECT add_compression_policy('"quickdid.http.request.duration_ms"', INTERVAL '7 days');
+
+-- Manually compress old chunks
+SELECT compress_chunk(format('%I.%I', c.chunk_schema, c.chunk_name)::regclass)
+FROM timescaledb_information.chunks c
+WHERE c.hypertable_name = 'quickdid.http.request.duration_ms'
+AND c.range_end < NOW() - INTERVAL '7 days'
+AND NOT c.is_compressed;
+
+-- Check compression status
+SELECT 
+    hypertable_name,
+    uncompressed_total_bytes,
+    compressed_total_bytes,
+    compression_ratio
+FROM timescaledb_information.hypertable_compression_stats
+WHERE hypertable_name LIKE 'quickdid%';
+```
+
+## Monitoring and Maintenance
+
+### Health Checks
+
+```sql
+-- Check chunk distribution
 SELECT 
     hypertable_name,
     chunk_name,
     range_start,
     range_end,
-    is_compressed
+    is_compressed,
+    pg_size_pretty(total_bytes) as size
 FROM timescaledb_information.chunks
-WHERE hypertable_name = 'quickdid.http.request.duration_ms'
-ORDER BY range_start DESC
-LIMIT 10;
+WHERE hypertable_name LIKE 'quickdid%'
+ORDER BY hypertable_name, range_start DESC
+LIMIT 20;
 
--- Check compression status
+-- Check background jobs
+SELECT 
+    job_id,
+    application_name,
+    job_type,
+    schedule_interval,
+    last_run_started_at,
+    last_successful_finish,
+    next_scheduled_run
+FROM timescaledb_information.job_stats
+ORDER BY job_id;
+
+-- Check table sizes
 SELECT 
     hypertable_name,
-    uncompressed_total_bytes,
-    compressed_total_bytes,
-    compression_ratio
-FROM timescaledb_information.hypertable_compression_stats;
+    chunks_total_size,
+    chunks_compressed_size,
+    chunks_uncompressed_size
+FROM timescaledb_information.hypertables
+WHERE hypertable_name LIKE 'quickdid%';
 ```
 
 ### Troubleshooting
 
-1. **Telegraf can't connect to TimescaleDB:**
-   - Check that TimescaleDB is healthy: `docker-compose ps`
-   - Verify credentials in `.env` file
-   - Check network connectivity between containers
+1. **Tables not being created:**
+   - Check Telegraf logs: `docker-compose logs telegraf | grep -i error`
+   - Verify PostgreSQL connectivity: `docker exec telegraf telegraf --test`
+   - Ensure metrics are being received: `docker-compose logs telegraf | grep statsd`
 
-2. **Metrics not appearing in database:**
-   - Check Telegraf logs: `docker-compose logs telegraf`
-   - Verify StatsD metrics are being received: `docker-compose logs telegraf | grep statsd`
-   - Check table creation: Connect to database and run `\dt telegraf.*`
+2. **Queries returning no data:**
+   - Verify tables exist: `\dt "quickdid*"` in psql
+   - Check table contents: `SELECT COUNT(*) FROM "quickdid.http.request.count";`
+   - Verify time ranges in WHERE clauses
 
 3. **Performance issues:**
-   - Check chunk sizes and adjust chunk_time_interval if needed
-   - Verify compression is working
-   - Consider creating appropriate indexes on frequently queried columns
+   - Check if hypertables are created: Query `timescaledb_information.hypertables`
+   - Verify compression is working if enabled
+   - Consider creating appropriate indexes on JSONB paths:
+   ```sql
+   CREATE INDEX idx_http_method ON "quickdid.http.request.duration_ms" ((tags->>'method'));
+   CREATE INDEX idx_http_path ON "quickdid.http.request.duration_ms" ((tags->>'path'));
+   ```
 
 ## Integration with QuickDID
 
···
 
 QuickDID will automatically send metrics to Telegraf, which will store them in TimescaleDB for analysis.
 
-## Backup and Restore
-
-### Backup
+## Key Differences from Generic Metrics Table Approach
 
-Create a backup of your metrics data:
-
-```bash
-# Backup entire database
-docker exec timescaledb pg_dump -U postgres metrics_db > metrics_backup.sql
-
-# Backup specific time range
-docker exec timescaledb pg_dump -U postgres \
-  --table='telegraf.metrics' \
-  --data-only \
-  --where="time >= '2024-01-01' AND time < '2024-02-01'" \
-  metrics_db > metrics_january.sql
-```
-
-### Restore
+This configuration creates **individual tables per metric** instead of a single generic metrics table. Benefits include:
 
-Restore from backup:
+1. **Better performance**: Each metric has its own optimized schema
+2. **Clearer data model**: Tables directly represent metrics
+3. **Easier querying**: No need to filter by metric name
+4. **Type safety**: Each metric's fields have appropriate types
+5. **Efficient compression**: Per-metric compression strategies
 
-```bash
-# Restore database
-docker exec -i timescaledb psql -U postgres metrics_db < metrics_backup.sql
-```
+Trade-offs:
+- More tables to manage (mitigated by TimescaleDB automation)
+- Need to know metric names upfront for queries
+- Schema changes require table alterations
 
 ## Security Considerations
 
 1. **Use strong passwords:** Update the default passwords in `.env`
 2. **Enable SSL:** Configure `sslmode=require` in production
 3. **Network isolation:** Use Docker networks to isolate services
-4. **Access control:** Limit database user permissions to minimum required
+4. **Access control:** Create separate database users with minimal permissions:
+   ```sql
+   CREATE USER metrics_reader WITH PASSWORD 'readonly_password';
+   GRANT CONNECT ON DATABASE metrics TO metrics_reader;
+   GRANT USAGE ON SCHEMA public TO metrics_reader;
+   GRANT SELECT ON ALL TABLES IN SCHEMA public TO metrics_reader;
+   ```
 5. **Regular updates:** Keep Docker images updated for security patches
 
 ## Performance Tuning
 
-### TimescaleDB Configuration
+### PostgreSQL/TimescaleDB Settings
 
-Add these settings to optimize performance:
+The docker-compose.yml includes optimized settings. Adjust based on your hardware:
 
-```yaml
-# In docker-compose.yml under timescaledb service
-command:
-  - postgres
-  - -c
-  - shared_buffers=1GB
-  - -c
-  - effective_cache_size=3GB
-  - -c
-  - maintenance_work_mem=512MB
-  - -c
-  - work_mem=32MB
-  - -c
-  - timescaledb.max_background_workers=8
-  - -c
-  - max_parallel_workers_per_gather=2
-  - -c
-  - max_parallel_workers=8
-```
+- `shared_buffers`: 25% of system RAM
+- `effective_cache_size`: 75% of system RAM
+- `maintenance_work_mem`: 5% of system RAM
+- `work_mem`: RAM / max_connections / 2
 
 ### Telegraf Buffer Settings
 
-Adjust buffer settings based on your metric volume:
+For high-volume metrics, adjust in telegraf.conf:
 
 ```toml
 [agent]
-  metric_batch_size = 5000     # Increase for high volume
+  metric_batch_size = 5000      # Increase for high volume
   metric_buffer_limit = 100000  # Increase buffer size
-  flush_interval = "5s"         # Decrease for more frequent writes
+  flush_interval = "5s"          # Decrease for more frequent writes
 ```
 
 ## Conclusion
 
-This setup provides a robust metrics collection and storage solution using:
-- **Telegraf** for flexible metric collection via StatsD
-- **TimescaleDB** for efficient time-series data storage
-- **Docker Compose** for easy deployment and management
+This setup provides a robust metrics collection and storage solution with:
+- **Individual metric tables** for optimal performance and clarity
+- **JSONB tag storage** for flexible querying
+- **TimescaleDB hypertables** for efficient time-series storage
+- **Comprehensive test suite** to verify functionality
+- **Production-ready configuration** with compression and retention policies
 
-The system is production-ready with support for compression, retention policies, and continuous aggregates for optimal performance at scale.
+The system correctly handles StatsD metrics from QuickDID and provides powerful querying capabilities through PostgreSQL's JSONB support and TimescaleDB's time-series functions.

test-scripts/docker-test.sh

···
+#!/bin/bash
+
+# Comprehensive test script for Telegraf/TimescaleDB metrics setup
+# This script validates the entire metrics pipeline
+
+set -e
+
+echo "========================================="
+echo "Telegraf/TimescaleDB Metrics Test Suite"
+echo "========================================="
+echo ""
+
+# Check if Docker is running
+if ! docker info > /dev/null 2>&1; then
+    echo "❌ Docker is not running. Please start Docker first."
+    exit 1
+fi
+
+# Function to wait for a service to be healthy
+wait_for_service() {
+    local service=$1
+    local max_attempts=30
+    local attempt=1
+    
+    echo -n "Waiting for $service to be healthy"
+    while [ $attempt -le $max_attempts ]; do
+        if docker-compose ps $service | grep -q "healthy"; then
+            echo " ✅"
+            return 0
+        fi
+        echo -n "."
+        sleep 2
+        attempt=$((attempt + 1))
+    done
+    echo " ❌"
+    echo "Service $service failed to become healthy after $max_attempts attempts"
+    return 1
+}
+
+# Function to run SQL query
+run_query() {
+    docker exec -i timescaledb psql -U postgres -d metrics -t -c "$1" 2>/dev/null
+}
+
+# Function to check table exists
+check_table() {
+    local table=$1
+    local result=$(run_query "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = '$table');")
+    if [[ "$result" =~ "t" ]]; then
+        echo "✅ Table '$table' exists"
+        return 0
+    else
+        echo "❌ Table '$table' does not exist"
+        return 1
+    fi
+}
+
+# Navigate to the metrics-stack directory (create if needed)
+if [ ! -d "metrics-stack" ]; then
+    echo "Creating metrics-stack directory..."
+    mkdir -p metrics-stack/telegraf
+    mkdir -p metrics-stack/test-scripts
+    mkdir -p metrics-stack/init-scripts
+fi
+
+cd metrics-stack
+
+# Create .env file if it doesn't exist
+if [ ! -f ".env" ]; then
+    echo "Creating .env file..."
+    cat > .env << 'EOF'
+# PostgreSQL/TimescaleDB Configuration
+POSTGRES_DB=metrics
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=secretpassword
+
+# Telegraf Database User
+TELEGRAF_DB_USER=postgres
+TELEGRAF_DB_PASSWORD=secretpassword
+
+# TimescaleDB Settings
+TIMESCALE_TELEMETRY=off
+EOF
+fi
+
+# Copy configuration files if they don't exist
+if [ ! -f "telegraf/telegraf.conf" ]; then
+    echo "Creating telegraf.conf..."
+    cat > telegraf/telegraf.conf << 'EOF'
+[agent]
+  interval = "10s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  precision = ""
+  debug = false
+  quiet = false
+  hostname = "telegraf-agent"
+  omit_hostname = false
+
+[[inputs.statsd]]
+  service_address = ":8125"
+  protocol = "udp"
+  delete_gauges = true
+  delete_counters = true
+  delete_sets = true
+  delete_timings = true
+  percentiles = [50, 90, 95, 99]
+  metric_separator = "."
+  allowed_pending_messages = 10000
+  datadog_extensions = true
+  datadog_distributions = true
+
+[[outputs.postgresql]]
+  connection = "host=timescaledb user=${TELEGRAF_DB_USER} password=${TELEGRAF_DB_PASSWORD} dbname=${POSTGRES_DB} sslmode=disable"
+  schema = "public"
+  create_templates = [
+    '''CREATE TABLE IF NOT EXISTS {{.table}} ({{.columns}})''',
+    '''SELECT create_hypertable({{.table|quoteLiteral}}, 'time', if_not_exists => TRUE)''',
+  ]
+  tags_as_jsonb = true
+  fields_as_jsonb = false
+EOF
+fi
+
+# Copy docker-compose.yml if it doesn't exist
+if [ ! -f "docker-compose.yml" ]; then
+    echo "Creating docker-compose.yml..."
+    cat > docker-compose.yml << 'EOF'
+version: '3.8'
+
+services:
+  timescaledb:
+    image: timescale/timescaledb:latest-pg17
+    container_name: timescaledb
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      TIMESCALE_TELEMETRY: ${TIMESCALE_TELEMETRY}
+    ports:
+      - "5442:5432"
+    volumes:
+      - timescale_data:/home/postgres/pgdata/data
+      - ./init-scripts:/docker-entrypoint-initdb.d:ro
+    command:
+      - postgres
+      - -c
+      - shared_buffers=256MB
+      - -c
+      - effective_cache_size=1GB
+      - -c
+      - maintenance_work_mem=64MB
+      - -c
+      - work_mem=8MB
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - metrics_network
+
+  telegraf:
+    image: telegraf:1.35
+    container_name: telegraf
+    restart: unless-stopped
+    environment:
+      TELEGRAF_DB_USER: ${TELEGRAF_DB_USER}
+      TELEGRAF_DB_PASSWORD: ${TELEGRAF_DB_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    ports:
+      - "8125:8125/udp"
+    volumes:
+      - ./telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    depends_on:
+      timescaledb:
+        condition: service_healthy
+    networks:
+      - metrics_network
+    command: ["telegraf", "--config", "/etc/telegraf/telegraf.conf"]
+
+networks:
+  metrics_network:
+    driver: bridge
+
+volumes:
+  timescale_data:
+EOF
+fi
+
+# Create init script
+if [ ! -f "init-scripts/01-init.sql" ]; then
+    echo "Creating init script..."
+    cat > init-scripts/01-init.sql << 'EOF'
+-- Enable TimescaleDB extension
+CREATE EXTENSION IF NOT EXISTS timescaledb;
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+EOF
+fi
+
+echo ""
+echo "Step 1: Starting Docker services..."
+echo "========================================="
+docker-compose down -v 2>/dev/null || true
+docker-compose up -d
+
+echo ""
+echo "Step 2: Waiting for services to be healthy..."
+echo "========================================="
+wait_for_service timescaledb
+sleep 5  # Extra time for Telegraf to connect
+
+echo ""
+echo "Step 3: Sending test metrics..."
+echo "========================================="
+
+# Send various types of metrics
+echo "Sending counter metrics..."
+for i in {1..5}; do
+  echo "quickdid.http.request.count:1|c|#method:GET,path:/resolve,status:200" | nc -u -w0 localhost 8125
+  echo "quickdid.http.request.count:1|c|#method:POST,path:/api,status:201" | nc -u -w0 localhost 8125
+done
+
+echo "Sending gauge metrics..."
+echo "quickdid.resolver.rate_limit.available_permits:10|g" | nc -u -w0 localhost 8125
+sleep 1
+echo "quickdid.resolver.rate_limit.available_permits:5|g" | nc -u -w0 localhost 8125
+
+echo "Sending timing metrics..."
+for i in {1..10}; do
+  duration=$((RANDOM % 100 + 10))
+  echo "quickdid.http.request.duration_ms:${duration}|ms|#method:GET,path:/resolve,status:200" | nc -u -w0 localhost 8125
+done
+
+echo "Sending histogram metrics..."
+for i in {1..5}; do
+  resolution_time=$((RANDOM % 500 + 50))
+  echo "quickdid.resolver.resolution_time:${resolution_time}|h|#resolver:redis" | nc -u -w0 localhost 8125
+done
+
+echo "Waiting 15 seconds for Telegraf to flush metrics..."
+sleep 15
+
+echo ""
+echo "Step 4: Verifying table creation..."
+echo "========================================="
+
+# Check if tables were created
+check_table "quickdid.http.request.count"
+check_table "quickdid.http.request.duration_ms"
+check_table "quickdid.resolver.rate_limit.available_permits"
+check_table "quickdid.resolver.resolution_time"
+
+echo ""
+echo "Step 5: Verifying data insertion..."
+echo "========================================="
+
+# Check row counts
+for table in "quickdid.http.request.count" "quickdid.http.request.duration_ms" "quickdid.resolver.rate_limit.available_permits" "quickdid.resolver.resolution_time"; do
+    count=$(run_query "SELECT COUNT(*) FROM \"$table\";" | tr -d ' ')
+    if [ "$count" -gt 0 ]; then
+        echo "✅ Table '$table' has $count rows"
+    else
+        echo "❌ Table '$table' is empty"
+    fi
+done
+
+echo ""
+echo "Step 6: Testing JSONB tag queries..."
+echo "========================================="
+
+# Test JSONB tag filtering
+result=$(run_query "SELECT COUNT(*) FROM \"quickdid.http.request.count\" WHERE tags->>'method' = 'GET';" | tr -d ' ')
+if [ "$result" -gt 0 ]; then
+    echo "✅ JSONB tag filtering works (found $result GET requests)"
+else
+    echo "❌ JSONB tag filtering failed"
+fi
+
+echo ""
+echo "Step 7: Testing TimescaleDB functions..."
+echo "========================================="
+
+# Test time_bucket function
+result=$(run_query "SELECT COUNT(*) FROM (SELECT time_bucket('1 minute', time) FROM \"quickdid.http.request.count\" GROUP BY 1) t;" | tr -d ' ')
+if [ "$result" -gt 0 ]; then
+    echo "✅ time_bucket function works"
+else
+    echo "❌ time_bucket function failed"
+fi
+
+# Check if hypertables were created
+hypertable_count=$(run_query "SELECT COUNT(*) FROM timescaledb_information.hypertables WHERE hypertable_name LIKE 'quickdid%';" | tr -d ' ')
+if [ "$hypertable_count" -gt 0 ]; then
+    echo "✅ Found $hypertable_count hypertables"
+else
+    echo "❌ No hypertables found"
+fi
+
+echo ""
+echo "Step 8: Running comprehensive query tests..."
+echo "========================================="
+
+# Run the verify-queries.sql script if it exists
+if [ -f "../test-scripts/verify-queries.sql" ]; then
+    echo "Running verify-queries.sql..."
+    docker exec -i timescaledb psql -U postgres -d metrics < ../test-scripts/verify-queries.sql > query_results.txt 2>&1
+    if [ $? -eq 0 ]; then
+        echo "✅ All queries executed successfully"
+        echo "   Results saved to query_results.txt"
+    else
+        echo "❌ Some queries failed. Check query_results.txt for details"
+    fi
+else
+    echo "⚠️  verify-queries.sql not found, skipping comprehensive query tests"
+fi
+
+echo ""
+echo "========================================="
+echo "Test Summary"
+echo "========================================="
+
+# Generate summary
+failures=0
+successes=0
+
+# Count successes and failures from the output
+if check_table "quickdid.http.request.count" > /dev/null 2>&1; then
+    successes=$((successes + 1))
+else
+    failures=$((failures + 1))
+fi
+
+if [ "$hypertable_count" -gt 0 ]; then
+    successes=$((successes + 1))
+else
+    failures=$((failures + 1))
+fi
+
+echo ""
+if [ $failures -eq 0 ]; then
+    echo "✅ All tests passed successfully!"
+    echo ""
+    echo "You can now:"
+    echo "1. Connect to the database: docker exec -it timescaledb psql -U postgres -d metrics"
+    echo "2. View logs: docker-compose logs -f"
+    echo "3. Send more metrics: echo 'metric.name:value|type|#tag:value' | nc -u -w0 localhost 8125"
+    echo "4. Stop services: docker-compose down"
+else
+    echo "⚠️  Some tests failed. Please check the output above for details."
+    echo ""
+    echo "Troubleshooting tips:"
+    echo "1. Check Telegraf logs: docker-compose logs telegraf"
+    echo "2. Check TimescaleDB logs: docker-compose logs timescaledb"
+    echo "3. Verify connectivity: docker exec telegraf telegraf --test"
+fi
+
+echo ""
+echo "Test complete!"

test-scripts/send-metrics.sh

···
+#!/bin/bash
+
+# Send test metrics to StatsD/Telegraf
+
+echo "Sending test metrics to StatsD on localhost:8125..."
+
+# Counter metrics
+for i in {1..10}; do
+  echo "quickdid.http.request.count:1|c|#method:GET,path:/resolve,status:200" | nc -u -w0 localhost 8125
+  echo "quickdid.http.request.count:1|c|#method:POST,path:/api,status:201" | nc -u -w0 localhost 8125
+  echo "quickdid.http.request.count:1|c|#method:GET,path:/resolve,status:404" | nc -u -w0 localhost 8125
+done
+
+# Gauge metrics
+echo "quickdid.resolver.rate_limit.available_permits:10|g" | nc -u -w0 localhost 8125
+echo "quickdid.resolver.rate_limit.available_permits:8|g" | nc -u -w0 localhost 8125
+echo "quickdid.resolver.rate_limit.available_permits:5|g" | nc -u -w0 localhost 8125
+
+# Timing metrics (in milliseconds)
+for i in {1..20}; do
+  duration=$((RANDOM % 100 + 10))
+  echo "quickdid.http.request.duration_ms:${duration}|ms|#method:GET,path:/resolve,status:200" | nc -u -w0 localhost 8125
+done
+
+for i in {1..10}; do
+  duration=$((RANDOM % 200 + 50))
+  echo "quickdid.http.request.duration_ms:${duration}|ms|#method:POST,path:/api,status:201" | nc -u -w0 localhost 8125
+done
+
+# Histogram metrics
+for i in {1..15}; do
+  resolution_time=$((RANDOM % 500 + 50))
+  echo "quickdid.resolver.resolution_time:${resolution_time}|h|#resolver:redis" | nc -u -w0 localhost 8125
+  echo "quickdid.resolver.resolution_time:$((resolution_time * 2))|h|#resolver:base" | nc -u -w0 localhost 8125
+done
+
+# Cache metrics
+echo "quickdid.cache.hit.count:45|c|#cache_type:redis" | nc -u -w0 localhost 8125
+echo "quickdid.cache.miss.count:5|c|#cache_type:redis" | nc -u -w0 localhost 8125
+echo "quickdid.cache.size:1024|g|#cache_type:memory" | nc -u -w0 localhost 8125
+
+echo "Metrics sent! Wait 15 seconds for Telegraf to flush..."
+sleep 15
+echo "Done!"

test-scripts/verify-queries.sql

···
+-- Test script to verify all metrics queries work correctly
+-- Run this after sending test metrics with send-metrics.sh
+
+\echo '===== CHECKING AVAILABLE TABLES ====='
+SELECT table_name 
+FROM information_schema.tables 
+WHERE table_schema = 'public' 
+  AND table_name LIKE 'quickdid%'
+ORDER BY table_name;
+
+\echo ''
+\echo '===== CHECKING TABLE STRUCTURES ====='
+\echo 'Structure of quickdid.http.request.count table:'
+\d "quickdid.http.request.count"
+
+\echo ''
+\echo 'Structure of quickdid.http.request.duration_ms table:'
+\d "quickdid.http.request.duration_ms"
+
+\echo ''
+\echo '===== QUERY 1: Recent HTTP Request Counts ====='
+SELECT 
+    time,
+    tags,
+    tags->>'method' as method,
+    tags->>'path' as path,
+    tags->>'status' as status,
+    value
+FROM "quickdid.http.request.count"
+WHERE time > NOW() - INTERVAL '1 hour'
+ORDER BY time DESC
+LIMIT 10;
+
+\echo ''
+\echo '===== QUERY 2: HTTP Request Duration Statistics by Endpoint ====='
+SELECT 
+    time_bucket('1 minute', time) AS minute,
+    tags->>'method' as method,
+    tags->>'path' as path,
+    tags->>'status' as status,
+    COUNT(*) as request_count,
+    AVG(mean) as avg_duration_ms,
+    MAX(p99) as p99_duration_ms,
+    MIN(mean) as min_duration_ms
+FROM "quickdid.http.request.duration_ms"
+WHERE time > NOW() - INTERVAL '1 hour'
+  AND tags IS NOT NULL
+GROUP BY minute, tags->>'method', tags->>'path', tags->>'status'
+ORDER BY minute DESC
+LIMIT 10;
+
+\echo ''
+\echo '===== QUERY 3: Rate Limiter Status Over Time ====='
+SELECT 
+    time,
+    value as available_permits
+FROM "quickdid.resolver.rate_limit.available_permits"
+WHERE time > NOW() - INTERVAL '1 hour'
+ORDER BY time DESC
+LIMIT 10;
+
+\echo ''
+\echo '===== QUERY 4: Resolver Performance Comparison ====='
+SELECT 
+    tags->>'resolver' as resolver_type,
+    COUNT(*) as sample_count,
+    AVG(mean) as avg_resolution_time_ms,
+    MAX(p99) as p99_resolution_time_ms,
+    MIN(mean) as min_resolution_time_ms
+FROM "quickdid.resolver.resolution_time"
+WHERE time > NOW() - INTERVAL '1 hour'
+  AND tags->>'resolver' IS NOT NULL
+GROUP BY tags->>'resolver'
+ORDER BY avg_resolution_time_ms;
+
+\echo ''
+\echo '===== QUERY 5: Cache Hit Rate Analysis ====='
+WITH cache_stats AS (
+    SELECT 
+        'hits' as metric_type,
+        SUM(value) as total_count
+    FROM "quickdid.cache.hit.count"
+    WHERE time > NOW() - INTERVAL '1 hour'
+    UNION ALL
+    SELECT 
+        'misses' as metric_type,
+        SUM(value) as total_count
+    FROM "quickdid.cache.miss.count"
+    WHERE time > NOW() - INTERVAL '1 hour'
+)
+SELECT 
+    SUM(CASE WHEN metric_type = 'hits' THEN total_count ELSE 0 END) as total_hits,
+    SUM(CASE WHEN metric_type = 'misses' THEN total_count ELSE 0 END) as total_misses,
+    CASE 
+        WHEN SUM(total_count) > 0 THEN 
+            ROUND(100.0 * SUM(CASE WHEN metric_type = 'hits' THEN total_count ELSE 0 END) / SUM(total_count), 2)
+        ELSE 0 
+    END as hit_rate_percentage
+FROM cache_stats;
+
+\echo ''
+\echo '===== QUERY 6: Hypertable Information ====='
+SELECT 
+    hypertable_schema,
+    hypertable_name,
+    owner,
+    num_dimensions,
+    num_chunks,
+    compression_enabled
+FROM timescaledb_information.hypertables
+WHERE hypertable_name LIKE 'quickdid%'
+ORDER BY hypertable_name;
+
+\echo ''
+\echo '===== QUERY 7: HTTP Error Rate by Endpoint ====='
+WITH status_counts AS (
+    SELECT 
+        time_bucket('5 minutes', time) as period,
+        tags->>'path' as path,
+        CASE 
+            WHEN (tags->>'status')::int >= 400 THEN 'error'
+            ELSE 'success'
+        END as status_category,
+        SUM(value) as request_count
+    FROM "quickdid.http.request.count"
+    WHERE time > NOW() - INTERVAL '1 hour'
+    GROUP BY period, path, status_category
+)
+SELECT 
+    period,
+    path,
+    SUM(CASE WHEN status_category = 'error' THEN request_count ELSE 0 END) as error_count,
+    SUM(CASE WHEN status_category = 'success' THEN request_count ELSE 0 END) as success_count,
+    CASE 
+        WHEN SUM(request_count) > 0 THEN 
+            ROUND(100.0 * SUM(CASE WHEN status_category = 'error' THEN request_count ELSE 0 END) / SUM(request_count), 2)
+        ELSE 0 
+    END as error_rate_percentage
+FROM status_counts
+GROUP BY period, path
+HAVING SUM(request_count) > 0
+ORDER BY period DESC, error_rate_percentage DESC;
+
+\echo ''
+\echo '===== TEST COMPLETED ====='