package query import ( "context" "encoding/json" "fmt" "io" "net/http" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "go.iscode.ca/mats/internal/catalog" "go.iscode.ca/mats/internal/cold" "go.iscode.ca/mats/internal/handover" "go.iscode.ca/mats/internal/hot" "go.iscode.ca/mats/internal/ingest" "go.iscode.ca/mats/internal/types" ) func TestAPIServer(t *testing.T) { baseDir := t.TempDir() catDir := filepath.Join(baseDir, "catalog") hotDir := filepath.Join(baseDir, "hot") cat, err := catalog.Open(catDir) if err != nil { t.Fatalf("failed opening catalog: %v", err) } defer func() { _ = cat.Close() }() hotBuf, err := hot.Open(hotDir) if err != nil { t.Fatalf("failed opening hot buffer: %v", err) } defer func() { _ = hotBuf.Close() }() storage, err := cold.NewLocalStorage(baseDir) if err != nil { t.Fatalf("failed creating local storage: %v", err) } disp := ingest.NewDispatcher(cat, hotBuf, time.Hour, 15*time.Minute) qEngine := NewEngine(cat, hotBuf, storage, time.Hour) server := NewServer("127.0.0.1:0", qEngine, disp) if err := server.Start(); err != nil { t.Fatalf("failed starting HTTP server: %v", err) } defer func() { _ = server.Stop(context.Background()) }() baseURL := "http://" + server.Addr().String() // Seed one series and point ctx := context.Background() mID, err := cat.GetOrCreate(ctx, "router", "ping.latency", "") if err != nil { t.Fatalf("failed seeding series: %v", err) } nowMS := time.Now().UnixMilli() _ = hotBuf.Put(types.BucketID((nowMS/1000)/3600), mID, nowMS, 15.4) // 1. Test /api/v1/query resp, err := http.Get(baseURL + "/api/v1/query?host=router&service=ping.latency") if err != nil { t.Fatalf("GET /api/v1/query failed: %v", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { t.Fatalf("expected 200 OK, got %d", resp.StatusCode) } var results []SeriesResult if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { t.Fatalf("failed decoding query response: %v", err) } if len(results) != 1 || len(results[0].Values) != 1 { t.Fatalf("expected 1 result with 1 value, got %+v", results) } // 2. Test /api/v1/render?format=svg svgResp, err := http.Get(baseURL + "/api/v1/render?host=router&service=ping.latency&format=svg") if err != nil { t.Fatalf("GET /api/v1/render?format=svg failed: %v", err) } defer func() { _ = svgResp.Body.Close() }() if ct := svgResp.Header.Get("Content-Type"); !strings.Contains(ct, "image/svg+xml") { t.Errorf("expected SVG content-type, got %s", ct) } svgBytes, _ := io.ReadAll(svgResp.Body) if !strings.HasPrefix(string(svgBytes), " must bypass cache and be X-Cache: MISS req4, _ := http.NewRequest("GET", renderURL, nil) req4.Header.Set("Cache-Control", "no-cache") resp4, err := http.DefaultClient.Do(req4) if err != nil { t.Fatalf("bypass request failed: %v", err) } defer func() { _ = resp4.Body.Close() }() if resp4.Header.Get("X-Cache") != "MISS" { t.Errorf("expected X-Cache: MISS with no-cache header, got %s", resp4.Header.Get("X-Cache")) } // 5. Verify /api/v1/stats includes cache metrics statsResp, err := http.Get(baseURL + "/api/v1/stats") if err != nil { t.Fatalf("stats failed: %v", err) } defer func() { _ = statsResp.Body.Close() }() var stats map[string]uint64 if err := json.NewDecoder(statsResp.Body).Decode(&stats); err != nil { t.Fatalf("stats decode failed: %v", err) } if stats["cache_hits"] < 2 { t.Errorf("expected at least 2 cache hits, got %d", stats["cache_hits"]) } if stats["cache_items"] == 0 { t.Errorf("expected cached items > 0, got %d", stats["cache_items"]) } } func TestAPIServer_Downsampling(t *testing.T) { baseDir := t.TempDir() catDir := filepath.Join(baseDir, "catalog") hotDir := filepath.Join(baseDir, "hot") cat, err := catalog.Open(catDir) if err != nil { t.Fatalf("failed opening catalog: %v", err) } defer func() { _ = cat.Close() }() hotBuf, err := hot.Open(hotDir) if err != nil { t.Fatalf("failed opening hot buffer: %v", err) } defer func() { _ = hotBuf.Close() }() storage, err := cold.NewLocalStorage(baseDir) if err != nil { t.Fatalf("failed creating local storage: %v", err) } disp := ingest.NewDispatcher(cat, hotBuf, 24*time.Hour, 15*time.Minute) qEngine := NewEngine(cat, hotBuf, storage, 24*time.Hour) server := NewServer("127.0.0.1:0", qEngine, disp, WithDefaultMaxPoints(1500)) if err := server.Start(); err != nil { t.Fatalf("failed starting HTTP server: %v", err) } defer func() { _ = server.Stop(context.Background()) }() baseURL := "http://" + server.Addr().String() // Seed 3000 points (1 point per second for 3000 seconds) ctx := context.Background() mID, err := cat.GetOrCreate(ctx, "srv-downsample", "sensor.val", "") if err != nil { t.Fatalf("seeding series failed: %v", err) } now := time.Now().UnixMilli() startTS := now - 3000*1000 bucketID := types.BucketID((startTS / 1000) / 86400) for i := range 3000 { ts := startTS + int64(i*1000) _ = hotBuf.Put(bucketID, mID, ts, float64(i%100)) } queryBase := fmt.Sprintf("%s/api/v1/query?host=srv-downsample&service=sensor.val&from=%d&to=%d&exact=1", baseURL, startTS, now) // 1. Default max_points: 1500 resp1, err := http.Get(queryBase) if err != nil { t.Fatalf("default query failed: %v", err) } defer func() { _ = resp1.Body.Close() }() var res1 []SeriesResult if err := json.NewDecoder(resp1.Body).Decode(&res1); err != nil { t.Fatalf("decoding res1: %v", err) } if len(res1) != 1 || len(res1[0].Timestamps) != 1500 { t.Fatalf("expected 1500 downsampled points, got %d", len(res1[0].Timestamps)) } // 2. Explicit max_points=250 resp2, err := http.Get(queryBase + "&max_points=250") if err != nil { t.Fatalf("max_points=250 query failed: %v", err) } defer func() { _ = resp2.Body.Close() }() var res2 []SeriesResult if err := json.NewDecoder(resp2.Body).Decode(&res2); err != nil { t.Fatalf("decoding res2: %v", err) } if len(res2) != 1 || len(res2[0].Timestamps) != 250 { t.Fatalf("expected 250 downsampled points, got %d", len(res2[0].Timestamps)) } // 3. Raw query (raw=1 bypasses downsampling) resp3, err := http.Get(queryBase + "&raw=1") if err != nil { t.Fatalf("raw query failed: %v", err) } defer func() { _ = resp3.Body.Close() }() var res3 []SeriesResult if err := json.NewDecoder(resp3.Body).Decode(&res3); err != nil { t.Fatalf("decoding res3: %v", err) } if len(res3) != 1 || len(res3[0].Timestamps) != 3000 { t.Fatalf("expected 3000 raw points, got %d", len(res3[0].Timestamps)) } // 4. Time-bucket step downsampling (step=10s) resp4, err := http.Get(queryBase + "&step=10s&method=avg") if err != nil { t.Fatalf("step query failed: %v", err) } defer func() { _ = resp4.Body.Close() }() var res4 []SeriesResult if err := json.NewDecoder(resp4.Body).Decode(&res4); err != nil { t.Fatalf("decoding res4: %v", err) } // 3000s / 10s = ~300 buckets if len(res4) != 1 || len(res4[0].Timestamps) < 290 || len(res4[0].Timestamps) > 310 { t.Fatalf("expected ~300 bucketed points, got %d", len(res4[0].Timestamps)) } // 5. Render SVG respects downsampling renderURL := fmt.Sprintf("%s/api/v1/render?host=srv-downsample&service=sensor.val&from=%d&to=%d&format=svg&w=400&h=200", baseURL, startTS, now) renderResp, err := http.Get(renderURL) if err != nil { t.Fatalf("render request failed: %v", err) } defer func() { _ = renderResp.Body.Close() }() if renderResp.StatusCode != http.StatusOK { t.Errorf("expected 200 OK for SVG render, got %d", renderResp.StatusCode) } } func TestAPIServer_SkipEmpty(t *testing.T) { baseDir := t.TempDir() catDir := filepath.Join(baseDir, "catalog") hotDir := filepath.Join(baseDir, "hot") cat, err := catalog.Open(catDir) if err != nil { t.Fatalf("failed opening catalog: %v", err) } defer func() { _ = cat.Close() }() hotBuf, err := hot.Open(hotDir) if err != nil { t.Fatalf("failed opening hot buffer: %v", err) } defer func() { _ = hotBuf.Close() }() storage, err := cold.NewLocalStorage(baseDir) if err != nil { t.Fatalf("failed creating local storage: %v", err) } disp := ingest.NewDispatcher(cat, hotBuf, 24*time.Hour, 15*time.Minute) qEngine := NewEngine(cat, hotBuf, storage, 24*time.Hour) server := NewServer("127.0.0.1:0", qEngine, disp) if err := server.Start(); err != nil { t.Fatalf("failed starting HTTP server: %v", err) } defer func() { _ = server.Stop(context.Background()) }() baseURL := "http://" + server.Addr().String() ctx := context.Background() // 1. Create active series with a point mID1, _ := cat.GetOrCreate(ctx, "srv-1", "active.metric", "") nowMS := time.Now().UnixMilli() _ = hotBuf.Put(types.BucketID((nowMS/1000)/86400), mID1, nowMS, 99.0) // 2. Create dormant series with NO points _, _ = cat.GetOrCreate(ctx, "srv-2", "dormant.metric", "") // Default query: includes both series respAll, err := http.Get(baseURL + "/api/v1/query?host=.*&service=.*") if err != nil { t.Fatalf("query failed: %v", err) } defer func() { _ = respAll.Body.Close() }() var allResults []SeriesResult _ = json.NewDecoder(respAll.Body).Decode(&allResults) if len(allResults) != 2 { t.Fatalf("expected 2 series in default query, got %d", len(allResults)) } // Query with skip_empty=1: returns only active series respFiltered, err := http.Get(baseURL + "/api/v1/query?host=.*&service=.*&skip_empty=1") if err != nil { t.Fatalf("query with skip_empty failed: %v", err) } defer func() { _ = respFiltered.Body.Close() }() var filteredResults []SeriesResult _ = json.NewDecoder(respFiltered.Body).Decode(&filteredResults) if len(filteredResults) != 1 { t.Fatalf("expected 1 series with skip_empty=1, got %d", len(filteredResults)) } if filteredResults[0].Service != "active.metric" { t.Errorf("expected active.metric, got %s", filteredResults[0].Service) } } type slowStorage struct { delay time.Duration fetchCount atomic.Int32 storage cold.StorageClient } func (s *slowStorage) PutObject(ctx context.Context, key string, r io.Reader, size int64) error { return s.storage.PutObject(ctx, key, r, size) } func (s *slowStorage) GetObject(ctx context.Context, key string) (io.ReadCloser, error) { s.fetchCount.Add(1) select { case <-time.After(s.delay): return s.storage.GetObject(ctx, key) case <-ctx.Done(): return nil, ctx.Err() } } func (s *slowStorage) GetObjectRange(ctx context.Context, key string, start, end int64) (io.ReadCloser, error) { s.fetchCount.Add(1) select { case <-time.After(s.delay): return s.storage.GetObjectRange(ctx, key, start, end) case <-ctx.Done(): return nil, ctx.Err() } } func (s *slowStorage) DeleteObject(ctx context.Context, key string) error { return s.storage.DeleteObject(ctx, key) } func (s *slowStorage) StatObject(ctx context.Context, key string) (*cold.ObjectMeta, error) { return s.storage.StatObject(ctx, key) } func (s *slowStorage) ListObjects(ctx context.Context, prefix string) ([]cold.ObjectMeta, error) { return s.storage.ListObjects(ctx, prefix) } func TestAPIServer_QueryTimeout(t *testing.T) { baseDir := t.TempDir() catDir := filepath.Join(baseDir, "catalog") hotDir := filepath.Join(baseDir, "hot") scratchDir := filepath.Join(baseDir, "scratch") cat, err := catalog.Open(catDir) if err != nil { t.Fatalf("failed opening catalog: %v", err) } defer func() { _ = cat.Close() }() hotBuf, err := hot.Open(hotDir) if err != nil { t.Fatalf("failed opening hot buffer: %v", err) } defer func() { _ = hotBuf.Close() }() storage, err := cold.NewLocalStorage(baseDir) if err != nil { t.Fatalf("failed creating local storage: %v", err) } flusher, err := handover.NewFlusher(hotBuf, cat, storage, scratchDir, time.Hour, 15*time.Minute) if err != nil { t.Fatalf("failed creating flusher: %v", err) } ctx := context.Background() mID, err := cat.GetOrCreate(ctx, "router", "bandwidth.in", "") if err != nil { t.Fatalf("failed creating series: %v", err) } b100 := types.BucketID(100) b100TS := int64(100 * 3600 * 1000) pts := []types.DataPoint{ {MetricID: mID, Timestamp: b100TS + 100, Value: 100.0}, } if err := hotBuf.PutBatch(pts, b100); err != nil { t.Fatalf("failed putting batch: %v", err) } if _, err := flusher.FlushBucket(ctx, b100); err != nil { t.Fatalf("failed flushing bucket: %v", err) } slow := &slowStorage{ delay: 300 * time.Millisecond, storage: storage, } disp := ingest.NewDispatcher(cat, hotBuf, time.Hour, 15*time.Minute) qEngine := NewEngine(cat, hotBuf, slow, time.Hour) server := NewServer("127.0.0.1:0", qEngine, disp, WithQueryTimeout(50*time.Millisecond)) if err := server.Start(); err != nil { t.Fatalf("failed starting HTTP server: %v", err) } defer func() { _ = server.Stop(context.Background()) }() baseURL := "http://" + server.Addr().String() // 1. /api/v1/query should return 504 Gateway Timeout rather than dropping the TCP connection queryURL := fmt.Sprintf("%s/api/v1/query?host=router&service=bandwidth.in&from=%d&to=%d", baseURL, b100TS, b100TS+3600000) resp, err := http.Get(queryURL) if err != nil { t.Fatalf("request failed unexpectedly: %v", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusGatewayTimeout { t.Fatalf("expected 504 Gateway Timeout, got %d", resp.StatusCode) } body, _ := io.ReadAll(resp.Body) if !strings.Contains(string(body), "query timeout exceeded") { t.Fatalf("expected 'query timeout exceeded' in response, got %s", string(body)) } // 2. /api/v1/render should also return 504 Gateway Timeout renderURL := fmt.Sprintf("%s/api/v1/render?host=router&service=bandwidth.in&from=%d&to=%d&format=ascii", baseURL, b100TS, b100TS+3600000) rResp, err := http.Get(renderURL) if err != nil { t.Fatalf("render request failed unexpectedly: %v", err) } defer func() { _ = rResp.Body.Close() }() if rResp.StatusCode != http.StatusGatewayTimeout { t.Fatalf("expected 504 Gateway Timeout for render, got %d", rResp.StatusCode) } } func TestAPIServer_SingleflightCoalescing(t *testing.T) { baseDir := t.TempDir() catDir := filepath.Join(baseDir, "catalog") hotDir := filepath.Join(baseDir, "hot") scratchDir := filepath.Join(baseDir, "scratch") cat, err := catalog.Open(catDir) if err != nil { t.Fatalf("failed opening catalog: %v", err) } defer func() { _ = cat.Close() }() hotBuf, err := hot.Open(hotDir) if err != nil { t.Fatalf("failed opening hot buffer: %v", err) } defer func() { _ = hotBuf.Close() }() storage, err := cold.NewLocalStorage(baseDir) if err != nil { t.Fatalf("failed creating local storage: %v", err) } flusher, err := handover.NewFlusher(hotBuf, cat, storage, scratchDir, time.Hour, 15*time.Minute) if err != nil { t.Fatalf("failed creating flusher: %v", err) } ctx := context.Background() mID, err := cat.GetOrCreate(ctx, "router", "coalesce.test", "") if err != nil { t.Fatalf("failed creating series: %v", err) } b200 := types.BucketID(200) b200TS := int64(200 * 3600 * 1000) pts := []types.DataPoint{ {MetricID: mID, Timestamp: b200TS + 500, Value: 42.0}, } if err := hotBuf.PutBatch(pts, b200); err != nil { t.Fatalf("failed putting batch: %v", err) } if _, err := flusher.FlushBucket(ctx, b200); err != nil { t.Fatalf("failed flushing bucket: %v", err) } slow := &slowStorage{ delay: 100 * time.Millisecond, storage: storage, } disp := ingest.NewDispatcher(cat, hotBuf, time.Hour, 15*time.Minute) qEngine := NewEngine(cat, hotBuf, slow, time.Hour) server := NewServer("127.0.0.1:0", qEngine, disp, WithQueryTimeout(5*time.Second)) if err := server.Start(); err != nil { t.Fatalf("failed starting HTTP server: %v", err) } defer func() { _ = server.Stop(context.Background()) }() baseURL := "http://" + server.Addr().String() queryURL := fmt.Sprintf("%s/api/v1/query?host=router&service=coalesce.test&from=%d&to=%d&exact=true", baseURL, b200TS, b200TS+3600000) // Launch 5 concurrent queries for the same uncached query const concurrency = 5 var wg sync.WaitGroup wg.Add(concurrency) errorsChan := make(chan error, concurrency) for range concurrency { go func() { defer wg.Done() resp, err := http.Get(queryURL) if err != nil { errorsChan <- err return } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { errorsChan <- fmt.Errorf("expected 200 OK, got %d", resp.StatusCode) return } var res []SeriesResult if err := json.NewDecoder(resp.Body).Decode(&res); err != nil { errorsChan <- fmt.Errorf("failed decoding: %w", err) return } if len(res) != 1 || len(res[0].Values) != 1 || res[0].Values[0] != 42.0 { errorsChan <- fmt.Errorf("unexpected results: %+v", res) return } }() } wg.Wait() close(errorsChan) for err := range errorsChan { t.Fatalf("concurrent query failure: %v", err) } // Because of singleflight coalescing, the slow storage GetObject was called only 1 time if count := slow.fetchCount.Load(); count != 1 { t.Fatalf("expected singleflight to coalesce 5 requests to 1 storage fetch, got %d", count) } }