package query import ( "context" "path/filepath" "testing" "time" "go.iscode.ca/mats/internal/catalog" "go.iscode.ca/mats/internal/cold" "go.iscode.ca/mats/internal/handover" "go.iscode.ca/mats/internal/hot" "go.iscode.ca/mats/internal/types" ) func TestEngine_HybridQuery(t *testing.T) { baseDir := t.TempDir() catDir := filepath.Join(baseDir, "catalog") hotDir := filepath.Join(baseDir, "hot") scratchDir := filepath.Join(baseDir, "scratch") cat, err := catalog.Open(catDir) if err != nil { t.Fatalf("failed opening catalog: %v", err) } defer func() { _ = cat.Close() }() hotBuf, err := hot.Open(hotDir) if err != nil { t.Fatalf("failed opening hot buffer: %v", err) } defer func() { _ = hotBuf.Close() }() storage, err := cold.NewLocalStorage(baseDir) if err != nil { t.Fatalf("failed creating local storage: %v", err) } bucketDur := time.Hour flusher, err := handover.NewFlusher(hotBuf, cat, storage, scratchDir, bucketDur, 15*time.Minute) if err != nil { t.Fatalf("failed creating flusher: %v", err) } ctx := context.Background() mID, err := cat.GetOrCreate(ctx, "nas-01", "disk.usage", "") if err != nil { t.Fatalf("failed creating series: %v", err) } // Bucket 100 start time in milliseconds (100 * 3600 * 1000) b100TS := int64(100 * 3600 * 1000) b101TS := int64(101 * 3600 * 1000) // 1. Write points for Bucket 100 and flush to Cold Tier b100 := types.BucketID(100) coldPoints := []types.DataPoint{ {MetricID: mID, Timestamp: b100TS + 1000, Value: 10.0}, {MetricID: mID, Timestamp: b100TS + 1010, Value: 11.0}, {MetricID: mID, Timestamp: b100TS + 1020, Value: 12.0}, } if err := hotBuf.PutBatch(coldPoints, b100); err != nil { t.Fatalf("failed writing cold points to hotBuf: %v", err) } if _, err := flusher.FlushBucket(ctx, b100); err != nil { t.Fatalf("failed flushing bucket 100: %v", err) } // 2. Write points for Bucket 101 staying in Hot Tier b101 := types.BucketID(101) hotPoints := []types.DataPoint{ {MetricID: mID, Timestamp: b101TS + 10, Value: 20.0}, {MetricID: mID, Timestamp: b101TS + 20, Value: 21.0}, {MetricID: mID, Timestamp: b101TS + 30, Value: 22.0}, } if err := hotBuf.PutBatch(hotPoints, b101); err != nil { t.Fatalf("failed writing hot points: %v", err) } // 3. Execute hybrid query spanning both tiers [b100TS+1005, b101TS+25] qEngine := NewEngine(cat, hotBuf, storage, bucketDur) results, err := qEngine.Query(ctx, types.SeriesFilter{ HostRegex: "nas.*", ServiceRegex: "disk.*", }, b100TS+1005, b101TS+25) if err != nil { t.Fatalf("Query failed: %v", err) } if len(results) != 1 { t.Fatalf("expected 1 series result, got %d", len(results)) } res := results[0] if res.MetricID != mID || res.Host != "nas-01" || res.Service != "disk.usage" { t.Errorf("unexpected series metadata: %+v", res) } // Expected timestamps in range: // From Cold: b100TS+1010, b100TS+1020 // From Hot: b101TS+10, b101TS+20 expectedTS := []int64{b100TS + 1010, b100TS + 1020, b101TS + 10, b101TS + 20} expectedVals := []float64{11.0, 12.0, 20.0, 21.0} if len(res.Timestamps) != len(expectedTS) { t.Fatalf("expected %d points, got %d: ts=%v vals=%v", len(expectedTS), len(res.Timestamps), res.Timestamps, res.Values) } for i := range expectedTS { if res.Timestamps[i] != expectedTS[i] { t.Errorf("point %d timestamp mismatch: got %d, expected %d", i, res.Timestamps[i], expectedTS[i]) } if res.Values[i] != expectedVals[i] { t.Errorf("point %d value mismatch: got %f, expected %f", i, res.Values[i], expectedVals[i]) } } } func TestEngine_MultiNodeBucketQuery(t *testing.T) { baseDir := t.TempDir() catDir := filepath.Join(baseDir, "catalog") scratchDir := filepath.Join(baseDir, "scratch") storage, err := cold.NewLocalStorage(baseDir) if err != nil { t.Fatalf("failed creating local storage: %v", err) } cat, err := catalog.Open(catDir) if err != nil { t.Fatalf("failed opening catalog: %v", err) } defer func() { _ = cat.Close() }() ctx := context.Background() mID, err := cat.GetOrCreate(ctx, "srv-cluster", "cpu.util", "") if err != nil { t.Fatalf("failed creating series: %v", err) } bucketDur := time.Hour b100TS := int64(100 * 3600 * 1000) b100 := types.BucketID(100) // Ingress Node 1 flushes chunk for bucket 100 hot1Dir := filepath.Join(baseDir, "hot1") hotBuf1, _ := hot.Open(hot1Dir) flusher1, _ := handover.NewFlusher(hotBuf1, cat, storage, scratchDir, bucketDur, 15*time.Minute, "node-1") _ = hotBuf1.PutBatch([]types.DataPoint{ {MetricID: mID, Timestamp: b100TS + 10, Value: 1.0}, {MetricID: mID, Timestamp: b100TS + 30, Value: 3.0}, {MetricID: mID, Timestamp: b100TS + 50, Value: 5.0}, }, b100) _, _ = flusher1.FlushBucket(ctx, b100) _ = hotBuf1.Close() // Ingress Node 2 flushes interleaved chunk for bucket 100 hot2Dir := filepath.Join(baseDir, "hot2") hotBuf2, _ := hot.Open(hot2Dir) flusher2, _ := handover.NewFlusher(hotBuf2, cat, storage, scratchDir, bucketDur, 15*time.Minute, "node-2") _ = hotBuf2.PutBatch([]types.DataPoint{ {MetricID: mID, Timestamp: b100TS + 20, Value: 2.0}, {MetricID: mID, Timestamp: b100TS + 40, Value: 4.0}, {MetricID: mID, Timestamp: b100TS + 60, Value: 6.0}, }, b100) _, _ = flusher2.FlushBucket(ctx, b100) _ = hotBuf2.Close() // Query node querying both cold chunks emptyHotDir := filepath.Join(baseDir, "empty_hot") emptyHot, _ := hot.Open(emptyHotDir) defer func() { _ = emptyHot.Close() }() qEngine := NewEngine(cat, emptyHot, storage, bucketDur) results, err := qEngine.Query(ctx, types.SeriesFilter{ HostRegex: "srv-cluster", ServiceRegex: "cpu.*", }, b100TS, b100TS+100) if err != nil { t.Fatalf("query failed: %v", err) } if len(results) != 1 { t.Fatalf("expected 1 result, got %d", len(results)) } res := results[0] if len(res.Timestamps) != 6 { t.Fatalf("expected 6 merged points, got %d: %v", len(res.Timestamps), res.Timestamps) } expectedTS := []int64{b100TS + 10, b100TS + 20, b100TS + 30, b100TS + 40, b100TS + 50, b100TS + 60} expectedVals := []float64{1.0, 2.0, 3.0, 4.0, 5.0, 6.0} for i := range expectedTS { if res.Timestamps[i] != expectedTS[i] { t.Errorf("timestamp %d mismatch: got %d, want %d", i, res.Timestamps[i], expectedTS[i]) } if res.Values[i] != expectedVals[i] { t.Errorf("value %d mismatch: got %f, want %f", i, res.Values[i], expectedVals[i]) } } } func TestEngine_UnreadableColdChunkGracefulSkip(t *testing.T) { baseDir := t.TempDir() catDir := filepath.Join(baseDir, "catalog") hotDir := filepath.Join(baseDir, "hot") cat, err := catalog.Open(catDir) if err != nil { t.Fatalf("failed opening catalog: %v", err) } defer func() { _ = cat.Close() }() hotBuf, err := hot.Open(hotDir) if err != nil { t.Fatalf("failed opening hot buffer: %v", err) } defer func() { _ = hotBuf.Close() }() storage, err := cold.NewLocalStorage(baseDir) if err != nil { t.Fatalf("failed creating local storage: %v", err) } ctx := context.Background() mID, err := cat.GetOrCreate(ctx, "srv-missing", "cpu.util", "") if err != nil { t.Fatalf("failed creating series: %v", err) } b100TS := int64(100 * 3600 * 1000) b100 := types.BucketID(100) // Save a manifest in catalog that points to a non-existent cold file manifest := &types.BucketManifest{ BucketID: b100, StorageKey: "cold/nonexistent_bucket_100.parquet", MinTS: b100TS, MaxTS: b100TS + 1000, RecordCount: 10, SizeBytes: 4096, State: types.BucketStateCommitted, } if err := cat.SaveManifest(ctx, manifest); err != nil { t.Fatalf("failed saving manifest: %v", err) } // Put a live point in hot buffer hotTS := b100TS + 500 _ = hotBuf.Put(b100, mID, hotTS, 42.0) qEngine := NewEngine(cat, hotBuf, storage, time.Hour) // Query across range intersecting the missing cold file results, err := qEngine.Query(ctx, types.SeriesFilter{ HostRegex: "srv-missing", ServiceRegex: "cpu.*", }, b100TS, b100TS+1000) if err != nil { t.Fatalf("expected query to succeed with warning, got error: %v", err) } if len(results) != 1 { t.Fatalf("expected 1 result, got %d", len(results)) } if len(results[0].Timestamps) != 1 || results[0].Timestamps[0] != hotTS { t.Errorf("expected hot point to be recovered despite missing cold chunk: %+v", results[0]) } }