package query import ( "context" "fmt" "io" "log" "runtime" "slices" "sort" "sync" "time" "github.com/parquet-go/parquet-go" "go.iscode.ca/mats/internal/catalog" "go.iscode.ca/mats/internal/cold" "go.iscode.ca/mats/internal/hot" "go.iscode.ca/mats/internal/types" ) // SeriesResult holds the columnar output for a single time-series. type SeriesResult struct { MetricID types.MetricID `json:"metric_id"` Host string `json:"host"` Service string `json:"service"` Tags string `json:"tags,omitempty"` Timestamps []int64 `json:"timestamps"` Values []float64 `json:"values"` } type chunkSeriesPoints struct { timestamps []int64 values []float64 } // Engine coordinates hybrid queries spanning cold Parquet files and hot Pebble LSM. type Engine struct { catalog *catalog.Catalog hot *hot.PebbleBuffer storage cold.StorageClient bucketDuration time.Duration } func NewEngine( cat *catalog.Catalog, hot *hot.PebbleBuffer, storage cold.StorageClient, bucketDur time.Duration, ) *Engine { return &Engine{ catalog: cat, hot: hot, storage: storage, bucketDuration: bucketDur, } } // Query executes a hybrid query across both cold storage and the hot tier, // with optional server-side downsampling. func (e *Engine) Query(ctx context.Context, filter types.SeriesFilter, startTS, endTS int64, downsample ...DownsampleOptions) ([]SeriesResult, error) { startTime := time.Now() // 1. Resolve matching series IDs metricIDs, err := e.catalog.ResolveIDs(ctx, filter) if err != nil { return nil, fmt.Errorf("resolving metric IDs: %w", err) } if len(metricIDs) == 0 { return nil, nil } resultsMap := make(map[types.MetricID]*SeriesResult, len(metricIDs)) for _, id := range metricIDs { meta, _ := e.catalog.GetMetadata(ctx, id) resultsMap[id] = &SeriesResult{ MetricID: id, Host: meta.Host, Service: meta.Service, Tags: meta.Tags, } } var singleTarget types.MetricID hasSingleTarget := len(metricIDs) == 1 if hasSingleTarget { singleTarget = metricIDs[0] } // 2. Query Cold Tier (Committed Parquet manifests) manifests, err := e.catalog.ListCommitted(ctx, startTS, endTS) if err != nil { return nil, fmt.Errorf("listing manifests: %w", err) } coldMaxTS := int64(0) if len(manifests) > 0 { maxPointsTarget := 0 if len(downsample) > 0 && downsample[0].MaxPoints > 0 { maxPointsTarget = downsample[0].MaxPoints } targetPerChunk := 0 if maxPointsTarget > 0 && len(manifests) > 2 { targetPerChunk = max(150, (maxPointsTarget*3)/len(manifests)) } chunkResults := make([]map[types.MetricID]*chunkSeriesPoints, len(manifests)) validChunk := make([]bool, len(manifests)) numWorkers := min(len(manifests), min(8, max(2, runtime.GOMAXPROCS(0)))) jobs := make(chan int, len(manifests)) for i := range manifests { jobs <- i } close(jobs) var wg sync.WaitGroup wg.Add(numWorkers) for range numWorkers { go func() { defer wg.Done() for idx := range jobs { if ctx.Err() != nil { return } m := manifests[idx] if m.RecordCount == 0 || m.SizeBytes <= 0 { continue } readerAt, closer, openErr := cold.OpenReaderAt(ctx, e.storage, m.StorageKey, m.SizeBytes) if openErr != nil { if ctx.Err() == nil { log.Printf("[query] warning: skipping unreadable cold parquet chunk %s: %v", m.StorageKey, openErr) } continue } pf, pfErr := parquet.OpenFile(readerAt, m.SizeBytes) if pfErr != nil { _ = closer.Close() if ctx.Err() == nil { log.Printf("[query] warning: skipping unreadable cold parquet chunk %s: %v", m.StorageKey, pfErr) } continue } allInclusive := m.MinTS >= startTS && m.MaxTS <= endTS localPoints, scanErr := e.scanChunkPoints(ctx, pf, resultsMap, hasSingleTarget, singleTarget, startTS, endTS, allInclusive) _ = closer.Close() if scanErr != nil { if ctx.Err() == nil { log.Printf("[query] warning: error reading parquet chunk %s: %v", m.StorageKey, scanErr) } continue } if localPoints == nil { continue } // Intermediate per-chunk downsampling to bound memory usage if targetPerChunk > 0 { for _, pts := range localPoints { if len(pts.timestamps) > targetPerChunk { pts.timestamps, pts.values = DownsampleLTTB(pts.timestamps, pts.values, targetPerChunk) } } } chunkResults[idx] = localPoints validChunk[idx] = true } }() } wg.Wait() if ctx.Err() != nil { return nil, ctx.Err() } // Merge chunk results in original manifest order for idx := range manifests { if !validChunk[idx] { continue } m := manifests[idx] if m.MaxTS > coldMaxTS { coldMaxTS = m.MaxTS } cpMap := chunkResults[idx] if cpMap == nil { continue } for mID, pts := range cpMap { if len(pts.timestamps) > 0 { res := resultsMap[mID] res.Timestamps = append(res.Timestamps, pts.timestamps...) res.Values = append(res.Values, pts.values...) } } } } // 3. Query Hot Tier (Pebble LSM) // Query hot tier only for points newer than coldMaxTS (to prevent duplicate reads at the boundary) hotStartTS := startTS if coldMaxTS > 0 && coldMaxTS >= startTS { hotStartTS = coldMaxTS + 1 } if hotStartTS <= endTS { if ctx.Err() != nil { return nil, ctx.Err() } bucketDurSec := int64(e.bucketDuration.Seconds()) startBucket := types.BucketID((hotStartTS / 1000) / bucketDurSec) endBucket := types.BucketID((endTS / 1000) / bucketDurSec) for _, mID := range metricIDs { hotPoints, hotErr := e.hot.ScanMetricRange(ctx, mID, startBucket, endBucket, hotStartTS, endTS) if hotErr != nil { return nil, fmt.Errorf("scanning hot tier for metric %d: %w", mID, hotErr) } res := resultsMap[mID] for _, pt := range hotPoints { res.Timestamps = append(res.Timestamps, pt.Timestamp) res.Values = append(res.Values, pt.Value) } } } // 4. Finalize sort ordering for each series var results []SeriesResult for _, id := range metricIDs { res := resultsMap[id] if len(res.Timestamps) > 0 { if !slices.IsSorted(res.Timestamps) { sortPoints(res.Timestamps, res.Values) } if len(downsample) > 0 { res.Downsample(downsample[0]) } } results = append(results, *res) } if time.Since(startTime) > 3*time.Second { log.Printf("[query] completed scan of %d manifests in %v for %d series", len(manifests), time.Since(startTime), len(metricIDs)) } return results, nil } // scanChunkPoints extracts data points from a Parquet file using direct columnar decoding, // with automatic fallback to generic row reading if necessary. func (e *Engine) scanChunkPoints( ctx context.Context, pf *parquet.File, resultsMap map[types.MetricID]*SeriesResult, hasSingleTarget bool, singleTarget types.MetricID, startTS, endTS int64, allInclusive bool, ) (map[types.MetricID]*chunkSeriesPoints, error) { // Optimization 1: Embedded series dictionary in Parquet footer if dict, err := cold.ExtractSeriesDictionary(pf); err == nil && len(dict) > 0 { hasMetric := false for _, s := range dict { if hasSingleTarget { if s.MetricID == singleTarget { hasMetric = true break } } else if _, ok := resultsMap[s.MetricID]; ok { hasMetric = true break } } if !hasMetric { return nil, nil } } localPoints := make(map[types.MetricID]*chunkSeriesPoints) rowGroups := pf.RowGroups() if len(rowGroups) == 0 { return localPoints, nil } canUseColumnar := true for _, rg := range rowGroups { if len(rg.ColumnChunks()) != 3 { canUseColumnar = false break } } if canUseColumnar { intBuf := make([]int64, 4096) valBuf := make([]parquet.Value, 4096) doubleBuf := make([]float64, 4096) for _, rg := range rowGroups { if ctx.Err() != nil { return nil, ctx.Err() } numRows := rg.NumRows() if numRows == 0 { continue } chunks := rg.ColumnChunks() // 1. Scan Column 0 (MetricID) p0 := chunks[0].Pages() metricIDs := make([]int64, 0, numRows) for { p, err := p0.ReadPage() if err != nil { break } v := p.Values() if r, ok := v.(parquet.Int64Reader); ok { for { n, rErr := r.ReadInt64s(intBuf) if n > 0 { metricIDs = append(metricIDs, intBuf[:n]...) } if rErr != nil { break } } } else { for { n, rErr := v.ReadValues(valBuf) for i := range n { metricIDs = append(metricIDs, valBuf[i].Int64()) } if rErr != nil { break } } } } _ = p0.Close() if len(metricIDs) == 0 { canUseColumnar = false break } // Fast pruning: check if this row group contains ANY target metric hasTargetInRG := false for _, m := range metricIDs { mID := types.MetricID(m) if hasSingleTarget { if mID == singleTarget { hasTargetInRG = true break } } else if _, ok := resultsMap[mID]; ok { hasTargetInRG = true break } } if !hasTargetInRG { continue // Skip Column 1 and Column 2 completely! } // 2. Scan Column 1 (Timestamp) p1 := chunks[1].Pages() timestamps := make([]int64, 0, numRows) for { p, err := p1.ReadPage() if err != nil { break } v := p.Values() if r, ok := v.(parquet.Int64Reader); ok { for { n, rErr := r.ReadInt64s(intBuf) if n > 0 { timestamps = append(timestamps, intBuf[:n]...) } if rErr != nil { break } } } else { for { n, rErr := v.ReadValues(valBuf) for i := range n { timestamps = append(timestamps, valBuf[i].Int64()) } if rErr != nil { break } } } } _ = p1.Close() // 3. Scan Column 2 (Value) p2 := chunks[2].Pages() values := make([]float64, 0, numRows) for { p, err := p2.ReadPage() if err != nil { break } v := p.Values() if r, ok := v.(parquet.DoubleReader); ok { for { n, rErr := r.ReadDoubles(doubleBuf) if n > 0 { values = append(values, doubleBuf[:n]...) } if rErr != nil { break } } } else { for { n, rErr := v.ReadValues(valBuf) for i := range n { values = append(values, valBuf[i].Double()) } if rErr != nil { break } } } } _ = p2.Close() limit := min(len(metricIDs), len(timestamps), len(values)) for i := range limit { mID := types.MetricID(metricIDs[i]) if hasSingleTarget { if mID != singleTarget { continue } } else if _, ok := resultsMap[mID]; !ok { continue } ts := timestamps[i] if !allInclusive && (ts < startTS || ts > endTS) { continue } pts := localPoints[mID] if pts == nil { pts = &chunkSeriesPoints{} localPoints[mID] = pts } pts.timestamps = append(pts.timestamps, ts) pts.values = append(pts.values, values[i]) } } if canUseColumnar { return localPoints, nil } } // Fallback to row-by-row GenericReader localPoints = make(map[types.MetricID]*chunkSeriesPoints) reader := parquet.NewGenericReader[cold.ParquetRow](pf) defer func() { _ = reader.Close() }() batch := make([]cold.ParquetRow, 8192) for { if ctx.Err() != nil { return nil, ctx.Err() } n, readErr := reader.Read(batch) for i := range n { mID := types.MetricID(batch[i].MetricID) if hasSingleTarget { if mID == singleTarget { ts := batch[i].Timestamp if allInclusive || (ts >= startTS && ts <= endTS) { pts := localPoints[singleTarget] if pts == nil { pts = &chunkSeriesPoints{} localPoints[singleTarget] = pts } pts.timestamps = append(pts.timestamps, ts) pts.values = append(pts.values, batch[i].Value) } } } else if _, ok := resultsMap[mID]; ok { ts := batch[i].Timestamp if allInclusive || (ts >= startTS && ts <= endTS) { pts := localPoints[mID] if pts == nil { pts = &chunkSeriesPoints{} localPoints[mID] = pts } pts.timestamps = append(pts.timestamps, ts) pts.values = append(pts.values, batch[i].Value) } } } if readErr != nil { if readErr != io.EOF { return nil, readErr } break } } return localPoints, nil } type pointSorter struct { ts []int64 val []float64 } func (s pointSorter) Len() int { return len(s.ts) } func (s pointSorter) Less(i, j int) bool { return s.ts[i] < s.ts[j] } func (s pointSorter) Swap(i, j int) { s.ts[i], s.ts[j] = s.ts[j], s.ts[i] s.val[i], s.val[j] = s.val[j], s.val[i] } func sortPoints(ts []int64, val []float64) { sort.Sort(pointSorter{ts: ts, val: val}) }