Something went wrong. Try again.
Metric and Time-Series Engine: A Minimal Archiving Time Series Storage Service
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535package query
import ( "context" "fmt" "io" "log" "runtime" "slices" "sort" "sync" "time"
"github.com/parquet-go/parquet-go" "go.iscode.ca/mats/internal/catalog" "go.iscode.ca/mats/internal/cold" "go.iscode.ca/mats/internal/hot" "go.iscode.ca/mats/internal/types")
// SeriesResult holds the columnar output for a single time-series.type SeriesResult struct { MetricID types.MetricID `json:"metric_id"` Host string `json:"host"` Service string `json:"service"` Tags string `json:"tags,omitempty"` Timestamps []int64 `json:"timestamps"` Values []float64 `json:"values"`}
type chunkSeriesPoints struct { timestamps []int64 values []float64}
// Engine coordinates hybrid queries spanning cold Parquet files and hot Pebble LSM.type Engine struct { catalog *catalog.Catalog hot *hot.PebbleBuffer storage cold.StorageClient bucketDuration time.Duration}
func NewEngine( cat *catalog.Catalog, hot *hot.PebbleBuffer, storage cold.StorageClient, bucketDur time.Duration,) *Engine { return &Engine{ catalog: cat, hot: hot, storage: storage, bucketDuration: bucketDur, }}
// Query executes a hybrid query across both cold storage and the hot tier,// with optional server-side downsampling.func (e *Engine) Query(ctx context.Context, filter types.SeriesFilter, startTS, endTS int64, downsample ...DownsampleOptions) ([]SeriesResult, error) { startTime := time.Now() // 1. Resolve matching series IDs metricIDs, err := e.catalog.ResolveIDs(ctx, filter) if err != nil { return nil, fmt.Errorf("resolving metric IDs: %w", err) } if len(metricIDs) == 0 { return nil, nil }
resultsMap := make(map[types.MetricID]*SeriesResult, len(metricIDs)) for _, id := range metricIDs { meta, _ := e.catalog.GetMetadata(ctx, id) resultsMap[id] = &SeriesResult{ MetricID: id, Host: meta.Host, Service: meta.Service, Tags: meta.Tags, } }
var singleTarget types.MetricID hasSingleTarget := len(metricIDs) == 1 if hasSingleTarget { singleTarget = metricIDs[0] }
// 2. Query Cold Tier (Committed Parquet manifests) manifests, err := e.catalog.ListCommitted(ctx, startTS, endTS) if err != nil { return nil, fmt.Errorf("listing manifests: %w", err) }
coldMaxTS := int64(0) if len(manifests) > 0 { maxPointsTarget := 0 if len(downsample) > 0 && downsample[0].MaxPoints > 0 { maxPointsTarget = downsample[0].MaxPoints }
targetPerChunk := 0 if maxPointsTarget > 0 && len(manifests) > 2 { targetPerChunk = max(150, (maxPointsTarget*3)/len(manifests)) }
chunkResults := make([]map[types.MetricID]*chunkSeriesPoints, len(manifests)) validChunk := make([]bool, len(manifests))
numWorkers := min(len(manifests), min(8, max(2, runtime.GOMAXPROCS(0))))
jobs := make(chan int, len(manifests)) for i := range manifests { jobs <- i } close(jobs)
var wg sync.WaitGroup wg.Add(numWorkers)
for range numWorkers { go func() { defer wg.Done()
for idx := range jobs { if ctx.Err() != nil { return }
m := manifests[idx] if m.RecordCount == 0 || m.SizeBytes <= 0 { continue }
readerAt, closer, openErr := cold.OpenReaderAt(ctx, e.storage, m.StorageKey, m.SizeBytes) if openErr != nil { if ctx.Err() == nil { log.Printf("[query] warning: skipping unreadable cold parquet chunk %s: %v", m.StorageKey, openErr) } continue }
pf, pfErr := parquet.OpenFile(readerAt, m.SizeBytes) if pfErr != nil { _ = closer.Close() if ctx.Err() == nil { log.Printf("[query] warning: skipping unreadable cold parquet chunk %s: %v", m.StorageKey, pfErr) } continue }
allInclusive := m.MinTS >= startTS && m.MaxTS <= endTS localPoints, scanErr := e.scanChunkPoints(ctx, pf, resultsMap, hasSingleTarget, singleTarget, startTS, endTS, allInclusive) _ = closer.Close()
if scanErr != nil { if ctx.Err() == nil { log.Printf("[query] warning: error reading parquet chunk %s: %v", m.StorageKey, scanErr) } continue } if localPoints == nil { continue }
// Intermediate per-chunk downsampling to bound memory usage if targetPerChunk > 0 { for _, pts := range localPoints { if len(pts.timestamps) > targetPerChunk { pts.timestamps, pts.values = DownsampleLTTB(pts.timestamps, pts.values, targetPerChunk) } } }
chunkResults[idx] = localPoints validChunk[idx] = true } }() }
wg.Wait()
if ctx.Err() != nil { return nil, ctx.Err() }
// Merge chunk results in original manifest order for idx := range manifests { if !validChunk[idx] { continue } m := manifests[idx] if m.MaxTS > coldMaxTS { coldMaxTS = m.MaxTS }
cpMap := chunkResults[idx] if cpMap == nil { continue } for mID, pts := range cpMap { if len(pts.timestamps) > 0 { res := resultsMap[mID] res.Timestamps = append(res.Timestamps, pts.timestamps...) res.Values = append(res.Values, pts.values...) } } } }
// 3. Query Hot Tier (Pebble LSM) // Query hot tier only for points newer than coldMaxTS (to prevent duplicate reads at the boundary) hotStartTS := startTS if coldMaxTS > 0 && coldMaxTS >= startTS { hotStartTS = coldMaxTS + 1 }
if hotStartTS <= endTS { if ctx.Err() != nil { return nil, ctx.Err() } bucketDurSec := int64(e.bucketDuration.Seconds()) startBucket := types.BucketID((hotStartTS / 1000) / bucketDurSec) endBucket := types.BucketID((endTS / 1000) / bucketDurSec)
for _, mID := range metricIDs { hotPoints, hotErr := e.hot.ScanMetricRange(ctx, mID, startBucket, endBucket, hotStartTS, endTS) if hotErr != nil { return nil, fmt.Errorf("scanning hot tier for metric %d: %w", mID, hotErr) } res := resultsMap[mID] for _, pt := range hotPoints { res.Timestamps = append(res.Timestamps, pt.Timestamp) res.Values = append(res.Values, pt.Value) } } }
// 4. Finalize sort ordering for each series var results []SeriesResult for _, id := range metricIDs { res := resultsMap[id] if len(res.Timestamps) > 0 { if !slices.IsSorted(res.Timestamps) { sortPoints(res.Timestamps, res.Values) } if len(downsample) > 0 { res.Downsample(downsample[0]) } } results = append(results, *res) }
if time.Since(startTime) > 3*time.Second { log.Printf("[query] completed scan of %d manifests in %v for %d series", len(manifests), time.Since(startTime), len(metricIDs)) }
return results, nil}
// scanChunkPoints extracts data points from a Parquet file using direct columnar decoding,// with automatic fallback to generic row reading if necessary.func (e *Engine) scanChunkPoints( ctx context.Context, pf *parquet.File, resultsMap map[types.MetricID]*SeriesResult, hasSingleTarget bool, singleTarget types.MetricID, startTS, endTS int64, allInclusive bool,) (map[types.MetricID]*chunkSeriesPoints, error) { // Optimization 1: Embedded series dictionary in Parquet footer if dict, err := cold.ExtractSeriesDictionary(pf); err == nil && len(dict) > 0 { hasMetric := false for _, s := range dict { if hasSingleTarget { if s.MetricID == singleTarget { hasMetric = true break } } else if _, ok := resultsMap[s.MetricID]; ok { hasMetric = true break } } if !hasMetric { return nil, nil } }
localPoints := make(map[types.MetricID]*chunkSeriesPoints) rowGroups := pf.RowGroups() if len(rowGroups) == 0 { return localPoints, nil }
canUseColumnar := true for _, rg := range rowGroups { if len(rg.ColumnChunks()) != 3 { canUseColumnar = false break } }
if canUseColumnar { intBuf := make([]int64, 4096) valBuf := make([]parquet.Value, 4096) doubleBuf := make([]float64, 4096)
for _, rg := range rowGroups { if ctx.Err() != nil { return nil, ctx.Err() } numRows := rg.NumRows() if numRows == 0 { continue }
chunks := rg.ColumnChunks()
// 1. Scan Column 0 (MetricID) p0 := chunks[0].Pages() metricIDs := make([]int64, 0, numRows) for { p, err := p0.ReadPage() if err != nil { break } v := p.Values() if r, ok := v.(parquet.Int64Reader); ok { for { n, rErr := r.ReadInt64s(intBuf) if n > 0 { metricIDs = append(metricIDs, intBuf[:n]...) } if rErr != nil { break } } } else { for { n, rErr := v.ReadValues(valBuf) for i := range n { metricIDs = append(metricIDs, valBuf[i].Int64()) } if rErr != nil { break } } } } _ = p0.Close()
if len(metricIDs) == 0 { canUseColumnar = false break }
// Fast pruning: check if this row group contains ANY target metric hasTargetInRG := false for _, m := range metricIDs { mID := types.MetricID(m) if hasSingleTarget { if mID == singleTarget { hasTargetInRG = true break } } else if _, ok := resultsMap[mID]; ok { hasTargetInRG = true break } } if !hasTargetInRG { continue // Skip Column 1 and Column 2 completely! }
// 2. Scan Column 1 (Timestamp) p1 := chunks[1].Pages() timestamps := make([]int64, 0, numRows) for { p, err := p1.ReadPage() if err != nil { break } v := p.Values() if r, ok := v.(parquet.Int64Reader); ok { for { n, rErr := r.ReadInt64s(intBuf) if n > 0 { timestamps = append(timestamps, intBuf[:n]...) } if rErr != nil { break } } } else { for { n, rErr := v.ReadValues(valBuf) for i := range n { timestamps = append(timestamps, valBuf[i].Int64()) } if rErr != nil { break } } } } _ = p1.Close()
// 3. Scan Column 2 (Value) p2 := chunks[2].Pages() values := make([]float64, 0, numRows) for { p, err := p2.ReadPage() if err != nil { break } v := p.Values() if r, ok := v.(parquet.DoubleReader); ok { for { n, rErr := r.ReadDoubles(doubleBuf) if n > 0 { values = append(values, doubleBuf[:n]...) } if rErr != nil { break } } } else { for { n, rErr := v.ReadValues(valBuf) for i := range n { values = append(values, valBuf[i].Double()) } if rErr != nil { break } } } } _ = p2.Close()
limit := min(len(metricIDs), len(timestamps), len(values)) for i := range limit { mID := types.MetricID(metricIDs[i]) if hasSingleTarget { if mID != singleTarget { continue } } else if _, ok := resultsMap[mID]; !ok { continue } ts := timestamps[i] if !allInclusive && (ts < startTS || ts > endTS) { continue } pts := localPoints[mID] if pts == nil { pts = &chunkSeriesPoints{} localPoints[mID] = pts } pts.timestamps = append(pts.timestamps, ts) pts.values = append(pts.values, values[i]) } }
if canUseColumnar { return localPoints, nil } }
// Fallback to row-by-row GenericReader localPoints = make(map[types.MetricID]*chunkSeriesPoints) reader := parquet.NewGenericReader[cold.ParquetRow](pf) defer func() { _ = reader.Close() }()
batch := make([]cold.ParquetRow, 8192) for { if ctx.Err() != nil { return nil, ctx.Err() } n, readErr := reader.Read(batch) for i := range n { mID := types.MetricID(batch[i].MetricID) if hasSingleTarget { if mID == singleTarget { ts := batch[i].Timestamp if allInclusive || (ts >= startTS && ts <= endTS) { pts := localPoints[singleTarget] if pts == nil { pts = &chunkSeriesPoints{} localPoints[singleTarget] = pts } pts.timestamps = append(pts.timestamps, ts) pts.values = append(pts.values, batch[i].Value) } } } else if _, ok := resultsMap[mID]; ok { ts := batch[i].Timestamp if allInclusive || (ts >= startTS && ts <= endTS) { pts := localPoints[mID] if pts == nil { pts = &chunkSeriesPoints{} localPoints[mID] = pts } pts.timestamps = append(pts.timestamps, ts) pts.values = append(pts.values, batch[i].Value) } } } if readErr != nil { if readErr != io.EOF { return nil, readErr } break } } return localPoints, nil}
type pointSorter struct { ts []int64 val []float64}
func (s pointSorter) Len() int { return len(s.ts) }func (s pointSorter) Less(i, j int) bool { return s.ts[i] < s.ts[j] }func (s pointSorter) Swap(i, j int) { s.ts[i], s.ts[j] = s.ts[j], s.ts[i] s.val[i], s.val[j] = s.val[j], s.val[i]}
func sortPoints(ts []int64, val []float64) { sort.Sort(pointSorter{ts: ts, val: val})}