Something went wrong. Try again.
Metric and Time-Series Engine: A Minimal Archiving Time Series Storage Service
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619package query
import ( "bytes" "context" "encoding/json" "errors" "fmt" "io/fs" "net" "net/http" "strconv" "sync" "time"
"go.iscode.ca/mats/internal/ingest" "go.iscode.ca/mats/internal/types" "go.iscode.ca/mats/web")
// ServerOption configures the HTTP query server.type ServerOption func(*Server)
// WithCache sets a custom response cache.func WithCache(cache *ResponseCache) ServerOption { return func(s *Server) { s.cache = cache }}
// WithCacheSize sets the capacity of the default response cache.func WithCacheSize(maxEntries int) ServerOption { return func(s *Server) { s.cache = NewResponseCache(maxEntries) }}
// WithDisabledCache disables the in-memory response cache.func WithDisabledCache() ServerOption { return func(s *Server) { s.cache = nil }}
// WithDefaultMaxPoints configures the default max points returned per series for /api/v1/query.func WithDefaultMaxPoints(pts int) ServerOption { return func(s *Server) { s.defaultMaxPoints = pts }}
// WithQueryTimeout configures the execution timeout for query and render requests.func WithQueryTimeout(timeout time.Duration) ServerOption { return func(s *Server) { if timeout > 0 { s.queryTimeout = timeout } }}
type singleflightCall struct { done chan struct{} val any err error cancel context.CancelFunc mu sync.Mutex refCount int}
type singleflightGroup struct { mu sync.Mutex m map[string]*singleflightCall}
func (g *singleflightGroup) Do(ctx context.Context, key string, fn func(callCtx context.Context) (any, error)) (any, error) { g.mu.Lock() if g.m == nil { g.m = make(map[string]*singleflightCall) } if c, ok := g.m[key]; ok { c.mu.Lock() c.refCount++ c.mu.Unlock() g.mu.Unlock()
select { case <-c.done: return c.val, c.err case <-ctx.Done(): c.mu.Lock() c.refCount-- if c.refCount == 0 && c.cancel != nil { c.cancel() } c.mu.Unlock() return nil, ctx.Err() } }
callCtx, cancel := context.WithCancel(context.Background()) c := &singleflightCall{ done: make(chan struct{}), cancel: cancel, refCount: 1, } g.m[key] = c g.mu.Unlock()
go func() { defer func() { if r := recover(); r != nil { c.err = fmt.Errorf("panic in singleflight execution: %v", r) } cancel() close(c.done)
g.mu.Lock() delete(g.m, key) g.mu.Unlock() }()
c.val, c.err = fn(callCtx) }()
select { case <-c.done: return c.val, c.err case <-ctx.Done(): c.mu.Lock() c.refCount-- if c.refCount == 0 { cancel() } c.mu.Unlock() return nil, ctx.Err() }}
// Server provides the HTTP Query API and serves the embedded web dashboard.type Server struct { addr string engine *Engine dispatcher *ingest.Dispatcher server *http.Server listener net.Listener cache *ResponseCache defaultMaxPoints int queryTimeout time.Duration flight singleflightGroup stopOnce sync.Once}
func NewServer(addr string, engine *Engine, disp *ingest.Dispatcher, opts ...ServerOption) *Server { s := &Server{ addr: addr, engine: engine, dispatcher: disp, cache: NewResponseCache(512), // default 512 entries defaultMaxPoints: 1500, // default 1500 points per series queryTimeout: 120 * time.Second, // default 120s query execution timeout }
for _, opt := range opts { opt(s) }
mux := http.NewServeMux() mux.HandleFunc("GET /api/v1/query", s.handleQuery) mux.HandleFunc("GET /api/v1/render", s.handleRender) mux.HandleFunc("GET /api/v1/stats", s.handleStats)
// Serve embedded web interface staticFS, err := fs.Sub(web.Assets, ".") if err == nil { fileServer := http.FileServer(http.FS(staticFS)) mux.Handle("GET /", fileServer) }
writeTimeout := max(120*time.Second, s.queryTimeout+30*time.Second) s.server = &http.Server{ Handler: mux, ReadTimeout: 30 * time.Second, WriteTimeout: writeTimeout, }
return s}
// Cache returns the server's response cache instance, if enabled.func (s *Server) Cache() *ResponseCache { return s.cache}
func (s *Server) Start() error { var lc net.ListenConfig l, err := lc.Listen(context.Background(), "tcp", s.addr) if err != nil { return fmt.Errorf("failed binding http on %s: %w", s.addr, err) } s.listener = l
go func() { _ = s.server.Serve(l) }() return nil}
func (s *Server) Addr() net.Addr { if s.listener != nil { return s.listener.Addr() } return nil}
func (s *Server) Stop(ctx context.Context) error { var err error s.stopOnce.Do(func() { if s.server != nil { err = s.server.Shutdown(ctx) } }) return err}
func (s *Server) handleQuery(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() host := q.Get("host") service := q.Get("service")
now := time.Now().UnixMilli() to := now if toStr := q.Get("to"); toStr != "" { if val, err := strconv.ParseInt(toStr, 10, 64); err == nil { to = val } }
from := to - (3600 * 1000) // Default 1 hour if fromStr := q.Get("from"); fromStr != "" { if val, err := strconv.ParseInt(fromStr, 10, 64); err == nil { from = val } }
bypassCache := r.Header.Get("Cache-Control") == "no-cache" || q.Get("nocache") == "1" exact := q.Get("exact") == "true" || q.Get("exact") == "1"
var queryFrom, queryTo int64 var ttl time.Duration if exact { queryFrom, queryTo, ttl = from, to, 10*time.Second } else { queryFrom, queryTo, ttl, _ = computeQuantization(from, to, now) }
maxPoints := s.defaultMaxPoints if mpStr := q.Get("max_points"); mpStr != "" { if mpStr == "0" || mpStr == "none" || mpStr == "raw" { maxPoints = 0 } else if val, err := strconv.Atoi(mpStr); err == nil && val >= 0 { maxPoints = val } } else if ptsStr := q.Get("points"); ptsStr != "" { if val, err := strconv.Atoi(ptsStr); err == nil && val >= 0 { maxPoints = val } } else if q.Get("raw") == "1" || q.Get("raw") == "true" { maxPoints = 0 }
var stepMS int64 if stepStr := q.Get("step"); stepStr != "" { if d, err := time.ParseDuration(stepStr); err == nil && d > 0 { stepMS = d.Milliseconds() } else if val, err := strconv.ParseInt(stepStr, 10, 64); err == nil && val > 0 { stepMS = val } }
method := q.Get("method") if method == "" { method = q.Get("agg") } if method == "" { method = "lttb" }
downsampleOpts := DownsampleOptions{ MaxPoints: maxPoints, StepMS: stepMS, Method: method, }
skipEmpty := q.Get("skip_empty") == "1" || q.Get("non_empty") == "1" || q.Get("empty") == "0"
cacheKey := fmt.Sprintf("query:%s:%s:%d:%d:%d:%d:%s:%t", host, service, queryFrom, queryTo, maxPoints, stepMS, method, skipEmpty)
if !bypassCache && s.cache != nil { if entry, ok := s.cache.Get(cacheKey); ok { if r.Header.Get("If-None-Match") == entry.ETag { w.WriteHeader(http.StatusNotModified) return } w.Header().Set("Content-Type", entry.ContentType) w.Header().Set("ETag", entry.ETag) w.Header().Set("X-Cache", "HIT") maxAge := max(1, int(time.Until(entry.ExpiresAt).Seconds())) w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) w.WriteHeader(entry.StatusCode) _, _ = w.Write(entry.Body) return } }
filter := types.SeriesFilter{ HostRegex: host, ServiceRegex: service, }
reqCtx, reqCancel := context.WithTimeout(r.Context(), s.queryTimeout) defer reqCancel()
val, err := s.flight.Do(reqCtx, cacheKey, func(callCtx context.Context) (any, error) { if !bypassCache && s.cache != nil { if entry, ok := s.cache.Get(cacheKey); ok { return entry, nil } }
workCtx, workCancel := context.WithTimeout(callCtx, s.queryTimeout) defer workCancel()
results, qErr := s.engine.Query(workCtx, filter, queryFrom, queryTo, downsampleOpts) if qErr != nil { return nil, qErr }
if results == nil { results = []SeriesResult{} }
if skipEmpty { filtered := make([]SeriesResult, 0, len(results)) for _, r := range results { if len(r.Timestamps) > 0 { filtered = append(filtered, r) } } results = filtered }
data, mErr := json.Marshal(results) if mErr != nil { return nil, mErr }
etag := computeETag(data) entry := CacheEntry{ ContentType: "application/json", Body: data, StatusCode: http.StatusOK, ExpiresAt: time.Now().Add(ttl), ETag: etag, }
if !bypassCache && s.cache != nil { s.cache.Set(cacheKey, entry) } return entry, nil })
if err != nil { if errors.Is(err, context.DeadlineExceeded) || reqCtx.Err() == context.DeadlineExceeded { http.Error(w, "query timeout exceeded", http.StatusGatewayTimeout) return } if errors.Is(err, context.Canceled) || reqCtx.Err() == context.Canceled { http.Error(w, "query canceled", http.StatusGatewayTimeout) return } http.Error(w, err.Error(), http.StatusInternalServerError) return }
entry := val.(CacheEntry) if r.Header.Get("If-None-Match") == entry.ETag { w.WriteHeader(http.StatusNotModified) return }
w.Header().Set("Content-Type", entry.ContentType) w.Header().Set("ETag", entry.ETag) w.Header().Set("X-Cache", "MISS") maxAge := max(1, int(time.Until(entry.ExpiresAt).Seconds())) w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) w.WriteHeader(entry.StatusCode) _, _ = w.Write(entry.Body)}
func (s *Server) handleRender(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() host := q.Get("host") service := q.Get("service") format := q.Get("format")
now := time.Now().UnixMilli() to := now if toStr := q.Get("to"); toStr != "" { if val, err := strconv.ParseInt(toStr, 10, 64); err == nil { to = val } }
from := to - (3600 * 1000) if fromStr := q.Get("from"); fromStr != "" { if val, err := strconv.ParseInt(fromStr, 10, 64); err == nil { from = val } }
bypassCache := r.Header.Get("Cache-Control") == "no-cache" || q.Get("nocache") == "1" exact := q.Get("exact") == "true" || q.Get("exact") == "1"
var queryFrom, queryTo int64 var ttl time.Duration if exact { queryFrom, queryTo, ttl = from, to, 10*time.Second } else { queryFrom, queryTo, ttl, _ = computeQuantization(from, to, now) }
width := 600 height := 200 if wStr := q.Get("w"); wStr != "" { if v, err := strconv.Atoi(wStr); err == nil && v > 0 { width = v } } if hStr := q.Get("h"); hStr != "" { if v, err := strconv.Atoi(hStr); err == nil && v > 0 { height = v } }
theme := q.Get("theme")
maxPoints := max(width*2, 1000) if format == "ascii" { maxPoints = 200 } if mpStr := q.Get("max_points"); mpStr != "" { if mpStr == "0" || mpStr == "none" || mpStr == "raw" { maxPoints = 0 } else if val, err := strconv.Atoi(mpStr); err == nil && val >= 0 { maxPoints = val } } else if ptsStr := q.Get("points"); ptsStr != "" { if val, err := strconv.Atoi(ptsStr); err == nil && val >= 0 { maxPoints = val } } else if q.Get("raw") == "1" || q.Get("raw") == "true" { maxPoints = 0 }
var stepMS int64 if stepStr := q.Get("step"); stepStr != "" { if d, err := time.ParseDuration(stepStr); err == nil && d > 0 { stepMS = d.Milliseconds() } else if val, err := strconv.ParseInt(stepStr, 10, 64); err == nil && val > 0 { stepMS = val } }
method := q.Get("method") if method == "" { method = q.Get("agg") } if method == "" { method = "lttb" }
downsampleOpts := DownsampleOptions{ MaxPoints: maxPoints, StepMS: stepMS, Method: method, }
cacheKey := fmt.Sprintf("render:%s:%s:%d:%d:%s:%d:%d:%s:%d:%d:%s", host, service, queryFrom, queryTo, format, width, height, theme, maxPoints, stepMS, method)
if !bypassCache && s.cache != nil { if entry, ok := s.cache.Get(cacheKey); ok { if r.Header.Get("If-None-Match") == entry.ETag { w.WriteHeader(http.StatusNotModified) return } w.Header().Set("Content-Type", entry.ContentType) w.Header().Set("ETag", entry.ETag) w.Header().Set("X-Cache", "HIT") maxAge := max(1, int(time.Until(entry.ExpiresAt).Seconds())) w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) w.WriteHeader(entry.StatusCode) _, _ = w.Write(entry.Body) return } }
filter := types.SeriesFilter{ HostRegex: host, ServiceRegex: service, }
reqCtx, reqCancel := context.WithTimeout(r.Context(), s.queryTimeout) defer reqCancel()
val, err := s.flight.Do(reqCtx, cacheKey, func(callCtx context.Context) (any, error) { if !bypassCache && s.cache != nil { if entry, ok := s.cache.Get(cacheKey); ok { return entry, nil } }
workCtx, workCancel := context.WithTimeout(callCtx, s.queryTimeout) defer workCancel()
results, qErr := s.engine.Query(workCtx, filter, queryFrom, queryTo, downsampleOpts) if qErr != nil { return nil, qErr }
title := fmt.Sprintf("%s / %s", host, service) var ts []int64 var vals []float64 if len(results) > 0 { title = fmt.Sprintf("%s / %s", results[0].Host, results[0].Service) ts = results[0].Timestamps vals = results[0].Values }
var buf bytes.Buffer var contentType string switch format { case "ascii": contentType = "text/plain; charset=utf-8" _ = RenderASCII(&buf, title, ts, vals) default: // svg contentType = "image/svg+xml" _ = RenderSVG(&buf, title, ts, vals, width, height, theme) }
body := buf.Bytes() etag := computeETag(body) entry := CacheEntry{ ContentType: contentType, Body: body, StatusCode: http.StatusOK, ExpiresAt: time.Now().Add(ttl), ETag: etag, }
if !bypassCache && s.cache != nil { s.cache.Set(cacheKey, entry) } return entry, nil })
if err != nil { if errors.Is(err, context.DeadlineExceeded) || reqCtx.Err() == context.DeadlineExceeded { http.Error(w, "query timeout exceeded", http.StatusGatewayTimeout) return } if errors.Is(err, context.Canceled) || reqCtx.Err() == context.Canceled { http.Error(w, "query canceled", http.StatusGatewayTimeout) return } http.Error(w, err.Error(), http.StatusInternalServerError) return }
entry := val.(CacheEntry) if r.Header.Get("If-None-Match") == entry.ETag { w.WriteHeader(http.StatusNotModified) return }
w.Header().Set("Content-Type", entry.ContentType) w.Header().Set("ETag", entry.ETag) w.Header().Set("X-Cache", "MISS") maxAge := max(1, int(time.Until(entry.ExpiresAt).Seconds())) w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) w.WriteHeader(entry.StatusCode) _, _ = w.Write(entry.Body)}
func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { var ingested, droppedLate, droppedNonNum, droppedErrors uint64 if s.dispatcher != nil { ingested, droppedLate, droppedNonNum, droppedErrors = s.dispatcher.Stats() }
stats := map[string]uint64{ "ingested": ingested, "dropped_late": droppedLate, "dropped_non_numeric": droppedNonNum, "dropped_errors": droppedErrors, }
if s.cache != nil { cStats := s.cache.Stats() stats["cache_hits"] = cStats.Hits stats["cache_misses"] = cStats.Misses stats["cache_evictions"] = cStats.Evictions stats["cache_items"] = uint64(cStats.Items) stats["cache_size_bytes"] = uint64(cStats.SizeBytes) }
w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(stats)}