Something went wrong. Try again.
Metric and Time-Series Engine: A Minimal Archiving Time Series Storage Service
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541package proxy
import ( "context" "encoding/json" "errors" "fmt" "io" "io/fs" "log" "net" "net/http" "strings" "sync" "sync/atomic" "time"
"go.iscode.ca/mats/web")
// Server is the public-facing gateway proxy.type Server struct { cfg Config gate *SecurityGate cache *ResponseCache flight singleflightGroup upstreamSem chan struct{} // bounds concurrent upstream queries client *http.Client httpServer *http.Server listener net.Listener rateLimiter *ipRateLimiter snapshotMu sync.RWMutex snapshotEntry CacheEntry blockedQueries uint64 rateLimitedReqs uint64 stopOnce sync.Once stopChan chan struct{}}
// NewServer initializes a new mats-proxy server.func NewServer(cfg Config) (*Server, error) { if err := cfg.Validate(); err != nil { return nil, err }
gate := NewSecurityGate(cfg.AllowedHosts, cfg.AllowedServices, cfg.MaxQueryWindow) cache := NewResponseCache(cfg.CacheSize)
client := &http.Client{ Timeout: cfg.WriteTimeout, Transport: &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 20, IdleConnTimeout: 90 * time.Second, }, }
s := &Server{ cfg: cfg, gate: gate, cache: cache, upstreamSem: make(chan struct{}, 4), // max 4 concurrent uncached queries upstream client: client, rateLimiter: newIPRateLimiter(cfg.RateLimitRPS, float64(cfg.RateLimitBurst)), stopChan: make(chan struct{}), }
return s, nil}
// Start binds and starts serving HTTP traffic on the configured listener.func (s *Server) Start(lp ListenerProvider) error { ctx := context.Background() rawListener, err := lp.Listen(ctx, "tcp", s.cfg.ListenAddr) if err != nil { return fmt.Errorf("failed binding listener on %s: %w", s.cfg.ListenAddr, err) }
// Wrap with concurrency connection limiter limitedListener := newLimitListener(rawListener, s.cfg.MaxConnections) s.listener = limitedListener
mux := http.NewServeMux() mux.HandleFunc("GET /api/v1/query", s.handleQuery) mux.HandleFunc("GET /api/v1/render", s.handleRender) mux.HandleFunc("GET /api/v1/stats", s.handleStats) mux.HandleFunc("GET /api/v1/snapshot", s.handleSnapshot)
// Serve embedded web UI staticFS, err := fs.Sub(web.Assets, ".") if err == nil { fileServer := http.FileServer(http.FS(staticFS)) mux.Handle("GET /", fileServer) }
s.httpServer = &http.Server{ Handler: mux, ReadTimeout: s.cfg.ReadTimeout, WriteTimeout: s.cfg.WriteTimeout, IdleTimeout: s.cfg.IdleTimeout, MaxHeaderBytes: 16 * 1024, // 16 KB max request header }
// Start background pre-render snapshot worker if enabled if s.cfg.EnableSnapshot { go s.snapshotWorker() }
go func() { _ = s.httpServer.Serve(limitedListener) }()
return nil}
// Addr returns the bound listener address.func (s *Server) Addr() net.Addr { if s.listener != nil { return s.listener.Addr() } return nil}
// Stop gracefully shuts down the server.func (s *Server) Stop(ctx context.Context) error { var err error s.stopOnce.Do(func() { close(s.stopChan) if s.httpServer != nil { err = s.httpServer.Shutdown(ctx) } }) return err}
func (s *Server) checkRateLimit(w http.ResponseWriter, r *http.Request) bool { ip := extractClientIP(r) if !s.rateLimiter.allow(ip) { atomic.AddUint64(&s.rateLimitedReqs, 1) w.Header().Set("Retry-After", "1") http.Error(w, "rate limit exceeded: too many requests", http.StatusTooManyRequests) return false } return true}
func (s *Server) handleQuery(w http.ResponseWriter, r *http.Request) { if !s.checkRateLimit(w, r) { return }
now := time.Now().UnixMilli() sanitized, ttl, err := s.gate.SanitizeQuery(r.URL.Query(), now) if err != nil { atomic.AddUint64(&s.blockedQueries, 1) http.Error(w, fmt.Sprintf("forbidden: %v", err), http.StatusForbidden) return }
cacheKey := s.gate.CanonicalCacheKey("query", sanitized)
// 1. Check in-memory cache if entry, ok := s.cache.Get(cacheKey); ok { s.respondWithEntry(w, r, entry) return }
// 2. Coalesce concurrent misses val, err := s.flight.Do(cacheKey, func() (any, error) { // Double check cache if entry, ok := s.cache.Get(cacheKey); ok { return entry, nil }
// Acquire upstream execution semaphore select { case s.upstreamSem <- struct{}{}: defer func() { <-s.upstreamSem }() case <-r.Context().Done(): return nil, r.Context().Err() case <-time.After(s.cfg.WriteTimeout): return nil, errors.New("upstream gateway busy, query timeout") }
targetURL := fmt.Sprintf("%s/api/v1/query?%s", strings.TrimSuffix(s.cfg.BackendURL, "/"), sanitized.Encode()) req, reqErr := http.NewRequestWithContext(r.Context(), http.MethodGet, targetURL, nil) if reqErr != nil { return nil, reqErr }
resp, respErr := s.client.Do(req) if respErr != nil { return nil, fmt.Errorf("upstream query failed: %w", respErr) } defer func() { _ = resp.Body.Close() }()
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024)) // 10MB safety ceiling if readErr != nil { return nil, fmt.Errorf("reading upstream response: %w", readErr) }
if resp.StatusCode != http.StatusOK { return CacheEntry{ ContentType: resp.Header.Get("Content-Type"), Body: body, StatusCode: resp.StatusCode, ExpiresAt: time.Now().Add(5 * time.Second), ETag: computeETag(body), }, nil }
// Defense-in-depth: scrub output to strictly allow-listed series scrubbedBody, scrubErr := s.gate.ScrubSeriesResults(body) if scrubErr == nil { body = scrubbedBody }
entry := CacheEntry{ ContentType: "application/json", Body: body, StatusCode: http.StatusOK, ExpiresAt: time.Now().Add(ttl), ETag: computeETag(body), }
s.cache.Set(cacheKey, entry) return entry, nil })
if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return }
s.respondWithEntry(w, r, val.(CacheEntry))}
func (s *Server) handleRender(w http.ResponseWriter, r *http.Request) { if !s.checkRateLimit(w, r) { return }
now := time.Now().UnixMilli() sanitized, ttl, err := s.gate.SanitizeQuery(r.URL.Query(), now) if err != nil { atomic.AddUint64(&s.blockedQueries, 1) http.Error(w, fmt.Sprintf("forbidden: %v", err), http.StatusForbidden) return }
format := r.URL.Query().Get("format") if format == "" { format = "svg" } sanitized.Set("format", format) if wStr := r.URL.Query().Get("width"); wStr != "" { sanitized.Set("width", wStr) } if hStr := r.URL.Query().Get("height"); hStr != "" { sanitized.Set("height", hStr) }
cacheKey := s.gate.CanonicalCacheKey("render", sanitized)
if entry, ok := s.cache.Get(cacheKey); ok { s.respondWithEntry(w, r, entry) return }
val, err := s.flight.Do(cacheKey, func() (any, error) { if entry, ok := s.cache.Get(cacheKey); ok { return entry, nil }
select { case s.upstreamSem <- struct{}{}: defer func() { <-s.upstreamSem }() case <-r.Context().Done(): return nil, r.Context().Err() case <-time.After(s.cfg.WriteTimeout): return nil, errors.New("upstream busy, render timeout") }
targetURL := fmt.Sprintf("%s/api/v1/render?%s", strings.TrimSuffix(s.cfg.BackendURL, "/"), sanitized.Encode()) req, reqErr := http.NewRequestWithContext(r.Context(), http.MethodGet, targetURL, nil) if reqErr != nil { return nil, reqErr }
resp, respErr := s.client.Do(req) if respErr != nil { return nil, fmt.Errorf("upstream render failed: %w", respErr) } defer func() { _ = resp.Body.Close() }()
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 5*1024*1024)) if readErr != nil { return nil, fmt.Errorf("reading upstream response: %w", readErr) }
contentType := resp.Header.Get("Content-Type") if contentType == "" { if format == "ascii" { contentType = "text/plain; charset=utf-8" } else { contentType = "image/svg+xml" } }
entry := CacheEntry{ ContentType: contentType, Body: body, StatusCode: resp.StatusCode, ExpiresAt: time.Now().Add(ttl), ETag: computeETag(body), }
s.cache.Set(cacheKey, entry) return entry, nil })
if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return }
s.respondWithEntry(w, r, val.(CacheEntry))}
func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { cStats := s.cache.Stats()
stats := map[string]any{ "cache_hits": cStats.Hits, "cache_misses": cStats.Misses, "cache_evictions": cStats.Evictions, "cache_items": cStats.Items, "blocked_queries": atomic.LoadUint64(&s.blockedQueries), "rate_limited_reqs": atomic.LoadUint64(&s.rateLimitedReqs), "max_query_window": s.cfg.MaxQueryWindow.String(), }
w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(stats)}
func (s *Server) handleSnapshot(w http.ResponseWriter, r *http.Request) { s.snapshotMu.RLock() entry := s.snapshotEntry s.snapshotMu.RUnlock()
if entry.Body == nil { http.Error(w, "snapshot warming in progress", http.StatusServiceUnavailable) return }
s.respondWithEntry(w, r, entry)}
func (s *Server) respondWithEntry(w http.ResponseWriter, r *http.Request, entry CacheEntry) { if r.Header.Get("If-None-Match") == entry.ETag { w.WriteHeader(http.StatusNotModified) return }
w.Header().Set("Content-Type", entry.ContentType) w.Header().Set("ETag", entry.ETag) remaining := max(1, int(time.Until(entry.ExpiresAt).Seconds())) w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", remaining)) w.WriteHeader(entry.StatusCode) _, _ = w.Write(entry.Body)}
func (s *Server) snapshotWorker() { ticker := time.NewTicker(s.cfg.SnapshotInterval) defer ticker.Stop()
// Initial warmup s.refreshSnapshot()
for { select { case <-s.stopChan: return case <-ticker.C: s.refreshSnapshot() } }}
func (s *Server) refreshSnapshot() { ctx, cancel := context.WithTimeout(context.Background(), s.cfg.WriteTimeout) defer cancel()
now := time.Now().UnixMilli() sanitized, _, err := s.gate.SanitizeQuery(nil, now) if err != nil { return }
targetURL := fmt.Sprintf("%s/api/v1/query?%s", strings.TrimSuffix(s.cfg.BackendURL, "/"), sanitized.Encode()) req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) if err != nil { return }
resp, err := s.client.Do(req) if err != nil { log.Printf("[proxy-snapshot] warning: failed refreshing snapshot: %v", err) return } defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024)) if err != nil || resp.StatusCode != http.StatusOK { return }
scrubbed, _ := s.gate.ScrubSeriesResults(body) if scrubbed != nil { body = scrubbed }
entry := CacheEntry{ ContentType: "application/json", Body: body, StatusCode: http.StatusOK, ExpiresAt: time.Now().Add(s.cfg.SnapshotInterval * 2), ETag: computeETag(body), }
s.snapshotMu.Lock() s.snapshotEntry = entry s.snapshotMu.Unlock()}
func extractClientIP(r *http.Request) string { if cfIP := r.Header.Get("CF-Connecting-IP"); cfIP != "" { return strings.TrimSpace(cfIP) } if xff := r.Header.Get("X-Forwarded-For"); xff != "" { parts := strings.Split(xff, ",") if len(parts) > 0 { return strings.TrimSpace(parts[0]) } } host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { return r.RemoteAddr } return host}
// ipRateLimiter implements a lightweight per-IP token bucket limiter.type ipRateLimiter struct { mu sync.Mutex buckets map[string]*bucket rate float64 burst float64}
type bucket struct { tokens float64 lastRefill time.Time}
func newIPRateLimiter(rate, burst float64) *ipRateLimiter { return &ipRateLimiter{ buckets: make(map[string]*bucket), rate: rate, burst: burst, }}
func (l *ipRateLimiter) allow(ip string) bool { l.mu.Lock() defer l.mu.Unlock()
now := time.Now() b, ok := l.buckets[ip] if !ok { l.buckets[ip] = &bucket{ tokens: l.burst - 1.0, lastRefill: now, } // Periodically clean up entries if map grows if len(l.buckets) > 10000 { for k, v := range l.buckets { if now.Sub(v.lastRefill) > 10*time.Minute { delete(l.buckets, k) } } } return true }
elapsed := now.Sub(b.lastRefill).Seconds() b.tokens = min(l.burst, b.tokens+(elapsed*l.rate)) b.lastRefill = now
if b.tokens >= 1.0 { b.tokens -= 1.0 return true } return false}
// limitListener restricts concurrent active connections.type limitListener struct { net.Listener sem chan struct{}}
func newLimitListener(l net.Listener, maxConns int) net.Listener { return &limitListener{ Listener: l, sem: make(chan struct{}, maxConns), }}
func (l *limitListener) Accept() (net.Conn, error) { l.sem <- struct{}{} c, err := l.Listener.Accept() if err != nil { <-l.sem return nil, err } return &limitConn{Conn: c, release: func() { <-l.sem }}, nil}
type limitConn struct { net.Conn release func() onceRelease sync.Once}
func (c *limitConn) Close() error { c.onceRelease.Do(c.release) return c.Conn.Close()}