Something went wrong. Try again.
Metric and Time-Series Engine: A Minimal Archiving Time Series Storage Service
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483package proxy
import ( "encoding/json" "fmt" "net/url" "regexp" "sort" "strconv" "strings" "time"
"go.iscode.ca/mats/internal/query")
// allowRule represents a compiled allow-list pattern rule.type allowRule struct { pattern string // original pattern string, e.g. ".*/humidity" or "dyson/humidity" isRegex bool // true if rule contains regex metacharacters re *regexp.Regexp // compiled regex matching full identifier}
func isRegexPattern(s string) bool { return strings.ContainsAny(s, "*+?()|[]{}^$")}
func cleanPatternRule(s string) (pattern string, isRegex bool, re *regexp.Regexp) { s = strings.TrimSpace(s) inner := s for { prev := inner inner = strings.TrimPrefix(inner, "^") inner = strings.TrimSuffix(inner, "$") if inner == prev { break } } if isRegexPattern(inner) { reStr := "^(?:" + inner + ")$" compiled, _ := regexp.Compile(reStr) return inner, true, compiled } compiled, _ := regexp.Compile("^(?:" + regexp.QuoteMeta(inner) + ")$") return inner, false, compiled}
// SecurityGate enforces PII allow-lists, query bounds, parameter sanitization,// and scrubbing on public requests.type SecurityGate struct { allowedHosts map[string]struct{} allowedServices map[string]struct{} allowedHostRules []allowRule allowedServiceRules []allowRule allowedHostsRegex string allowedServicesRegex string maxWindowMs int64}
// NewSecurityGate initializes an authoritative allow-list security gate.func NewSecurityGate(allowedHosts, allowedServices []string, maxWindow time.Duration) *SecurityGate { g := &SecurityGate{ allowedHosts: make(map[string]struct{}, len(allowedHosts)), allowedServices: make(map[string]struct{}, len(allowedServices)), allowedHostRules: make([]allowRule, 0, len(allowedHosts)), allowedServiceRules: make([]allowRule, 0, len(allowedServices)), maxWindowMs: maxWindow.Milliseconds(), }
cleanHosts := make([]string, 0, len(allowedHosts)) for _, h := range allowedHosts { h = strings.TrimSpace(h) if h == "" { continue } pattern, isRegex, re := cleanPatternRule(h) rule := allowRule{pattern: pattern, isRegex: isRegex, re: re} if isRegex { cleanHosts = append(cleanHosts, "(?:"+pattern+")") } else { g.allowedHosts[pattern] = struct{}{} cleanHosts = append(cleanHosts, regexp.QuoteMeta(pattern)) } g.allowedHostRules = append(g.allowedHostRules, rule) } if len(cleanHosts) > 0 { sort.Strings(cleanHosts) g.allowedHostsRegex = "^(" + strings.Join(cleanHosts, "|") + ")$" }
cleanServices := make([]string, 0, len(allowedServices)) for _, s := range allowedServices { s = strings.TrimSpace(s) if s == "" { continue } pattern, isRegex, re := cleanPatternRule(s) rule := allowRule{pattern: pattern, isRegex: isRegex, re: re} if isRegex { cleanServices = append(cleanServices, "(?:"+pattern+")") } else { g.allowedServices[pattern] = struct{}{} cleanServices = append(cleanServices, regexp.QuoteMeta(pattern)) } g.allowedServiceRules = append(g.allowedServiceRules, rule) } if len(cleanServices) > 0 { sort.Strings(cleanServices) g.allowedServicesRegex = "^(" + strings.Join(cleanServices, "|") + ")$" }
return g}
// normalizePattern extracts constituent identifier tokens from anchored or grouped regexes// (e.g., "^io$", "^(io)$", "^(io|weather)$", "^load\\.short$").func normalizePattern(val string) []string { val = strings.TrimSpace(val) if val == "" { return nil }
for { prev := val val = strings.TrimPrefix(val, "^") val = strings.TrimSuffix(val, "$") val = strings.TrimPrefix(val, "(") val = strings.TrimSuffix(val, ")") val = strings.TrimPrefix(val, "^") val = strings.TrimSuffix(val, "$") if val == prev { break } }
parts := strings.Split(val, "|") res := make([]string, 0, len(parts)) for _, p := range parts { p = strings.TrimSpace(p) p = strings.ReplaceAll(p, `\`, "") // unescape regex escapes like \. if p != "" { res = append(res, p) } } return res}
// IsHostAllowed checks if an explicit host name or anchored host pattern is permitted.func (g *SecurityGate) IsHostAllowed(host string) bool { if len(g.allowedHosts) == 0 && len(g.allowedHostRules) == 0 { return true } if _, ok := g.allowedHosts[host]; ok { return true } for _, rule := range g.allowedHostRules { if rule.re != nil && rule.re.MatchString(host) { return true } }
parts := normalizePattern(host) if len(parts) == 0 { return false } for _, p := range parts { if _, ok := g.allowedHosts[p]; !ok { matched := false for _, rule := range g.allowedHostRules { if rule.re != nil && rule.re.MatchString(p) { matched = true break } } if !matched { return false } } } return true}
// IsServiceAllowed checks if an explicit service name or anchored service pattern is permitted.func (g *SecurityGate) IsServiceAllowed(service string) bool { if len(g.allowedServices) == 0 && len(g.allowedServiceRules) == 0 { return true } if _, ok := g.allowedServices[service]; ok { return true } for _, rule := range g.allowedServiceRules { if rule.re != nil && rule.re.MatchString(service) { return true } }
parts := normalizePattern(service) if len(parts) == 0 { return false } for _, p := range parts { if _, ok := g.allowedServices[p]; !ok { matchedRule := false for _, rule := range g.allowedServiceRules { if rule.re != nil && rule.re.MatchString(p) { matchedRule = true break } } if !matchedRule { // Suffix / prefix matching on literal allow-rules for allowedRule := range g.allowedServices { if p == allowedRule || strings.HasSuffix(p, "/"+allowedRule) || strings.HasPrefix(p, allowedRule+"/") { matchedRule = true break } } } if !matchedRule { return false } } } return true}
// IsAllowed returns true if both host and service are allowed.func (g *SecurityGate) IsAllowed(host, service string) bool { return g.IsHostAllowed(host) && g.IsServiceAllowed(service)}
// matchAllowed resolves a query pattern against the allow-list rules.// It returns the subset of allowed rules that match the query pattern.// If zero allowed items match, it returns an empty slice.func (g *SecurityGate) matchAllowed(pattern string, allowedMap map[string]struct{}, rules []allowRule) ([]string, error) { if len(allowedMap) == 0 && len(rules) == 0 { return nil, nil }
pattern = strings.TrimSpace(pattern) if pattern == "" || pattern == ".*" { res := make([]string, 0, len(rules)) for _, r := range rules { res = append(res, r.pattern) } sort.Strings(res) return res, nil }
// 1. Direct exact match in literal map if _, ok := allowedMap[pattern]; ok { return []string{pattern}, nil }
// 2. Exact anchored/grouped regex match (e.g., "^io$", "^(io)$", "^load\\.short$") parts := normalizePattern(pattern) if len(parts) > 0 { allInAllowed := true for _, p := range parts { if _, ok := allowedMap[p]; !ok { matched := false for _, r := range rules { if r.re != nil && r.re.MatchString(p) { matched = true break } } if !matched { allInAllowed = false break } } } if allInAllowed { sort.Strings(parts) return parts, nil } }
// 3. Regex / substring match re, err := regexp.Compile(pattern) if err != nil { re, err = regexp.Compile(regexp.QuoteMeta(pattern)) if err != nil { return nil, fmt.Errorf("invalid regex %q: %w", pattern, err) } }
var matched []string seen := make(map[string]struct{}) for _, r := range rules { isMatch := false if r.re != nil && r.re.MatchString(pattern) { isMatch = true } else if re.MatchString(r.pattern) { isMatch = true } if isMatch { if _, ok := seen[r.pattern]; !ok { seen[r.pattern] = struct{}{} matched = append(matched, r.pattern) } } } sort.Strings(matched) return matched, nil}
// SanitizeQuery checks, validates, and rewrites incoming query parameters.// Returns sanitized parameters, the computed cache TTL, or an error if forbidden.func (g *SecurityGate) SanitizeQuery(q url.Values, now int64) (url.Values, time.Duration, error) { sanitized := make(url.Values)
// 1. Host validation & rewriting host := strings.TrimSpace(q.Get("host")) if len(g.allowedHosts) > 0 || len(g.allowedHostRules) > 0 { if host == "" || host == ".*" { sanitized.Set("host", g.allowedHostsRegex) } else { matched, err := g.matchAllowed(host, g.allowedHosts, g.allowedHostRules) if err != nil { return nil, 0, err } if len(matched) == 0 { return nil, 0, fmt.Errorf("host %q is not in the public allow-list", host) } if g.IsHostAllowed(host) { sanitized.Set("host", host) } else if len(matched) == 1 { m := matched[0] if isRegexPattern(m) { sanitized.Set("host", "^(?:"+m+")$") } else { sanitized.Set("host", "^("+regexp.QuoteMeta(m)+")$") } } else { quoted := make([]string, len(matched)) for i, m := range matched { if isRegexPattern(m) { quoted[i] = "(?:" + m + ")" } else { quoted[i] = regexp.QuoteMeta(m) } } sanitized.Set("host", "^("+strings.Join(quoted, "|")+")$") } } } else if host != "" { sanitized.Set("host", host) }
// 2. Service validation & rewriting service := strings.TrimSpace(q.Get("service")) if len(g.allowedServices) > 0 || len(g.allowedServiceRules) > 0 { if service == "" || service == ".*" { sanitized.Set("service", g.allowedServicesRegex) } else { matched, err := g.matchAllowed(service, g.allowedServices, g.allowedServiceRules) if err != nil { return nil, 0, err } if len(matched) == 0 { return nil, 0, fmt.Errorf("service %q is not in the public allow-list", service) } if g.IsServiceAllowed(service) { sanitized.Set("service", service) } else if len(matched) == 1 { m := matched[0] if isRegexPattern(m) { sanitized.Set("service", "^(?:"+m+")$") } else { sanitized.Set("service", "^("+regexp.QuoteMeta(m)+")$") } } else { quoted := make([]string, len(matched)) for i, m := range matched { if isRegexPattern(m) { quoted[i] = "(?:" + m + ")" } else { quoted[i] = regexp.QuoteMeta(m) } } sanitized.Set("service", "^("+strings.Join(quoted, "|")+")$") } } } else if service != "" { sanitized.Set("service", service) }
// 3. Time window parsing and clamping to := now if toStr := q.Get("to"); toStr != "" { if val, err := strconv.ParseInt(toStr, 10, 64); err == nil && val > 0 { to = val } }
from := to - (3600 * 1000) // Default 1 hour if fromStr := q.Get("from"); fromStr != "" { if val, err := strconv.ParseInt(fromStr, 10, 64); err == nil && val > 0 { from = val } }
if from > to { from = to - (3600 * 1000) }
if (to - from) > g.maxWindowMs { from = to - g.maxWindowMs }
// 4. Forced Modulus Quantization qFrom, qTo, ttl, _ := query.ComputeQuantization(from, to, now) sanitized.Set("from", strconv.FormatInt(qFrom, 10)) sanitized.Set("to", strconv.FormatInt(qTo, 10))
// 5. Downsampling limits (protect against multi-megabyte payloads) maxPoints := 1500 if mpStr := q.Get("max_points"); mpStr != "" { if val, err := strconv.Atoi(mpStr); err == nil && val > 0 { maxPoints = min(val, 5000) // Upper ceiling at 5,000 points } } else if ptsStr := q.Get("points"); ptsStr != "" { if val, err := strconv.Atoi(ptsStr); err == nil && val > 0 { maxPoints = min(val, 5000) } } sanitized.Set("max_points", strconv.Itoa(maxPoints))
if step := q.Get("step"); step != "" { sanitized.Set("step", step) } if method := q.Get("method"); method != "" { sanitized.Set("method", method) } if q.Get("skip_empty") == "1" || q.Get("non_empty") == "1" { sanitized.Set("skip_empty", "1") }
return sanitized, ttl, nil}
// CanonicalCacheKey generates a deterministic, sorted cache key.func (g *SecurityGate) CanonicalCacheKey(prefix string, vals url.Values) string { keys := make([]string, 0, len(vals)) for k := range vals { keys = append(keys, k) } sort.Strings(keys)
var sb strings.Builder sb.WriteString(prefix) for _, k := range keys { sb.WriteByte(':') sb.WriteString(k) sb.WriteByte('=') sb.WriteString(vals.Get(k)) } return sb.String()}
// ScrubSeriesResults unmarshals JSON query responses from upstream and removes// any series whose host or service is not permitted by the allow-list.func (g *SecurityGate) ScrubSeriesResults(data []byte) ([]byte, error) { if len(g.allowedHostRules) == 0 && len(g.allowedServiceRules) == 0 { return data, nil }
var results []query.SeriesResult if err := json.Unmarshal(data, &results); err != nil { return data, nil // Return as-is if not standard JSON series results }
scrubbed := make([]query.SeriesResult, 0, len(results)) for _, s := range results { if g.IsAllowed(s.Host, s.Service) { scrubbed = append(scrubbed, s) } }
return json.Marshal(scrubbed)}