package sseproxy import ( "bufio" "context" "encoding/binary" "encoding/json" "errors" "fmt" "io" "log" "net" "net/http" "strconv" "strings" "sync" "sync/atomic" "time" pb "go.iscode.ca/mats/internal/proto/riemann" "google.golang.org/protobuf/proto" ) // Config holds options for the Riemann SSE to Protobuf forwarding proxy. type Config struct { SSEURL string // e.g. "http://riemann:5556/index?query=true" TargetAddr string // e.g. "127.0.0.1:5555" Protocol string // "tcp" or "udp" BatchSize int // Max events per forwarded message (default: 100) BatchTimeout time.Duration // Max wait before sending partial batch (default: 100ms) BufferSize int // Max queue size before dropping events (default: 10000) HTTPClient *http.Client Verbose bool // Enable verbose debugging output } // Proxy streams events from Riemann's HTTP SSE API and forwards them // as framed Protobuf messages (TCP) or datagrams (UDP) to a downstream receiver. type Proxy struct { cfg Config eventChan chan *pb.Event wg sync.WaitGroup // Metrics receivedCount atomic.Uint64 forwardedCount atomic.Uint64 droppedQueueFull atomic.Uint64 parseErrors atomic.Uint64 sendErrors atomic.Uint64 } func NewProxy(cfg Config) *Proxy { if cfg.BatchSize <= 0 { cfg.BatchSize = 100 } if cfg.BatchTimeout <= 0 { cfg.BatchTimeout = 100 * time.Millisecond } if cfg.BufferSize <= 0 { cfg.BufferSize = 10000 } if cfg.Protocol == "" { cfg.Protocol = "tcp" } if cfg.HTTPClient == nil { cfg.HTTPClient = &http.Client{ Timeout: 0, // Infinite timeout for long-lived SSE streaming } } return &Proxy{ cfg: cfg, eventChan: make(chan *pb.Event, cfg.BufferSize), } } // Stats returns operational counters. func (p *Proxy) Stats() (received, forwarded, droppedQueueFull, parseErrors, sendErrors uint64) { return p.receivedCount.Load(), p.forwardedCount.Load(), p.droppedQueueFull.Load(), p.parseErrors.Load(), p.sendErrors.Load() } // Start launches the background SSE ingestion and downstream sender goroutines. func (p *Proxy) Start(ctx context.Context) { p.wg.Add(2) go p.runSSEReader(ctx) go p.runBatchSender(ctx) } // Wait blocks until all goroutines exit cleanly. func (p *Proxy) Wait() { p.wg.Wait() } func (p *Proxy) runSSEReader(ctx context.Context) { defer p.wg.Done() backoff := 500 * time.Millisecond for { select { case <-ctx.Done(): return default: } err := p.streamSSE(ctx) if err != nil && ctx.Err() == nil { log.Printf("[sse-proxy] SSE stream disconnected (%v); reconnecting in %v...", err, backoff) select { case <-ctx.Done(): return case <-time.After(backoff): backoff *= 2 if backoff > 10*time.Second { backoff = 10 * time.Second } } } else { backoff = 500 * time.Millisecond } } } func (p *Proxy) streamSSE(ctx context.Context) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.cfg.SSEURL, nil) if err != nil { return err } req.Header.Set("Accept", "text/event-stream") req.Header.Set("Cache-Control", "no-cache") resp, err := p.cfg.HTTPClient.Do(req) if err != nil { return err } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status %d from riemann sse", resp.StatusCode) } log.Printf("[sse-proxy] Connected to Riemann SSE stream at %s (HTTP %s)", p.cfg.SSEURL, resp.Status) reader := bufio.NewReader(resp.Body) for { line, err := reader.ReadString('\n') if err != nil { return err } line = strings.TrimSpace(line) if !strings.HasPrefix(line, "data:") { continue } payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) if payload == "" || payload == "[DONE]" { continue } pbEv, err := parseSSEPayload([]byte(payload)) if err != nil { n := p.parseErrors.Add(1) if n <= 10 || p.cfg.Verbose { log.Printf("[sse-proxy] Warning: failed to parse SSE event #%d: %v (payload: %.120s)", n, err, payload) } continue } if p.cfg.Verbose { metricVal := "" if pbEv.MetricD != nil { metricVal = fmt.Sprintf("%f", *pbEv.MetricD) } log.Printf("[sse-proxy] [event] host=%s service=%s metric=%s time=%d", pbEv.GetHost(), pbEv.GetService(), metricVal, pbEv.GetTime()) } // Non-blocking enqueue to protect upstream Riemann from backpressure select { case p.eventChan <- pbEv: p.receivedCount.Add(1) default: // Buffer full: drop oldest event and enqueue newest select { case <-p.eventChan: default: } p.eventChan <- pbEv p.droppedQueueFull.Add(1) } } } func parseSSEPayload(payload []byte) (*pb.Event, error) { var raw map[string]any if err := json.Unmarshal(payload, &raw); err != nil { return nil, fmt.Errorf("invalid json: %w", err) } ev := &pb.Event{} for k, v := range raw { if v == nil { continue } switch k { case "host": if s, ok := v.(string); ok { ev.Host = new(s) } case "service": if s, ok := v.(string); ok { ev.Service = new(s) } case "state": if s, ok := v.(string); ok { ev.State = new(s) } case "description": if s, ok := v.(string); ok { ev.Description = new(s) } case "ttl": if f, ok := toFloat64(v); ok { ev.Ttl = new(float32(f)) } case "time": if ts, ok := parseTimestamp(v); ok { ev.Time = new(ts) } case "time_micros": if tm, ok := toInt64(v); ok { ev.TimeMicros = new(tm) } case "metric": if f, ok := toFloat64(v); ok { ev.MetricD = new(f) } case "tags": if arr, ok := v.([]any); ok { for _, tag := range arr { if s, ok := tag.(string); ok && s != "" { ev.Tags = append(ev.Tags, s) } } } case "attributes": if m, ok := v.(map[string]any); ok { for attrK, attrV := range m { if attrV != nil { ev.Attributes = append(ev.Attributes, &pb.Attribute{ Key: new(attrK), Value: new(fmt.Sprintf("%v", attrV)), }) } } } case "extra": if m, ok := v.(map[string]any); ok { for attrK, attrV := range m { if attrV != nil { ev.Attributes = append(ev.Attributes, &pb.Attribute{ Key: new(attrK), Value: new(fmt.Sprintf("%v", attrV)), }) } } } default: // Custom attributes flattened at top-level switch val := v.(type) { case string: ev.Attributes = append(ev.Attributes, &pb.Attribute{ Key: new(k), Value: new(val), }) case float64, float32, int, int64, bool: ev.Attributes = append(ev.Attributes, &pb.Attribute{ Key: new(k), Value: new(fmt.Sprintf("%v", val)), }) } } } // Default timestamp if missing or zero if ev.Time == nil || *ev.Time == 0 { ev.Time = new(time.Now().Unix()) } if ev.GetHost() == "" && ev.GetService() == "" { return nil, errors.New("event missing both host and service") } return ev, nil } func parseTimestamp(v any) (int64, bool) { switch t := v.(type) { case float64: sec := int64(t) if sec > 1e11 { sec /= 1000 } return sec, true case int64: if t > 1e11 { t /= 1000 } return t, true case int: return int64(t), true case string: str := strings.TrimSpace(t) if num, err := strconv.ParseInt(str, 10, 64); err == nil { if num > 1e11 { num /= 1000 } return num, true } layouts := []string{ time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.000Z", "2006-01-02T15:04:05Z", "2006-01-02 15:04:05", "2006-01-02T15:04:05", } for _, layout := range layouts { if parsed, err := time.Parse(layout, str); err == nil { return parsed.Unix(), true } } return 0, false default: return 0, false } } func toFloat64(v any) (float64, bool) { switch n := v.(type) { case float64: return n, true case float32: return float64(n), true case int64: return float64(n), true case int: return float64(n), true case json.Number: if f, err := n.Float64(); err == nil { return f, true } case string: if f, err := strconv.ParseFloat(strings.TrimSpace(n), 64); err == nil { return f, true } } return 0, false } func toInt64(v any) (int64, bool) { switch n := v.(type) { case float64: return int64(n), true case int64: return n, true case int: return int64(n), true case string: if i, err := strconv.ParseInt(strings.TrimSpace(n), 10, 64); err == nil { return i, true } } return 0, false } func (p *Proxy) runBatchSender(ctx context.Context) { defer p.wg.Done() ticker := time.NewTicker(p.cfg.BatchTimeout) defer ticker.Stop() var batch []*pb.Event flush := func() { if len(batch) == 0 { return } p.sendBatch(ctx, batch) batch = nil } for { select { case <-ctx.Done(): // Drain remaining events in channel for { select { case ev := <-p.eventChan: batch = append(batch, ev) if len(batch) >= p.cfg.BatchSize { flush() } default: flush() return } } case ev := <-p.eventChan: batch = append(batch, ev) if len(batch) >= p.cfg.BatchSize { flush() } case <-ticker.C: flush() } } } func (p *Proxy) sendBatch(ctx context.Context, events []*pb.Event) { msg := &pb.Msg{ Events: events, } payload, err := proto.Marshal(msg) if err != nil { p.sendErrors.Add(1) return } if strings.ToLower(p.cfg.Protocol) == "udp" { p.sendUDP(payload) } else { p.sendTCP(ctx, payload) } } func (p *Proxy) sendUDP(payload []byte) { conn, err := net.Dial("udp", p.cfg.TargetAddr) if err != nil { p.sendErrors.Add(1) return } defer func() { _ = conn.Close() }() if _, err := conn.Write(payload); err != nil { p.sendErrors.Add(1) return } p.forwardedCount.Add(1) } func (p *Proxy) sendTCP(ctx context.Context, payload []byte) { var d net.Dialer conn, err := d.DialContext(ctx, "tcp", p.cfg.TargetAddr) if err != nil { n := p.sendErrors.Add(1) if n <= 10 || p.cfg.Verbose { log.Printf("[sse-proxy] Target connection error (%s): %v", p.cfg.TargetAddr, err) } return } defer func() { _ = conn.Close() }() var frame [4]byte binary.BigEndian.PutUint32(frame[:], uint32(len(payload))) _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) // Send length prefix + payload if _, err := conn.Write(frame[:]); err != nil { p.sendErrors.Add(1) return } if _, err := conn.Write(payload); err != nil { p.sendErrors.Add(1) return } // Read ACK var ackLenBuf [4]byte if _, err := io.ReadFull(conn, ackLenBuf[:]); err != nil { p.sendErrors.Add(1) return } ackLen := binary.BigEndian.Uint32(ackLenBuf[:]) ackPayload := make([]byte, ackLen) if _, err := io.ReadFull(conn, ackPayload); err != nil { p.sendErrors.Add(1) return } p.forwardedCount.Add(1) }