Something went wrong. Try again.
Metric and Time-Series Engine: A Minimal Archiving Time Series Storage Service
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233package sseproxy
import ( "context" "encoding/binary" "fmt" "io" "net" "net/http" "net/http/httptest" "sync" "testing" "time"
pb "go.iscode.ca/mats/internal/proto/riemann" "google.golang.org/protobuf/proto")
func TestParseSSEPayload(t *testing.T) { tests := []struct { name string payload string wantHost string wantService string wantMetric float64 wantTime int64 wantAttrKey string wantAttrVal string }{ { name: "Riemann ISO8601 millisecond timestamp with flattened attributes", payload: `{"host":"pr300.site2.prd","service":"lxcnetstat-wp/gauge-TcpExt_TCPTimeouts","state":"ok","description":null,"metric":4271.0,"tags":["gw:default","gw:prd"],"time":"2026-06-27T15:03:10.000Z","ttl":900,"gwalias":"@wp","ds_index":"0","ds_name":"value","ds_type":"gauge","type_instance":"TcpExt_TCPTimeouts","type":"gauge","plugin_instance":"wp","plugin":"lxcnetstat"}`, wantHost: "pr300.site2.prd", wantService: "lxcnetstat-wp/gauge-TcpExt_TCPTimeouts", wantMetric: 4271.0, wantTime: 1782572590, // 2026-06-27T15:03:10Z wantAttrKey: "plugin", wantAttrVal: "lxcnetstat", }, { name: "RFC3339 timestamp", payload: `{"host":"nas-01","service":"cpu.idle","metric":98.5,"time":"2026-09-05T20:52:48Z"}`, wantHost: "nas-01", wantService: "cpu.idle", wantMetric: 98.5, wantTime: 1788641568, // 2026-09-05T20:52:48Z }, { name: "Unix timestamp integer", payload: `{"host":"nas-01","service":"mem.free","metric":1024.0,"time":1725537600}`, wantHost: "nas-01", wantService: "mem.free", wantMetric: 1024.0, wantTime: 1725537600, }, { name: "String metric and nested attributes", payload: `{"host":"server-01","service":"load.one","metric":"2.5","attributes":{"cluster":"prod"}}`, wantHost: "server-01", wantService: "load.one", wantMetric: 2.5, wantAttrKey: "cluster", wantAttrVal: "prod", }, }
for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ev, err := parseSSEPayload([]byte(tc.payload)) if err != nil { t.Fatalf("failed to parse payload: %v", err) } if ev.GetHost() != tc.wantHost { t.Errorf("expected host %q, got %q", tc.wantHost, ev.GetHost()) } if ev.GetService() != tc.wantService { t.Errorf("expected service %q, got %q", tc.wantService, ev.GetService()) } if ev.MetricD == nil || *ev.MetricD != tc.wantMetric { t.Errorf("expected metric %f, got %v", tc.wantMetric, ev.MetricD) } if tc.wantTime > 0 && ev.GetTime() != tc.wantTime { t.Errorf("expected time %d, got %d", tc.wantTime, ev.GetTime()) } if tc.wantAttrKey != "" { found := false for _, attr := range ev.GetAttributes() { if attr.GetKey() == tc.wantAttrKey && attr.GetValue() == tc.wantAttrVal { found = true break } } if !found { t.Errorf("expected attribute %s=%s in %+v", tc.wantAttrKey, tc.wantAttrVal, ev.GetAttributes()) } } }) }}
func TestProxy_EndToEnd(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel()
// 1. Mock Riemann HTTP SSE Server with real Riemann JSON formats sseEvents := []string{ `{"host":"nas-01","service":"cpu.idle","metric":98.5,"time":"2026-09-05T20:52:48Z","plugin":"cpu"}`, `{"host":"nas-01","service":"mem.free","metric":1024.0,"time":"2026-09-05T20:52:48.000Z","plugin":"memory"}`, }
sseServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.WriteHeader(http.StatusOK)
flusher, ok := w.(http.Flusher) if !ok { return }
for _, ev := range sseEvents { _, _ = fmt.Fprintf(w, "data: %s\n\n", ev) flusher.Flush() time.Sleep(10 * time.Millisecond) }
<-r.Context().Done() })) defer sseServer.Close()
// 2. Mock Downstream Riemann TCP Listener (like mats engine) var ( receivedEvents []*pb.Event mu sync.Mutex )
tcpListener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("failed starting mock TCP listener: %v", err) } defer func() { _ = tcpListener.Close() }()
go func() { for { conn, err := tcpListener.Accept() if err != nil { return } go func(c net.Conn) { defer func() { _ = c.Close() }() var lenBuf [4]byte ackMsg := &pb.Msg{Ok: new(true)} ackBytes, _ := proto.Marshal(ackMsg) var ackFrame [4]byte binary.BigEndian.PutUint32(ackFrame[:], uint32(len(ackBytes)))
for { if _, err := io.ReadFull(c, lenBuf[:]); err != nil { return } msgLen := binary.BigEndian.Uint32(lenBuf[:]) payload := make([]byte, msgLen) if _, err := io.ReadFull(c, payload); err != nil { return }
var msg pb.Msg if err := proto.Unmarshal(payload, &msg); err != nil { return }
mu.Lock() receivedEvents = append(receivedEvents, msg.Events...) mu.Unlock()
// Send framed ACK _, _ = c.Write(ackFrame[:]) _, _ = c.Write(ackBytes) } }(conn) } }()
// 3. Start Proxy cfg := Config{ SSEURL: sseServer.URL, TargetAddr: tcpListener.Addr().String(), Protocol: "tcp", BatchSize: 2, BatchTimeout: 50 * time.Millisecond, BufferSize: 100, }
proxy := NewProxy(cfg) proxy.Start(ctx)
// 4. Wait for delivery deadline := time.Now().Add(3 * time.Second) for time.Now().Before(deadline) { mu.Lock() count := len(receivedEvents) mu.Unlock() if count >= 2 { break } time.Sleep(20 * time.Millisecond) }
// 5. Clean teardown before assertions cancel() proxy.Wait()
mu.Lock() defer mu.Unlock()
if len(receivedEvents) != 2 { t.Fatalf("expected 2 received events downstream, got %d", len(receivedEvents)) }
if receivedEvents[0].GetHost() != "nas-01" || receivedEvents[0].GetService() != "cpu.idle" { t.Errorf("unexpected event 0: %+v", receivedEvents[0]) } if receivedEvents[1].GetHost() != "nas-01" || receivedEvents[1].GetService() != "mem.free" { t.Errorf("unexpected event 1: %+v", receivedEvents[1]) }
recv, fwd, drop, parseErrs, sendErrs := proxy.Stats() if recv != 2 || fwd == 0 || drop != 0 || parseErrs != 0 || sendErrs != 0 { t.Errorf("unexpected stats: recv=%d, fwd=%d, drop=%d, parseErrs=%d, sendErrs=%d", recv, fwd, drop, parseErrs, sendErrs) }}