appview,knotclient,knotserver,spindle: pass ctx throughout

Signed-off-by: Anirudh Oppiliappan <[e-mail]>

appview/state/knotstream.go

···
 package state
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"slices"
···
 	"github.com/posthog/posthog-go"
 )
 
-func KnotstreamConsumer(c *config.Config, d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client) (*kc.EventConsumer, error) {
+func KnotstreamConsumer(ctx context.Context, c *config.Config, d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client) (*kc.EventConsumer, error) {
 	knots, err := db.GetCompletedRegistrations(d)
 	if err != nil {
 		return nil, err
···
 
 	cfg := kc.ConsumerConfig{
 		Sources:           srcs,
-		ProcessFunc:       knotstreamIngester(d, enforcer, posthog, c.Core.Dev),
+		ProcessFunc:       knotstreamIngester(ctx, d, enforcer, posthog, c.Core.Dev),
 		RetryInterval:     c.Knotstream.RetryInterval,
 		MaxRetryInterval:  c.Knotstream.MaxRetryInterval,
 		ConnectionTimeout: c.Knotstream.ConnectionTimeout,
···
 	return kc.NewEventConsumer(cfg), nil
 }
 
-func knotstreamIngester(d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client, dev bool) kc.ProcessFunc {
-	return func(source kc.EventSource, msg kc.Message) error {
+func knotstreamIngester(ctx context.Context, d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client, dev bool) kc.ProcessFunc {
+	return func(ctx context.Context, source kc.EventSource, msg kc.Message) error {
 		switch msg.Nsid {
 		case tangled.GitRefUpdateNSID:
 			return ingestRefUpdate(d, enforcer, posthog, dev, source, msg)

appview/state/state.go

···
 	knotstream   *knotclient.EventConsumer
 }
 
-func Make(config *config.Config) (*State, error) {
+func Make(ctx context.Context, config *config.Config) (*State, error) {
 	d, err := db.Make(config.Core.DbPath)
 	if err != nil {
 		return nil, err
···
 	if err != nil {
 		return nil, fmt.Errorf("failed to create jetstream client: %w", err)
 	}
-	err = jc.StartJetstream(context.Background(), appview.Ingest(wrapper, enforcer))
+	err = jc.StartJetstream(ctx, appview.Ingest(wrapper, enforcer))
 	if err != nil {
 		return nil, fmt.Errorf("failed to start jetstream watcher: %w", err)
 	}
 
-	knotstream, err := KnotstreamConsumer(config, d, enforcer, posthog)
+	knotstream, err := KnotstreamConsumer(ctx, config, d, enforcer, posthog)
 	if err != nil {
 		return nil, fmt.Errorf("failed to start knotstream consumer: %w", err)
 	}
-	knotstream.Start(context.Background())
+	knotstream.Start(ctx)
 
 	state := &State{
 		d,

cmd/appview/main.go

···
 func main() {
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, nil)))
 
-	c, err := config.LoadConfig(context.Background())
+	ctx := context.Background()
+
+	c, err := config.LoadConfig(ctx)
 	if err != nil {
 		log.Println("failed to load config", "error", err)
 		return
 	}
 
-	state, err := state.Make(c)
+	state, err := state.Make(ctx, c)
 
 	if err != nil {
 		log.Fatal(err)

cmd/eventconsumer/main.go

···
 		return
 	}
 
-	var srcs []knotclient.EventSource
-	for k := range strings.SplitSeq(*knots, ",") {
-		srcs = append(srcs, knotclient.EventSource{k})
-	}
-
-	consumer := knotclient.NewEventConsumer(knotclient.ConsumerConfig{
-		Sources:          srcs,
+	ccfg := knotclient.ConsumerConfig{
 		ProcessFunc:      processEvent,
 		RetryInterval:    *retryFlag,
 		MaxRetryInterval: *maxRetryFlag,
 		WorkerCount:      *workerCount,
 		Dev:              true,
-	})
+	}
+	for k := range strings.SplitSeq(*knots, ",") {
+		ccfg.AddEventSource(knotclient.NewEventSource(k))
+	}
+
+	consumer := knotclient.NewEventConsumer(ccfg)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	consumer.Start(ctx)
···
 	consumer.Stop()
 }
 
-func processEvent(source knotclient.EventSource, msg knotclient.Message) error {
+func processEvent(_ context.Context, source knotclient.EventSource, msg knotclient.Message) error {
 	fmt.Printf("From %s (%s, %s): %s\n", source.Knot, msg.Rkey, msg.Nsid, string(msg.EventJson))
 	return nil
 }

knotserver/events.go

···
 	}
 
 	// complete backfill first before going to live data
+	l.Info("going through backfill", "cursor", cursor)
 	l.Debug("going through backfill", "cursor", cursor)
 	if err := h.streamOps(conn, &cursor); err != nil {
 		l.Error("failed to backfill", "err", err)
 		return
 	}
-
 	for {
 		// wait for new data or timeout
 		select {

spindle/exec.go

···
-package spindle
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-
-	"tangled.sh/tangled.sh/core/api/tangled"
-)
-
-func (s *Spindle) exec(ctx context.Context, src string, msg []byte) error {
-	pipeline := tangled.Pipeline{}
-	data := map[string]any{}
-	err := json.Unmarshal(msg, &data)
-	if err != nil {
-		fmt.Println("error unmarshalling", err)
-		return err
-	}
-
-	if data["nsid"] == tangled.PipelineNSID {
-		event, ok := data["event"]
-		if !ok {
-			s.l.Error("no event in message")
-			return nil
-		}
-
-		rawEvent, err := json.Marshal(event)
-		if err != nil {
-			return err
-		}
-
-		err = json.Unmarshal(rawEvent, &pipeline)
-		if err != nil {
-			return err
-		}
-
-		rkey, ok := data["rkey"].(string)
-		if !ok {
-			s.l.Error("no rkey in message")
-			return nil
-		}
-
-		err = s.eng.SetupPipeline(ctx, &pipeline, rkey)
-		if err != nil {
-			return err
-		}
-		err = s.eng.StartWorkflows(ctx, &pipeline, rkey)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

spindle/server.go

···
 package spindle
 
 import (
+	"encoding/json"
 	"fmt"
 	"log/slog"
 	"net/http"
···
 
 	go func() {
 		logger.Info("starting event consumer")
-		ec := knotclient.NewEventConsumer(knotclient.ConsumerConfig{
-			Sources:     []string{"ws://localhost:5555/events"},
+		knotEventSource := knotclient.NewEventSource("localhost:5555")
+		ccfg := knotclient.ConsumerConfig{
 			Logger:      logger,
 			ProcessFunc: spindle.exec,
-		})
+		}
+		ccfg.AddEventSource(knotEventSource)
+
+		ec := knotclient.NewEventConsumer(ccfg)
 
 		ec.Start(ctx)
 	}()
···
 	mux.HandleFunc("/events", s.Events)
 	return mux
 }
+
+func (s *Spindle) exec(ctx context.Context, src knotclient.EventSource, msg knotclient.Message) error {
+	pipeline := tangled.Pipeline{}
+	err := json.Unmarshal(msg.EventJson, &pipeline)
+	if err != nil {
+		fmt.Println("error unmarshalling", err)
+		return err
+	}
+
+	if msg.Nsid == tangled.PipelineNSID {
+		err = s.eng.SetupPipeline(ctx, &pipeline, msg.Rkey)
+		if err != nil {
+			return err
+		}
+		err = s.eng.StartWorkflows(ctx, &pipeline, msg.Rkey)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}