spindle/queue: enqueue pipeline jobs

Signed-off-by: Anirudh Oppiliappan <[e-mail]>

spindle/queue/queue.go

···
+package queue
+
+type Job struct {
+	Run    func() error
+	OnFail func(error)
+}
+
+type Queue struct {
+	jobs chan Job
+}
+
+func NewQueue(size int) *Queue {
+	return &Queue{
+		jobs: make(chan Job, size),
+	}
+}
+
+func (q *Queue) Enqueue(job Job) bool {
+	select {
+	case q.jobs <- job:
+		return true
+	default:
+		return false
+	}
+}
+
+func (q *Queue) StartRunner() {
+	go func() {
+		for job := range q.jobs {
+			if err := job.Run(); err != nil {
+				if job.OnFail != nil {
+					job.OnFail(err)
+				}
+			}
+		}
+	}()
+}

spindle/server.go

···
 package spindle
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"log/slog"
 	"net/http"
 
 	"github.com/go-chi/chi/v5"
-	"golang.org/x/net/context"
 	"tangled.sh/tangled.sh/core/api/tangled"
 	"tangled.sh/tangled.sh/core/jetstream"
 	"tangled.sh/tangled.sh/core/knotclient"
···
 	"tangled.sh/tangled.sh/core/spindle/config"
 	"tangled.sh/tangled.sh/core/spindle/db"
 	"tangled.sh/tangled.sh/core/spindle/engine"
+	"tangled.sh/tangled.sh/core/spindle/queue"
 )
 
 type Spindle struct {
···
 	l   *slog.Logger
 	n   *notifier.Notifier
 	eng *engine.Engine
+	jq  *queue.Queue
 }
 
 func Run(ctx context.Context) error {
···
 		return err
 	}
 
+	jq := queue.NewQueue(100)
+
+	// starts a job queue runner in the background
+	jq.StartRunner()
+
 	spindle := Spindle{
 		jc:  jc,
 		e:   e,
···
 		l:   logger,
 		n:   &n,
 		eng: eng,
+		jq:  jq,
 	}
 
+	// for each incoming sh.tangled.pipeline, we execute
+	// spindle.processPipeline, which in turn enqueues the pipeline
+	// job in the above registered queue.
 	go func() {
 		logger.Info("starting event consumer")
 		knotEventSource := knotclient.NewEventSource("localhost:5555")
···
 		ccfg := knotclient.NewConsumerConfig()
 		ccfg.Logger = logger
 		ccfg.Dev = cfg.Server.Dev
-		ccfg.ProcessFunc = spindle.exec
+		ccfg.ProcessFunc = spindle.processPipeline
 		ccfg.AddEventSource(knotEventSource)
 
 		ec := knotclient.NewEventConsumer(*ccfg)
···
 	return mux
 }
 
-func (s *Spindle) exec(ctx context.Context, src knotclient.EventSource, msg knotclient.Message) error {
+func (s *Spindle) processPipeline(ctx context.Context, src knotclient.EventSource, msg knotclient.Message) error {
 	if msg.Nsid == tangled.PipelineNSID {
 		pipeline := tangled.Pipeline{}
 		err := json.Unmarshal(msg.EventJson, &pipeline)
···
 			return err
 		}
 
-		// this is a "fake" at uri for now
-		pipelineAtUri := fmt.Sprintf("at://%s/did:web:%s/%s", tangled.PipelineNSID, pipeline.TriggerMetadata.Repo.Knot, msg.Rkey)
+		ok := s.jq.Enqueue(queue.Job{
+			Run: func() error {
+				// this is a "fake" at uri for now
+				pipelineAtUri := fmt.Sprintf("at://%s/did:web:%s/%s", tangled.PipelineNSID, pipeline.TriggerMetadata.Repo.Knot, msg.Rkey)
 
-		rkey := TID()
-		err = s.eng.SetupPipeline(ctx, &pipeline, pipelineAtUri, rkey)
-		if err != nil {
-			return err
-		}
-		err = s.eng.StartWorkflows(ctx, &pipeline, rkey)
-		if err != nil {
-			return err
+				rkey := TID()
+				err = s.eng.SetupPipeline(ctx, &pipeline, pipelineAtUri, rkey)
+				if err != nil {
+					return err
+				}
+				return s.eng.StartWorkflows(ctx, &pipeline, rkey)
+			},
+			OnFail: func(error) {
+				s.l.Error("pipeline setup failed", "error", err)
+			},
+		})
+		if ok {
+			s.l.Info("pipeline enqueued successfully", "id", msg.Rkey)
+		} else {
+			s.l.Error("failed to enqueue pipeline: queue is full")
 		}
 	}