appview,spindle: collapse system introduced steps

Signed-off-by: oppiliappan <[e-mail]>

appview/pages/pages.go

···
 	return p.executeRepo("repo/pipelines/pipelines", w, params)
 }
 
+type LogBlockParams struct {
+	Id        int
+	Name      string
+	Command   string
+	Collapsed bool
+}
+
+func (p *Pages) LogBlock(w io.Writer, params LogBlockParams) error {
+	return p.executePlain("repo/pipelines/fragments/logBlock", w, params)
+}
+
+type LogLineParams struct {
+	Id      int
+	Content string
+}
+
+func (p *Pages) LogLine(w io.Writer, params LogLineParams) error {
+	return p.executePlain("repo/pipelines/fragments/logLine", w, params)
+}
+
 type WorkflowParams struct {
 	LoggedInUser *oauth.User
 	RepoInfo     repoinfo.RepoInfo

appview/pages/templates/repo/pipelines/fragments/logBlock.html

···
+{{ define "repo/pipelines/fragments/logBlock" }}
+<div id="lines" hx-swap-oob="beforeend">
+  <details id="step-{{ .Id }}" {{if not .Collapsed}}open{{end}} class="group bg-gray-100 px-2 dark:bg-gray-900">
+    <summary class="sticky top-0 py-1 list-none cursor-pointer py-2 bg-gray-100 dark:bg-gray-900 hover:text-gray-500 hover:dark:text-gray-400">
+      <div class="group-open:hidden flex items-center gap-1">
+        {{ i "chevron-right" "w-4 h-4" }} {{ .Name }}
+      </div>
+      <div class="hidden group-open:flex items-center gap-1">
+        {{ i "chevron-down" "w-4 h-4" }} {{ .Name }}
+      </div>
+    </summary>
+    <div class="text-blue-600 dark:text-blue-300 font-mono">{{ .Command }}</div>
+    <div id="step-body-{{ .Id }}" class="font-mono"></div>
+  </details>
+</div>
+{{ end }}

appview/pages/templates/repo/pipelines/fragments/logLine.html

···
+{{ define "repo/pipelines/fragments/logLine" }}
+  <div id="step-body-{{ .Id }}" hx-swap-oob="beforeend">
+    <p>{{ .Content }}</p>
+  </div>
+{{ end }}
+

appview/pages/templates/repo/pipelines/pipelines.html

···
 
 {{ define "repoContent" }}
 <div class="flex justify-between items-center gap-4">
-  <div class="flex gap-4">
-  </div>
-
-</div>
-<div class="error" id="issues"></div>
-{{ end }}
-
-{{ define "repoAfter" }}
-  <section
-      class="w-full flex flex-col gap-2 mt-2"
-  >
+  <div class="w-full flex flex-col gap-2">
   {{ range .Pipelines }}
     {{ block "pipeline" (list $ .) }} {{ end }}
   {{ else }}
···
       No pipelines run for this repository.
     </p>
   {{ end }}
-  </section>
+  </div>
+</div>
 {{ end }}
+
 
 {{ define "pipeline" }}
   {{ $root := index . 0 }}
   {{ $p := index . 1 }}
-  <div class="py-4 px-6 bg-white dark:bg-gray-800 dark:text-white">
+  <div class="py-2 bg-white dark:bg-gray-800 dark:text-white">
     {{ block "pipelineHeader" $ }} {{ end }}
   </div>
 {{ end }}

appview/pages/templates/repo/pipelines/workflow.html

···
   {{ $active := .Workflow }}
   {{ with .Pipeline }}
     {{ $id := .Id }}
-    <div class="grid grid-cols-1 rounded border border-gray-200 dark:border-gray-700 divide-y divide-gray-200 dark:divide-gray-700">
+    <div class="sticky top-2 grid grid-cols-1 rounded border border-gray-200 dark:border-gray-700 divide-y divide-gray-200 dark:divide-gray-700">
       {{ range $name, $all := .Statuses }}
       <a href="/{{ $.RepoInfo.FullName }}/pipelines/{{ $id }}/workflow/{{ $name }}" class="no-underline hover:no-underline hover:bg-gray-100/25 hover:dark:bg-gray-700/25">
         <div
···
 
 {{ define "logs" }}
   <div id="log-stream"
-       class="p-2 bg-gray-100 dark:bg-gray-900 font-mono text-sm min-h-96 max-h-screen overflow-auto flex flex-col-reverse [overflow-anchor:auto_!important]"
+       class="text-sm"
        hx-ext="ws"
        ws-connect="/{{ $.RepoInfo.FullName }}/pipelines/{{ .Pipeline.Id }}/workflow/{{ .Workflow }}/logs">
-    <div id="lines">
-      <!-- Each log line should be rendered with class="item" like below -->
-      <!-- <div class="item">[INFO] Log line here</div> -->
+    <div id="lines" class="flex flex-col gap-2">
     </div>
   </div>
 {{ end }}

appview/pipelines/pipelines.go

···
 package pipelines
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
-	"html"
 	"log/slog"
 	"net/http"
 	"strings"
···
 
 	ctx, cancel := context.WithCancel(r.Context())
 	defer cancel()
-	go func() {
-		for {
-			if _, _, err := clientConn.NextReader(); err != nil {
-				l.Error("failed to read", "err", err)
-				cancel()
-				return
-			}
-		}
-	}()
 
 	user := p.oauth.GetUser(r)
 	f, err := p.repoResolver.Resolve(r)
···
 	defer spindleConn.Close()
 
 	// create a channel for incoming messages
-	msgChan := make(chan []byte, 10)
-	errChan := make(chan error, 1)
-
+	evChan := make(chan logEvent, 100)
 	// start a goroutine to read from spindle
-	go func() {
-		defer close(msgChan)
-		defer close(errChan)
-
-		for {
-			_, msg, err := spindleConn.ReadMessage()
-			if err != nil {
-				if websocket.IsCloseError(err,
-					websocket.CloseNormalClosure,
-					websocket.CloseGoingAway,
-					websocket.CloseAbnormalClosure) {
-					errChan <- nil // signal graceful end
-				} else {
-					errChan <- err
-				}
-				return
-			}
-			msgChan <- msg
-		}
-	}()
+	go readLogs(spindleConn, evChan)
 
 	stepIdx := 0
+	var fragment bytes.Buffer
 	for {
 		select {
 		case <-ctx.Done():
 			l.Info("client disconnected")
 			return
-		case err := <-errChan:
-			if err != nil {
-				l.Error("error reading from spindle", "err", err)
+
+		case ev, ok := <-evChan:
+			if !ok {
+				continue
 			}
 
-			if err == nil {
-				l.Info("log tail complete")
+			if ev.err != nil && ev.isCloseError() {
+				l.Debug("graceful shutdown, tail complete", "err", err)
+				return
+			}
+			if ev.err != nil {
+				l.Error("error reading from spindle", "err", err)
+				return
 			}
 
-			return
-		case msg := <-msgChan:
 			var logLine spindlemodel.LogLine
-			if err = json.Unmarshal(msg, &logLine); err != nil {
+			if err = json.Unmarshal(ev.msg, &logLine); err != nil {
 				l.Error("failed to parse logline", "err", err)
 				continue
 			}
 
-			var fragment []byte
+			fragment.Reset()
+
 			switch logLine.Kind {
 			case spindlemodel.LogKindControl:
 				// control messages create a new step block
 				stepIdx++
-				fragment = fmt.Appendf(nil, `
-					<div id="lines" hx-swap-oob="beforeend">
-						<details id="step-%d" open>
-							<summary>%s</summary>
-							<div id="step-body-%d"></div>
-						</details>
-					</div>
-				`, stepIdx, logLine.Content, stepIdx)
+				collapsed := false
+				if logLine.StepKind == spindlemodel.StepKindSystem {
+					collapsed = true
+				}
+				err = p.pages.LogBlock(&fragment, pages.LogBlockParams{
+					Id:        stepIdx,
+					Name:      logLine.Content,
+					Command:   logLine.StepCommand,
+					Collapsed: collapsed,
+				})
 			case spindlemodel.LogKindData:
 				// data messages simply insert new log lines into current step
-				escaped := html.EscapeString(logLine.Content)
-				fragment = fmt.Appendf(nil, `
-					<div id="step-body-%d" hx-swap-oob="beforeend">
-						<p>%s</p>
-					</div>
-				`, stepIdx, escaped)
+				err = p.pages.LogLine(&fragment, pages.LogLineParams{
+					Id:      stepIdx,
+					Content: logLine.Content,
+				})
+			}
+			if err != nil {
+				l.Error("failed to render log line", "err", err)
+				return
 			}
 
-			if err = clientConn.WriteMessage(websocket.TextMessage, fragment); err != nil {
+			if err = clientConn.WriteMessage(websocket.TextMessage, fragment.Bytes()); err != nil {
 				l.Error("error writing to client", "err", err)
 				return
 			}
+
 		case <-time.After(30 * time.Second):
 			l.Debug("sent keepalive")
 			if err = clientConn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second)); err != nil {
 				l.Error("failed to write control", "err", err)
+				return
 			}
 		}
 	}
 }
+
+// either a message or an error
+type logEvent struct {
+	msg []byte
+	err error
+}
+
+func (ev *logEvent) isCloseError() bool {
+	return websocket.IsCloseError(
+		ev.err,
+		websocket.CloseNormalClosure,
+		websocket.CloseGoingAway,
+		websocket.CloseAbnormalClosure,
+	)
+}
+
+// read logs from spindle and pass through to chan
+func readLogs(conn *websocket.Conn, ch chan logEvent) {
+	defer close(ch)
+
+	for {
+		if conn == nil {
+			return
+		}
+
+		_, msg, err := conn.ReadMessage()
+		if err != nil {
+			ch <- logEvent{err: err}
+			return
+		}
+		ch <- logEvent{msg: msg}
+	}
+}

spindle/engine/engine.go

···
 	}
 	defer wfLogger.Close()
 
-	ctl := wfLogger.ControlWriter()
-	ctl.Write([]byte(step.Command))
+	ctl := wfLogger.ControlWriter(stepIdx, step)
+	ctl.Write([]byte(step.Name))
 
 	logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{
 		Follow:     true,

spindle/engine/logger.go

···
 
 func (l *WorkflowLogger) DataWriter(stream string) io.Writer {
 	// TODO: emit stream
-	return &jsonWriter{logger: l, kind: models.LogKindData}
+	return &dataWriter{
+		logger: l,
+		stream: stream,
+	}
 }
 
-func (l *WorkflowLogger) ControlWriter() io.Writer {
-	return &jsonWriter{logger: l, kind: models.LogKindControl}
+func (l *WorkflowLogger) ControlWriter(idx int, step models.Step) io.Writer {
+	return &controlWriter{
+		logger: l,
+		idx:    idx,
+		step:   step,
+	}
 }
 
-type jsonWriter struct {
+type dataWriter struct {
 	logger *WorkflowLogger
-	kind   models.LogKind
+	stream string
 }
 
-func (w *jsonWriter) Write(p []byte) (int, error) {
+func (w *dataWriter) Write(p []byte) (int, error) {
 	line := strings.TrimRight(string(p), "\r\n")
+	entry := models.NewDataLogLine(line, w.stream)
+	if err := w.logger.encoder.Encode(entry); err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
 
-	entry := models.LogLine{
-		Kind:    w.kind,
-		Content: line,
-	}
+type controlWriter struct {
+	logger *WorkflowLogger
+	idx    int
+	step   models.Step
+}
 
+func (w *controlWriter) Write(_ []byte) (int, error) {
+	entry := models.NewControlLogLine(w.idx, w.step)
 	if err := w.logger.encoder.Encode(entry); err != nil {
 		return 0, err
 	}
-
-	return len(p), nil
+	return len(w.step.Name), nil
 }

spindle/models/models.go

···
 	Stream string `json:"stream,omitempty"`
 
 	// fields if kind is "control"
+	StepId      int      `json:"step_id,omitempty"`
+	StepKind    StepKind `json:"step_kind,omitempty"`
+	StepCommand string   `json:"step_command,omitempty"`
+}
+
+func NewDataLogLine(content, stream string) LogLine {
+	return LogLine{
+		Kind:    LogKindData,
+		Content: content,
+		Stream:  stream,
+	}
+}
+
+func NewControlLogLine(idx int, step Step) LogLine {
+	return LogLine{
+		Kind:        LogKindControl,
+		Content:     step.Name,
+		StepId:      idx,
+		StepKind:    step.Kind,
+		StepCommand: step.Command,
+	}
 }

spindle/models/pipeline.go

···
 	Command     string
 	Name        string
 	Environment map[string]string
+	Kind        StepKind
 }
+
+type StepKind int
+
+const (
+	// steps injected by the CI runner
+	StepKindSystem StepKind = iota
+	// steps defined by the user in the original pipeline
+	StepKindUser
+)
 
 type Workflow struct {
 	Steps       []Step
···
 			sstep.Environment = stepEnvToMap(tstep.Environment)
 			sstep.Command = tstep.Command
 			sstep.Name = tstep.Name
+			sstep.Kind = StepKindUser
 			swf.Steps = append(swf.Steps, sstep)
 		}
 		swf.Name = twf.Name
···
 		setup.addStep(nixConfStep())
 		setup.addStep(cloneStep(*twf, *pl.TriggerMetadata.Repo, cfg.Server.Dev))
 		setup.addStep(checkoutStep(*twf, *pl.TriggerMetadata))
-		setup.addStep(dependencyStep(*twf))
+		// this step could be empty
+		if s := dependencyStep(*twf); s != nil {
+			setup.addStep(*s)
+		}
 
 		// append setup steps in order to the start of workflow steps
 		swf.Steps = append(*setup, swf.Steps...)

spindle/models/setup_steps.go

···
 // For dependencies using a custom registry (i.e. not nixpkgs), it collects
 // all packages and adds a single 'nix profile install' step to the
 // beginning of the workflow's step list.
-func dependencyStep(twf tangled.Pipeline_Workflow) Step {
+func dependencyStep(twf tangled.Pipeline_Workflow) *Step {
 	var customPackages []string
 
 	for _, d := range twf.Dependencies {
···
 				"NIX_SHOW_DOWNLOAD_PROGRESS": "0",
 			},
 		}
-		return installStep
+		return &installStep
 	}
-	return Step{}
+	return nil
 }