spindle/models: Pipline model and setup steps

Signed-off-by: Anirudh Oppiliappan <[e-mail]>

spindle/models/pipeline.go

···
+package models
+
+import (
+	"path"
+
+	"tangled.sh/tangled.sh/core/api/tangled"
+)
+
+type Pipeline struct {
+	Workflows []Workflow
+}
+
+type Step struct {
+	Command     string
+	Name        string
+	Environment map[string]string
+}
+
+type Workflow struct {
+	Steps       []Step
+	Environment map[string]string
+	Name        string
+	Image       string
+}
+
+// setupSteps get added to start of Steps
+type setupSteps []Step
+
+// addStep adds a step to the beginning of the workflow's steps.
+func (ss *setupSteps) addStep(step Step) {
+	*ss = append(*ss, step)
+}
+
+// ToPipeline converts a tangled.Pipeline into a model.Pipeline.
+// In the process, dependencies are resolved: nixpkgs deps
+// are constructed atop nixery and set as the Workflow.Image,
+// and ones from custom registries
+func ToPipeline(pl tangled.Pipeline, dev bool) *Pipeline {
+	workflows := []Workflow{}
+
+	for _, twf := range pl.Workflows {
+		swf := &Workflow{}
+		for _, tstep := range twf.Steps {
+			sstep := Step{}
+			sstep.Environment = stepEnvToMap(tstep.Environment)
+			sstep.Command = tstep.Command
+			sstep.Name = tstep.Name
+			swf.Steps = append(swf.Steps, sstep)
+		}
+		swf.Name = twf.Name
+		swf.Environment = workflowEnvToMap(twf.Environment)
+		swf.Image = workflowImage(twf.Dependencies)
+
+		swf.addNixProfileToPath()
+		setup := &setupSteps{}
+
+		setup.addStep(cloneStep(*twf, *pl.TriggerMetadata.Repo, dev))
+		setup.addStep(checkoutStep(*twf, *pl.TriggerMetadata))
+		setup.addStep(dependencyStep(*twf))
+
+		// append setup steps in order to the start of workflow steps
+		swf.Steps = append(*setup, swf.Steps...)
+
+		workflows = append(workflows, *swf)
+	}
+	return &Pipeline{Workflows: workflows}
+}
+
+func workflowEnvToMap(envs []*tangled.Pipeline_Workflow_Environment_Elem) map[string]string {
+	envMap := map[string]string{}
+	for _, env := range envs {
+		envMap[env.Key] = env.Value
+	}
+	return envMap
+}
+
+func stepEnvToMap(envs []*tangled.Pipeline_Step_Environment_Elem) map[string]string {
+	envMap := map[string]string{}
+	for _, env := range envs {
+		envMap[env.Key] = env.Value
+	}
+	return envMap
+}
+
+func workflowImage(deps []tangled.Pipeline_Dependencies_Elem) string {
+	var dependencies string
+	for _, d := range deps {
+		if d.Registry == "nixpkgs" {
+			dependencies = path.Join(d.Packages...)
+		}
+	}
+
+	// load defaults from somewhere else
+	dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix")
+
+	// TODO: this should use nixery from the config
+	return path.Join("nixery.dev", dependencies)
+}
+
+func (wf *Workflow) addNixProfileToPath() {
+	wf.Environment["PATH"] = "$PATH:/.nix-profile/bin"
+}

spindle/models/setup_steps.go

···
+package models
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"tangled.sh/tangled.sh/core/api/tangled"
+)
+
+// checkoutStep checks out the specified ref in the cloned repository.
+func checkoutStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMetadata) Step {
+	if twf.Clone.Skip {
+		return Step{}
+	}
+
+	var ref string
+	switch tr.Kind {
+	case "push":
+		ref = tr.Push.Ref
+	case "pull_request":
+		ref = tr.PullRequest.TargetBranch
+
+	// TODO: this needs to be specified in lexicon
+	case "manual":
+		ref = tr.Repo.DefaultBranch
+	}
+
+	checkoutCmd := fmt.Sprintf("git config advice.detachedHead false; git checkout --progress --force %s", ref)
+
+	return Step{
+		Command: checkoutCmd,
+		Name:    "Checkout ref " + ref,
+	}
+}
+
+// cloneOptsAsSteps processes clone options and adds corresponding steps
+// to the beginning of the workflow's step list if cloning is not skipped.
+func cloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerRepo, dev bool) Step {
+	if twf.Clone.Skip {
+		return Step{}
+	}
+
+	uri := "https://"
+	if dev {
+		uri = "http://"
+		tr.Knot = strings.ReplaceAll(tr.Knot, "localhost", "host.docker.internal")
+	}
+
+	cloneUrl := uri + path.Join(tr.Knot, tr.Did, tr.Repo)
+	cloneCmd := []string{"git", "clone", cloneUrl, "."}
+
+	// default clone depth is 1
+	cloneDepth := 1
+	if twf.Clone.Depth > 1 {
+		cloneDepth = int(twf.Clone.Depth)
+		cloneCmd = append(cloneCmd, []string{"--depth", fmt.Sprintf("%d", cloneDepth)}...)
+	}
+
+	if twf.Clone.Submodules {
+		cloneCmd = append(cloneCmd, "--recursive")
+	}
+
+	fmt.Println(strings.Join(cloneCmd, " "))
+
+	cloneStep := Step{
+		Command: strings.Join(cloneCmd, " "),
+		Name:    "Clone repository into workspace",
+	}
+	return cloneStep
+}
+
+// dependencyStep processes dependencies defined in the workflow.
+// For dependencies using a custom registry (i.e. not nixpkgs), it collects
+// all packages and adds a single 'nix profile install' step to the
+// beginning of the workflow's step list.
+func dependencyStep(twf tangled.Pipeline_Workflow) Step {
+	var customPackages []string
+
+	for _, d := range twf.Dependencies {
+		registry := d.Registry
+		packages := d.Packages
+
+		if registry == "nixpkgs" {
+			continue
+		}
+
+		// collect packages from custom registries
+		for _, pkg := range packages {
+			customPackages = append(customPackages, fmt.Sprintf("'%s#%s'", registry, pkg))
+		}
+	}
+
+	if len(customPackages) > 0 {
+		installCmd := "nix --extra-experimental-features nix-command --extra-experimental-features flakes profile install"
+		cmd := fmt.Sprintf("%s %s", installCmd, strings.Join(customPackages, " "))
+		installStep := Step{
+			Command: cmd,
+			Name:    "Install custom dependencies",
+			Environment: map[string]string{
+				"NIX_NO_COLOR":               "1",
+				"NIX_SHOW_DOWNLOAD_PROGRESS": "0",
+			},
+		}
+		return installStep
+	}
+	return Step{}
+}