package engine import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "git.kilimanjaro.io/ci/internal/lex" "git.kilimanjaro.io/ci/internal/manifest" "git.kilimanjaro.io/ci/internal/runlog" ) // DefaultMaxTransitions is the loop guard (DESIGN.md ยง8 rule 4). const DefaultMaxTransitions = 50 // Options change how a run executes. type Options struct { // Stubs maps an action to a result file used instead of running it. Stubs map[string]string // Until stops the run before this action would run. Until string // NoFollow runs the first action only and reports what would follow. NoFollow bool // MaxTransitions overrides DefaultMaxTransitions. MaxTransitions int // Gates decides gates as they are reached; nil pauses at every gate. Gates Gatekeeper // Detach pauses at every gate even with a Gatekeeper (the runner's // mode; locally `ci run --detach`). Detach bool // LocksDir holds concurrency locks; default /.locks. LocksDir string // Locker overrides the file locks in LocksDir (the runner's, for local // runs that must not overlap CI deploys). Locker Locker } // Engine runs one workflow run. type Engine struct { M *manifest.Manifest Dir Dir Exec Executor Log *runlog.Writer BaseEnv []string Opts Options router *Router st *State locks map[string]func() } // Create starts a new run in dir: it writes the manifest copy, state.json and // the first event. Close the engine's Log when done. func Create(m *manifest.Manifest, dir Dir, meta Meta, ex Executor, opts Options) (*Engine, error) { if err := dir.Create(); err != nil { return nil, err } if _, err := os.Stat(dir.StateFile()); err == nil { return nil, fmt.Errorf("run %s already exists", dir) } if err := os.WriteFile(dir.ManifestFile(), m.Raw, 0o644); err != nil { return nil, err } log, err := runlog.Open(dir.Events(), meta.Run) if err != nil { return nil, err } t := now() e := &Engine{ M: m, Dir: dir, Exec: ex, Log: log, BaseEnv: osEnviron(), Opts: opts, router: &Router{M: m, Detached: meta.Parent != ""}, st: &State{Version: StateVersion, Meta: meta, Status: StatusRunning, Created: t}, } for _, c := range meta.Checkouts { e.emit(lex.RunEvent{Kind: lex.KindCheckout, Repo: c.Repo, Rev: c.Rev, Path: c.Path}) } detail := fmt.Sprintf("%s %s", m.Name, meta.Repo) if meta.Sha != "" { detail += "@" + short(meta.Sha) } if meta.Dirty { detail += " (working copy with changes)" } e.emit(lex.RunEvent{Kind: lex.KindRun, Status: StatusRunning, Content: detail}) if err := e.st.Save(dir); err != nil { return nil, err } return e, nil } func short(sha string) string { if len(sha) > 12 { return sha[:12] } return sha } // State returns the run's current state. func (e *Engine) State() *State { return e.st } // Run runs the workflow from its first step. func (e *Engine) Run(ctx context.Context) (*State, error) { return e.drive(ctx, e.router.Start()) } // RunFrom runs the workflow starting at action, as if a person had chosen // it; routing then follows that action's result. func (e *Engine) RunFrom(ctx context.Context, action string) (*State, error) { if _, ok := e.M.Actions[action]; !ok { return nil, fmt.Errorf("no action %q in %s", action, e.M.Name) } return e.drive(ctx, e.router.Begin(action)) } func (e *Engine) maxTransitions() int { if e.Opts.MaxTransitions > 0 { return e.Opts.MaxTransitions } return DefaultMaxTransitions } // drive follows decisions until the run ends. func (e *Engine) drive(ctx context.Context, d Decision) (*State, error) { for { if ctx.Err() != nil { return e.finish(End{StatusCancelled, lex.ReasonCancel, "interrupted"}, "") } switch { case d.End != nil: return e.finish(*d.End, e.lastAction()) case d.Gate != nil: next, paused, err := e.reachGate(ctx, d.Gate) if err != nil { return e.finish(End{StatusFailure, lex.ReasonError, err.Error()}, e.lastAction()) } if paused { return e.st, nil } d = next case d.Run != nil: p := *d.Run if e.st.Seq >= e.maxTransitions() { return e.finish(End{StatusFailure, lex.ReasonLimit, fmt.Sprintf("more than %d transitions; is a result routing in a loop?", e.maxTransitions())}, p.From) } if p.Action == e.Opts.Until { return e.finish(End{StatusCancelled, lex.ReasonUntil, "stopped before " + p.Action + " (--until)"}, p.From) } e.transition(lex.Transition{From: p.From, To: p.Action, Then: p.Then, Reason: p.Reason}) res, file, err := e.execute(ctx, p) if err != nil { if ctx.Err() != nil { return e.finish(End{StatusCancelled, lex.ReasonCancel, "interrupted during " + p.Action}, p.Action) } return e.finish(End{StatusFailure, lex.ReasonError, err.Error()}, p.Action) } next := e.router.After(e.st, p, res, file) if e.Opts.NoFollow { status := StatusSuccess if res.Status == lex.Failure { status = StatusFailure } return e.finish(End{status, lex.ReasonStop, "not followed (--no-follow): would " + next.String()}, p.Action) } d = next default: return e.finish(End{StatusFailure, lex.ReasonError, "router returned no decision"}, e.lastAction()) } } } func (e *Engine) lastAction() string { if x := e.st.LastExec(); x != nil { return x.Action } return "" } func (e *Engine) transition(t lex.Transition) { e.st.Seq++ t.Pipeline, t.Run, t.Workflow, t.Seq, t.CreatedAt = e.st.Meta.Pipeline, e.st.Meta.Run, e.st.Meta.Workflow, e.st.Seq, now() e.emit(lex.RunEvent{Kind: lex.KindTransition, Transition: &t}) } func (e *Engine) emit(ev lex.RunEvent) { if err := e.Log.Emit(ev); err != nil { fmt.Fprintf(os.Stderr, "ci: writing %s: %v\n", e.Dir.Events(), err) } } // finish ends the run. func (e *Engine) finish(end End, from string) (*State, error) { e.releaseLocks() _ = os.Remove(e.Dir.Answer()) e.transition(lex.Transition{From: from, Reason: end.Reason, Detail: end.Detail}) e.st.End = &end e.st.Status = end.Status e.st.Gate = nil e.emit(lex.RunEvent{Kind: lex.KindRun, Status: end.Status, Content: end.Detail}) return e.st, e.st.Save(e.Dir) } // execute runs one action and returns its (validated) result and the // result file's path relative to the run dir. An error means the run can't // go on (cancellation, an engine fault); a failing action is a result. func (e *Engine) execute(ctx context.Context, p Pending) (lex.Result, string, error) { e.st.Execs++ n := e.st.Execs if i, ok := e.M.IsStep(p.Action); ok { e.st.Cursor = i + 1 } a := e.M.Actions[p.Action] root := e.st.Meta.Root dir := filepath.Clean(filepath.Join(root, a.Dir)) base := fmt.Sprintf("%03d-%s", n, p.Action) resultRel := filepath.Join("results", base+".json") resultFile := filepath.Join(e.Dir.Path(), resultRel) inputsFile := filepath.Join(e.Dir.Results(), base+".inputs.json") inputs := p.Inputs if len(inputs) == 0 { inputs = json.RawMessage(`{}`) } if err := os.WriteFile(inputsFile, append(inputs, '\n'), 0o644); err != nil { return lex.Result{}, "", err } triggerFile := "" if p.TriggerResult != "" { triggerFile = filepath.Join(e.Dir.Path(), p.TriggerResult) } _ = os.Remove(resultFile) x := Exec{N: n, Action: p.Action, Then: p.Then, Reason: p.Reason, Result: resultRel, Started: now()} e.st.History = append(e.st.History, x) if err := e.st.Save(e.Dir); err != nil { return lex.Result{}, "", err } if a.Concurrency != nil { if err := e.acquire(ctx, a.Concurrency.Group); err != nil { return lex.Result{}, "", errors.Join(err, e.recordExec(x, StatusCancelled, -1)) } } e.emit(lex.RunEvent{Kind: lex.KindStart, Action: p.Action, Attempt: n, Command: a.Command}) code, runErr := 0, error(nil) if stub, ok := e.Opts.Stubs[p.Action]; ok { x.Stubbed = true code, runErr = e.stub(stub, resultFile, p.Action, n) } else { stdout := e.Log.Lines(p.Action, n, "stdout") stderr := e.Log.Lines(p.Action, n, "stderr") code, runErr = e.Exec.Exec(ctx, Spec{ Root: root, Dir: dir, Command: a.Command, Env: e.actionEnv(p, n, resultFile, inputsFile, triggerFile), Stdout: stdout, Stderr: stderr, }) stdout.Close() stderr.Close() } if a.Input != nil { _ = os.Remove(e.Dir.Answer()) } if runErr != nil && ctx.Err() != nil { if err := e.recordExec(x, StatusCancelled, -1); err != nil { return lex.Result{}, "", errors.Join(runErr, err) } return lex.Result{}, "", runErr } var res lex.Result if runErr != nil { res = lex.Result{Version: lex.ResultVersion, Status: lex.Failure, Summary: "could not run: " + runErr.Error()} code = -1 } else { var err error res, err = ReadResult(e.M, resultFile, code, e.Dir.Artifacts()) if err != nil { res = ContractFailure(err) } } if err := mergeOutputs(filepath.Join(e.stateDir(), "outputs.json"), res.Outputs); err != nil { res = ContractFailure(fmt.Errorf("outputs: %w", err)) } exit := code e.emit(lex.RunEvent{Kind: lex.KindEnd, Action: p.Action, Attempt: n, ExitCode: &exit, Status: res.Status}) e.emit(lex.RunEvent{Kind: lex.KindResult, Action: p.Action, Attempt: n, Result: &res}) if err := e.recordExec(x, res.Status, code); err != nil { return lex.Result{}, "", err } return res, resultRel, nil } func (e *Engine) recordExec(x Exec, status string, code int) error { x.Status, x.ExitCode, x.Finished = status, code, now() e.st.History[len(e.st.History)-1] = x return e.st.Save(e.Dir) } // stub stands in for an action: the fixture becomes its result, and its // exit code follows the fixture's status. func (e *Engine) stub(fixture, resultFile, action string, n int) (int, error) { b, err := os.ReadFile(fixture) if err != nil { return -1, fmt.Errorf("stub for %s: %w", action, err) } if err := os.WriteFile(resultFile, b, 0o644); err != nil { return -1, err } e.emit(lex.RunEvent{Kind: lex.KindLog, Action: action, Attempt: n, Stream: "stderr", Content: "ci: stubbed with " + fixture}) var probe struct { Status string `json:"status"` } if err := json.Unmarshal(b, &probe); err == nil && probe.Status == lex.Failure { return 1, nil } return 0, nil } // ErrNoRun is returned when a run directory has no state. var ErrNoRun = errors.New("no such run")