From 4ef97d6937df414a71aca7f1899e63be421db688 Mon Sep 17 00:00:00 2001 From: Eric Rodrigues Pires Date: Sat, 12 Sep 2026 14:07:19 -0300 Subject: [PATCH] Derive eval ID from the workflow source and make more paths async --- Cargo.lock | 68 ++++++++++++++++++- Cargo.toml | 1 + build.rs | 4 +- src/builder/local.rs | 78 ++++++++++------------ src/builder/mod.rs | 17 +++-- src/builder/remote.rs | 149 +++++++++++++++--------------------------- src/environment.rs | 65 ++++++++++++++---- src/job.rs | 36 ++++++---- src/main.rs | 30 +++++---- src/serde.rs | 22 +++---- src/subscriber.rs | 18 ++--- src/utils.rs | 71 ++++++++++++++++---- src/workflow.rs | 79 ++++++++++++++-------- 13 files changed, 387 insertions(+), 251 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2fadfb1..aa54e9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,6 +299,15 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -360,7 +369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core", ] @@ -492,6 +501,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -535,6 +553,16 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctrlc" version = "3.5.2" @@ -591,6 +619,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dispatch2" version = "0.3.1" @@ -825,6 +863,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -1111,6 +1159,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "shlex", "smol", "supports-color 3.0.2", @@ -1508,6 +1557,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1766,6 +1826,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-general-category" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index df85b08..d77b1d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ rand = "0.10.2" regex = "1.13.1" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" +sha2 = "0.10.9" shlex = "2.0.1" smol = "2.0.2" supports-color = "3.0.2" diff --git a/build.rs b/build.rs index ecfd722..b28d73b 100644 --- a/build.rs +++ b/build.rs @@ -40,8 +40,8 @@ fn main() { { let file_path = dir_entry.path(); tar.append_file( - &file_path, - &mut std::fs::File::open(&file_path).expect("file should be readable"), + file_path, + &mut std::fs::File::open(file_path).expect("file should be readable"), ) .expect("should write file to archive"); } diff --git a/src/builder/local.rs b/src/builder/local.rs index ab00265..15491ad 100644 --- a/src/builder/local.rs +++ b/src/builder/local.rs @@ -18,7 +18,6 @@ use std::{ collections::{HashMap, HashSet}, env::temp_dir, ffi::{OsStr, OsString}, - io::Write as _, num::NonZeroUsize, os::unix::ffi::OsStrExt, path::{Path, PathBuf}, @@ -28,7 +27,6 @@ use async_trait::async_trait; use futures::{AsyncWriteExt, stream::FuturesUnordered}; use smol::{ channel::{self, Receiver}, - io::AsyncReadExt, lock::{ RwLock, RwLockReadGuard, RwLockWriteGuard, Semaphore, SemaphoreGuard, futures::{Acquire, Read, Write}, @@ -43,7 +41,7 @@ use crate::{ remote::RemoteBuilder, }, environment::NowEnvironment, - utils::{get_random_string, pipe_outputs_to_stderr}, + utils::{get_random_string, wait_for_output, write_output_to_stderr}, workflow::{NowCheckout, NowJob}, }; @@ -76,14 +74,15 @@ pub(crate) struct RunnerGuard<'a> { } impl LocalBuilder { - pub(crate) fn new( + pub(crate) async fn new( environment: &NowEnvironment, builders: Option, local_only: bool, remote_only: bool, cores: Option, ) -> color_eyre::Result { - let output = std::process::Command::new("nix") + let mut command = Command::new("nix"); + command .args([ "--extra-experimental-features", "nix-command flakes", @@ -91,16 +90,18 @@ impl LocalBuilder { "show", "--json", ]) - .output()?; + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); - if !output.status.success() { - let mut stderr = std::io::stderr(); - stderr.write_all(&output.stderr)?; - stderr.flush()?; + let mut child = command.spawn()?; + let output = wait_for_output(&mut child, None).await?; + let config: NixConfig = if output.status.success() { + serde_json::from_slice(&output.stdout)? + } else { + write_output_to_stderr(&output)?; return Err(color_eyre::eyre::eyre!("Failed to fetch Nix config")); - } - - let config: NixConfig = serde_json::from_slice(&output.stdout)?; + }; let remote_builders = if local_only { vec![] @@ -109,7 +110,8 @@ impl LocalBuilder { &config, builders, environment.nix_project_source.as_ref(), - )? + ) + .await? }; if remote_only && remote_builders.is_empty() { return Err(color_eyre::eyre::eyre!("No remote builders available")); @@ -119,10 +121,14 @@ impl LocalBuilder { let cores = match cores { Some(cores) => cores.into(), - None => sysinfo::System::physical_core_count().unwrap_or(1), + None => smol::unblock(sysinfo::System::physical_core_count) + .await + .unwrap_or(1), }; - let hostname = sysinfo::System::host_name().unwrap_or_else(|| "localhost".into()); + let hostname = smol::unblock(sysinfo::System::host_name) + .await + .unwrap_or_else(|| "localhost".into()); Ok(Self { cancellation, @@ -325,7 +331,7 @@ impl NowBuilder for LocalBuilder { child, stdin_future, })), - PathBuf::from(tmpdir), + tmpdir, )) } NowCheckout::CloneAll => { @@ -380,29 +386,16 @@ impl NowBuilder for LocalBuilder { .stderr(Stdio::piped()); let mut child = command.spawn()?; - let result = smol::future::race( - async { - let _ = cancellation.recv().await; - Err(color_eyre::eyre::eyre!("Runner aborted")) - }, - async { - if child.status().await?.success() { - let mut stdout = child.stdout.take().expect("stdout is piped"); - let mut buf = Vec::new(); - stdout.read_to_end(&mut buf).await?; - Ok(PathBuf::from(OsStr::from_bytes(buf.trim_ascii()))) - } else { - pipe_outputs_to_stderr(&mut child).await?; - Err(color_eyre::eyre::eyre!( - "Failed to realize derivation '{}'", - derivation.to_string_lossy(), - )) - } - }, - ) - .await; - let _ = child.kill(); - result + let output = wait_for_output(&mut child, Some(cancellation)).await?; + if output.status.success() { + Ok(PathBuf::from(OsStr::from_bytes(output.stdout.trim_ascii()))) + } else { + write_output_to_stderr(&output)?; + Err(color_eyre::eyre::eyre!( + "Failed to realize derivation '{}'", + derivation.to_string_lossy(), + )) + } } async fn download( @@ -454,10 +447,11 @@ impl NowBuilder for LocalBuilder { command.arg("-rf").arg(path); let mut child = command.spawn()?; - if child.status().await?.success() { + let output = wait_for_output(&mut child, None).await?; + if output.status.success() { Ok(()) } else { - pipe_outputs_to_stderr(&mut child).await?; + write_output_to_stderr(&output)?; Err(color_eyre::eyre::eyre!( "Failed to remove locally checked out directory" )) diff --git a/src/builder/mod.rs b/src/builder/mod.rs index e1834b2..2731a91 100644 --- a/src/builder/mod.rs +++ b/src/builder/mod.rs @@ -31,7 +31,10 @@ use smol::{ use async_trait::async_trait; use serde::Deserialize; -use crate::{utils::pipe_outputs_to_stderr, workflow::NowCheckout}; +use crate::{ + utils::{wait_for_output, write_output_to_stderr}, + workflow::NowCheckout, +}; pub(crate) mod local; pub(crate) mod remote; @@ -56,11 +59,11 @@ impl CheckoutTask for CommandCheckoutTask { &'a mut self, ) -> std::pin::Pin> + 'a>> { Box::pin(async { - let status = self.child.status().await?; - if status.success() { + let output = wait_for_output(&mut self.child, None).await?; + if output.status.success() { Ok(()) } else { - pipe_outputs_to_stderr(&mut self.child).await?; + write_output_to_stderr(&output)?; Err(color_eyre::eyre::eyre!( "Failed to checkout current directory to {}", self.builder @@ -87,11 +90,11 @@ impl CheckoutTask for RsyncCheckoutTask { Box::pin( smol::future::zip( async { - let status = self.child.status().await?; - if status.success() { + let output = wait_for_output(&mut self.child, None).await?; + if output.status.success() { Ok(()) } else { - pipe_outputs_to_stderr(&mut self.child).await?; + write_output_to_stderr(&output)?; Err(color_eyre::eyre::eyre!( "Failed to checkout current directory to {}", self.builder diff --git a/src/builder/remote.rs b/src/builder/remote.rs index 6d12d47..81546c8 100644 --- a/src/builder/remote.rs +++ b/src/builder/remote.rs @@ -24,7 +24,7 @@ use std::{ use async_trait::async_trait; use smol::{ channel, - io::{AsyncReadExt, AsyncWriteExt}, + io::AsyncWriteExt, lock::{ RwLock, Semaphore, futures::{Acquire, Read, Write}, @@ -34,7 +34,7 @@ use smol::{ use crate::{ builder::{CheckoutTask, CommandCheckoutTask, NixConfig, NowBuilder, RsyncCheckoutTask}, - utils::{get_random_string, pipe_outputs_to_stderr}, + utils::{get_random_string, wait_for_output, write_output_to_stderr}, workflow::NowCheckout, }; @@ -54,7 +54,7 @@ pub(crate) struct RemoteBuilder { pub(crate) fn ssh_options(control_path: &Path) -> impl Iterator { let mut control_path_option: OsString = "ControlPath=".into(); - control_path_option.push(&control_path); + control_path_option.push(control_path); [ "-o".into(), "ControlMaster=auto".into(), @@ -69,7 +69,7 @@ pub(crate) fn ssh_options(control_path: &Path) -> impl Iterator } impl RemoteBuilder { - pub(crate) fn get_remote_builders( + pub(crate) async fn get_remote_builders( config: &NixConfig, builders: Option, project_source: &Path, @@ -125,7 +125,7 @@ impl RemoteBuilder { let maximum_builds = if let Some(maximum_builds) = iter.next() && maximum_builds != "-" { - usize::from_str_radix(maximum_builds, 10)? + str::parse(maximum_builds)? } else { 1 }; @@ -161,7 +161,7 @@ impl RemoteBuilder { let control_path = project_source.join(format!("ssh-{}", get_random_string(10))); // Get host system for remote - let mut command = std::process::Command::new("ssh"); + let mut command = Command::new("ssh"); if let Some(ssh_identity) = ssh_identity.as_ref() { command.arg("-i").arg(ssh_identity); } @@ -183,11 +183,13 @@ impl RemoteBuilder { .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let output = command.output()?; + + let mut child = command.spawn()?; + let output = wait_for_output(&mut child, None).await?; let host_system = if output.status.success() { String::from_utf8(output.stdout)? } else { - eprintln!("{}", String::from_utf8_lossy(&output.stderr)); + write_output_to_stderr(&output)?; return Err(color_eyre::eyre::eyre!("Failed to connect to {}", ssh_uri)); }; @@ -380,27 +382,16 @@ impl NowBuilder for RemoteBuilder { .stderr(Stdio::piped()); let mut child = command.spawn()?; - let result = smol::future::race( - async { - let _ = cancellation.recv().await; - Err(color_eyre::eyre::eyre!("Runner aborted")) - }, - async { - if child.status().await?.success() { - Ok(()) - } else { - pipe_outputs_to_stderr(&mut child).await?; - Err(color_eyre::eyre::eyre!( - "Failed to copy '{}' derivations to {}", - job_name, - self.ssh_uri - )) - } - }, - ) - .await; - let _ = child.kill(); - result + let output = wait_for_output(&mut child, Some(cancellation)).await?; + if !output.status.success() { + write_output_to_stderr(&output)?; + return Err(color_eyre::eyre::eyre!( + "Failed to copy '{}' derivations to {}", + job_name, + self.ssh_uri + )); + } + Ok(()) } async fn realize_derivation( @@ -426,30 +417,17 @@ impl NowBuilder for RemoteBuilder { .stdout(Stdio::piped()) .stderr(Stdio::piped()); let mut child = command.spawn()?; - let result = smol::future::race( - async { - let _ = cancellation.recv().await; - Err(color_eyre::eyre::eyre!("Runner aborted")) - }, - async { - if child.status().await?.success() { - let mut stdout = child.stdout.take().expect("stdout is piped"); - let mut buf = Vec::new(); - stdout.read_to_end(&mut buf).await?; - Ok(PathBuf::from(OsStr::from_bytes(buf.trim_ascii()))) - } else { - pipe_outputs_to_stderr(&mut child).await?; - Err(color_eyre::eyre::eyre!( - "Failed to realize derivation '{}' in {}", - derivation.to_string_lossy(), - self.ssh_uri - )) - } - }, - ) - .await; - let _ = child.kill(); - result + let output = wait_for_output(&mut child, Some(cancellation)).await?; + if output.status.success() { + Ok(PathBuf::from(OsStr::from_bytes(output.stdout.trim_ascii()))) + } else { + write_output_to_stderr(&output)?; + Err(color_eyre::eyre::eyre!( + "Failed to realize derivation '{}' in {}", + derivation.to_string_lossy(), + self.ssh_uri + )) + } } async fn download( @@ -479,26 +457,15 @@ impl NowBuilder for RemoteBuilder { .stderr(Stdio::piped()); let mut child = command.spawn()?; - let result = smol::future::race( - async { - let _ = cancellation.recv().await; - Err(color_eyre::eyre::eyre!("Runner aborted")) - }, - async { - if child.status().await?.success() { - Ok(()) - } else { - pipe_outputs_to_stderr(&mut child).await?; - Err(color_eyre::eyre::eyre!( - "Failed to copy uploads to {}", - self.ssh_uri - )) - } - }, - ) - .await; - let _ = child.kill(); - result + let output = wait_for_output(&mut child, Some(cancellation)).await?; + if !output.status.success() { + write_output_to_stderr(&output)?; + return Err(color_eyre::eyre::eyre!( + "Failed to copy uploads to {}", + self.ssh_uri + )); + } + Ok(()) } fn run_derivation( @@ -566,27 +533,16 @@ impl NowBuilder for RemoteBuilder { .stderr(Stdio::piped()); let mut child = command.spawn()?; - let result = smol::future::race( - async { - let _ = cancellation.recv().await; - Err(color_eyre::eyre::eyre!("Runner aborted")) - }, - async { - if child.status().await?.success() { - Ok(()) - } else { - pipe_outputs_to_stderr(&mut child).await?; - Err(color_eyre::eyre::eyre!( - "Failed to copy '{}' derivation from {}", - derivation.to_string_lossy(), - self.ssh_uri - )) - } - }, - ) - .await; - let _ = child.kill(); - result + let output = wait_for_output(&mut child, Some(cancellation)).await?; + if !output.status.success() { + write_output_to_stderr(&output)?; + return Err(color_eyre::eyre::eyre!( + "Failed to copy '{}' derivation from {}", + derivation.to_string_lossy(), + self.ssh_uri + )); + } + Ok(()) } async fn undo_checkout(&self, checkout: NowCheckout, path: &Path) -> color_eyre::Result<()> { @@ -614,10 +570,11 @@ impl NowBuilder for RemoteBuilder { .stderr(Stdio::piped()); let mut child = command.spawn()?; - if child.status().await?.success() { + let output = wait_for_output(&mut child, None).await?; + if output.status.success() { Ok(()) } else { - pipe_outputs_to_stderr(&mut child).await?; + write_output_to_stderr(&output)?; Err(color_eyre::eyre::eyre!( "Failed to remove checked out directory in {}", self.ssh_uri diff --git a/src/environment.rs b/src/environment.rs index 3bfce26..70149da 100644 --- a/src/environment.rs +++ b/src/environment.rs @@ -20,20 +20,63 @@ use std::{ io::Write, os::unix::ffi::OsStringExt, path::{Path, PathBuf}, - sync::{LazyLock, Mutex}, + process::Stdio, + sync::{Mutex, OnceLock}, }; +use sha2::{Digest, Sha256}; use smol::{channel::Receiver, process::Command}; use tracing::instrument; use crate::{ project::{ProjectSource, create_nix_project_source}, secret::SecretString, - utils::get_random_string, + utils::{wait_for_output, write_output_to_stderr}, workflow::{NowJob, NowJobContainer, NowStepEnvVar, NowWorkflow, WorkflowSource}, }; -pub(crate) static EVAL_ID: LazyLock = LazyLock::new(|| get_random_string(10)); +pub(crate) static EVAL_ID: OnceLock = OnceLock::new(); + +pub(crate) fn eval_id() -> &'static str { + EVAL_ID + .get() + .expect("EVAL_ID initialized before evaluation") +} + +async fn eval_id_from_workflow_source(workflow: &WorkflowSource) -> color_eyre::Result { + let mut command = smol::process::Command::new("nix"); + command + .args([ + "--extra-experimental-features", + "nix-command flakes", + "eval", + "--impure", + "--raw", + "--expr", + ]) + .arg(workflow.nix_expression()?) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = command.spawn()?; + let output = wait_for_output(&mut child, None).await?; + let data = if output.status.success() { + output.stdout + } else { + write_output_to_stderr(&output)?; + return Err(color_eyre::eyre::eyre!( + "Failed to get workflow source from {}", + workflow + )); + }; + + let digest = Sha256::digest(data); + Ok(digest[..16] + .iter() + .map(|byte| format!("{byte:02x}")) + .collect()) +} pub(crate) struct NowEnvironment { pub(crate) nix_project_source: ProjectSource, @@ -69,6 +112,8 @@ impl NowEnvironment { }; env_vars.extend(std::env::vars_os()); + let _ = EVAL_ID.set(eval_id_from_workflow_source(workflow).await?); + smol::future::or( async { let _ = ctrl_c.recv().await; @@ -84,9 +129,7 @@ impl NowEnvironment { .secrets .into_iter() .filter_map(|secret| { - let Some(value) = env_vars.remove(&secret) else { - return None; - }; + let value = env_vars.remove(&secret)?; let key = match secret.into_string() { Ok(secret) => secret, Err(os_string) => { @@ -114,9 +157,7 @@ impl NowEnvironment { .vars .into_iter() .filter_map(|var| { - let Some(value) = env_vars.remove(&var) else { - return None; - }; + let value = env_vars.remove(&var)?; let key = match var.into_string() { Ok(var) => var, Err(os_string) => { @@ -172,10 +213,10 @@ impl NowEnvironment { .ok_or_else(|| color_eyre::eyre::eyre!("non-UTF8 path"))?; let nix_env_path = format!("(/. + {})", serde_json::to_string(&nix_env_str)?); - let eval_id = serde_json::to_string(&*EVAL_ID)?; + let eval_id_json = serde_json::to_string(eval_id())?; let nix_command = format!( - "import {nix_env_path} {{ }} {{ workflow = {workflow_path}; evalId = {eval_id}; }}" + "import {nix_env_path} {{ }} {{ workflow = {workflow_path}; evalId = {eval_id_json}; }}" ); let mut command = Command::new("nix"); @@ -201,7 +242,7 @@ impl NowEnvironment { let mut secrets: HashSet = HashSet::new(); - let vars_regex = regex::bytes::Regex::new(&format!("@@__nowVar_{}_([^@]+)@@", *EVAL_ID)) + let vars_regex = regex::bytes::Regex::new(&format!("@@__nowVar_{}_([^@]+)@@", eval_id())) .expect("valid regex"); let vars: HashSet = vars_regex .captures_iter(&output.stdout) diff --git a/src/job.rs b/src/job.rs index b079b71..f6bf71b 100644 --- a/src/job.rs +++ b/src/job.rs @@ -81,8 +81,8 @@ impl NowEnvironment { let mut realize_futs: FuturesOrdered<_> = job .steps .iter() - .cloned() .map(|step| async { + let step = step.clone(); let (_lock, _guard, receiver, builder) = match local_builder.get_builder(&job).await? { Some(BuilderGuard { @@ -111,10 +111,14 @@ impl NowEnvironment { r#type = "step-teardown-realize", ); builder - .copy_derivations(&job.name, &[teardown_drv.clone()], &receiver) + .copy_derivations( + &job.name, + std::slice::from_ref(teardown_drv), + receiver, + ) .await?; - let teardown = builder.realize_derivation(teardown_drv, &receiver).await?; - builder.fetch_derivation(&teardown, &receiver).await?; + let teardown = builder.realize_derivation(teardown_drv, receiver).await?; + builder.fetch_derivation(&teardown, receiver).await?; Some(teardown) } else { None @@ -127,10 +131,14 @@ impl NowEnvironment { r#type = "step-run-realize", ); builder - .copy_derivations(&job.name, &[step.run_drv.clone()], &receiver) + .copy_derivations( + &job.name, + std::slice::from_ref(&step.run_drv), + receiver, + ) .await?; - let run = builder.realize_derivation(&step.run_drv, &receiver).await?; - builder.fetch_derivation(&run, &receiver).await?; + let run = builder.realize_derivation(&step.run_drv, receiver).await?; + builder.fetch_derivation(&run, receiver).await?; run }; Ok((step, run, teardown)) @@ -188,7 +196,7 @@ impl NowEnvironment { } runner - .copy_derivations(&job.name, &derivations, &receiver) + .copy_derivations(&job.name, &derivations, receiver) .await?; for (step, run, teardown) in steps { @@ -214,7 +222,9 @@ impl NowEnvironment { } } } - runner.download(&downloads, &receiver).await?; + if !downloads.is_empty() { + runner.download(&downloads, receiver).await?; + } if let Some(teardown) = teardown { teardown_stack.push((step.name.clone(), teardown, step.env.clone())); @@ -259,7 +269,7 @@ impl NowEnvironment { let mut buf = Vec::new(); stdout.read_to_end(&mut buf).await?; let upload_path = PathBuf::from(OsStr::from_bytes(buf.trim_ascii())); - runner.fetch_derivation(&upload_path, &receiver).await?; + runner.fetch_derivation(&upload_path, receiver).await?; info!( runner = runner_name, is_remote, @@ -279,7 +289,7 @@ impl NowEnvironment { let mut result: Result<(), JobError> = smol::future::or(steps_fut, async { if let Some(timeout) = job.timeout { - smol::Timer::after(timeout.into()).await; + smol::Timer::after(timeout).await; Err(color_eyre::eyre::eyre!( "Job '{}' timed out after {}", job.name, @@ -429,9 +439,7 @@ impl NowEnvironment { let (fail_fast, no_fail_fast) = smol::future::zip( async move { while let Some(future) = fail_fast.next().await { - if future.is_err() { - return future; - } + future?; } Ok(()) }, diff --git a/src/main.rs b/src/main.rs index d4be769..dfd1f82 100644 --- a/src/main.rs +++ b/src/main.rs @@ -426,7 +426,7 @@ fn main() -> color_eyre::Result<()> { smol::block_on(async { let environment = NowEnvironment::get(&workflow, ctrl_c, env_file.as_ref(), None).await?; - let evaluated = environment.evaluate_workflow(&workflow)?; + let evaluated = environment.evaluate_workflow(&workflow).await?; println!("{}", serde_json::to_string(&evaluated)?); Ok::<(), color_eyre::Report>(()) })?; @@ -482,19 +482,21 @@ fn main() -> color_eyre::Result<()> { let mut environment = NowEnvironment::get(&workflow, ctrl_c.clone(), env_file.as_ref(), gcroot_dir) .await?; - environment.run_workflow(NowWorkflowParams { - workflow, - ctrl_c, - abort, - timeout: timeout.map(|timeout| timeout.into()), - jobs, - all_jobs, - builders, - cores, - local_only, - remote_only, - skip, - }) + environment + .run_workflow(NowWorkflowParams { + workflow, + ctrl_c, + abort, + timeout: timeout.map(|timeout| timeout.into()), + jobs, + all_jobs, + builders, + cores, + local_only, + remote_only, + skip, + }) + .await })?; } } diff --git a/src/serde.rs b/src/serde.rs index 03547bf..5c72889 100644 --- a/src/serde.rs +++ b/src/serde.rs @@ -19,7 +19,7 @@ use std::{collections::HashMap, path::PathBuf}; use serde::{Deserialize, Serialize, de::Visitor, ser::SerializeStruct}; use crate::{ - environment::EVAL_ID, + environment::eval_id, workflow::{NowJobContainer, NowStep, NowStepDownload, NowStepEnvVar, NowStepSecret}, }; @@ -53,10 +53,7 @@ impl<'de> Deserialize<'de> for NowJobContainer { where A: serde::de::SeqAccess<'de>, { - let mut job_vec = seq - .size_hint() - .map(|capacity| Vec::with_capacity(capacity)) - .unwrap_or_default(); + let mut job_vec = seq.size_hint().map(Vec::with_capacity).unwrap_or_default(); while let Some(job) = seq.next_element()? { job_vec.push(job); } @@ -122,7 +119,7 @@ impl<'de> Deserialize<'de> for NowStep { "runDrv" => run_drv = Some(map.next_value()?), "teardownDrv" => teardown_drv = map.next_value()?, "env" => env = Some(map.next_value()?), - _ if matches!(key.split_once(&*EVAL_ID), Some(("__nowUpload_", ""))) => { + _ if matches!(key.split_once(eval_id()), Some(("__nowUpload_", ""))) => { upload_key = map.next_value()? } _ => {} // Ignore unknown keys @@ -183,7 +180,7 @@ impl<'de> Deserialize<'de> for NowStepSecret { let key = map .next_key::()? .ok_or_else(|| serde::de::Error::custom("missing key for map"))?; - match key.split_once(&EVAL_ID.to_string()) { + match key.split_once(eval_id()) { Some(("__nowSecret_", "")) => Ok(NowStepSecret { secret_name: map.next_value()?, }), @@ -228,7 +225,7 @@ impl<'de> Deserialize<'de> for NowStepDownload { let key = map .next_key::()? .ok_or_else(|| serde::de::Error::custom("missing key for map"))?; - match key.split_once(&EVAL_ID.to_string()) { + match key.split_once(eval_id()) { Some(("__nowDownload_", "")) => Ok(NowStepDownload { download_name: map.next_value()?, }), @@ -257,9 +254,9 @@ pub(crate) mod now_job_timeout { match value { Some(value) => serializer.serialize_some( &jiff::Span::try_from(*value) - .map_err(|error| serde::ser::Error::custom(error))? + .map_err(serde::ser::Error::custom)? .round(jiff::SpanRound::new().largest(jiff::Unit::Hour)) - .map_err(|error| serde::ser::Error::custom(error))?, + .map_err(serde::ser::Error::custom)?, ), None => serializer.serialize_none(), } @@ -312,10 +309,13 @@ pub(crate) mod now_job_timeout { mod serde_tests { use std::str::FromStr; + use crate::environment::EVAL_ID; + use super::*; #[test] fn deserialize_job_single() { + EVAL_ID.get_or_init(|| "test".into()); let json = format!( "{{ \"buildSystem\": \"x86_64-linux\", @@ -338,7 +338,7 @@ mod serde_tests { \"strategy\": null, \"timeout\": null }}", - *EVAL_ID + eval_id() ); let job: NowJobContainer = serde_json::from_str(&json).unwrap(); assert!(matches!(job, NowJobContainer::Single(_))) diff --git a/src/subscriber.rs b/src/subscriber.rs index c75618b..467b02c 100644 --- a/src/subscriber.rs +++ b/src/subscriber.rs @@ -107,16 +107,14 @@ impl<'a> NowSubscriberVisitor<'a> { impl<'a> Visit for NowSubscriberVisitor<'a> { fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn core::fmt::Debug) { - match field.name() { - "message" => self.message = Some(format!("{:?}", value)), - _ => (), + if field.name() == "message" { + self.message = Some(format!("{:?}", value)); } } fn record_bool(&mut self, field: &tracing::field::Field, value: bool) { - match field.name() { - "is_remote" => self.is_remote = Some(value), - _ => self.record_debug(field, &value), + if field.name() == "is_remote" { + self.is_remote = Some(value); } } @@ -131,9 +129,7 @@ impl<'a> Visit for NowSubscriberVisitor<'a> { impl<'a> tracing_subscriber::field::VisitOutput> for NowSubscriberVisitor<'a> { fn finish(self) -> Option { - let Some(message) = self.message else { - return None; - }; + let message = self.message?; let Some(runner) = self.runner else { return Some(message); }; @@ -148,7 +144,7 @@ impl<'a> tracing_subscriber::field::VisitOutput> for NowSubscribe if let Some(step) = self.step { Some(format!( "{} {}", - format!("{} step[{}]>", &builder.short_name, step) + format!("{} step[{}]>", builder.short_name, step) .if_supports_color(owo_colors::Stream::Stderr, |text| text .style(builder.style)), message @@ -156,7 +152,7 @@ impl<'a> tracing_subscriber::field::VisitOutput> for NowSubscribe } else { Some(format!( "{} {}", - format!("{}>", &builder.short_name) + format!("{}>", builder.short_name) .if_supports_color(owo_colors::Stream::Stderr, |text| text .style(builder.style)), message diff --git a/src/utils.rs b/src/utils.rs index ad0efff..be83f46 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -14,24 +14,67 @@ // You should have received a copy of the GNU Affero General Public License along // with this program. If not, see . -use std::io::Write; +use std::{io::Write, process::Output}; use rand::distr::SampleString; -use smol::{io::AsyncReadExt, process::Child}; +use smol::{channel::Receiver, future::zip, io::AsyncReadExt, process::Child}; -pub(crate) async fn pipe_outputs_to_stderr(child: &mut Child) -> color_eyre::Result<()> { +/// Waits for `child` to exit while draining its pipe buffers concurrently. +pub(crate) async fn wait_for_output( + child: &mut Child, + cancellation: Option<&Receiver<()>>, +) -> color_eyre::Result { + let mut stdout = child.stdout.take(); + let mut stderr = child.stderr.take(); + let mut stdout_buf = Vec::new(); + let mut stderr_buf = Vec::new(); + + let run = async { + let ((status, out), err) = zip( + zip(child.status(), async { + if let Some(pipe) = stdout.as_mut() { + pipe.read_to_end(&mut stdout_buf).await?; + } + Ok::<(), color_eyre::Report>(()) + }), + async { + if let Some(pipe) = stderr.as_mut() { + pipe.read_to_end(&mut stderr_buf).await?; + } + Ok::<(), color_eyre::Report>(()) + }, + ) + .await; + out?; + err?; + Ok(Output { + status: status?, + stdout: stdout_buf, + stderr: stderr_buf, + }) + }; + + let result = if let Some(cancellation) = cancellation { + smol::future::race( + async { + let _ = cancellation.recv().await; + Err(color_eyre::eyre::eyre!("Runner aborted")) + }, + run, + ) + .await + } else { + run.await + }; + let _ = child.kill(); + result +} + +pub(crate) fn write_output_to_stderr(output: &Output) -> color_eyre::Result<()> { let mut stderr = std::io::stderr(); - if let Some(mut pipe) = child.stdout.take() { - let mut buf = Vec::new(); - pipe.read_to_end(&mut buf).await?; - stderr.write_all(&buf)?; - } - if let Some(mut pipe) = child.stderr.take() { - let mut buf = Vec::new(); - pipe.read_to_end(&mut buf).await?; - stderr.write_all(&buf)?; - } - Ok(stderr.flush()?) + stderr.write_all(&output.stderr)?; + stderr.flush()?; + Ok(()) } pub(crate) fn get_random_string(len: usize) -> String { diff --git a/src/workflow.rs b/src/workflow.rs index bbe3d70..fb18414 100644 --- a/src/workflow.rs +++ b/src/workflow.rs @@ -16,11 +16,11 @@ use std::{ collections::{BTreeSet, HashMap, HashSet}, - io::Write, + fmt::{Display, Write}, num::NonZeroUsize, path::PathBuf, pin::Pin, - process::Command, + process::Stdio, time::Duration, }; @@ -31,14 +31,15 @@ use petgraph::{ visit::EdgeRef, }; use serde::{Deserialize, Serialize}; -use smol::{channel::Receiver, stream::StreamExt}; +use smol::{channel::Receiver, process::Command, stream::StreamExt}; use tracing::{debug, info, instrument, warn}; use crate::{ builder::{NowBuilder, local::LocalBuilder}, - environment::{EVAL_ID, NowEnvironment}, + environment::{NowEnvironment, eval_id}, job::{JobError, JobResult}, serde::now_job_timeout, + utils::{wait_for_output, write_output_to_stderr}, }; #[derive(Debug, Serialize, Deserialize)] @@ -120,6 +121,19 @@ pub(crate) enum WorkflowSource { Flake { path: String, attribute: String }, } +impl Display for WorkflowSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WorkflowSource::Path(path_buf) => f.write_str(&path_buf.to_string_lossy()), + WorkflowSource::Flake { path, attribute } => { + f.write_str(path)?; + f.write_char('#')?; + f.write_str(attribute) + } + } + } +} + impl WorkflowSource { pub(crate) fn nix_expression(&self) -> color_eyre::Result { match self { @@ -176,7 +190,7 @@ impl NowEnvironment { skip, ) )] - pub(crate) fn run_workflow( + pub(crate) async fn run_workflow( &mut self, NowWorkflowParams { workflow, @@ -192,7 +206,7 @@ impl NowEnvironment { skip, }: NowWorkflowParams, ) -> color_eyre::Result<()> { - let builder = LocalBuilder::new(self, builders, local_only, remote_only, cores)?; + let builder = LocalBuilder::new(self, builders, local_only, remote_only, cores).await?; let runner = builder.get_name(); info!( @@ -201,7 +215,7 @@ impl NowEnvironment { "Evaluating workflow '{}'...", String::from(&workflow) ); - let workflow = self.evaluate_workflow(&workflow)?; + let workflow = self.evaluate_workflow(&workflow).await?; debug!("$duper.workflow" = duper::serde::ser::to_string_compact(&workflow)?); if let Some(name) = workflow.name.as_ref() { @@ -370,11 +384,13 @@ impl NowEnvironment { } }); - smol::future::block_on(executor.run(smol::future::or(workflow_task, abort_task))) + executor + .run(smol::future::or(workflow_task, abort_task)) + .await } #[instrument(skip(self))] - pub(crate) fn evaluate_workflow( + pub(crate) async fn evaluate_workflow( &self, workflow: &WorkflowSource, ) -> color_eyre::Result { @@ -388,31 +404,40 @@ impl NowEnvironment { let nix_workflow_path = format!("(/. + {})", serde_json::to_string(&nix_workflow_str)?); let vars_json = serde_json::to_string(&serde_json::to_string(&self.vars)?)?; - let eval_id = serde_json::to_string(&*EVAL_ID)?; + let eval_id = serde_json::to_string(eval_id())?; let nix_command = format!( "(import {nix_workflow_path} {{ }}) {{ workflow = {workflow_path}; vars = builtins.fromJSON {vars_json}; evalId = {eval_id}; }}" ); let mut command = Command::new("nix"); - command.env("NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM", "1").args([ - "--extra-experimental-features", - "nix-command flakes", - "eval", - "--impure", - "--json", - "--keep-derivations", - ]); - let output = command.arg("--expr").arg(nix_command).output()?; - - if !output.status.success() { - let mut stderr = std::io::stderr(); - stderr.write_all(&output.stderr)?; - stderr.flush()?; - return Err(color_eyre::eyre::eyre!("Failed to evaluate workflow")); + command + .env("NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM", "1") + .args([ + "--extra-experimental-features", + "nix-command flakes", + "eval", + "--impure", + "--json", + "--keep-derivations", + "--expr", + ]) + .arg(nix_command) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = command.spawn()?; + let output = wait_for_output(&mut child, None).await?; + if output.status.success() { + Ok(serde_json::from_slice(&output.stdout)?) + } else { + write_output_to_stderr(&output)?; + Err(color_eyre::eyre::eyre!( + "Failed to evaluate workflow at {}", + workflow + )) } - - Ok(serde_json::from_slice(&output.stdout)?) } } -- knotmirror 0.0.1