Something went wrong. Try again.
Personal fork of prime agent. Mirror of: https://mnemosyne.sigilzero.dev/sigilzero/prime-agent
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399//! Kernel Python environment bootstrap: the public entry point [`ensure_kernel_python`]//! resolves (and on first run builds) the venv that runs `python -m rlm.repl`,//! deduplicating concurrent bootstraps; the venv and version-file machinery lives in//! [`venv`], the runtime bootstrap code in [`runtime_code`].
pub(crate) mod dir_lock;mod disk_memo;mod runtime_code;pub(crate) mod venv;
use std::fmt::Write as _;use std::path::PathBuf;use std::sync::{Arc, Mutex};
use anyhow::anyhow;
use dir_lock::acquire_bootstrap_lock;pub use runtime_code::{ build_rlm_bootstrap_code, parse_unavailable_python_skills, UnavailablePythonSkills, PYTHON_SKILL_IMPORT_ERROR_REPORT_MARKER,};use venv::{ ensure_uv, expand_home, has_prime_agent_runtime, missing_python_skill_import_labels, missing_rlm_extra_import_labels, normalize_python_skills, resolve_writable_kernel_venv_dir, sync_python_skills, BootstrapPythonSkill,};pub use venv::{invalidate_runtime_probe_cache, kernel_venv_dir, kernel_venv_python};use venv::{kernel_base_ready, kernel_ready};
/// One Python skill the kernel should import at bootstrap.#[derive(Debug, Clone, PartialEq, Eq)]pub struct KernelPythonSkill { pub name: String, pub import_name: String, pub package_path: PathBuf, pub pyproject_path: PathBuf,}
/// The default extra packages pre-installed in the kernel venv and promised to/// the model in the system prompt.pub const DEFAULT_RLM_EXTRA_PACKAGES: [(&str, &str, &str); 12] = [ // (uvArg, importName, promptLabel) ("requests", "requests", "requests"), ("httpx", "httpx", "httpx"), ("pyyaml", "yaml", "yaml (PyYAML)"), ("tomli", "tomli", "tomli"), ("python-dotenv", "dotenv", "dotenv (python-dotenv)"), ("pandas", "pandas", "pandas"), ("numpy", "numpy", "numpy"), ("scipy", "scipy", "scipy"), ("beautifulsoup4", "bs4", "bs4 (Beautiful Soup)"), ("lxml", "lxml", "lxml"), ("pydantic", "pydantic", "pydantic"), ("tyro", "tyro", "tyro"),];
/// Whether `python` can serve as the kernel interpreter: it must carry a current/// `prime-agent-runtime` with the callable RLM surface. Callers that gate on a live/// kernel ask this instead of testing that the interpreter merely exists, because an/// interpreter that exists and cannot import `rlm` starts nothing.#[must_use]pub fn kernel_python_has_runtime(python: &str) -> bool { venv::has_prime_agent_runtime(python)}
#[must_use]pub fn default_rlm_extra_uv_args() -> Vec<&'static str> { DEFAULT_RLM_EXTRA_PACKAGES .iter() .map(|(uv, _, _)| *uv) .collect()}
#[must_use]pub fn default_rlm_extra_import_names() -> Vec<&'static str> { DEFAULT_RLM_EXTRA_PACKAGES .iter() .map(|(_, import, _)| *import) .collect()}
#[must_use]pub fn default_rlm_extra_import_labels() -> Vec<&'static str> { DEFAULT_RLM_EXTRA_PACKAGES .iter() .map(|(_, _, label)| *label) .collect()}
/// Progress callback for the bootstrap (`ensure_kernel_python`).pub type KernelBootstrapProgressHandler = std::sync::Arc<dyn Fn(&str) + Send + Sync>;
#[derive(Default)]pub struct EnsureKernelPythonOptions { pub python_skills: Vec<KernelPythonSkill>, pub on_progress: Option<KernelBootstrapProgressHandler>,}
impl EnsureKernelPythonOptions { pub(crate) fn report(&self, message: &str) { match &self.on_progress { Some(handler) => handler(message), None => eprintln!("{message}"), } }}
fn format_bootstrap_failure(error: &anyhow::Error) -> anyhow::Error { let mut message = format!( "Failed to set up the Python kernel runtime. {error:#}\n\ First-time setup needs the bundled local runtime and internet to install uv, Python, and default Python packages; once set up, Prime Agent runs offline. \ An interrupted runtime upgrade needs network once more, so re-run this while online. \ Set PRIME_AGENT_KERNEL_PYTHON to a Python with a current prime-agent-runtime and default Python packages installed to skip auto-bootstrap." ); // The packaged exe-adjacent sidecar is the kernel runtime source; when it is missing everywhere // (packaged layout and source checkout), name it. if venv::packaged_runtime_dir().is_none() { let package = venv::package_dir(); let _ = write!(message, "\nThe packaged prime-agent-runtime directory was not found (looked next to the executable at {} and PI_PACKAGE_DIR); reinstall prime-agent so the kernel runtime ships beside the binary.", package.display() ); } anyhow!(message)}
/// One in-flight bootstrap per unique options set, joined by concurrent callers.type InFlightBootstrap = Option<( String, Arc<tokio::sync::Mutex<Option<anyhow::Result<PathBuf>>>>,)>;
static IN_FLIGHT: Mutex<InFlightBootstrap> = Mutex::new(None);
/// Resolve the Python interpreter for the kernel: the `PRIME_AGENT_KERNEL_PYTHON`/// override when valid, else the auto-bootstrapped venv python.////// # Errors////// Returns an error when the override Python is invalid or the venv bootstrap fails.////// # Panics////// The in-flight promise stores its outcome under the same lock, so the `expect` is unreachable.pub async fn ensure_kernel_python(options: EnsureKernelPythonOptions) -> anyhow::Result<PathBuf> { let python_skills = normalize_python_skills(&options.python_skills); let key = [ std::env::var("PRIME_AGENT_KERNEL_PYTHON").unwrap_or_default(), std::env::var("PRIME_AGENT_KERNEL_VENV").unwrap_or_default(), std::env::var("HOME").unwrap_or_default(), std::env::var("XDG_DATA_HOME").unwrap_or_default(), serde_json::to_string(&python_skills).unwrap_or_default(), ] .join("\u{0}");
let shared = { let mut in_flight = IN_FLIGHT .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); match in_flight.as_ref() { Some((existing_key, promise)) if *existing_key == key => promise.clone(), _ => { let promise = Arc::new(tokio::sync::Mutex::new(None)); *in_flight = Some((key, promise.clone())); promise } } }; let mut guard = shared.lock().await; if guard.is_none() { *guard = Some(ensure_kernel_python_uncached(&options, &python_skills).await); } let outcome = guard.take().expect("outcome was just stored"); // Drop the registry entry so a later call re-validates instead of reusing // a cached rejection forever. let mut in_flight = IN_FLIGHT .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); if matches!(in_flight.as_ref(), Some((_, promise)) if Arc::ptr_eq(promise, &shared)) { *in_flight = None; } outcome}
async fn ensure_kernel_python_uncached( options: &EnsureKernelPythonOptions, python_skills: &[BootstrapPythonSkill],) -> anyhow::Result<PathBuf> { if let Ok(override_python) = std::env::var("PRIME_AGENT_KERNEL_PYTHON") { if !override_python.is_empty() { let python = expand_home(&override_python); let python_str = python.to_string_lossy().to_string(); let mut missing = Vec::new(); if !has_prime_agent_runtime(&python_str) { missing.push( "a current prime-agent-runtime with callable rlm.spawn, rlm.create_session, rlm.host_request, rlm.progress_note, and rlm.rename".to_string(), ); } if missing.is_empty() { let missing_extras = missing_rlm_extra_import_labels(&python_str); if !missing_extras.is_empty() { missing.push(format!( "default Python packages ({})", missing_extras.join(", ") )); } } if missing.is_empty() && !options.python_skills.is_empty() { let missing_skills = missing_python_skill_import_labels(&python_str, &options.python_skills); if !missing_skills.is_empty() { options.report(&format!( "Warning: Python skills unavailable in PRIME_AGENT_KERNEL_PYTHON and will be disabled: {}", missing_skills.join(", ") )); } } if missing.is_empty() { return Ok(python); } return Err(anyhow!( "PRIME_AGENT_KERNEL_PYTHON points to a Python missing {}: {}", missing.join(" and "), python.display() )); } }
let venv = resolve_writable_kernel_venv_dir()?; ensure_kernel_venv(options, python_skills, venv).await}
async fn ensure_kernel_venv( options: &EnsureKernelPythonOptions, python_skills: &[BootstrapPythonSkill], venv: PathBuf,) -> anyhow::Result<PathBuf> { let python = kernel_venv_python(&venv); let python_str = python.to_string_lossy().to_string(); // Validate the local source before probing any installed runtime or // removing an existing venv. Missing payloads must fail closed. let source_dir = venv::require_runtime_source_dir().map_err(|error| format_bootstrap_failure(&error))?; let runtime_identity = venv::hash_runtime_source(&source_dir).map_err(|error| format_bootstrap_failure(&error))?; if !venv::recovery::intent_path(&venv).exists() && kernel_ready(&python_str, &venv, &runtime_identity, python_skills) { return Ok(python); }
// A caller can cancel its startup future while uv still runs on a // blocking thread. Keep the transaction and its lock alive until that // child finishes, rather than admitting a concurrent venv writer. let options = EnsureKernelPythonOptions { python_skills: options.python_skills.clone(), on_progress: options.on_progress.clone(), }; let python_skills = python_skills.to_vec(); let lock_venv = venv.clone(); finish_locked_bootstrap(lock_venv, async move { let result = async { venv::recovery::recover_interrupted_upgrade(&venv, &runtime_identity)?; if kernel_ready(&python_str, &venv, &runtime_identity, &python_skills) { return Ok(python); } if kernel_base_ready(&python_str, &venv, &runtime_identity) { let uv = ensure_uv()?; sync_python_skills( &uv, &venv, &python, &source_dir, &runtime_identity, &python_skills, &options, ) .await?; return Ok(python); } if venv::recovery::restore(&venv, &runtime_identity)? { if !kernel_ready(&python_str, &venv, &runtime_identity, &python_skills) { sync_python_skills( &ensure_uv()?, &venv, &python, &source_dir, &runtime_identity, &python_skills, &options, ) .await?; } return Ok(python); } options.report("› setting up python kernel (one-time, ~30s)…"); venv::rebuild_kernel_venv( &ensure_uv()?, &venv, &source_dir, &runtime_identity, &python_skills, &options, ) .await?; Ok(python) } .await; if result.is_ok() { options.report("✓ ready"); } result.map_err(|error| format_bootstrap_failure(&error)) }) .await}
async fn finish_locked_bootstrap( venv: PathBuf, action: impl std::future::Future<Output = anyhow::Result<PathBuf>> + Send + 'static,) -> anyhow::Result<PathBuf> { tokio::spawn(async move { let _lock = acquire_bootstrap_lock(&venv).await?; action.await }) .await .map_err(|error| anyhow!("kernel bootstrap task failed: {error}"))?}
#[cfg(test)]mod tests { use super::*;
#[tokio::test(start_paused = true)] async fn cancelled_caller_does_not_unlock_a_running_bootstrap() { let root = tempfile::tempdir().unwrap(); let venv = root.path().join("kernel-venv"); let lock = root.path().join("kernel-venv.bootstrap.lock"); let done = root.path().join("mutation-completed"); let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); let (release_tx, release_rx) = tokio::sync::oneshot::channel(); let result = venv.clone(); let finished = done.clone(); let caller = tokio::spawn(finish_locked_bootstrap(venv.clone(), async move { entered_tx.send(()).unwrap(); release_rx.await.unwrap(); std::fs::write(finished, "done")?; Ok(result) })); entered_rx.await.unwrap(); caller.abort(); assert!(caller.await.unwrap_err().is_cancelled()); assert!(lock.is_file()); let mut contender = Box::pin(acquire_bootstrap_lock(&venv)); assert!(matches!( futures::poll!(&mut contender), std::task::Poll::Pending )); release_tx.send(()).unwrap(); let guard = contender.await.unwrap(); assert_eq!(std::fs::read_to_string(done).unwrap(), "done"); drop(guard); assert!(!lock.exists()); }
#[tokio::test(start_paused = true)] async fn lock_timeout_prevents_venv_rebuild() { let root = tempfile::tempdir().unwrap(); let venv = root.path().join("kernel-venv"); std::fs::create_dir(&venv).unwrap(); let sentinel = venv.join("owner-state"); std::fs::write(&sentinel, "another bootstrap owns this venv").unwrap(); let owner = acquire_bootstrap_lock(&venv).await.unwrap(); let options = EnsureKernelPythonOptions { // Fail before any destructive operation or network access if the // caller regresses and enters setup without acquiring the lock. on_progress: Some(Arc::new(|message| { panic!("bootstrap must stop at the lock error, got progress: {message}"); })), ..Default::default() };
let error = tokio::time::timeout( std::time::Duration::from_secs(121), ensure_kernel_venv(&options, &[], venv), ) .await .expect("lock acquisition must have a bounded wait") .expect_err("a lock timeout must propagate to the caller");
assert!(error.to_string().contains("Timed out"), "{error}"); assert_eq!( std::fs::read_to_string(sentinel).unwrap(), "another bootstrap owns this venv" ); assert!(root.path().join("kernel-venv.bootstrap.lock").exists()); drop(owner); }}