diff --git a/.changes/rlm-children-survive-daemon-restart.md b/.changes/rlm-children-survive-daemon-restart.md new file mode 100644 index 000000000..3c496e933 --- /dev/null +++ b/.changes/rlm-children-survive-daemon-restart.md @@ -0,0 +1 @@ +- After a daemon restart or update, a parent session lists its subagents again, and messaging one wakes it as the parent's subagent, instead of rlm.list_subagents() coming back empty. diff --git a/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs b/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs index 30a2a8aab..da83747d0 100644 --- a/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs +++ b/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs @@ -297,9 +297,9 @@ fn release_settled_child_kernel_defers_to_scheduled_jobs_and_fires_the_release_p /// The whole-worker idle passivation gate accepts exactly when the /// kernel release would fire under the shared settled gates, plus the /// worker-only registry rule: a child record keeps the worker resident -/// (a revival starts with an empty registry) while the kernel release -/// still fires — releasing a kernel keeps the worker and its registry -/// resident. +/// (a revival rebuilds only settled ledger rows; the passivation stop +/// closes every row) while the kernel release still fires — releasing +/// a kernel keeps the worker and its registry resident. #[test] fn can_passivate_worker_mirrors_the_release_gates_and_adds_the_registry_rule() { let dir = tempfile::TempDir::new().unwrap(); @@ -377,9 +377,10 @@ fn can_passivate_worker_mirrors_the_release_gates_and_adds_the_registry_rule() { .registered_jobs_probe .lock() .expect("registered jobs probe lock") = None; - // A settled child record keeps the worker resident (a revival starts - // with an empty registry) while the kernel release still fires: the - // registry rule is whole-worker only. + // A settled child record keeps the worker resident (a revival + // rebuilds only settled ledger rows; the passivation stop closes + // every row) while the kernel release still fires: the registry + // rule is whole-worker only. let children = engine.children.clone().expect("children registry"); engine.runtime.block_on(async { children diff --git a/crates/pa-daemon/src/branch_navigation.rs b/crates/pa-daemon/src/branch_navigation.rs index fb9729c62..83a76656a 100644 --- a/crates/pa-daemon/src/branch_navigation.rs +++ b/crates/pa-daemon/src/branch_navigation.rs @@ -617,7 +617,7 @@ impl Worker { // session's scheduled jobs rebind onto the forked // session file — future restores target the fork, not the // source branch. - self.refresh_replaced_session_state(); + self.refresh_replaced_session_state().await; self.reseed_service_tier_for_replacement(); self.bind_scheduled_jobs().await; self.prewarm_replacement_session(); diff --git a/crates/pa-daemon/src/engine.rs b/crates/pa-daemon/src/engine.rs index 4ebbb6f5d..310eded70 100644 --- a/crates/pa-daemon/src/engine.rs +++ b/crates/pa-daemon/src/engine.rs @@ -86,12 +86,13 @@ pub trait SessionEngine: Send + Sync { /// `idleEvictionMinutes` consumer re-checks them before asking the /// supervisor for the graceful stop): the engine's settled gates /// plus an empty RLM child registry. The registry rule holds - /// because the port does not rebuild a revived session's subagent - /// registry from the spawn ledger (TS `listPassiveRlmSubagents` - /// does), so the stop would discard state the revival cannot - /// restore. The default `false` keeps scripted harness engines and - /// kernel-less embeddings resident — the same conservative arm as - /// the release default. + /// because a revival rebuilds the registry from the spawn ledger as + /// settled rows only — live run state (answer previews, collect + /// envelopes, in-flight watchers) does not come back, so the stop + /// would still discard state the revival cannot restore. The + /// default `false` keeps scripted harness engines and kernel-less + /// embeddings resident — the same conservative arm as the release + /// default. fn can_passivate_worker( &self, ) -> std::pin::Pin + Send + '_>> { diff --git a/crates/pa-daemon/src/messaging.rs b/crates/pa-daemon/src/messaging.rs index 0affe8d7e..46f966f74 100644 --- a/crates/pa-daemon/src/messaging.rs +++ b/crates/pa-daemon/src/messaging.rs @@ -246,7 +246,7 @@ impl Supervisor { // child selector falls back to its live edges (child id, // session id, or name) and wakes the child's own file. Ok(None) => { - return match self.wake_ledger_child(selector, &sessions_dir).await { + return match self.wake_ledger_child(selector).await { Some(outcome) => outcome, None => WakeOutcome::Unknown, } @@ -331,18 +331,13 @@ impl Supervisor { /// a selector the saved-session catalog missed against the spawn /// ledger's live child edges (the child id, the child's session-id /// file stem, or the child's name), then wake one worker over the - /// child's session file. `None` keeps the caller's unknown-session - /// error; `Some(Failed)` carries the wake's own error (an ambiguous - /// selector outranks the miss, like the catalog's). - async fn wake_ledger_child( - self: &Arc, - selector: &str, - sessions_dir: &std::path::Path, - ) -> Option { - let ledger = match self - .rlm_spawn_ledger_for(Some(&sessions_dir.to_string_lossy())) - .await - { + /// child's session file - the daemon-default ledger the spawn + /// admission appends to (TS `rlmSpawnLedger()`). `None` keeps the + /// caller's unknown-session error; `Some(Failed)` carries the wake's + /// own error (an ambiguous selector outranks the miss, like the + /// catalog's). + async fn wake_ledger_child(self: &Arc, selector: &str) -> Option { + let ledger = match self.rlm_spawn_ledger_for(None).await { Ok(ledger) => ledger, Err(error) => return Some(WakeOutcome::Failed(error.to_string())), }; @@ -409,7 +404,7 @@ impl Supervisor { // fence sees a root and never re-passivates. config: Some(json!({ "cwd": cwd, "rlmDepth": depth })), telemetry_disabled: None, - runtime_metadata: Some(json!({ "rlmChildId": child_id })), + runtime_metadata: Some(json!({ "kind": "subagent", "rlmChildId": child_id })), lifecycle: None, env: None, launch_env: None, diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rs index 378baefef..98b5c99a3 100644 --- a/crates/pa-daemon/src/rlm_children.rs +++ b/crates/pa-daemon/src/rlm_children.rs @@ -205,8 +205,10 @@ struct ChildRecord { session_file: Option, /// Rows of [`ChildRecord::session_file`] already folded into the /// parent's attribution rows. The walk resumes here, so repeated - /// observation never double-bills a child. - attributed_rows: usize, + /// observation never double-bills a child. `None` on a reseeded row: + /// nothing has been observed since the reseed, and the first delivery + /// primes the cursor at the file's tail. + attributed_rows: Option, /// A follow-up usage watcher is live for this retained child /// (delayed agent messaging after the task run settled). usage_watch_live: bool, @@ -537,6 +539,12 @@ impl SupervisorChildSessions { *self.inner.identity.lock().expect("identity lock") = identity; } + /// Rebuild the children registry from the spawn ledger (a restarted + /// parent lists its ledger children again). + pub async fn reseed_from_ledger(&self) { + self.inner.reseed_from_ledger().await; + } + /// The inherited RLM depth bound (TS `getRlmMaxDepthStatus().maxDepth` /// before any chat override). /// @@ -701,7 +709,7 @@ impl SupervisorChildSessions { error: None, closed_by_parent: false, session_file: None, - attributed_rows: 0, + attributed_rows: Some(0), usage_watch_live: false, usage_rearm: false, emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())), diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rs index 783686ed7..3e6d1791d 100644 --- a/crates/pa-daemon/src/rlm_children/host.rs +++ b/crates/pa-daemon/src/rlm_children/host.rs @@ -145,7 +145,7 @@ impl RlmSubagentHost for SupervisorChildSessions { error: None, closed_by_parent: false, session_file: created.session_file.clone(), - attributed_rows: 0, + attributed_rows: Some(0), usage_watch_live: false, usage_rearm: false, emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())), diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rs index 08642ab47..0c39f8aea 100644 --- a/crates/pa-daemon/src/rlm_children/lifecycle.rs +++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs @@ -436,6 +436,33 @@ impl SupervisorChildSessionsInner { // notice rides the parent's follow-up route (TS: the run // task's `finally` flushes pending usage at settlement). self.emit_child_usage(record).await; + // Only a successful run completes the display (TS + // `completeRlmSubagentRuntime`); a cancelled or failed run + // stays `running`, which a restart relists as `error`. + let completed = { + let record = record.lock().await; + (record.settled_status == Some("done")) + .then(|| (record.session_dir.clone(), record.rlm_child_id.clone())) + }; + if let Some((session_dir, child_id)) = completed { + if let Err(error) = tokio::task::spawn_blocking(move || { + let Some(mut display) = + crate::rlm_ledger::read_rlm_subagent_display(Path::new(&session_dir)) + else { + return Ok(()); + }; + if display.child_id != child_id || display.status != "running" { + return Ok(()); + } + display.status = "completed".to_string(); + crate::rlm_ledger::write_rlm_subagent_display(&display).map(|_| ()) + }) + .await + .unwrap_or_else(|error| Err(anyhow!(error))) + { + eprintln!("pa-daemon: RLM child display completion failed: {error:#}"); + } + } self.deliver_settle_notice(record).await; // A settled child releases an owed goal continuation (TS // `_maybeResumeGoalContinuationAfterRlmWork` at the child diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rs index 6ef480998..2dfe59513 100644 --- a/crates/pa-daemon/src/rlm_children/registry.rs +++ b/crates/pa-daemon/src/rlm_children/registry.rs @@ -1,11 +1,14 @@ //! The registry-mutation concern: run cancellation, inactive-child -//! deletes with their tombstone receipts, the close walk, and the -//! target lookup/resolution; the close-failure no-op marker is -//! registry-only. +//! deletes with their tombstone receipts, the close walk, the target +//! lookup/resolution, and the ledger reseed; the close-failure no-op +//! marker is registry-only. +use std::path::{Path, PathBuf}; + use super::{ - bail, json, Arc, ChildCloseReason, ChildRecord, Context, DaemonCommand, DeletedChild, Map, - Mutex, Result, SupervisorChildSessionsInner, KILL_TIMEOUT_MS, + bail, json, rlm_child_label, Arc, ChildCloseReason, ChildRecord, Context, DaemonCommand, + DeletedChild, Map, Mutex, Result, SupervisorChildSessionsInner, KILL_TIMEOUT_MS, }; +use crate::lease::canonical_session_path; /// The already-gone marker inside a close failure (the supervisor's /// `Unknown active session` route failure): TS `closeSessionOnce` treats a @@ -274,4 +277,138 @@ impl SupervisorChildSessionsInner { ), } } + + /// Rebuild the children registry from the spawn ledger (TS + /// `listPassiveRlmSubagents`): a restarted parent worker lists its + /// non-deleted ledger children again, addressed by their durable + /// session ids. Every reseeded row is settled (nothing is owed), and + /// its usage cursor starts lazy: the first delivery primes it at the + /// file's tail. + pub(super) async fn reseed_from_ledger(&self) { + let Some(parent_file) = self + .identity + .lock() + .expect("identity lock") + .session_file + .clone() + else { + return; + }; + let agent_dir = self.agent_dir.clone(); + let supervisor_socket = self.link.socket_path().clone(); + let parent_file_read = parent_file.clone(); + let records = tokio::task::spawn_blocking(move || { + ledger_child_records(&agent_dir, &supervisor_socket, Path::new(&parent_file_read)) + }) + .await + .unwrap_or_default(); + let mut children = self.children.lock().await; + // A swap rebound the identity mid-read: its own reseed lists the new session's children. + if self + .identity + .lock() + .expect("identity lock") + .session_file + .as_deref() + != Some(parent_file.as_str()) + { + return; + } + children.extend( + records + .into_iter() + .map(|record| Arc::new(Mutex::new(record))), + ); + } +} + +/// One parent file's non-deleted ledger edges as settled registry rows +/// (TS `listPassiveRlmSubagents`), read from the ledger the supervisor +/// writes (its persisted default sessions dir): the child file's stem +/// addresses each row both ways - resolve for a resident child, wake +/// for a passive one; an unreadable ledger reads as empty. +fn ledger_child_records( + agent_dir: &Path, + supervisor_socket: &Path, + parent_file: &Path, +) -> Vec { + let Some(sessions_dir) = crate::descriptor::load_supervisor_config( + &crate::descriptor::descriptor_dir(agent_dir, supervisor_socket) + .join(crate::descriptor::SUPERVISOR_CONFIG_FILE_NAME), + supervisor_socket, + ) + .and_then(|config| config.default_session_dir) else { + return Vec::new(); + }; + let ledger = + crate::rlm_ledger::RlmSpawnLedger::new(agent_dir, Path::new(&sessions_dir), |_| {}); + let edges = ledger.live_edges().unwrap_or_else(|error| { + eprintln!("pa-daemon: RLM ledger reseed skipped: {error:#}"); + Vec::new() + }); + let parent_file = canonical_session_path(parent_file); + let mut records = Vec::new(); + for edge in edges { + if canonical_session_path(Path::new(&edge.parent)) != parent_file { + continue; + } + let child = PathBuf::from(&edge.child); + let session_id = child + .file_stem() + .map(|stem| stem.to_string_lossy().into_owned()); + let display = child + .parent() + .and_then(crate::rlm_ledger::read_rlm_subagent_display) + .filter(|display| display.child_id == edge.child_id); + records.push(ChildRecord { + rlm_child_id: edge.child_id, + session_name: edge.name, + active_session_id: session_id.clone().unwrap_or_default(), + session_id, + session_dir: child + .parent() + .map(|dir| dir.to_string_lossy().into_owned()) + .unwrap_or_default(), + label: rlm_child_label( + display + .as_ref() + .and_then(|display| display.prompt.as_deref()) + .unwrap_or_default(), + ), + started_at_ms: display.as_ref().map_or_else( + || { + std::fs::metadata(&child) + .ok() + .and_then(|metadata| metadata.created().ok()) + .and_then(|created| created.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_millis() as u64) + }, + |display| display.created_at, + ), + settled_status: Some( + if display + .as_ref() + .is_some_and(|display| display.status == "running") + { + "error" + } else { + "done" + }, + ), + settled: true, + answer_preview: None, + answer_captured: false, + replied_since_task: false, + notice_delivered: true, + prompt_admitted: true, + error: None, + closed_by_parent: false, + session_file: Some(edge.child), + attributed_rows: None, + usage_watch_live: false, + usage_rearm: false, + emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())), + }); + } + records } diff --git a/crates/pa-daemon/src/rlm_children/usage.rs b/crates/pa-daemon/src/rlm_children/usage.rs index d8d7cf9ab..c55bb3b32 100644 --- a/crates/pa-daemon/src/rlm_children/usage.rs +++ b/crates/pa-daemon/src/rlm_children/usage.rs @@ -51,6 +51,10 @@ impl SupervisorChildSessionsInner { let Some(session_file) = session_file else { return; }; + // A reseeded row nothing has observed yet: nothing to bill until a delivery primes it. + let Some(from) = from else { + return; + }; // The open+parse is a blocking read of a file that can reach tens // of megabytes: run it on the blocking pool, never the async // worker (a slow file read must not stall unrelated tasks on the @@ -68,7 +72,7 @@ impl SupervisorChildSessionsInner { let (batches, next) = crate::rlm_child_usage::child_usage_batches(store.entries(), from); { let mut record_guard = record.lock().await; - record_guard.attributed_rows = next; + record_guard.attributed_rows = Some(next); } if batches.is_empty() { return; @@ -102,11 +106,34 @@ impl SupervisorChildSessionsInner { /// watcher covers the follow-up turn's usage. It never touches the /// run status, notices, or the settle hook. pub(super) async fn arm_usage_watch(this: &Arc, record: &Arc>) { + if record.lock().await.closed_by_parent { + return; + } + // Prime a lazy cursor at the tail so the pre-restart history is never billed. + if record.lock().await.attributed_rows.is_none() { + // A local, not an if-let scrutinee: the scrutinee's guard would + // live through the body and deadlock the re-lock below. + let session_file = record + .lock() + .await + .session_file + .clone() + .filter(|path| !path.is_empty()); + if let Some(session_file) = session_file { + let path = PathBuf::from(session_file); + let tail = tokio::task::spawn_blocking(move || { + crate::session_store::SessionFile::open(&path) + .map(|store| store.entries().len()) + }) + .await; + // A failed read stays lazy: a zero cursor would re-bill the whole history. + if let Ok(Ok(tail)) = tail { + record.lock().await.attributed_rows.get_or_insert(tail); + } + } + } { let mut record = record.lock().await; - if record.closed_by_parent { - return; - } if record.usage_watch_live { // A watcher is already observing this child (armed for an // earlier delivery): ask IT to observe this delivery's diff --git a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs index 9572f0cd4..ded279563 100644 --- a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs +++ b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs @@ -45,23 +45,23 @@ fn record_with_file(child_id: &str, session_file: &Path) -> Arc SupervisorChildSessions { +fn registry(agent_dir: &Path) -> SupervisorChildSessions { let tmp = tempfile::tempdir().unwrap(); let socket = tmp.path().join("supervisor.sock"); std::mem::forget(tmp); SupervisorChildSessions::new( Arc::new(SupervisorLink::new(socket)), - std::path::PathBuf::from("/agent"), + agent_dir.to_path_buf(), "parent-live".to_string(), std::sync::Arc::new(crate::model_allowlist::ModelRefusalTelemetry::new( - std::path::PathBuf::from("/agent"), + agent_dir.to_path_buf(), /*telemetry_disabled*/ true, )), ) @@ -91,7 +91,7 @@ fn child_file(dir: &Path) -> PathBuf { async fn emit_reads_once_and_advances_the_cursor() { let tmp = tempfile::tempdir().unwrap(); let file = child_file(tmp.path()); - let sessions = registry(); + let sessions = registry(Path::new("/agent")); let sink = Arc::new(CapturingSink::default()); sessions.set_usage_sink(sink.clone()); let record = record_with_file("sub-emit1", &file); @@ -108,7 +108,7 @@ async fn emit_reads_once_and_advances_the_cursor() { assert_eq!(usage.output, 2_929); assert!((usage.cost.total.as_f64() - 0.008_995_7).abs() < 1e-9); let consumed = record.lock().await.attributed_rows; - assert!(consumed > 0); + assert!(consumed.is_some_and(|rows| rows > 0)); // The cursor consumed the rows: nothing re-delivers. sessions.inner.emit_child_usage(&record).await; @@ -123,11 +123,11 @@ async fn emit_reads_once_and_advances_the_cursor() { async fn emit_without_a_sink_consumes_nothing() { let tmp = tempfile::tempdir().unwrap(); let file = child_file(tmp.path()); - let sessions = registry(); + let sessions = registry(Path::new("/agent")); let record = record_with_file("sub-emit2", &file); sessions.inner.emit_child_usage(&record).await; - assert_eq!(record.lock().await.attributed_rows, 0); + assert_eq!(record.lock().await.attributed_rows, Some(0)); } /// A record without a session file (the test seam's shape) observes @@ -136,7 +136,7 @@ async fn emit_without_a_sink_consumes_nothing() { #[tokio::test] async fn emit_tolerates_missing_and_absent_files() { let tmp = tempfile::tempdir().unwrap(); - let sessions = registry(); + let sessions = registry(Path::new("/agent")); let sink = Arc::new(CapturingSink::default()); sessions.set_usage_sink(sink.clone()); let missing = record_with_file("sub-emit3", &tmp.path().join("absent.jsonl")); @@ -148,3 +148,154 @@ async fn emit_tolerates_missing_and_absent_files() { sessions.inner.emit_child_usage(&no_file).await; assert_eq!(sink.0.lock().expect("reports lock").len(), 0); } + +/// The reseed lists the parent's live ledger children as settled rows +/// (tombstones/torn/`meta`/foreign parents excluded), the cursor lazy +/// until a delivery primes it at the tail. +#[tokio::test] +async fn reseed_lists_live_ledger_children_settled_and_bills_only_after_a_delivery() { + let tmp = tempfile::tempdir().unwrap(); + let agent = tmp.path().join("agent"); + let daemon_sessions = tmp.path().join("daemon-sessions"); + let proto = child_file(tmp.path()); + let file_at = |path: PathBuf| { + std::fs::create_dir_all(path.parent().expect("parent dir")).unwrap(); + std::fs::copy(&proto, &path).unwrap(); + path.display().to_string() + }; + let parent_one = file_at(daemon_sessions.join("P.jsonl")); + let parent_two = file_at(daemon_sessions.join("Q.jsonl")); + let child_one = file_at(agent.join("session-artifacts/P/sub-a/sess-a.jsonl")); + let child_two = file_at(agent.join("session-artifacts/P/sub-b/sess-b.jsonl")); + let child_legacy = file_at(agent.join("session-artifacts/P/sub-legacy/sess-legacy.jsonl")); + let child_three = file_at(agent.join("session-artifacts/P/sub-c/sess-c.jsonl")); + let child_four = file_at(agent.join("session-artifacts/Q/sub-d/sess-d.jsonl")); + for (child_id, session_file, status, prompt) in [ + ("sub-a", &child_one, "completed", Some("finished task")), + ("sub-b", &child_two, "running", None), + ] { + crate::rlm_ledger::write_rlm_subagent_display( + &crate::rlm_ledger::RlmSubagentDisplayEntry { + type_tag: "rlm_subagent".to_string(), + child_id: child_id.to_string(), + session_name: child_id.to_string(), + session_dir: Path::new(session_file) + .parent() + .unwrap() + .display() + .to_string(), + session_file: session_file.clone(), + rlm_parent_node_id: None, + prompt: prompt.map(str::to_string), + spawn_code: None, + model: None, + status: status.to_string(), + created_at: 1234, + }, + ) + .unwrap(); + } + let spawn = |child_id: &str, parent: &str, child: &str| { + format!( + r#"{{"v":1,"op":"spawn","at":"2026-09-30T00:00:00Z","childId":"{child_id}","parent":"{parent}","child":"{child}","depth":1,"name":"{child_id}"}}"# + ) + }; + let ledger_path = crate::rlm_ledger::rlm_ledger_path(&agent, &daemon_sessions); + std::fs::create_dir_all(ledger_path.parent().expect("ledger dir")).unwrap(); + std::fs::write( + ledger_path, + [ + r#"{"v":1,"op":"meta","at":"2026-09-30T00:00:00Z"}"#, + spawn("sub-a", &parent_one, &child_one).as_str(), + r#"{"v":1,"op":"spawn","at":"#, + spawn("sub-b", &parent_one, &child_two).as_str(), + spawn("sub-legacy", &parent_one, &child_legacy).as_str(), + spawn("sub-c", &parent_one, &child_three).as_str(), + format!( + r#"{{"v":1,"op":"delete","at":"2026-09-30T00:00:01Z","childId":"sub-c","child":"{child_three}","reason":"user"}}"# + ) + .as_str(), + spawn("sub-d", &parent_two, &child_four).as_str(), + ] + .join("\n"), + ) + .unwrap(); + let sessions = registry(&agent); + let supervisor_socket = sessions.inner.link.socket_path().clone(); + crate::descriptor::persist_supervisor_config( + &crate::descriptor::descriptor_dir(&agent, &supervisor_socket) + .join(crate::descriptor::SUPERVISOR_CONFIG_FILE_NAME), + &crate::descriptor::PersistedSupervisorConfig { + version: 1, + socket_path: supervisor_socket.to_string_lossy().to_string(), + default_session_dir: Some(daemon_sessions.display().to_string()), + }, + ) + .unwrap(); + sessions.set_identity(ParentIdentity { + session_file: Some(parent_one.clone()), + ..ParentIdentity::with_default_depth() + }); + let sink = Arc::new(CapturingSink::default()); + sessions.set_usage_sink(sink.clone()); + sessions.reseed_from_ledger().await; + let roster = sessions.list_subagents().await.expect("the roster read"); + let rows: Vec<_> = roster + .iter() + .map(|row| { + ( + row.rlm_child_id.as_str(), + row.status, + row.active_session_id.as_deref(), + row.label.as_deref(), + ) + }) + .collect(); + assert_eq!( + rows, + [ + ("sub-a", "completed", Some("sess-a"), Some("finished task")), + ("sub-b", "error", Some("sess-b"), Some("child agent")), + ( + "sub-legacy", + "completed", + Some("sess-legacy"), + Some("child agent") + ), + ], + "completed and interrupted children keep their display verdicts" + ); + + let legacy = sessions + .inner + .find_record("sub-legacy") + .await + .expect("ledger-only row"); + assert!(legacy.lock().await.started_at_ms > 0); + + let record = sessions + .inner + .find_record("sub-a") + .await + .expect("the reseeded row"); + // Nothing is owed: the pre-restart history never re-bills. + sessions.inner.emit_child_usage(&record).await; + assert_eq!(sink.0.lock().expect("reports lock").len(), 0); + // A delivery primes the lazy cursor at the tail; the delivered turn + // appends its row; the next emit bills only that row. + SupervisorChildSessionsInner::arm_usage_watch(&sessions.inner, &record).await; + assert_eq!(record.lock().await.attributed_rows, Some(2)); + let mut content = std::fs::read_to_string(Path::new(&child_one)).expect("child file"); + content.push_str( + r#"{"type":"message","id":"a2","parentId":"a1","timestamp":"2026-09-23T00:00:03.000Z","message":{"role":"assistant","content":[],"stopReason":"toolUse","usage":{"input":10,"output":5,"cacheRead":0,"cacheWrite":0,"totalTokens":15,"cost":{"input":0.1,"output":0.05,"cacheRead":0,"cacheWrite":0,"total":0.15}}}}"#, + ); + content.push('\n'); + std::fs::write(Path::new(&child_one), content).expect("append the row"); + sessions.inner.emit_child_usage(&record).await; + let reports = sink.0.lock().expect("reports lock").clone(); + assert_eq!(reports.len(), 1); + let [(_, usage)] = reports[0].batches[..] else { + panic!("one batch: {:?}", reports[0].batches); + }; + assert_eq!((usage.input, usage.output), (10, 5)); +} diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rs index cfa56936e..a5adc8e84 100644 --- a/crates/pa-daemon/src/rlm_children/watch_tests.rs +++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs @@ -395,6 +395,41 @@ async fn a_worker_leaving_inside_the_settle_grace_keeps_the_verdict() { .is_some_and(|content| content.contains("the child final answer"))); } +/// A cancelled run's watcher settle leaves the display `running`, so a +/// restart relists the child as `error` instead of `completed`. +#[tokio::test] +async fn a_cancelled_child_keeps_its_display_running_through_the_watcher_settle() { + let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel(); + let (sessions, _kill_rx) = + sessions_with_fake_supervisor(follow_up_tx, 500, FakeKill::Success, FakeChild::Healthy) + .await; + let (hook_tx, mut hook_rx) = mpsc::unbounded_channel(); + sessions.set_settle_hook(Arc::new(move || { + let _ = hook_tx.send(()); + })); + let handle = spawn_child(&sessions).await; + let display_file = Path::new(&handle.session_dir).join("rlm-subagent.json"); + std::fs::write( + &display_file, + json!({ "type": "rlm_subagent", "childId": handle.rlm_child_id, + "sessionDir": handle.session_dir, "status": "running" }) + .to_string(), + ) + .unwrap(); + sessions.notify_turn_done(); + assert!(sessions.cancel_child_run(&handle.rlm_child_id).await); + // One settle hook from the cancel, one from the watcher's settle tail. + for _ in 0..2 { + tokio::time::timeout(Duration::from_secs(10), hook_rx.recv()) + .await + .expect("both settle hooks fire") + .expect("hook channel open"); + } + let display = crate::rlm_ledger::read_rlm_subagent_display(Path::new(&handle.session_dir)) + .expect("display entry stays readable"); + assert_eq!(display.status, "running"); +} + /// The `follow_up` carries exactly the TS failure row (`createRlmChildFailureMessage`). fn assert_failure_row(follow_up: &Value, child_id: &str, error: &str) { let custom = &follow_up["customMessage"]; @@ -980,7 +1015,7 @@ fn an_already_settled_child_never_re_scores_as_an_unreachable_error() { error: None, closed_by_parent: false, session_file: None, - attributed_rows: 0, + attributed_rows: Some(0), usage_watch_live: false, usage_rearm: false, emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())), diff --git a/crates/pa-daemon/src/session_navigation.rs b/crates/pa-daemon/src/session_navigation.rs index f805ed066..77702449c 100644 --- a/crates/pa-daemon/src/session_navigation.rs +++ b/crates/pa-daemon/src/session_navigation.rs @@ -407,7 +407,7 @@ impl Worker { } match self.navigation.replace_session(target.file).await { Ok(()) => { - self.refresh_replaced_session_state(); + self.refresh_replaced_session_state().await; self.reseed_service_tier_for_replacement(); self.bind_scheduled_jobs().await; self.prewarm_replacement_session(); diff --git a/crates/pa-daemon/src/worker/create.rs b/crates/pa-daemon/src/worker/create.rs index ef21194a1..c0dd01e5d 100644 --- a/crates/pa-daemon/src/worker/create.rs +++ b/crates/pa-daemon/src/worker/create.rs @@ -646,6 +646,7 @@ impl Worker { }) { return response_failure(None, "create", &error.to_string(), None); } + self.reseed_rlm_children().await; // The built-in Herdr connector binds here, per session: the pane // identity comes from the create payload's client env (the client // that owns the pane sent it), never from this process's ambient diff --git a/crates/pa-daemon/src/worker/lifecycle.rs b/crates/pa-daemon/src/worker/lifecycle.rs index 20645fa48..abd1160f7 100644 --- a/crates/pa-daemon/src/worker/lifecycle.rs +++ b/crates/pa-daemon/src/worker/lifecycle.rs @@ -237,6 +237,20 @@ impl Worker { } } + /// Relist the bound session's ledger children (TS + /// `listPassiveRlmSubagents`): awaited where the identity is bound - + /// create, and the replacement rebind inside the gate - so the scan is + /// ordered with the close walk that follows a later replacement. + pub(crate) async fn reseed_rlm_children(&self) { + if let Some(children) = self + .agent_engine + .as_ref() + .and_then(|engine| engine.children.clone()) + { + children.reseed_from_ledger().await; + } + } + /// Wait until the replacement teardown can retire the runtime: no /// turn and no compaction in flight. Like the navigation settle, the /// park rides a timeout backstop - the turn runner notifies the idle @@ -314,7 +328,7 @@ impl Worker { /// catalog rebind runs separately (`bind_scheduled_jobs`), like the /// TS dispatch handlers that call `rebindCronJobsToState` after the /// runtime call. - pub(crate) fn refresh_replaced_session_state(&self) { + pub(crate) async fn refresh_replaced_session_state(&self) { let (rlm_depth, summary, child_script) = { let mut core = self .core @@ -340,7 +354,7 @@ impl Worker { // child engine file rides along: TS children inherit the // replacement runtime's `sessionConfig`, which the runtime keeps // across its swaps. - if let Err(error) = self + match self .engine .configure_rlm_identity(crate::engine::RlmSessionIdentity { rlm_depth, @@ -350,9 +364,9 @@ impl Worker { session_file: summary.session_file.clone(), thinking: None, child_script, - }) - { - eprintln!("pa-daemon: replacement identity rebind failed: {error:#}"); + }) { + Ok(()) => self.reseed_rlm_children().await, + Err(error) => eprintln!("pa-daemon: replacement identity rebind failed: {error:#}"), } if let Ok(summary_value) = serde_json::to_value(&summary) { self.engine.set_session_summary(summary_value); diff --git a/crates/pa-daemon/tests/rlm_children_parent_death_e2e.rs b/crates/pa-daemon/tests/rlm_children_parent_death_e2e.rs index 3b1b4eec6..d388be312 100644 --- a/crates/pa-daemon/tests/rlm_children_parent_death_e2e.rs +++ b/crates/pa-daemon/tests/rlm_children_parent_death_e2e.rs @@ -426,9 +426,10 @@ fn sigkill(pid: u32) { /// A parent worker killed with SIGKILL cannot run any teardown (#246's /// closes all live in the worker), so the supervisor's death monitoring /// closes its resident children: the spawned child's worker leaves the -/// roster, its session archives, the respawned parent's registry reads -/// empty, and the child stays visible as a passive ledger row (the close -/// is a plain stop - the spawn edge survives, no delete record). +/// roster, its session archives, the respawned parent relists the child +/// as a settled row from the surviving spawn edge, and the child stays +/// visible as a passive ledger row (the close is a plain stop - the +/// spawn edge survives, no delete record). #[test] fn sigkill_closes_the_spawned_child_and_passivates_the_row() { let Some(kernel_python) = kernel_python() else { @@ -534,21 +535,13 @@ fn sigkill_closes_the_spawned_child_and_passivates_the_row() { "the parent-death close keeps the child's resume entry (no archive): {child_session}" ); - // The respawned parent's registry is fresh: the parent died before it - // could track anything, and the death close owns its children now. - // (get_rlm_children fails while the worker restarts; poll for the - // respawned answer.) + // The respawned parent reseeds the spawn edge the death close kept. + // Its display still says running, so TS exposes the interrupted task + // as error rather than presenting it as a completed child. wait_until(&mut client, Duration::from_mins(1), |client| { - client.send_command( - "g3", - &json!({ "type": "get_rlm_children", "activeSessionId": parent_id }), - ); - let response = client.read_response("g3"); - (response["success"] == true - && response["data"]["children"] - .as_array() - .is_some_and(std::vec::Vec::is_empty)) - .then_some(()) + rlm_children_rows(client, "g3", &parent_id) + .into_iter() + .find(|row| row["id"] == json!(child_id) && row["status"] == "error") }); // The passive row per TS: the spawn edge survived the close (a stop, diff --git a/crates/pa-daemon/tests/rlm_children_replacement_e2e.rs b/crates/pa-daemon/tests/rlm_children_replacement_e2e.rs index a38edb2d8..ca5d683a4 100644 --- a/crates/pa-daemon/tests/rlm_children_replacement_e2e.rs +++ b/crates/pa-daemon/tests/rlm_children_replacement_e2e.rs @@ -314,6 +314,16 @@ fn roster_cell(receipt: &Path, error_receipt: &Path) -> String { ) } +/// The probe cell: the restarted parent's roster rows, then a send to the child. +fn probe_cell(list_receipt: &Path, send_receipt: &Path, error_receipt: &Path) -> String { + format!( + "import inspect, json, traceback\ntry:\n roster = rlm.list_subagents()\n if inspect.isawaitable(roster):\n roster = await roster\n rows = [[s.rlm_child_id, s.status, s.active_session_id] for s in roster]\n open({list:?}, \"w\").write(json.dumps(rows))\n receipt = await agent_message.send(\"boot-ping\", receiver_role=\"child\", receiver_name=\"kid\")\n open({send:?}, \"w\").write(json.dumps(receipt))\n print(rows, receipt)\nexcept Exception:\n open({error:?}, \"w\").write(traceback.format_exc())\n raise", + list = list_receipt.display().to_string(), + send = send_receipt.display().to_string(), + error = error_receipt.display().to_string(), + ) +} + /// The kernel cell that creates one depth-0 resident root session through /// the product `rlm.create_session` surface. fn create_session_cell(receipt: &Path, error_receipt: &Path) -> String { @@ -370,23 +380,25 @@ fn create_parent( dir: &Path, parent_script: &Path, child_script: &Path, + session_path: Option<&Path>, id: &str, ) -> Value { let sessions_dir = dir.join("sessions"); std::fs::create_dir_all(&sessions_dir).expect("sessions dir"); - client.send_command( - id, - &json!({ - "type": "create", - "name": "parent", - "config": { - "cwd": dir.to_string_lossy(), - "sessionDir": sessions_dir.to_string_lossy(), - "script": parent_script.to_string_lossy(), - "childScript": child_script.to_string_lossy(), - }, - }), - ); + let mut create = json!({ + "type": "create", + "name": "parent", + "config": { + "cwd": dir.to_string_lossy(), + "sessionDir": sessions_dir.to_string_lossy(), + "script": parent_script.to_string_lossy(), + "childScript": child_script.to_string_lossy(), + }, + }); + if let Some(path) = session_path { + create["sessionPath"] = json!(path.to_string_lossy()); + } + client.send_command(id, &create); let created = client.read_response(id); assert_eq!(created["success"], true, "create parent failed: {created}"); created["data"].clone() @@ -426,7 +438,14 @@ fn new_session_closes_the_spawned_child_and_empties_the_roster() { wait_socket_ready(&socket); let (mut client, hello) = Client::connect(&socket); assert_eq!(hello["type"], "daemon_hello"); - let parent = create_parent(&mut client, dir.path(), &parent_script, &child_script, "c1"); + let parent = create_parent( + &mut client, + dir.path(), + &parent_script, + &child_script, + None, + "c1", + ); let parent_id = parent["activeSessionId"] .as_str() .or_else(|| parent["id"].as_str()) @@ -558,7 +577,14 @@ fn new_session_keeps_a_created_root_session_running() { wait_socket_ready(&socket); let (mut client, hello) = Client::connect(&socket); assert_eq!(hello["type"], "daemon_hello"); - let parent = create_parent(&mut client, dir.path(), &parent_script, &child_script, "c1"); + let parent = create_parent( + &mut client, + dir.path(), + &parent_script, + &child_script, + None, + "c1", + ); let parent_id = parent["activeSessionId"] .as_str() .or_else(|| parent["id"].as_str()) @@ -609,3 +635,205 @@ fn new_session_keeps_a_created_root_session_running() { "the created root session is its own session, not the parent's" ); } + +/// A daemon restart must not lose the parent's RLM children: the +/// reattached parent lists its child again and messaging it wakes the child. +#[test] +fn a_daemon_restart_relists_and_wakes_the_parents_child() { + restart_relists_child(0, "completed"); +} + +#[test] +fn a_daemon_restart_marks_a_still_running_child_as_failed() { + restart_relists_child(30_000, "running"); +} + +fn restart_relists_child(delay_ms: u64, display_status: &str) { + let Some(kernel_python) = kernel_python() else { + return; + }; + let dir = tempfile::TempDir::new().expect("temp dir"); + let agent_dir = dir.path().join("agent"); + std::fs::create_dir_all(&agent_dir).expect("agent dir"); + let socket = dir.path().join("supervisor.sock"); + let receipts = dir.path().join("receipts"); + std::fs::create_dir_all(&receipts).expect("receipts dir"); + let spawn_receipt = receipts.join("spawn.json"); + let spawn_error = receipts.join("spawn.error"); + let list_receipt = receipts.join("list.json"); + let send_receipt = receipts.join("send.json"); + let probe_error = receipts.join("probe.error"); + + let child_script = dir.path().join("child.json"); + std::fs::write( + &child_script, + json!({ "responses": [ { "text": "kid done", "delayMs": delay_ms } ] }).to_string(), + ) + .expect("write child script"); + let parent_script = write_parent_script( + dir.path(), + &spawn_cell(&spawn_receipt, &spawn_error), + "pass", + ); + let mut daemon = spawn_supervisor(&socket, &agent_dir, &kernel_python); + wait_socket_ready(&socket); + let (mut client, hello) = Client::connect(&socket); + assert_eq!(hello["type"], "daemon_hello"); + let parent = create_parent( + &mut client, + dir.path(), + &parent_script, + &child_script, + None, + "c1", + ); + let parent_id = parent["activeSessionId"] + .as_str() + .or_else(|| parent["id"].as_str()) + .expect("parent active session id") + .to_string(); + let parent_session_id = parent["sessionId"].as_str().expect("parent session id"); + let parent_file = parent["sessionFile"] + .as_str() + .expect("parent session file") + .to_string(); + + run_turn(&mut client, &parent_id, "spawn the kid", "t1"); + let spawned: Value = + serde_json::from_str(&await_receipt(&spawn_receipt)).expect("spawn receipt json"); + let child_id = spawned["rlm_child_id"] + .as_str() + .expect("child id") + .to_string(); + assert!( + !spawn_error.exists(), + "the spawn cell failed: {}", + std::fs::read_to_string(&spawn_error).unwrap_or_default() + ); + let supervisor_row = wait_until(&mut client, Duration::from_secs(30), |client| { + roster_summaries(client, "l1").into_iter().find(|summary| { + summary["sessionName"] == json!("kid") && summary["runtimeKind"] == "subagent" + }) + }); + let child_session_id = supervisor_row["sessionId"] + .as_str() + .expect("child session id") + .to_string(); + let child_file = agent_dir + .join("session-artifacts") + .join(parent_session_id) + .join(&child_id) + .join(format!("{child_session_id}.jsonl")); + let display_file = child_file.parent().unwrap().join("rlm-subagent.json"); + if delay_ms > 0 { + wait_until(&mut client, Duration::from_secs(30), |client| { + let rows = rlm_children_rows(client, "g-running", &parent_id); + rows.iter() + .any(|row| row["id"] == child_id && row["status"] == "running") + .then_some(()) + }); + } + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let display: Value = + serde_json::from_slice(&std::fs::read(&display_file).expect("child display file")) + .expect("child display json"); + if display["status"] == display_status { + break; + } + assert!( + Instant::now() < deadline, + "child display never completed: {display}" + ); + std::thread::sleep(Duration::from_millis(100)); + } + + client.send_command("bye", &json!({ "type": "shutdown" })); + let deadline = Instant::now() + Duration::from_secs(30); + while daemon + .child + .try_wait() + .expect("wait for the supervisor exit") + .is_none() + { + assert!(Instant::now() < deadline, "the supervisor never exited"); + std::thread::sleep(Duration::from_millis(50)); + } + let _daemon = spawn_supervisor(&socket, &agent_dir, &kernel_python); + wait_socket_ready(&socket); + let (mut client, hello) = Client::connect(&socket); + assert_eq!(hello["type"], "daemon_hello"); + + let probe_script = write_parent_script( + dir.path(), + &probe_cell(&list_receipt, &send_receipt, &probe_error), + "pass", + ); + let parent = create_parent( + &mut client, + dir.path(), + &probe_script, + &child_script, + Some(Path::new(&parent_file)), + "c2", + ); + let new_parent_id = parent["activeSessionId"] + .as_str() + .or_else(|| parent["id"].as_str()) + .expect("parent active session id") + .to_string(); + + run_turn( + &mut client, + &new_parent_id, + "probe the restarted roster", + "t2", + ); + assert!( + !probe_error.exists(), + "the probe cell failed: {}", + std::fs::read_to_string(&probe_error).unwrap_or_default() + ); + // (a) relisted + let rows: Value = + serde_json::from_str(&await_receipt(&list_receipt)).expect("list receipt json"); + assert_eq!( + rows, + json!([[ + child_id, + if display_status == "completed" { + "completed" + } else { + "error" + }, + child_session_id + ]]), + "the restarted parent must relist its ledger child with its persisted status" + ); + // (b) send receipt + let _send: Value = + serde_json::from_str(&await_receipt(&send_receipt)).expect("send receipt json"); + // (c) wake oracle + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let content = std::fs::read_to_string(&child_file).unwrap_or_default(); + if content.contains("boot-ping") { + break; + } + assert!( + Instant::now() < deadline, + "the send never woke the child into its session file: {content}" + ); + std::thread::sleep(Duration::from_millis(100)); + } + // (d) child runtime + client.send_command( + "gs", + &json!({ "type": "get_state", "activeSessionId": child_session_id }), + ); + let state = client.read_response("gs"); + assert_eq!(state["success"], true, "get_state failed: {state}"); + assert_eq!(state["data"]["rlmChildId"], json!(child_id)); + assert_eq!(state["data"]["rlmDepth"], json!(1)); + assert_eq!(state["data"]["parentSessionPath"], json!(parent_file)); +}