From e22fcd5fbda61d7ebfb59f92fe39436a2a503e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Sat, 3 Oct 2026 10:14:31 -0700 Subject: [PATCH] a restarted parent lists and reaches its subagents again (#3268) * fix(pa-daemon): reseed a revived parent's RLM child registry from the spawn ledger * fix(pa-daemon): preserve child settle status in restarted roster * test(pa-daemon): expect interrupted child error after parent death * fix(pa-daemon): keep a cancelled or failed child's display running Only a successful run marks rlm-subagent.json completed (TS completeRlmSubagentRuntime). A cancelled run's watcher no longer overwrites the display, so a restart relists the child as error instead of completed. * fix(pa-daemon): await the ledger reseed where the session identity is bound The reseed ran as a detached task, so it could outlive the replacement gate and append the replaced session's children after the close walk. The worker now awaits it in create and in the replacement rebind, next to close_rlm_children, so it is ordered before any later teardown. --------- Co-authored-by: Kevin Thomas --- .../rlm-children-survive-daemon-restart.md | 1 + .../src/agent_engine/tests/rlm_children.rs | 13 +- crates/pa-daemon/src/branch_navigation.rs | 2 +- crates/pa-daemon/src/engine.rs | 13 +- crates/pa-daemon/src/messaging.rs | 23 +- crates/pa-daemon/src/rlm_children.rs | 14 +- crates/pa-daemon/src/rlm_children/host.rs | 2 +- .../pa-daemon/src/rlm_children/lifecycle.rs | 27 ++ crates/pa-daemon/src/rlm_children/registry.rs | 147 +++++++++- crates/pa-daemon/src/rlm_children/usage.rs | 35 ++- .../src/rlm_children/usage_emit_tests.rs | 169 +++++++++++- .../pa-daemon/src/rlm_children/watch_tests.rs | 37 ++- crates/pa-daemon/src/session_navigation.rs | 2 +- crates/pa-daemon/src/worker/create.rs | 1 + crates/pa-daemon/src/worker/lifecycle.rs | 24 +- .../tests/rlm_children_parent_death_e2e.rs | 27 +- .../tests/rlm_children_replacement_e2e.rs | 258 +++++++++++++++++- 17 files changed, 707 insertions(+), 88 deletions(-) create mode 100644 .changes/rlm-children-survive-daemon-restart.md diff --git a/.changes/rlm-children-survive-daemon-restart.md b/.changes/rlm-children-survive-daemon-restart.md new file mode 100644 index 000000000..3c496e933 --- /dev/null +++ b/.changes/rlm-children-survive-daemon-restart.md @@ -0,0 +1 @@ +- After a daemon restart or update, a parent session lists its subagents again, and messaging one wakes it as the parent's subagent, instead of rlm.list_subagents() coming back empty. diff --git a/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs b/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs index 30a2a8aab..da83747d0 100644 --- a/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs +++ b/crates/pa-daemon/src/agent_engine/tests/rlm_children.rs @@ -297,9 +297,9 @@ fn release_settled_child_kernel_defers_to_scheduled_jobs_and_fires_the_release_p /// The whole-worker idle passivation gate accepts exactly when the /// kernel release would fire under the shared settled gates, plus the /// worker-only registry rule: a child record keeps the worker resident -/// (a revival starts with an empty registry) while the kernel release -/// still fires — releasing a kernel keeps the worker and its registry -/// resident. +/// (a revival rebuilds only settled ledger rows; the passivation stop +/// closes every row) while the kernel release still fires — releasing +/// a kernel keeps the worker and its registry resident. #[test] fn can_passivate_worker_mirrors_the_release_gates_and_adds_the_registry_rule() { let dir = tempfile::TempDir::new().unwrap(); @@ -377,9 +377,10 @@ fn can_passivate_worker_mirrors_the_release_gates_and_adds_the_registry_rule() { .registered_jobs_probe .lock() .expect("registered jobs probe lock") = None; - // A settled child record keeps the worker resident (a revival starts - // with an empty registry) while the kernel release still fires: the - // registry rule is whole-worker only. + // A settled child record keeps the worker resident (a revival + // rebuilds only settled ledger rows; the passivation stop closes + // every row) while the kernel release still fires: the registry + // rule is whole-worker only. let children = engine.children.clone().expect("children registry"); engine.runtime.block_on(async { children diff --git a/crates/pa-daemon/src/branch_navigation.rs b/crates/pa-daemon/src/branch_navigation.rs index fb9729c62..83a76656a 100644 --- a/crates/pa-daemon/src/branch_navigation.rs +++ b/crates/pa-daemon/src/branch_navigation.rs @@ -617,7 +617,7 @@ impl Worker { // session's scheduled jobs rebind onto the forked // session file — future restores target the fork, not the // source branch. - self.refresh_replaced_session_state(); + self.refresh_replaced_session_state().await; self.reseed_service_tier_for_replacement(); self.bind_scheduled_jobs().await; self.prewarm_replacement_session(); diff --git a/crates/pa-daemon/src/engine.rs b/crates/pa-daemon/src/engine.rs index 4ebbb6f5d..310eded70 100644 --- a/crates/pa-daemon/src/engine.rs +++ b/crates/pa-daemon/src/engine.rs @@ -86,12 +86,13 @@ pub trait SessionEngine: Send + Sync { /// `idleEvictionMinutes` consumer re-checks them before asking the /// supervisor for the graceful stop): the engine's settled gates /// plus an empty RLM child registry. The registry rule holds - /// because the port does not rebuild a revived session's subagent - /// registry from the spawn ledger (TS `listPassiveRlmSubagents` - /// does), so the stop would discard state the revival cannot - /// restore. The default `false` keeps scripted harness engines and - /// kernel-less embeddings resident — the same conservative arm as - /// the release default. + /// because a revival rebuilds the registry from the spawn ledger as + /// settled rows only — live run state (answer previews, collect + /// envelopes, in-flight watchers) does not come back, so the stop + /// would still discard state the revival cannot restore. The + /// default `false` keeps scripted harness engines and kernel-less + /// embeddings resident — the same conservative arm as the release + /// default. fn can_passivate_worker( &self, ) -> std::pin::Pin + Send + '_>> { diff --git a/crates/pa-daemon/src/messaging.rs b/crates/pa-daemon/src/messaging.rs index 0affe8d7e..46f966f74 100644 --- a/crates/pa-daemon/src/messaging.rs +++ b/crates/pa-daemon/src/messaging.rs @@ -246,7 +246,7 @@ impl Supervisor { // child selector falls back to its live edges (child id, // session id, or name) and wakes the child's own file. Ok(None) => { - return match self.wake_ledger_child(selector, &sessions_dir).await { + return match self.wake_ledger_child(selector).await { Some(outcome) => outcome, None => WakeOutcome::Unknown, } @@ -331,18 +331,13 @@ impl Supervisor { /// a selector the saved-session catalog missed against the spawn /// ledger's live child edges (the child id, the child's session-id /// file stem, or the child's name), then wake one worker over the - /// child's session file. `None` keeps the caller's unknown-session - /// error; `Some(Failed)` carries the wake's own error (an ambiguous - /// selector outranks the miss, like the catalog's). - async fn wake_ledger_child( - self: &Arc, - selector: &str, - sessions_dir: &std::path::Path, - ) -> Option { - let ledger = match self - .rlm_spawn_ledger_for(Some(&sessions_dir.to_string_lossy())) - .await - { + /// child's session file - the daemon-default ledger the spawn + /// admission appends to (TS `rlmSpawnLedger()`). `None` keeps the + /// caller's unknown-session error; `Some(Failed)` carries the wake's + /// own error (an ambiguous selector outranks the miss, like the + /// catalog's). + async fn wake_ledger_child(self: &Arc, selector: &str) -> Option { + let ledger = match self.rlm_spawn_ledger_for(None).await { Ok(ledger) => ledger, Err(error) => return Some(WakeOutcome::Failed(error.to_string())), }; @@ -409,7 +404,7 @@ impl Supervisor { // fence sees a root and never re-passivates. config: Some(json!({ "cwd": cwd, "rlmDepth": depth })), telemetry_disabled: None, - runtime_metadata: Some(json!({ "rlmChildId": child_id })), + runtime_metadata: Some(json!({ "kind": "subagent", "rlmChildId": child_id })), lifecycle: None, env: None, launch_env: None, diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rs index 378baefef..98b5c99a3 100644 --- a/crates/pa-daemon/src/rlm_children.rs +++ b/crates/pa-daemon/src/rlm_children.rs @@ -205,8 +205,10 @@ struct ChildRecord { session_file: Option, /// Rows of [`ChildRecord::session_file`] already folded into the /// parent's attribution rows. The walk resumes here, so repeated - /// observation never double-bills a child. - attributed_rows: usize, + /// observation never double-bills a child. `None` on a reseeded row: + /// nothing has been observed since the reseed, and the first delivery + /// primes the cursor at the file's tail. + attributed_rows: Option, /// A follow-up usage watcher is live for this retained child /// (delayed agent messaging after the task run settled). usage_watch_live: bool, @@ -537,6 +539,12 @@ impl SupervisorChildSessions { *self.inner.identity.lock().expect("identity lock") = identity; } + /// Rebuild the children registry from the spawn ledger (a restarted + /// parent lists its ledger children again). + pub async fn reseed_from_ledger(&self) { + self.inner.reseed_from_ledger().await; + } + /// The inherited RLM depth bound (TS `getRlmMaxDepthStatus().maxDepth` /// before any chat override). /// @@ -701,7 +709,7 @@ impl SupervisorChildSessions { error: None, closed_by_parent: false, session_file: None, - attributed_rows: 0, + attributed_rows: Some(0), usage_watch_live: false, usage_rearm: false, emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())), diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rs index 783686ed7..3e6d1791d 100644 --- a/crates/pa-daemon/src/rlm_children/host.rs +++ b/crates/pa-daemon/src/rlm_children/host.rs @@ -145,7 +145,7 @@ impl RlmSubagentHost for SupervisorChildSessions { error: None, closed_by_parent: false, session_file: created.session_file.clone(), - attributed_rows: 0, + attributed_rows: Some(0), usage_watch_live: false, usage_rearm: false, emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())), diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rs index 08642ab47..0c39f8aea 100644 --- a/crates/pa-daemon/src/rlm_children/lifecycle.rs +++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs @@ -436,6 +436,33 @@ impl SupervisorChildSessionsInner { // notice rides the parent's follow-up route (TS: the run // task's `finally` flushes pending usage at settlement). self.emit_child_usage(record).await; + // Only a successful run completes the display (TS + // `completeRlmSubagentRuntime`); a cancelled or failed run + // stays `running`, which a restart relists as `error`. + let completed = { + let record = record.lock().await; + (record.settled_status == Some("done")) + .then(|| (record.session_dir.clone(), record.rlm_child_id.clone())) + }; + if let Some((session_dir, child_id)) = completed { + if let Err(error) = tokio::task::spawn_blocking(move || { + let Some(mut display) = + crate::rlm_ledger::read_rlm_subagent_display(Path::new(&session_dir)) + else { + return Ok(()); + }; + if display.child_id != child_id || display.status != "running" { + return Ok(()); + } + display.status = "completed".to_string(); + crate::rlm_ledger::write_rlm_subagent_display(&display).map(|_| ()) + }) + .await + .unwrap_or_else(|error| Err(anyhow!(error))) + { + eprintln!("pa-daemon: RLM child display completion failed: {error:#}"); + } + } self.deliver_settle_notice(record).await; // A settled child releases an owed goal continuation (TS // `_maybeResumeGoalContinuationAfterRlmWork` at the child diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rs index 6ef480998..2dfe59513 100644 --- a/crates/pa-daemon/src/rlm_children/registry.rs +++ b/crates/pa-daemon/src/rlm_children/registry.rs @@ -1,11 +1,14 @@ //! The registry-mutation concern: run cancellation, inactive-child -//! deletes with their tombstone receipts, the close walk, and the -//! target lookup/resolution; the close-failure no-op marker is -//! registry-only. +//! deletes with their tombstone receipts, the close walk, the target +//! lookup/resolution, and the ledger reseed; the close-failure no-op +//! marker is registry-only. +use std::path::{Path, PathBuf}; + use super::{ - bail, json, Arc, ChildCloseReason, ChildRecord, Context, DaemonCommand, DeletedChild, Map, - Mutex, Result, SupervisorChildSessionsInner, KILL_TIMEOUT_MS, + bail, json, rlm_child_label, Arc, ChildCloseReason, ChildRecord, Context, DaemonCommand, + DeletedChild, Map, Mutex, Result, SupervisorChildSessionsInner, KILL_TIMEOUT_MS, }; +use crate::lease::canonical_session_path; /// The already-gone marker inside a close failure (the supervisor's /// `Unknown active session` route failure): TS `closeSessionOnce` treats a @@ -274,4 +277,138 @@ impl SupervisorChildSessionsInner { ), } } + + /// Rebuild the children registry from the spawn ledger (TS + /// `listPassiveRlmSubagents`): a restarted parent worker lists its + /// non-deleted ledger children again, addressed by their durable + /// session ids. Every reseeded row is settled (nothing is owed), and + /// its usage cursor starts lazy: the first delivery primes it at the + /// file's tail. + pub(super) async fn reseed_from_ledger(&self) { + let Some(parent_file) = self + .identity + .lock() + .expect("identity lock") + .session_file + .clone() + else { + return; + }; + let agent_dir = self.agent_dir.clone(); + let supervisor_socket = self.link.socket_path().clone(); + let parent_file_read = parent_file.clone(); + let records = tokio::task::spawn_blocking(move || { + ledger_child_records(&agent_dir, &supervisor_socket, Path::new(&parent_file_read)) + }) + .await + .unwrap_or_default(); + let mut children = self.children.lock().await; + // A swap rebound the identity mid-read: its own reseed lists the new session's children. + if self + .identity + .lock() + .expect("identity lock") + .session_file + .as_deref() + != Some(parent_file.as_str()) + { + return; + } + children.extend( + records + .into_iter() + .map(|record| Arc::new(Mutex::new(record))), + ); + } +} + +/// One parent file's non-deleted ledger edges as settled registry rows +/// (TS `listPassiveRlmSubagents`), read from the ledger the supervisor +/// writes (its persisted default sessions dir): the child file's stem +/// addresses each row both ways - resolve for a resident child, wake +/// for a passive one; an unreadable ledger reads as empty. +fn ledger_child_records( + agent_dir: &Path, + supervisor_socket: &Path, + parent_file: &Path, +) -> Vec { + let Some(sessions_dir) = crate::descriptor::load_supervisor_config( + &crate::descriptor::descriptor_dir(agent_dir, supervisor_socket) + .join(crate::descriptor::SUPERVISOR_CONFIG_FILE_NAME), + supervisor_socket, + ) + .and_then(|config| config.default_session_dir) else { + return Vec::new(); + }; + let ledger = + crate::rlm_ledger::RlmSpawnLedger::new(agent_dir, Path::new(&sessions_dir), |_| {}); + let edges = ledger.live_edges().unwrap_or_else(|error| { + eprintln!("pa-daemon: RLM ledger reseed skipped: {error:#}"); + Vec::new() + }); + let parent_file = canonical_session_path(parent_file); + let mut records = Vec::new(); + for edge in edges { + if canonical_session_path(Path::new(&edge.parent)) != parent_file { + continue; + } + let child = PathBuf::from(&edge.child); + let session_id = child + .file_stem() + .map(|stem| stem.to_string_lossy().into_owned()); + let display = child + .parent() + .and_then(crate::rlm_ledger::read_rlm_subagent_display) + .filter(|display| display.child_id == edge.child_id); + records.push(ChildRecord { + rlm_child_id: edge.child_id, + session_name: edge.name, + active_session_id: session_id.clone().unwrap_or_default(), + session_id, + session_dir: child + .parent() + .map(|dir| dir.to_string_lossy().into_owned()) + .unwrap_or_default(), + label: rlm_child_label( + display + .as_ref() + .and_then(|display| display.prompt.as_deref()) + .unwrap_or_default(), + ), + started_at_ms: display.as_ref().map_or_else( + || { + std::fs::metadata(&child) + .ok() + .and_then(|metadata| metadata.created().ok()) + .and_then(|created| created.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_millis() as u64) + }, + |display| display.created_at, + ), + settled_status: Some( + if display + .as_ref() + .is_some_and(|display| display.status == "running") + { + "error" + } else { + "done" + }, + ), + settled: true, + answer_preview: None, + answer_captured: false, + replied_since_task: false, + notice_delivered: true, + prompt_admitted: true, + error: None, + closed_by_parent: false, + session_file: Some(edge.child), + attributed_rows: None, + usage_watch_live: false, + usage_rearm: false, + emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())), + }); + } + records } diff --git a/crates/pa-daemon/src/rlm_children/usage.rs b/crates/pa-daemon/src/rlm_children/usage.rs index d8d7cf9ab..c55bb3b32 100644 --- a/crates/pa-daemon/src/rlm_children/usage.rs +++ b/crates/pa-daemon/src/rlm_children/usage.rs @@ -51,6 +51,10 @@ impl SupervisorChildSessionsInner { let Some(session_file) = session_file else { return; }; + // A reseeded row nothing has observed yet: nothing to bill until a delivery primes it. + let Some(from) = from else { + return; + }; // The open+parse is a blocking read of a file that can reach tens // of megabytes: run it on the blocking pool, never the async // worker (a slow file read must not stall unrelated tasks on the @@ -68,7 +72,7 @@ impl SupervisorChildSessionsInner { let (batches, next) = crate::rlm_child_usage::child_usage_batches(store.entries(), from); { let mut record_guard = record.lock().await; - record_guard.attributed_rows = next; + record_guard.attributed_rows = Some(next); } if batches.is_empty() { return; @@ -102,11 +106,34 @@ impl SupervisorChildSessionsInner { /// watcher covers the follow-up turn's usage. It never touches the /// run status, notices, or the settle hook. pub(super) async fn arm_usage_watch(this: &Arc, record: &Arc>) { + if record.lock().await.closed_by_parent { + return; + } + // Prime a lazy cursor at the tail so the pre-restart history is never billed. + if record.lock().await.attributed_rows.is_none() { + // A local, not an if-let scrutinee: the scrutinee's guard would + // live through the body and deadlock the re-lock below. + let session_file = record + .lock() + .await + .session_file + .clone() + .filter(|path| !path.is_empty()); + if let Some(session_file) = session_file { + let path = PathBuf::from(session_file); + let tail = tokio::task::spawn_blocking(move || { + crate::session_store::SessionFile::open(&path) + .map(|store| store.entries().len()) + }) + .await; + // A failed read stays lazy: a zero cursor would re-bill the whole history. + if let Ok(Ok(tail)) = tail { + record.lock().await.attributed_rows.get_or_insert(tail); + } + } + } { let mut record = record.lock().await; - if record.closed_by_parent { - return; - } if record.usage_watch_live { // A watcher is already observing this child (armed for an // earlier delivery): ask IT to observe this delivery's diff --git a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs index 9572f0cd4..ded279563 100644 --- a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs +++ b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs @@ -45,23 +45,23 @@ fn record_with_file(child_id: &str, session_file: &Path) -> Arc SupervisorChildSessions { +fn registry(agent_dir: &Path) -> SupervisorChildSessions { let tmp = tempfile::tempdir().unwrap(); let socket = tmp.path().join("supervisor.sock"); std::mem::forget(tmp); SupervisorChildSessions::new( Arc::new(SupervisorLink::new(socket)), - std::path::PathBuf::from("/agent"), + agent_dir.to_path_buf(), "parent-live".to_string(), std::sync::Arc::new(crate::model_allowlist::ModelRefusalTelemetry::new( - std::path::PathBuf::from("/agent"), + agent_dir.to_path_buf(), /*telemetry_disabled*/ true, )), ) @@ -91,7 +91,7 @@ fn child_file(dir: &Path) -> PathBuf { async fn emit_reads_once_and_advances_the_cursor() { let tmp = tempfile::tempdir().unwrap(); let file = child_file(tmp.path()); - let sessions = registry(); + let sessions = registry(Path::new("/agent")); let sink = Arc::new(CapturingSink::default()); sessions.set_usage_sink(sink.clone()); let record = record_with_file("sub-emit1", &file); @@ -108,7 +108,7 @@ async fn emit_reads_once_and_advances_the_cursor() { assert_eq!(usage.output, 2_929); assert!((usage.cost.total.as_f64() - 0.008_995_7).abs() < 1e-9); let consumed = record.lock().await.attributed_rows; - assert!(consumed > 0); + assert!(consumed.is_some_and(|rows| rows > 0)); // The cursor consumed the rows: nothing re-delivers. sessions.inner.emit_child_usage(&record).await; @@ -123,11 +123,11 @@ async fn emit_reads_once_and_advances_the_cursor() { async fn emit_without_a_sink_consumes_nothing() { let tmp = tempfile::tempdir().unwrap(); let file = child_file(tmp.path()); - let sessions = registry(); + let sessions = registry(Path::new("/agent")); let record = record_with_file("sub-emit2", &file); sessions.inner.emit_child_usage(&record).await; - assert_eq!(record.lock().await.attributed_rows, 0); + assert_eq!(record.lock().await.attributed_rows, Some(0)); } /// A record without a session file (the test seam's shape) observes @@ -136,7 +136,7 @@ async fn emit_without_a_sink_consumes_nothing() { #[tokio::test] async fn emit_tolerates_missing_and_absent_files() { let tmp = tempfile::tempdir().unwrap(); - let sessions = registry(); + let sessions = registry(Path::new("/agent")); let sink = Arc::new(CapturingSink::default()); sessions.set_usage_sink(sink.clone()); let missing = record_with_file("sub-emit3", &tmp.path().join("absent.jsonl")); @@ -148,3 +148,154 @@ async fn emit_tolerates_missing_and_absent_files() { sessions.inner.emit_child_usage(&no_file).await; assert_eq!(sink.0.lock().expect("reports lock").len(), 0); } + +/// The reseed lists the parent's live ledger children as settled rows +/// (tombstones/torn/`meta`/foreign parents excluded), the cursor lazy +/// until a delivery primes it at the tail. +#[tokio::test] +async fn reseed_lists_live_ledger_children_settled_and_bills_only_after_a_delivery() { + let tmp = tempfile::tempdir().unwrap(); + let agent = tmp.path().join("agent"); + let daemon_sessions = tmp.path().join("daemon-sessions"); + let proto = child_file(tmp.path()); + let file_at = |path: PathBuf| { + std::fs::create_dir_all(path.parent().expect("parent dir")).unwrap(); + std::fs::copy(&proto, &path).unwrap(); + path.display().to_string() + }; + let parent_one = file_at(daemon_sessions.join("P.jsonl")); + let parent_two = file_at(daemon_sessions.join("Q.jsonl")); + let child_one = file_at(agent.join("session-artifacts/P/sub-a/sess-a.jsonl")); + let child_two = file_at(agent.join("session-artifacts/P/sub-b/sess-b.jsonl")); + let child_legacy = file_at(agent.join("session-artifacts/P/sub-legacy/sess-legacy.jsonl")); + let child_three = file_at(agent.join("session-artifacts/P/sub-c/sess-c.jsonl")); + let child_four = file_at(agent.join("session-artifacts/Q/sub-d/sess-d.jsonl")); + for (child_id, session_file, status, prompt) in [ + ("sub-a", &child_one, "completed", Some("finished task")), + ("sub-b", &child_two, "running", None), + ] { + crate::rlm_ledger::write_rlm_subagent_display( + &crate::rlm_ledger::RlmSubagentDisplayEntry { + type_tag: "rlm_subagent".to_string(), + child_id: child_id.to_string(), + session_name: child_id.to_string(), + session_dir: Path::new(session_file) + .parent() + .unwrap() + .display() + .to_string(), + session_file: session_file.clone(), + rlm_parent_node_id: None, + prompt: prompt.map(str::to_string), + spawn_code: None, + model: None, + status: status.to_string(), + created_at: 1234, + }, + ) + .unwrap(); + } + let spawn = |child_id: &str, parent: &str, child: &str| { + format!( + r#"{{"v":1,"op":"spawn","at":"2026-09-30T00:00:00Z","childId":"{child_id}","parent":"{parent}","child":"{child}","depth":1,"name":"{child_id}"}}"# + ) + }; + let ledger_path = crate::rlm_ledger::rlm_ledger_path(&agent, &daemon_sessions); + std::fs::create_dir_all(ledger_path.parent().expect("ledger dir")).unwrap(); + std::fs::write( + ledger_path, + [ + r#"{"v":1,"op":"meta","at":"2026-09-30T00:00:00Z"}"#, + spawn("sub-a", &parent_one, &child_one).as_str(), + r#"{"v":1,"op":"spawn","at":"#, + spawn("sub-b", &parent_one, &child_two).as_str(), + spawn("sub-legacy", &parent_one, &child_legacy).as_str(), + spawn("sub-c", &parent_one, &child_three).as_str(), + format!( + r#"{{"v":1,"op":"delete","at":"2026-09-30T00:00:01Z","childId":"sub-c","child":"{child_three}","reason":"user"}}"# + ) + .as_str(), + spawn("sub-d", &parent_two, &child_four).as_str(), + ] + .join("\n"), + ) + .unwrap(); + let sessions = registry(&agent); + let supervisor_socket = sessions.inner.link.socket_path().clone(); + crate::descriptor::persist_supervisor_config( + &crate::descriptor::descriptor_dir(&agent, &supervisor_socket) + .join(crate::descriptor::SUPERVISOR_CONFIG_FILE_NAME), + &crate::descriptor::PersistedSupervisorConfig { + version: 1, + socket_path: supervisor_socket.to_string_lossy().to_string(), + default_session_dir: Some(daemon_sessions.display().to_string()), + }, + ) + .unwrap(); + sessions.set_identity(ParentIdentity { + session_file: Some(parent_one.clone()), + ..ParentIdentity::with_default_depth() + }); + let sink = Arc::new(CapturingSink::default()); + sessions.set_usage_sink(sink.clone()); + sessions.reseed_from_ledger().await; + let roster = sessions.list_subagents().await.expect("the roster read"); + let rows: Vec<_> = roster + .iter() + .map(|row| { + ( + row.rlm_child_id.as_str(), + row.status, + row.active_session_id.as_deref(), + row.label.as_deref(), + ) + }) + .collect(); + assert_eq!( + rows, + [ + ("sub-a", "completed", Some("sess-a"), Some("finished task")), + ("sub-b", "error", Some("sess-b"), Some("child agent")), + ( + "sub-legacy", + "completed", + Some("sess-legacy"), + Some("child agent") + ), + ], + "completed and interrupted children keep their display verdicts" + ); + + let legacy = sessions + .inner + .find_record("sub-legacy") + .await + .expect("ledger-only row"); + assert!(legacy.lock().await.started_at_ms > 0); + + let record = sessions + .inner + .find_record("sub-a") + .await + .expect("the reseeded row"); + // Nothing is owed: the pre-restart history never re-bills. + sessions.inner.emit_child_usage(&record).await; + assert_eq!(sink.0.lock().expect("reports lock").len(), 0); + // A delivery primes the lazy cursor at the tail; the delivered turn + // appends its row; the next emit bills only that row. + SupervisorChildSessionsInner::arm_usage_watch(&sessions.inner, &record).await; + assert_eq!(record.lock().await.attributed_rows, Some(2)); + let mut content = std::fs::read_to_string(Path::new(&child_one)).expect("child file"); + content.push_str( + r#"{"type":"message","id":"a2","parentId":"a1","timestamp":"2026-09-23T00:00:03.000Z","message":{"role":"assistant","content":[],"stopReason":"toolUse","usage":{"input":10,"output":5,"cacheRead":0,"cacheWrite":0,"totalTokens":15,"cost":{"input":0.1,"output":0.05,"cacheRead":0,"cacheWrite":0,"total":0.15}}}}"#, + ); + content.push('\n'); + std::fs::write(Path::new(&child_one), content).expect("append the row"); + sessions.inner.emit_child_usage(&record).await; + let reports = sink.0.lock().expect("reports lock").clone(); + assert_eq!(reports.len(), 1); + let [(_, usage)] = reports[0].batches[..] else { + panic!("one batch: {:?}", reports[0].batches); + }; + assert_eq!((usage.input, usage.output), (10, 5)); +} diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rs index cfa56936e..a5adc8e84 100644 --- a/crates/pa-daemon/src/rlm_children/watch_tests.rs +++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs @@ -395,6 +395,41 @@ async fn a_worker_leaving_inside_the_settle_grace_keeps_the_verdict() { .is_some_and(|content| content.contains("the child final answer"))); } +/// A cancelled run's watcher settle leaves the display `running`, so a +/// restart relists the child as `error` instead of `completed`. +#[tokio::test] +async fn a_cancelled_child_keeps_its_display_running_through_the_watcher_settle() { + let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel(); + let (sessions, _kill_rx) = + sessions_with_fake_supervisor(follow_up_tx, 500, FakeKill::Success, FakeChild::Healthy) + .await; + let (hook_tx, mut hook_rx) = mpsc::unbounded_channel(); + sessions.set_settle_hook(Arc::new(move || { + let _ = hook_tx.send(()); + })); + let handle = spawn_child(&sessions).await; + let display_file = Path::new(&handle.session_dir).join("rlm-subagent.json"); + std::fs::write( + &display_file, + json!({ "type": "rlm_subagent", "childId": handle.rlm_child_id, + "sessionDir": handle.session_dir, "status": "running" }) + .to_string(), + ) + .unwrap(); + sessions.notify_turn_done(); + assert!(sessions.cancel_child_run(&handle.rlm_child_id).await); + // One settle hook from the cancel, one from the watcher's settle tail. + for _ in 0..2 { + tokio::time::timeout(Duration::from_secs(10), hook_rx.recv()) + .await + .expect("both settle hooks fire") + .expect("hook channel open"); + } + let display = crate::rlm_ledger::read_rlm_subagent_display(Path::new(&handle.session_dir)) + .expect("display entry stays readable"); + assert_eq!(display.status, "running"); +} + /// The `follow_up` carries exactly the TS failure row (`createRlmChildFailureMessage`). fn assert_failure_row(follow_up: &Value, child_id: &str, error: &str) { let custom = &follow_up["customMessage"]; @@ -980,7 +1015,7 @@ fn an_already_settled_child_never_re_scores_as_an_unreachable_error() { error: None, closed_by_parent: false, session_file: None, - attributed_rows: 0, + attributed_rows: Some(0), usage_watch_live: false, usage_rearm: false, emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())), diff --git a/crates/pa-daemon/src/session_navigation.rs b/crates/pa-daemon/src/session_navigation.rs index f805ed066..77702449c 100644 --- a/crates/pa-daemon/src/session_navigation.rs +++ b/crates/pa-daemon/src/session_navigation.rs @@ -407,7 +407,7 @@ impl Worker { } match self.navigation.replace_session(target.file).await { Ok(()) => { - self.refresh_replaced_session_state(); + self.refresh_replaced_session_state().await; self.reseed_service_tier_for_replacement(); self.bind_scheduled_jobs().await; self.prewarm_replacement_session(); diff --git a/crates/pa-daemon/src/worker/create.rs b/crates/pa-daemon/src/worker/create.rs index ef21194a1..c0dd01e5d 100644 --- a/crates/pa-daemon/src/worker/create.rs +++ b/crates/pa-daemon/src/worker/create.rs @@ -646,6 +646,7 @@ impl Worker { }) { return response_failure(None, "create", &error.to_string(), None); } + self.reseed_rlm_children().await; // The built-in Herdr connector binds here, per session: the pane // identity comes from the create payload's client env (the client // that owns the pane sent it), never from this process's ambient diff --git a/crates/pa-daemon/src/worker/lifecycle.rs b/crates/pa-daemon/src/worker/lifecycle.rs index 20645fa48..abd1160f7 100644 --- a/crates/pa-daemon/src/worker/lifecycle.rs +++ b/crates/pa-daemon/src/worker/lifecycle.rs @@ -237,6 +237,20 @@ impl Worker { } } + /// Relist the bound session's ledger children (TS + /// `listPassiveRlmSubagents`): awaited where the identity is bound - + /// create, and the replacement rebind inside the gate - so the scan is + /// ordered with the close walk that follows a later replacement. + pub(crate) async fn reseed_rlm_children(&self) { + if let Some(children) = self + .agent_engine + .as_ref() + .and_then(|engine| engine.children.clone()) + { + children.reseed_from_ledger().await; + } + } + /// Wait until the replacement teardown can retire the runtime: no /// turn and no compaction in flight. Like the navigation settle, the /// park rides a timeout backstop - the turn runner notifies the idle @@ -314,7 +328,7 @@ impl Worker { /// catalog rebind runs separately (`bind_scheduled_jobs`), like the /// TS dispatch handlers that call `rebindCronJobsToState` after the /// runtime call. - pub(crate) fn refresh_replaced_session_state(&self) { + pub(crate) async fn refresh_replaced_session_state(&self) { let (rlm_depth, summary, child_script) = { let mut core = self .core @@ -340,7 +354,7 @@ impl Worker { // child engine file rides along: TS children inherit the // replacement runtime's `sessionConfig`, which the runtime keeps // across its swaps. - if let Err(error) = self + match self .engine .configure_rlm_identity(crate::engine::RlmSessionIdentity { rlm_depth, @@ -350,9 +364,9 @@ impl Worker { session_file: summary.session_file.clone(), thinking: None, child_script, - }) - { - eprintln!("pa-daemon: replacement identity rebind failed: {error:#}"); + }) { + Ok(()) => self.reseed_rlm_children().await, + Err(error) => eprintln!("pa-daemon: replacement identity rebind failed: {error:#}"), } if let Ok(summary_value) = serde_json::to_value(&summary) { self.engine.set_session_summary(summary_value); diff --git a/crates/pa-daemon/tests/rlm_children_parent_death_e2e.rs b/crates/pa-daemon/tests/rlm_children_parent_death_e2e.rs index 3b1b4eec6..d388be312 100644 --- a/crates/pa-daemon/tests/rlm_children_parent_death_e2e.rs +++ b/crates/pa-daemon/tests/rlm_children_parent_death_e2e.rs @@ -426,9 +426,10 @@ fn sigkill(pid: u32) { /// A parent worker killed with SIGKILL cannot run any teardown (#246's /// closes all live in the worker), so the supervisor's death monitoring /// closes its resident children: the spawned child's worker leaves the -/// roster, its session archives, the respawned parent's registry reads -/// empty, and the child stays visible as a passive ledger row (the close -/// is a plain stop - the spawn edge survives, no delete record). +/// roster, its session archives, the respawned parent relists the child +/// as a settled row from the surviving spawn edge, and the child stays +/// visible as a passive ledger row (the close is a plain stop - the +/// spawn edge survives, no delete record). #[test] fn sigkill_closes_the_spawned_child_and_passivates_the_row() { let Some(kernel_python) = kernel_python() else { @@ -534,21 +535,13 @@ fn sigkill_closes_the_spawned_child_and_passivates_the_row() { "the parent-death close keeps the child's resume entry (no archive): {child_session}" ); - // The respawned parent's registry is fresh: the parent died before it - // could track anything, and the death close owns its children now. - // (get_rlm_children fails while the worker restarts; poll for the - // respawned answer.) + // The respawned parent reseeds the spawn edge the death close kept. + // Its display still says running, so TS exposes the interrupted task + // as error rather than presenting it as a completed child. wait_until(&mut client, Duration::from_mins(1), |client| { - client.send_command( - "g3", - &json!({ "type": "get_rlm_children", "activeSessionId": parent_id }), - ); - let response = client.read_response("g3"); - (response["success"] == true - && response["data"]["children"] - .as_array() - .is_some_and(std::vec::Vec::is_empty)) - .then_some(()) + rlm_children_rows(client, "g3", &parent_id) + .into_iter() + .find(|row| row["id"] == json!(child_id) && row["status"] == "error") }); // The passive row per TS: the spawn edge survived the close (a stop, diff --git a/crates/pa-daemon/tests/rlm_children_replacement_e2e.rs b/crates/pa-daemon/tests/rlm_children_replacement_e2e.rs index a38edb2d8..ca5d683a4 100644 --- a/crates/pa-daemon/tests/rlm_children_replacement_e2e.rs +++ b/crates/pa-daemon/tests/rlm_children_replacement_e2e.rs @@ -314,6 +314,16 @@ fn roster_cell(receipt: &Path, error_receipt: &Path) -> String { ) } +/// The probe cell: the restarted parent's roster rows, then a send to the child. +fn probe_cell(list_receipt: &Path, send_receipt: &Path, error_receipt: &Path) -> String { + format!( + "import inspect, json, traceback\ntry:\n roster = rlm.list_subagents()\n if inspect.isawaitable(roster):\n roster = await roster\n rows = [[s.rlm_child_id, s.status, s.active_session_id] for s in roster]\n open({list:?}, \"w\").write(json.dumps(rows))\n receipt = await agent_message.send(\"boot-ping\", receiver_role=\"child\", receiver_name=\"kid\")\n open({send:?}, \"w\").write(json.dumps(receipt))\n print(rows, receipt)\nexcept Exception:\n open({error:?}, \"w\").write(traceback.format_exc())\n raise", + list = list_receipt.display().to_string(), + send = send_receipt.display().to_string(), + error = error_receipt.display().to_string(), + ) +} + /// The kernel cell that creates one depth-0 resident root session through /// the product `rlm.create_session` surface. fn create_session_cell(receipt: &Path, error_receipt: &Path) -> String { @@ -370,23 +380,25 @@ fn create_parent( dir: &Path, parent_script: &Path, child_script: &Path, + session_path: Option<&Path>, id: &str, ) -> Value { let sessions_dir = dir.join("sessions"); std::fs::create_dir_all(&sessions_dir).expect("sessions dir"); - client.send_command( - id, - &json!({ - "type": "create", - "name": "parent", - "config": { - "cwd": dir.to_string_lossy(), - "sessionDir": sessions_dir.to_string_lossy(), - "script": parent_script.to_string_lossy(), - "childScript": child_script.to_string_lossy(), - }, - }), - ); + let mut create = json!({ + "type": "create", + "name": "parent", + "config": { + "cwd": dir.to_string_lossy(), + "sessionDir": sessions_dir.to_string_lossy(), + "script": parent_script.to_string_lossy(), + "childScript": child_script.to_string_lossy(), + }, + }); + if let Some(path) = session_path { + create["sessionPath"] = json!(path.to_string_lossy()); + } + client.send_command(id, &create); let created = client.read_response(id); assert_eq!(created["success"], true, "create parent failed: {created}"); created["data"].clone() @@ -426,7 +438,14 @@ fn new_session_closes_the_spawned_child_and_empties_the_roster() { wait_socket_ready(&socket); let (mut client, hello) = Client::connect(&socket); assert_eq!(hello["type"], "daemon_hello"); - let parent = create_parent(&mut client, dir.path(), &parent_script, &child_script, "c1"); + let parent = create_parent( + &mut client, + dir.path(), + &parent_script, + &child_script, + None, + "c1", + ); let parent_id = parent["activeSessionId"] .as_str() .or_else(|| parent["id"].as_str()) @@ -558,7 +577,14 @@ fn new_session_keeps_a_created_root_session_running() { wait_socket_ready(&socket); let (mut client, hello) = Client::connect(&socket); assert_eq!(hello["type"], "daemon_hello"); - let parent = create_parent(&mut client, dir.path(), &parent_script, &child_script, "c1"); + let parent = create_parent( + &mut client, + dir.path(), + &parent_script, + &child_script, + None, + "c1", + ); let parent_id = parent["activeSessionId"] .as_str() .or_else(|| parent["id"].as_str()) @@ -609,3 +635,205 @@ fn new_session_keeps_a_created_root_session_running() { "the created root session is its own session, not the parent's" ); } + +/// A daemon restart must not lose the parent's RLM children: the +/// reattached parent lists its child again and messaging it wakes the child. +#[test] +fn a_daemon_restart_relists_and_wakes_the_parents_child() { + restart_relists_child(0, "completed"); +} + +#[test] +fn a_daemon_restart_marks_a_still_running_child_as_failed() { + restart_relists_child(30_000, "running"); +} + +fn restart_relists_child(delay_ms: u64, display_status: &str) { + let Some(kernel_python) = kernel_python() else { + return; + }; + let dir = tempfile::TempDir::new().expect("temp dir"); + let agent_dir = dir.path().join("agent"); + std::fs::create_dir_all(&agent_dir).expect("agent dir"); + let socket = dir.path().join("supervisor.sock"); + let receipts = dir.path().join("receipts"); + std::fs::create_dir_all(&receipts).expect("receipts dir"); + let spawn_receipt = receipts.join("spawn.json"); + let spawn_error = receipts.join("spawn.error"); + let list_receipt = receipts.join("list.json"); + let send_receipt = receipts.join("send.json"); + let probe_error = receipts.join("probe.error"); + + let child_script = dir.path().join("child.json"); + std::fs::write( + &child_script, + json!({ "responses": [ { "text": "kid done", "delayMs": delay_ms } ] }).to_string(), + ) + .expect("write child script"); + let parent_script = write_parent_script( + dir.path(), + &spawn_cell(&spawn_receipt, &spawn_error), + "pass", + ); + let mut daemon = spawn_supervisor(&socket, &agent_dir, &kernel_python); + wait_socket_ready(&socket); + let (mut client, hello) = Client::connect(&socket); + assert_eq!(hello["type"], "daemon_hello"); + let parent = create_parent( + &mut client, + dir.path(), + &parent_script, + &child_script, + None, + "c1", + ); + let parent_id = parent["activeSessionId"] + .as_str() + .or_else(|| parent["id"].as_str()) + .expect("parent active session id") + .to_string(); + let parent_session_id = parent["sessionId"].as_str().expect("parent session id"); + let parent_file = parent["sessionFile"] + .as_str() + .expect("parent session file") + .to_string(); + + run_turn(&mut client, &parent_id, "spawn the kid", "t1"); + let spawned: Value = + serde_json::from_str(&await_receipt(&spawn_receipt)).expect("spawn receipt json"); + let child_id = spawned["rlm_child_id"] + .as_str() + .expect("child id") + .to_string(); + assert!( + !spawn_error.exists(), + "the spawn cell failed: {}", + std::fs::read_to_string(&spawn_error).unwrap_or_default() + ); + let supervisor_row = wait_until(&mut client, Duration::from_secs(30), |client| { + roster_summaries(client, "l1").into_iter().find(|summary| { + summary["sessionName"] == json!("kid") && summary["runtimeKind"] == "subagent" + }) + }); + let child_session_id = supervisor_row["sessionId"] + .as_str() + .expect("child session id") + .to_string(); + let child_file = agent_dir + .join("session-artifacts") + .join(parent_session_id) + .join(&child_id) + .join(format!("{child_session_id}.jsonl")); + let display_file = child_file.parent().unwrap().join("rlm-subagent.json"); + if delay_ms > 0 { + wait_until(&mut client, Duration::from_secs(30), |client| { + let rows = rlm_children_rows(client, "g-running", &parent_id); + rows.iter() + .any(|row| row["id"] == child_id && row["status"] == "running") + .then_some(()) + }); + } + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let display: Value = + serde_json::from_slice(&std::fs::read(&display_file).expect("child display file")) + .expect("child display json"); + if display["status"] == display_status { + break; + } + assert!( + Instant::now() < deadline, + "child display never completed: {display}" + ); + std::thread::sleep(Duration::from_millis(100)); + } + + client.send_command("bye", &json!({ "type": "shutdown" })); + let deadline = Instant::now() + Duration::from_secs(30); + while daemon + .child + .try_wait() + .expect("wait for the supervisor exit") + .is_none() + { + assert!(Instant::now() < deadline, "the supervisor never exited"); + std::thread::sleep(Duration::from_millis(50)); + } + let _daemon = spawn_supervisor(&socket, &agent_dir, &kernel_python); + wait_socket_ready(&socket); + let (mut client, hello) = Client::connect(&socket); + assert_eq!(hello["type"], "daemon_hello"); + + let probe_script = write_parent_script( + dir.path(), + &probe_cell(&list_receipt, &send_receipt, &probe_error), + "pass", + ); + let parent = create_parent( + &mut client, + dir.path(), + &probe_script, + &child_script, + Some(Path::new(&parent_file)), + "c2", + ); + let new_parent_id = parent["activeSessionId"] + .as_str() + .or_else(|| parent["id"].as_str()) + .expect("parent active session id") + .to_string(); + + run_turn( + &mut client, + &new_parent_id, + "probe the restarted roster", + "t2", + ); + assert!( + !probe_error.exists(), + "the probe cell failed: {}", + std::fs::read_to_string(&probe_error).unwrap_or_default() + ); + // (a) relisted + let rows: Value = + serde_json::from_str(&await_receipt(&list_receipt)).expect("list receipt json"); + assert_eq!( + rows, + json!([[ + child_id, + if display_status == "completed" { + "completed" + } else { + "error" + }, + child_session_id + ]]), + "the restarted parent must relist its ledger child with its persisted status" + ); + // (b) send receipt + let _send: Value = + serde_json::from_str(&await_receipt(&send_receipt)).expect("send receipt json"); + // (c) wake oracle + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let content = std::fs::read_to_string(&child_file).unwrap_or_default(); + if content.contains("boot-ping") { + break; + } + assert!( + Instant::now() < deadline, + "the send never woke the child into its session file: {content}" + ); + std::thread::sleep(Duration::from_millis(100)); + } + // (d) child runtime + client.send_command( + "gs", + &json!({ "type": "get_state", "activeSessionId": child_session_id }), + ); + let state = client.read_response("gs"); + assert_eq!(state["success"], true, "get_state failed: {state}"); + assert_eq!(state["data"]["rlmChildId"], json!(child_id)); + assert_eq!(state["data"]["rlmDepth"], json!(1)); + assert_eq!(state["data"]["parentSessionPath"], json!(parent_file)); +} -- knotmirror 0.0.1