diff --git a/crates/pa-core/src/session_engine/canonical_live.rs b/crates/pa-core/src/session_engine/canonical_live.rs index 9aea560a1..170b0b438 100644 --- a/crates/pa-core/src/session_engine/canonical_live.rs +++ b/crates/pa-core/src/session_engine/canonical_live.rs @@ -438,6 +438,12 @@ impl CanonicalLiveSession { self.blocked.store(true, Ordering::Release); } + /// Whether this view refuses every later projection. + #[must_use] + pub fn projection_blocked(&self) -> bool { + self.blocked.load(Ordering::Acquire) + } + /// Refuse new projections after any failed, uncertain or lost receipt. /// /// # Errors diff --git a/crates/pa-daemon/src/agent_engine/context_selection.rs b/crates/pa-daemon/src/agent_engine/context_selection.rs index a7c998945..4181c83aa 100644 --- a/crates/pa-daemon/src/agent_engine/context_selection.rs +++ b/crates/pa-daemon/src/agent_engine/context_selection.rs @@ -60,6 +60,13 @@ pub(crate) trait InitialContextSelectionStore: Send + Sync { fn canonical_read_refusal(&self) -> Option { None } + + /// Certify and retire a blocked SDK mirror's actual owner so the next build + /// attaches a successor epoch. Refusal keeps the cached runtime and its + /// blocked view; history is never rewritten. + fn retire_quarantined_owner(&self) -> anyhow::Result<()> { + anyhow::bail!("this session has no canonical owner to reopen") + } } impl super::AgentSessionEngine { diff --git a/crates/pa-daemon/src/agent_engine/lifecycle.rs b/crates/pa-daemon/src/agent_engine/lifecycle.rs index 5f76539b6..3f5bc83d3 100644 --- a/crates/pa-daemon/src/agent_engine/lifecycle.rs +++ b/crates/pa-daemon/src/agent_engine/lifecycle.rs @@ -887,6 +887,50 @@ impl AgentSessionEngine { Ok(()) } + /// Reopen a quarantined session runtime at a certified owner boundary. + /// + /// The actual owner must prove its last certified source unchanged and no + /// outstanding receipt before the cached SDK mirror is discarded. A refusal + /// keeps the cached runtime and its blocked view; nothing is retried. + pub(crate) async fn reopen_quarantined_session_runtime(&self) -> anyhow::Result<()> { + let mut completion_state = self.session_build.lock().await; + let store = self + .initial_context_selection_store + .lock() + .expect("initial context selection store lock") + .clone() + .ok_or_else(|| { + anyhow::anyhow!("persistent session has no canonical owner to reopen") + })?; + store.retire_quarantined_owner()?; + *completion_state = None; + self.clear_retired_runtime().await; + Ok(()) + } + + /// A prompt is a new demand, never a retry: a blocked canonical view is + /// reopened once at a certified boundary before this prompt is admitted. + pub(crate) fn reopen_blocked_canonical_runtime(&self) { + if self.session_is_closed() { + return; + } + let blocked = self + .session + .blocking_lock() + .as_deref() + .and_then(|engine| engine.session.canonical_live_session()) + .is_some_and(|live| live.projection_blocked()); + if !blocked { + return; + } + if let Err(error) = self + .runtime + .block_on(self.reopen_quarantined_session_runtime()) + { + eprintln!("pa-daemon: canonical runtime reopen refused: {error:#}"); + } + } + /// Authorized cancel-and-replace already aborts and settles at the worker. /// It does not grant permission to reuse the old physical owner. pub(crate) async fn discard_replaced_session_runtime(&self) { diff --git a/crates/pa-daemon/src/agent_engine/session_engine_impl.rs b/crates/pa-daemon/src/agent_engine/session_engine_impl.rs index 191c153aa..585b809ca 100644 --- a/crates/pa-daemon/src/agent_engine/session_engine_impl.rs +++ b/crates/pa-daemon/src/agent_engine/session_engine_impl.rs @@ -1001,9 +1001,6 @@ impl SessionEngine for AgentSessionEngine { } } }); - if let Some(children) = &self.children { - children.notify_turn_done(); - } } fn run_side_question( @@ -1455,6 +1452,11 @@ impl SessionEngine for AgentSessionEngine { "timestamp": now_millis(), })) }; + // A blocked canonical view is a quarantined mirror, not lost history: + // a new prompt certifies the owner boundary once and reopens cold + // before its accepted row persists. A refused certification keeps the + // view, and the prompt refuses exactly once like before. + self.reopen_blocked_canonical_runtime(); // A create-owned warmup may still be adopting/ACKing metadata. Hold // its existing gate through accepted-row persistence, without holding // any SDK/owner/core guard or triggering a truly lazy build. diff --git a/crates/pa-daemon/src/agent_engine/tests/session.rs b/crates/pa-daemon/src/agent_engine/tests/session.rs index c554c5d14..70b1b6893 100644 --- a/crates/pa-daemon/src/agent_engine/tests/session.rs +++ b/crates/pa-daemon/src/agent_engine/tests/session.rs @@ -551,3 +551,152 @@ fn a_bare_invocation_of_a_close_tagged_body_still_floors() { ); assert_eq!(assistant_texts(&events), vec!["ok"]); } + +#[test] +fn quarantined_runtime_settles_later_prompts_without_rewriting_history() { + let _faux = FAUX_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("session.jsonl"); + let mut store = + crate::session_store::SessionFile::create(dir.path().to_str().unwrap(), None, 0); + store.set_path(path.clone()); + store.rewrite().unwrap(); + let engine = AgentSessionEngine::new(AgentEngineConfig { + cwd: dir.path().to_path_buf(), + agent_dir: dir.path().join("agent"), + session_file: Some(path.clone()), + faux_script: Some(json!({"responses": [{"text": "warm"}]}).to_string()), + provider: None, + model: None, + api_key: None, + thinking: None, + session_dir: None, + supervisor_link: None, + telemetry_disabled: None, + cron_store: None, + queued_steering_probe: None, + }) + .unwrap(); + let _owner = own_canonical_store(&engine, store); + let mut events = Vec::new(); + admit(&engine, "warm".to_string(), &mut events); + assert!(events + .iter() + .any(|event| matches!(event, EngineEvent::Done(Ok(()))))); + let built = engine.session.blocking_lock().clone().unwrap(); + let live = built.session.canonical_live_session().unwrap(); + engine.runtime.block_on(async { + // An interrupted receipt cannot be declared reconciled by terminal cleanup. + let prepared = live + .prepare_mutation(pa_core::session::canonical::CanonicalMutationKind::OwnerMetadata) + .await + .unwrap(); + drop(prepared); + }); + let before = std::fs::read(&path).unwrap(); + for _ in 0..2 { + events.clear(); + admit(&engine, "must refuse".to_string(), &mut events); + assert_eq!( + events + .iter() + .filter(|event| matches!(event, EngineEvent::Done(Err(_)))) + .count(), + 1, + "each rejected prompt must settle exactly once: {events:?}", + ); + assert!(!events + .iter() + .any(|event| matches!(event, EngineEvent::AssistantMessage(_)))); + assert_eq!(std::fs::read(&path).unwrap(), before); + assert!(live.ensure_ready().is_err()); + assert!(std::sync::Arc::ptr_eq( + &built, + engine.session.blocking_lock().as_ref().unwrap(), + )); + } +} + +#[test] +fn a_blocked_view_reopens_cold_at_a_certified_owner_boundary() { + let _faux = FAUX_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("session.jsonl"); + let mut store = + crate::session_store::SessionFile::create(dir.path().to_str().unwrap(), None, 0); + store.set_path(path.clone()); + store.rewrite().unwrap(); + let engine = AgentSessionEngine::new(AgentEngineConfig { + cwd: dir.path().to_path_buf(), + agent_dir: dir.path().join("agent"), + session_file: Some(path.clone()), + faux_script: Some( + json!({"responses": [{"text": "warm"}, {"text": "reopened"}]}).to_string(), + ), + provider: None, + model: None, + api_key: None, + thinking: None, + session_dir: None, + supervisor_link: None, + telemetry_disabled: None, + cron_store: None, + queued_steering_probe: None, + }) + .unwrap(); + let _owner = own_canonical_store(&engine, store); + let mut events = Vec::new(); + admit(&engine, "warm".to_string(), &mut events); + assert!(events + .iter() + .any(|event| matches!(event, EngineEvent::Done(Ok(()))))); + let blocked = engine.session.blocking_lock().clone().unwrap(); + let live = blocked.session.canonical_live_session().unwrap(); + // Seal the runtime's canonical admission exactly like the replacement + // seam does: the view refuses every projection, the owner owes nothing. + blocked.session.close_runtime_admission_for_replacement(); + assert!(live.projection_blocked()); + let before = std::fs::read(&path).unwrap(); + // A new prompt is a new demand, not a retry. The certified reopen retires + // the actual owner at its unchanged cursor and rebuilds the mirror from + // the same durable rows before the prompt is admitted. + events.clear(); + admit(&engine, "reopened".to_string(), &mut events); + assert!( + events + .iter() + .any(|event| matches!(event, EngineEvent::Done(Ok(())))), + "the next prompt must be served after the certified reopen: {events:?}", + ); + assert!(!events + .iter() + .any(|event| matches!(event, EngineEvent::Done(Err(_))))); + let current = engine.session.blocking_lock().clone().unwrap(); + assert!( + !std::sync::Arc::ptr_eq(&blocked, ¤t), + "the blocked runtime is retired, never reused", + ); + // The blocked view stays closed: it can never serve the successor epoch. + assert!(live.projection_blocked()); + assert!(live.ensure_ready().is_err()); + // A cold reopen re-reads history: the pre-reopen bytes are an exact prefix + // and the new turn appended exactly one user and one assistant row. + let after = std::fs::read(&path).unwrap(); + assert!( + after.starts_with(&before), + "a cold reopen must not rewrite committed history", + ); + let appended = std::str::from_utf8(&after[before.len()..]).unwrap(); + assert_eq!( + appended + .lines() + .filter(|line| line.contains("\"role\":\"user\"")) + .count(), + 1, + "the refused prompt is never retried; only the new prompt is recorded: {appended}", + ); +} diff --git a/crates/pa-daemon/src/session_store/canonical_live.rs b/crates/pa-daemon/src/session_store/canonical_live.rs index b5e2f8b36..f9470e816 100644 --- a/crates/pa-daemon/src/session_store/canonical_live.rs +++ b/crates/pa-daemon/src/session_store/canonical_live.rs @@ -53,9 +53,29 @@ fn failure(detail: impl std::fmt::Display) -> CanonicalStoreError { } fn unresolved(detail: impl std::fmt::Display) -> CanonicalStoreError { + unresolved_with(detail, None) +} + +/// One refusal whose outcome is unresolved. `reason` overrides the fixed-detail +/// lookup with the caller's own fixed source-check label. +fn unresolved_with( + detail: impl std::fmt::Display, + reason: Option<&'static str>, +) -> CanonicalStoreError { let detail = detail.to_string(); // Report only fixed source-check labels, never the dynamic error payload. - let reason = match detail.as_str() { + let reason = reason.unwrap_or_else(|| label_of(&detail)); + eprintln!("pa-daemon: canonical refusal phase=owner reason={reason}"); + CanonicalStoreError { + kind: CanonicalFailureKind::AdoptionConflict, + requested: CanonicalRequestedWrite::Unknown, + source: CanonicalSourceDisposition::NeedsReconcile, + detail, + } +} + +fn label_of(detail: &str) -> &'static str { + match detail { "canonical owner adoption abandoned" => "canonical_owner_abandoned_pending_adoption", "canonical owner invalidated" => "canonical_owner_invalidated", "canonical owner authority poisoned" => "canonical_owner_authority_poisoned", @@ -67,6 +87,12 @@ fn unresolved(detail: impl std::fmt::Display) -> CanonicalStoreError { "canonical owner path/header/lease changed" => "canonical_owner_path_header_lease_changed", "canonical serving epoch is poisoned" => "canonical_serving_epoch_is_poisoned", "canonical owner mutex poisoned" => "canonical_owner_mutex_poisoned", + "canonical owner has prepared mutations" => "canonical_owner_prepared_mutations", + "canonical owner has admitted tools" => "canonical_owner_admitted_tools", + "canonical serving run is live" => "canonical_serving_run_live", + "canonical run has no terminal ACK" => "canonical_run_terminal_unacknowledged", + "canonical child usage I/O is unresolved" => "canonical_child_usage_unresolved", + "canonical caller has not completed" => "canonical_caller_incomplete", "actual tool executions still lack terminal adoption" => { "actual_tool_executions_still_lack_terminal_adoption" } @@ -103,6 +129,23 @@ fn unresolved(detail: impl std::fmt::Display) -> CanonicalStoreError { detail if detail.starts_with("delivery write start: ") => "delivery_write_start", detail if detail.starts_with("delivery finish: ") => "delivery_finish", "canonical source advanced outside owner" => "unowned_source_advance", + "canonical source changed outside owner" => "unowned_source_change", + "canonical Custom source changed outside owner" => "unowned_custom_source_change", + "canonical Custom admission source CAS conflict" => "custom_admission_source_changed", + "canonical user admission source CAS conflict" => "user_admission_source_changed", + "canonical prepare source/branch CAS conflict" => "prepare_source_branch_changed", + "canonical Custom write lacks exact single physical suffix" => { + "custom_write_physical_suffix" + } + "delivered Custom FIFO head differs from exact accepted intent" => { + "delivered_custom_intent_mismatch" + } + detail if detail.starts_with("Custom write admission: ") => "custom_write_admission", + detail if detail.starts_with("Custom write start: ") => "custom_write_start", + detail if detail.starts_with("Custom write finish: ") => "custom_write_finish", + detail if detail.starts_with("Custom preflight: ") => "custom_preflight", + detail if detail.starts_with("Custom delivery barrier: ") => "custom_delivery_barrier", + detail if detail.starts_with("Custom ledger: ") => "custom_ledger", "actual AgentStart belongs to another certified source" => "actual_run_source_mismatch", "actual AgentStart source changed outside owner" => "actual_run_external_source_change", "actual run was already registered" => "actual_run_duplicate_registration", @@ -110,13 +153,6 @@ fn unresolved(detail: impl std::fmt::Display) -> CanonicalStoreError { "actual_run_terminal_unacknowledged" } _ => "unclassified_owner_error", - }; - eprintln!("pa-daemon: canonical refusal phase=owner reason={reason}"); - CanonicalStoreError { - kind: CanonicalFailureKind::AdoptionConflict, - requested: CanonicalRequestedWrite::Unknown, - source: CanonicalSourceDisposition::NeedsReconcile, - detail, } } @@ -200,6 +236,36 @@ impl LiveCanonicalOwner { } } +/// Why a refused request is not a certified unstarted prior source. Each variant +/// is a fixed log label: the refusal must name the actual cause, never the +/// generic unclassified fallback. +#[derive(Clone, Copy)] +enum UnstartedRefusal { + Unavailable, + PreparedWork, + OwnerScope, + Session, + Path, + Header, + Branch, + Revision, +} + +impl UnstartedRefusal { + fn label(self) -> &'static str { + match self { + Self::Unavailable => "refused_unstarted_owner_unavailable", + Self::PreparedWork => "refused_unstarted_prepared_work", + Self::OwnerScope => "refused_unstarted_owner_scope_changed", + Self::Session => "refused_unstarted_session_changed", + Self::Path => "refused_unstarted_path_changed", + Self::Header => "refused_unstarted_header_changed", + Self::Branch => "refused_unstarted_branch_changed", + Self::Revision => "refused_unstarted_revision_ahead", + } + } +} + impl State { // Called only inside the verified lease mutation, before this request creates // an operation. A stale SDK source is not permission to ignore existing debt. @@ -208,31 +274,65 @@ impl State { expected: &pa_core::session::canonical::CanonicalCursor, detail: &str, ) -> CanonicalStoreError { - if self.certifies_unstarted_source(expected) { - CanonicalStoreError { + match self.certify_unstarted(expected) { + Ok(()) => CanonicalStoreError { kind: CanonicalFailureKind::SourceChanged, requested: CanonicalRequestedWrite::NotRequested, source: CanonicalSourceDisposition::ReadyForObservation, detail: detail.to_owned(), - } - } else { - unresolved(detail) + }, + Err(refusal) => unresolved_with(detail, Some(refusal.label())), + } + } + + /// Certify that `expected` is a previous source of this same owned journal and + /// that this request wrote nothing, so Core can re-observe and retry. + /// + /// Only the refusal's own debt matters: an unacknowledged write + /// (`available`), or a prepared mutation whose adoption Core never + /// committed. Unrelated work that is still settling — an admitted tool + /// awaiting its terminal, a live run, an unfinished caller — does not make + /// this unwritten refusal uncertain, because it is reached before any + /// operation, barrier or write exists. + fn certify_unstarted( + &self, + expected: &CanonicalCursor, + ) -> std::result::Result<(), UnstartedRefusal> { + self.available() + .map_err(|_| UnstartedRefusal::Unavailable)?; + if !self.prepared.is_empty() { + return Err(UnstartedRefusal::PreparedWork); } + let raw = &self.raw.cursor; + if expected.owner != raw.owner { + return Err(UnstartedRefusal::OwnerScope); + } + if expected.session_id != raw.session_id { + return Err(UnstartedRefusal::Session); + } + if expected.source_path != raw.source_path { + return Err(UnstartedRefusal::Path); + } + if expected.header_certificate != raw.header_certificate { + return Err(UnstartedRefusal::Header); + } + if expected.branch_revision != raw.branch_revision { + return Err(UnstartedRefusal::Branch); + } + if expected.canonical_revision > raw.canonical_revision { + return Err(UnstartedRefusal::Revision); + } + Ok(()) } + /// The strict run-opening form: no adoption, mutation, tool or caller debt may + /// remain. An `AgentStart` may only open on such a fully settled source. fn certifies_unstarted_source(&self, expected: &CanonicalCursor) -> bool { - self.available().is_ok() - && self.prepared.is_empty() + self.certify_unstarted(expected).is_ok() && self.tools.is_empty() && self.callers.iter().all(|signal| { signal.state() == pa_core::session::canonical::CanonicalCallerState::Completed }) - && expected.owner == self.raw.cursor.owner - && expected.session_id == self.raw.cursor.session_id - && expected.source_path == self.raw.cursor.source_path - && expected.header_certificate == self.raw.cursor.header_certificate - && expected.branch_revision == self.raw.cursor.branch_revision - && expected.canonical_revision <= self.raw.cursor.canonical_revision } fn terminal_metadata_suffix( @@ -2139,6 +2239,35 @@ impl SessionFile { }; lease.with_owned_mutation(&self.path, || Ok(())).is_err() } + + /// Retire the actual owner at a certified boundary after a quarantined SDK + /// view blocked every later projection. + /// + /// The source must still equal the owner's last certified cursor and no + /// receipt may remain: no prepared mutation, admitted tool, unfinished + /// caller, live run, or child usage I/O. The next build then attaches a + /// successor epoch that re-reads the same durable rows; history is never + /// rewritten and no unknown physical owner is reused. + pub(crate) fn retire_quarantined_canonical_owner(&mut self) -> Result<()> { + let owner = self + .canonical_live + .clone() + .ok_or_else(|| failure("canonical owner not attached"))?; + { + let state = owner + .0 + .lock() + .map_err(|_| unresolved("canonical owner mutex poisoned"))?; + state.available()?; + if state.callers.iter().any(|signal| { + signal.state() != pa_core::session::canonical::CanonicalCallerState::Completed + }) { + return Err(unresolved("canonical caller has not completed")); + } + } + let cursor = self.canonical_snapshot()?.checkpoint.cursor; + self.retire_completed_canonical_runtime(&cursor) + } } impl SessionFile { diff --git a/crates/pa-daemon/src/session_store/canonical_live_tests.rs b/crates/pa-daemon/src/session_store/canonical_live_tests.rs index eeb290d38..952c4ff8e 100644 --- a/crates/pa-daemon/src/session_store/canonical_live_tests.rs +++ b/crates/pa-daemon/src/session_store/canonical_live_tests.rs @@ -1121,3 +1121,28 @@ fn completed_owner_retirement_refuses_unacknowledged_mutation() { assert!(store.retire_completed_canonical_runtime(&cursor).is_err()); assert!(store.attach_canonical_owner().is_err()); } + +#[test] +fn accepted_delivery_refuses_external_rewrite_without_overwriting_history() { + let (_dir, mut store) = fixture(); + store + .persist_entry("custom", json!({"customType":"before", "data":"original"})) + .unwrap(); + store.attach_canonical_owner().unwrap(); + let original = std::fs::read_to_string(&store.path).unwrap(); + let rewritten = original.replace("original", "modified"); + assert_ne!(original, rewritten); + std::fs::write(&store.path, &rewritten).unwrap(); + let error = store + .persist_delivered_user(json!({"role":"user", "content":"must not append", "timestamp":1})) + .unwrap_err(); + assert_eq!( + error.kind, + pa_core::session::canonical::CanonicalFailureKind::AdoptionConflict + ); + assert_eq!( + error.detail, + "accepted delivery source changed outside owner" + ); + assert_eq!(std::fs::read_to_string(&store.path).unwrap(), rewritten); +} diff --git a/crates/pa-daemon/src/session_store/canonical_live_tests/runtime.rs b/crates/pa-daemon/src/session_store/canonical_live_tests/runtime.rs index c8538a92b..c7333262e 100644 --- a/crates/pa-daemon/src/session_store/canonical_live_tests/runtime.rs +++ b/crates/pa-daemon/src/session_store/canonical_live_tests/runtime.rs @@ -37,6 +37,7 @@ struct Faults { run_start: RunStartFault, tool_attribution: ToolAttribution, advance_at_provider: bool, + advance_at_custom: bool, lose_append_reply: bool, lose_applied_ack: bool, abandon_reply: AbandonReply, @@ -223,7 +224,13 @@ impl CanonicalLiveOwner for RealOwner { &self, request: CanonicalLiveCustomRequest, ) -> CanonicalFuture { - let result = self.store.lock().unwrap().admit_canonical_custom(request); + let mut store = self.store.lock().unwrap(); + if std::mem::take(&mut self.faults.lock().unwrap().advance_at_custom) { + store + .persist_owner_metadata("custom", json!({"customType":"child_finished", "data":{}})) + .unwrap(); + } + let result = store.admit_canonical_custom(request); Box::pin(async move { result }) } fn begin_provider_step( @@ -986,3 +993,113 @@ async fn goal_source_advance_before_first_write_keeps_real_owner_driver_retryabl .unwrap(); assert_eq!(provider.calls().len(), 1); } + +/// The production incident shape: child-usage attribution or a staged delivery +/// appends a legitimate owner row between Core's observation and the owner's +/// Custom admission check, while a tool execution is still outstanding. The +/// requested Custom row was never written, so the refusal must stay +/// recoverable instead of quarantining the live view. +#[tokio::test] +async fn custom_admission_behind_owner_append_during_tool_execution_stays_recoverable() { + struct CustomDuringTool { + live: Arc, + outcome: Arc>>>, + executions: Arc, + } + impl pa_agent::types::AgentTool for CustomDuringTool { + fn name(&self) -> &'static str { + "send_custom" + } + fn description(&self) -> &'static str { + "admit one Custom message while this tool runs" + } + fn parameters(&self) -> &serde_json::Value { + static PARAMETERS: std::sync::OnceLock = std::sync::OnceLock::new(); + PARAMETERS.get_or_init(|| json!({"type":"object"})) + } + fn execute( + self: Arc, + _tool_call_id: String, + _params: serde_json::Value, + _signal: pa_agent::abort::AbortSignal, + _on_update: pa_agent::types::AgentToolUpdateCallback, + ) -> pa_agent::BoxFut<'static, anyhow::Result> { + self.executions + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Box::pin(async move { + let message: pa_types::session::AgentMessage = serde_json::from_value(json!({ + "role":"custom", "customType":"agent_message", + "content":[{"type":"text","text":"queued while the tool runs"}], + "display":false, "timestamp":9 + }))?; + let pa_types::session::AgentMessage::Custom(message) = message else { + unreachable!("fixture must contain a tagged Custom") + }; + let result = self + .live + .persist_message(pa_types::session::AgentMessage::Custom(message)) + .await; + *self.outcome.lock().unwrap() = Some(result); + Ok(pa_agent::types::AgentToolResult::text("custom attempted")) + }) + } + } + + let (_dir, owner, engine, provider) = runtime().await; + // Consume both scripted text turns seeded by the runtime fixture so the next + // scripted turn is the actual tool call. + engine + .session + .prompt("warmup one", PromptOptions::default()) + .await + .unwrap(); + engine + .session + .prompt("warmup two", PromptOptions::default()) + .await + .unwrap(); + let live = engine.session.canonical_live_session().unwrap(); + let outcome = Arc::new(Mutex::new(None)); + let executions = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + engine + .session + .agent() + .set_tools(vec![Arc::new(CustomDuringTool { + live: live.clone(), + outcome: outcome.clone(), + executions: executions.clone(), + })]) + .await; + provider.push_tool_call_turn(None, vec![("custom-tool", "send_custom", json!({}))]); + provider.push_text_turn("tool finished"); + owner.faults.lock().unwrap().advance_at_custom = true; + engine + .session + .prompt("call tool", PromptOptions::default()) + .await + .unwrap(); + assert_eq!(executions.load(std::sync::atomic::Ordering::SeqCst), 1); + let error = outcome + .lock() + .unwrap() + .take() + .expect("the tool must attempt one Custom admission") + .expect_err("the owner must refuse the raced Custom admission"); + assert_eq!(error.kind, CanonicalFailureKind::SourceChanged); + assert_eq!(error.requested, CanonicalRequestedWrite::NotRequested); + assert_eq!( + error.source, + CanonicalSourceDisposition::ReadyForObservation + ); + assert_eq!( + error.detail, + "canonical Custom admission source CAS conflict" + ); + live.ensure_ready().unwrap(); + provider.push_text_turn("next prompt succeeds"); + engine + .session + .prompt("next", PromptOptions::default()) + .await + .unwrap(); +} diff --git a/crates/pa-daemon/src/worker/context_selection.rs b/crates/pa-daemon/src/worker/context_selection.rs index 043ef823f..93038937c 100644 --- a/crates/pa-daemon/src/worker/context_selection.rs +++ b/crates/pa-daemon/src/worker/context_selection.rs @@ -141,6 +141,19 @@ impl InitialContextSelectionStore for CoreInitialContextSelectionStore { })) } + fn retire_quarantined_owner(&self) -> anyhow::Result<()> { + let mut core = self + .0 + .lock() + .map_err(|_| anyhow::anyhow!("session core lock poisoned"))?; + let store = core + .store + .as_mut() + .ok_or_else(|| anyhow::anyhow!("canonical store is unavailable"))?; + store.retire_quarantined_canonical_owner()?; + Ok(()) + } + fn capture(&self, expected_file: &std::path::Path) -> Result { let core = self.0.lock().expect("session core lock"); let lease = current_lease(&core)?;