//// Incomplete native verdicts must retain cleanup ownership. E2E cannot make //// the signal helper return a partial response or duplicate a target identity. import albedo/clock import albedo/harness/kernel/driver import albedo/harness/kernel/launch import albedo/harness/kernel/link import albedo/harness/kernel/supervision import albedo/harness/kernel/termination import gleam/dict import gleam/dynamic import gleam/dynamic/decode import gleam/erlang/process import gleam/int import gleam/json import gleam/option.{None, Some} import gleam/string pub fn missing_group_confirmation_retains_cleanup_test() -> Nil { let assert termination.UnconfirmedCleanup(_) = termination.assess(["kernel", "job late"], [#("kernel", True, "")]) let assert termination.UnconfirmedCleanup(_) = termination.assess(["kernel"], []) Nil } pub fn duplicate_confirmation_is_not_an_acknowledgment_test() -> Nil { let assert termination.UnconfirmedCleanup(_) = termination.assess(["kernel"], [ #("kernel", True, ""), #("kernel", True, ""), ]) Nil } pub fn repeated_stop_requires_fresh_complete_confirmation_test() -> Nil { let targets = ["kernel", "job late"] let assert termination.UnconfirmedCleanup("job survived KILL") = termination.assess(targets, [ #("kernel", True, ""), #("job late", False, "job survived KILL"), ]) let assert termination.ConfirmedTermination = termination.assess(targets, [ #("kernel", True, ""), #("job late", True, ""), ]) Nil } /// A job transfer ordered after shutdown cannot be scheduled reliably in /// E2E. This drives the real driver loop behind a stand-in relay, and the /// real helper checks an absent process group. pub fn late_job_is_owned_across_failed_cleanup_and_repeated_stop_test() -> Nil { let listener = process.new_subject() let run_dir = "/tmp/albedo-shutdown-" <> int.to_string(clock.system_ms()) <> "-" <> int.to_string(int.random(1_000_000_000)) let _ = make_dir(run_dir) let attempts = atomics_new(1, []) let forget = fn() { case atomics_add_get(attempts, 1, 1) { 1 -> Error("injected forget failure") _ -> Ok(Nil) } } let own = fn(groups) { process.send(listener, groups) } let ledger = link.Link( persist: fn(_, _, _, _) { Ok(Nil) }, ack: fn(_) { Nil }, pending: fn() { [] }, call: fn(_) { link.Fresh }, reply: fn(_, _, _, _) { Ok(Nil) }, record: fn(_) { Nil }, forget:, own:, orphan: fn() { Ok(Nil) }, lose: fn(_) { Nil }, ) let ready = process.new_subject() let relay = process.spawn_unlinked(fn() { stand_in_relay(ready) }) let assert Ok(named) = process.receive(ready, 5000) let owner = process.self() let kernel = process.spawn_unlinked(fn() { driver.serve( driver.State( ..driver.detached(owner, ledger, fn(_, _) { "{}" }, "shutdown-test"), conn: Some(driver.Relay(relay)), run_dir:, ), ) }) process.send(named, kernel) let monitor = process.monitor(kernel) let first = driver.stop(kernel) let owned = case process.receive(listener, 1000) { Ok(groups) -> string.contains(groups, "\"late\"") Error(Nil) -> False } let second = driver.stop(kernel) let ended = process.new_selector() |> process.select_specific_monitor(monitor, fn(down) { down.reason == process.Normal }) |> process.selector_receive(2000) == Ok(True) process.kill(kernel) process.kill(relay) let _ = delete_tree(run_dir) let assert Error(_) = first assert owned assert second == Ok(Nil) assert ended } /// Answers the driver's shutdown frame with a job that started late, then /// hangs up, as a kernel's relay would. fn stand_in_relay(ready: process.Subject(process.Subject(process.Pid))) -> Nil { let named = process.new_subject() process.send(ready, named) let assert Ok(kernel) = process.receive(named, 5000) let to_driver = process.unsafely_create_subject(kernel, pid_dynamic(process.self())) case process.receive(launch.commands(process.self(), kernel), 5000) { Ok(launch.Command(_)) -> { let frame = "{\"seq\":1,\"frame\":{\"type\":\"job_start\",\"id\":\"late\",\"pid\":2147483647,\"pgid\":2147483647}}" process.send(to_driver, launch.Data(<>)) process.send(to_driver, launch.ExitStatus(0)) let _ = process.receive(launch.commands(process.self(), kernel), 5000) Nil } _ -> Nil } } /// A kernel found lost whose cleanup was not confirmed keeps owning the /// groups it could not prove gone. pub fn unconfirmed_cleanup_retains_job_groups_test() -> Nil { let group = supervision.Proc( pid: Some(2_147_483_647), pgid: Some(2_147_483_647), leader: None, command: "", service: False, cell: None, started: None, ) let ledger = link.Link( persist: fn(_, _, _, _) { Ok(Nil) }, ack: fn(_) { Nil }, pending: fn() { [] }, call: fn(_) { link.Fresh }, reply: fn(_, _, _, _) { Ok(Nil) }, record: fn(_) { Nil }, forget: fn() { Ok(Nil) }, own: fn(_) { panic as "ownership was replaced" }, orphan: fn() { Ok(Nil) }, lose: fn(_) { Nil }, ) let state = driver.State( ..driver.detached(process.self(), ledger, fn(_, _) { "{}" }, "k"), lost: True, groups: dict.from_list([#("missing", group)]), run_dir: "/tmp/missing-cleanup-listener", ) let kept = driver.record_lost(state, supervision.Unconfirmed("missing", [])) assert supervision.job_specs(kept.groups) == [ supervision.Target( "job missing", Some(2_147_483_647), Some(2_147_483_647), None, ), ] let assert termination.UnconfirmedCleanup(_) = termination.judge(["kernel"], [ verdict("{\"label\":\"kernel\",\"gone\":false,\"signals\":17}"), ]) Nil } type Atomics @external(erlang, "atomics", "new") fn atomics_new(size: Int, options: List(Nil)) -> Atomics @external(erlang, "atomics", "add_get") fn atomics_add_get(atomics: Atomics, index: Int, value: Int) -> Int @external(erlang, "file", "make_dir") fn make_dir(path: String) -> dynamic.Dynamic @external(erlang, "file", "del_dir_r") fn delete_tree(path: String) -> dynamic.Dynamic @external(erlang, "gleam_stdlib", "identity") fn pid_dynamic(pid: process.Pid) -> dynamic.Dynamic @external(erlang, "albedo_python_cleanup_test_support", "cleanup_failures") fn cleanup_failures() -> Bool pub fn incomplete_native_cleanup_retains_jobs_and_spare_retry_authority_test() -> Nil { assert cleanup_failures() } fn verdict(text: String) -> dynamic.Dynamic { let assert Ok(value) = json.parse(text, decode.dynamic) value } pub fn a_surviving_group_is_described_and_a_malformed_verdict_confirms_nothing_test() -> Nil { let assert termination.UnconfirmedCleanup( "kernel: process group 12 survived TERM+KILL (a; b) (zombie)", ) = termination.judge(["kernel"], [ verdict( "{\"label\":\"kernel\",\"gone\":false,\"pgid\":12,\"signals\":[\"TERM\",\"KILL\"],\"failures\":[\"a\",\"b\"],\"note\":\"zombie\"}", ), ]) let assert termination.UnconfirmedCleanup( "termination helper returned an invalid target verdict", ) = termination.judge([], [verdict("{\"label\":\"kernel\",\"gone\":\"no\"}")]) let assert Error("boom") = termination.answer(verdict("{\"ok\":false,\"error\":\"boom\"}")) Nil }