From 861ffe7734f958a8253652bd53a5ec318d2aa2cf Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Sat, 17 Oct 2026 18:11:50 +0300 Subject: [PATCH] remote: a lost connection wakes for its jobs, and a login shell need not be POSIX a connection that died cleared its live jobs but told nobody: a session idle on a remote job never woke, poll() kept answering None, and the daemon listed the job as running forever. the jobs no reference here claimed now wake the session once with their own notice, and a dead reference raises RemoteLost from poll(). in_login_shell quoted POSIX redirections into $SHELL, which nu and fish cannot parse: fd 1/2 were still on /dev/null when the login shell started, so every command exited 1 with no output. the script travels through a temporary file now and the login shell is only asked for one quote-free command, `/bin/sh `; it unlinks itself and the shell execs, so no process is left whose command line holds it. Signed-off-by: dawn <90008@klbr.net> --- priv/python/albedo_plugins/remote.py | 51 +++++++++++++++++++-- priv/python/albedo_ssh.py | 22 +++++++--- robot-docs/kernel.md | 27 +++++++++--- test/harness/remote_plugin_test.py | 25 +++++++++++ test/harness/ssh_login_shell_test.py | 66 ++++++++++++++++++++++++++++ 5 files changed, 176 insertions(+), 15 deletions(-) create mode 100644 test/harness/ssh_login_shell_test.py diff --git a/priv/python/albedo_plugins/remote.py b/priv/python/albedo_plugins/remote.py index f5ceeed7..61259008 100644 --- a/priv/python/albedo_plugins/remote.py +++ b/priv/python/albedo_plugins/remote.py @@ -37,7 +37,7 @@ from albedo_values import InvalidValue, decode import albedo_shell import albedo_ssh from albedo_plugins import hook -from albedo_plugins.run import notice +from albedo_plugins.run import NOTICE_RETRY, notice CONNECT_TIMEOUT = 60 # seconds before an unreachable target gives up HANDSHAKE_TIMEOUT = 60 # seconds for the remote kernel to say ready @@ -341,8 +341,13 @@ class RemoteRef: return result def poll(self) -> int | None: - """The remote job's exit status so far, from the mirrored stream.""" + """The remote job's exit status so far, from the mirrored stream. A job + whose connection died says so instead of running forever.""" state = self._mirror() + if state is None or state.get("exit_code") is None: + lost = self._connection.lost + if lost: + raise RemoteLost(lost) return state.get("exit_code") if state else None @property @@ -589,6 +594,9 @@ class RemoteConnection: self._reader: asyncio.Task[None] | None = None self._drain: asyncio.Task[None] | None = None self.closed = False + # Why the channel is gone, once it is: a dead connection's references + # say so rather than reporting a job that never ended. + self.lost: str | None = None self.degraded: RemoteBootError | None = None # --- boot --- @@ -753,7 +761,9 @@ class RemoteConnection: await self._stdin.drain() def _lost(self) -> None: - """The channel died: every pending call and reference learns it once.""" + """The channel died: every pending call and reference learns it once, + and the jobs nobody here claimed wake the session instead of running + forever in a supervisor that no longer has a connection.""" if self._handshake is not None and not self._handshake.done(): self._handshake.set_exception( RemoteBootError( @@ -767,9 +777,43 @@ class RemoteConnection: RemoteLost(f"the connection to {self.host} was lost") ) self.closed = True + self.lost = self.lost or f"the connection to {self.host} was lost" + orphaned = [job for id, job in self._live.items() if not self._claimed_here(id)] if self._live: self._live.clear() _report_live() + if orphaned: + try: + loop.create_task(self._announce_lost(orphaned)) + except RuntimeError: + pass # the loop is closing; no turn is left to wake + + async def _announce_lost(self, jobs: list[LiveJob]) -> None: + """Wake the session for the jobs a dead connection took with it; their + results are no longer readable, so this notice is all there is.""" + facts = [ + { + "id": job["id"], + "command": job["command"], + "where": f" on {self.host}", + "summary": "connection lost, exit status unknown", + "exit_code": None, + "timed_out": False, + "duration": None, + "host": self.host, + } + for job in jobs + ] + while True: + try: + await self._wake({"jobs": facts}) + return + except Exception as error: + # A busy session refuses rather than queueing, as it does for a + # local job's wake, so this retries until the run ends. + if getattr(error, "code", "") != "busy": + return # the daemon is gone, or the session is shutting down + await asyncio.sleep(NOTICE_RETRY) # --- calls --- @@ -1135,6 +1179,7 @@ class RemoteConnection: except asyncio.TimeoutError: self._process.kill() self.closed = True + self.lost = self.lost or f"the connection to {self.host} was closed" self._live.clear() _report_live() for future in self._pending.values(): diff --git a/priv/python/albedo_ssh.py b/priv/python/albedo_ssh.py index f966c40f..99805adf 100644 --- a/priv/python/albedo_ssh.py +++ b/priv/python/albedo_ssh.py @@ -171,13 +171,25 @@ def in_login_shell(script: str) -> str: user's tools. Profiles often print (a motd, a greeting), which would corrupt a framed stdout and fill the daemon's log on every command, so the login shell starts with both on /dev/null and the real ones kept on - fds 3 and 4, and `script` gets them back. The outer `/bin/sh` makes the - redirections work whatever the user's shell is. + fds 3 and 4, and `script` gets them back. + + `$SHELL` is not always a POSIX shell (`nu`, `fish` and `tcsh` are not), + so it never sees the script: `/bin/sh` writes it to a temporary file and + the login shell is asked for one quote-free command, `/bin/sh `. + Both the redirections and the script stay POSIX, and stdin is never + touched, which the bridge's frames travel on. """ - login = f"exec 1>&3 2>&4 3>&- 4>&-; {script}" + # The script deletes its own file before running: an unlinked script keeps + # reading from its open handle, and the exec below leaves no process behind + # whose command line still holds it. + login = f'rm -f "$0"; exec 1>&3 2>&4 3>&- 4>&-; {script}' outer = ( - "exec 3>&1 4>&2 1>/dev/null 2>&1; " - f'exec "${{SHELL:-/bin/sh}}" -l -c {shlex.quote(login)}' + "f=$(mktemp /tmp/albedo-remote-XXXXXX 2>/dev/null) || exit 1\n" + "cat > \"$f\" <<'ALBEDO-REMOTE' || exit 1\n" + f"{login}\n" + "ALBEDO-REMOTE\n" + "exec 3>&1 4>&2 1>/dev/null 2>&1\n" + 'exec "${SHELL:-/bin/sh}" -l -c "/bin/sh $f"\n' ) return f"/bin/sh -c {shlex.quote(outer)}" diff --git a/robot-docs/kernel.md b/robot-docs/kernel.md index 3d297622..345f3184 100644 --- a/robot-docs/kernel.md +++ b/robot-docs/kernel.md @@ -512,12 +512,19 @@ with backoff and `resume_kernels` treat an ssh drop as a bridge that exited. own ssh, colmena and git in `run` jobs: one sign-in (one hardware-key touch) per host. with none configured it is `/tmp/albedo-ssh-cm/%C` for 600 s. -- **clean stdout**: every remote command is `/bin/sh -c 'exec 3>&1 1>&2; - exec "$SHELL" -l -c "exec 1>&3 3>&-; "'`, quoted with shlex in - `albedo_ssh.in_login_shell` only. whatever the login profile prints (a - motd, a greeting) goes to stderr, and the command gets the real stdout - back, so the bridge's frames, the remote plugin's kernel and the probe's - lines stay clean whatever the user's shell is. +- **clean stdout**: every remote command is `/bin/sh -c ''`, quoted + with shlex in `albedo_ssh.in_login_shell` only. `` writes the + command to a temporary file, keeps the real stdout and stderr on fds 3 and + 4, sends the login shell's own output to /dev/null, and asks `$SHELL -l -c + "/bin/sh "`: whatever the login profile prints (a motd, a greeting) + is dropped, the command gets the real fds back, and the bridge's frames, + the remote plugin's kernel and the probe's lines stay clean. the shell is + only ever asked for that one quote-free command, so a login shell that is + not POSIX (`nu`, `fish`, `tcsh`) still runs it; the redirections and the + command itself stay POSIX, and stdin is never redirected (the bridge's + frames travel on it). the script unlinks itself before it runs and the + shell `exec`s the login shell, so no process is left whose command line + holds it (a test finds a host's bridge by its command line). - **fixed command lines**: the probe (and, with no network, `albedo_ssh.py commands `) answers the complete remote commands the daemon runs: `bridge` (`albedo_bridge.py --frame`), `signal` @@ -583,7 +590,13 @@ with backoff and `resume_kernels` treat an ssh drop as a bridge that exited. call still in flight (resolved in order), `{"__bytes__": b64}` for byte data, decoded remotely by `albedo_values`, so `rem.run(..., stdin=b"...")` and byte-sized results work. `rem.write` takes text or bytes; either travels - on the stdin of one ssh command, never in the command line. + on the stdin of one ssh command, never in the command line. a connection + that dies stops the jobs it was mirroring as far as this side can tell: its + live jobs leave the `/jobs` list, and every one no reference here awaited or + read wakes the session once, naming the job and "connection lost, exit status + unknown" (a busy session is retried, as for a local wake). a reference on a + dead connection raises `RemoteLost` from `poll()`/`exit_code` instead of + reporting `None`, which reads as a job that never ends. - **the model** gets a context line (python extension, `place.gleam`): "your python kernel and run jobs execute on chernobog (Linux aarch64)…", with the remote home. diff --git a/test/harness/remote_plugin_test.py b/test/harness/remote_plugin_test.py index b465b39f..1a570cc0 100644 --- a/test/harness/remote_plugin_test.py +++ b/test/harness/remote_plugin_test.py @@ -550,6 +550,31 @@ class RemotePluginTest(unittest.IsolatedAsyncioTestCase): self.assertEqual([e["live"] for e in job_events], [1, 0]) self.assertEqual(job_events[-1]["items"], []) self.assertIsNone(await remote.stop_remote_job("ghost-job")) + for _ in range(100): + if notices: + break + await asyncio.sleep(0.01) + self.assertEqual( + len(notices), 1, "a job lost with its connection must wake the session" + ) + [fact] = notices[0]["jobs"] + self.assertEqual(fact["id"], "ghost-job") + self.assertEqual(fact["summary"], "connection lost, exit status unknown") + self.assertEqual(fact["host"], "loopback") + self.assertIn("ghost-job", notices[0]["text"]) + + async def test_a_reference_on_a_lost_connection_says_so_instead_of_running(self): + connection, stream, call = self.reader_connection() + reference = remote.RemoteRef(connection, handle="h") + connection._mirrors["h"] = { + "handle": "h", + "job": "ghost-job", + "exit_code": None, + } + stream.feed_eof() + await self.assert_reader_lost(connection, call) + with self.assertRaises(remote.RemoteLost): + reference.poll() async def test_stop_remote_job_fallback_job_stops_locally(self): rem = await self.connect(modules=["definitely-not-a-module"]) diff --git a/test/harness/ssh_login_shell_test.py b/test/harness/ssh_login_shell_test.py new file mode 100644 index 00000000..174578ca --- /dev/null +++ b/test/harness/ssh_login_shell_test.py @@ -0,0 +1,66 @@ +"""A remote login shell that is not POSIX (`nu`, `fish`, `tcsh`) must still run +the command albedo sends it. Real hosts with those shells are not in the test +environment, so every remote command would fail there as it did on +`regent@astaroth`: `-c` quoted POSIX redirections into `$SHELL`, and nu cannot +parse `exec 1>&3 2>&4`, so a login shell exits 1 with no output at all.""" + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "priv" / "python")) +import albedo_ssh # noqa: E402 + +SCRIPT = "cat; echo to-stdout; echo to-stderr >&2; exit 3" + + +def stand_in_shell(directory: Path, *, motd: str = "") -> str: + """A shell that takes -l -c like nu and refuses POSIX redirection in it.""" + path = directory / "nu-stand-in" + lines = ["#!/bin/sh", "shift", "shift"] + if motd: + lines.append(f'echo "{motd}"') + lines.append('case "$1" in *">"*) echo "nu: refused: $1" >&2; exit 91 ;; esac') + lines.append('exec /bin/sh -c "$1"') + path.write_text("\n".join(lines) + "\n") + path.chmod(0o755) + return str(path) + + +class LoginShellTest(unittest.TestCase): + def run_command(self, shell, command): + return subprocess.run( + ["/bin/sh", "-c", command], + input="from-stdin\n", + text=True, + capture_output=True, + env={**os.environ, "SHELL": shell}, + timeout=30, + ) + + def test_a_login_shell_that_is_not_posix_still_runs_the_command(self): + with tempfile.TemporaryDirectory() as scratch: + shell = stand_in_shell(Path(scratch), motd="Welcome to astaroth") + done = self.run_command(shell, albedo_ssh.in_login_shell(SCRIPT)) + self.assertEqual(done.returncode, 3, done.stderr) + self.assertEqual(done.stdout, "from-stdin\nto-stdout\n") + self.assertEqual(done.stderr, "to-stderr\n") + + def test_a_posix_login_shell_still_passes_stdin_through(self): + done = self.run_command("/bin/sh", albedo_ssh.in_login_shell(SCRIPT)) + self.assertEqual(done.returncode, 3) + self.assertEqual(done.stdout, "from-stdin\nto-stdout\n") + self.assertEqual(done.stderr, "to-stderr\n") + + def test_the_script_leaves_no_temporary_file(self): + command = albedo_ssh.in_login_shell("true") + before = set(Path("/tmp").glob("albedo-remote-*")) + self.run_command("/bin/sh", command) + self.assertEqual(set(Path("/tmp").glob("albedo-remote-*")), before) + + +if __name__ == "__main__": + unittest.main() -- knotmirror 0.0.1