diff --git a/priv/python/albedo_plugins/remote.py b/priv/python/albedo_plugins/remote.py index f5ceeed7..61259008 100644 --- a/priv/python/albedo_plugins/remote.py +++ b/priv/python/albedo_plugins/remote.py @@ -37,7 +37,7 @@ from albedo_values import InvalidValue, decode import albedo_shell import albedo_ssh from albedo_plugins import hook -from albedo_plugins.run import notice +from albedo_plugins.run import NOTICE_RETRY, notice CONNECT_TIMEOUT = 60 # seconds before an unreachable target gives up HANDSHAKE_TIMEOUT = 60 # seconds for the remote kernel to say ready @@ -341,8 +341,13 @@ class RemoteRef: return result def poll(self) -> int | None: - """The remote job's exit status so far, from the mirrored stream.""" + """The remote job's exit status so far, from the mirrored stream. A job + whose connection died says so instead of running forever.""" state = self._mirror() + if state is None or state.get("exit_code") is None: + lost = self._connection.lost + if lost: + raise RemoteLost(lost) return state.get("exit_code") if state else None @property @@ -589,6 +594,9 @@ class RemoteConnection: self._reader: asyncio.Task[None] | None = None self._drain: asyncio.Task[None] | None = None self.closed = False + # Why the channel is gone, once it is: a dead connection's references + # say so rather than reporting a job that never ended. + self.lost: str | None = None self.degraded: RemoteBootError | None = None # --- boot --- @@ -753,7 +761,9 @@ class RemoteConnection: await self._stdin.drain() def _lost(self) -> None: - """The channel died: every pending call and reference learns it once.""" + """The channel died: every pending call and reference learns it once, + and the jobs nobody here claimed wake the session instead of running + forever in a supervisor that no longer has a connection.""" if self._handshake is not None and not self._handshake.done(): self._handshake.set_exception( RemoteBootError( @@ -767,9 +777,43 @@ class RemoteConnection: RemoteLost(f"the connection to {self.host} was lost") ) self.closed = True + self.lost = self.lost or f"the connection to {self.host} was lost" + orphaned = [job for id, job in self._live.items() if not self._claimed_here(id)] if self._live: self._live.clear() _report_live() + if orphaned: + try: + loop.create_task(self._announce_lost(orphaned)) + except RuntimeError: + pass # the loop is closing; no turn is left to wake + + async def _announce_lost(self, jobs: list[LiveJob]) -> None: + """Wake the session for the jobs a dead connection took with it; their + results are no longer readable, so this notice is all there is.""" + facts = [ + { + "id": job["id"], + "command": job["command"], + "where": f" on {self.host}", + "summary": "connection lost, exit status unknown", + "exit_code": None, + "timed_out": False, + "duration": None, + "host": self.host, + } + for job in jobs + ] + while True: + try: + await self._wake({"jobs": facts}) + return + except Exception as error: + # A busy session refuses rather than queueing, as it does for a + # local job's wake, so this retries until the run ends. + if getattr(error, "code", "") != "busy": + return # the daemon is gone, or the session is shutting down + await asyncio.sleep(NOTICE_RETRY) # --- calls --- @@ -1135,6 +1179,7 @@ class RemoteConnection: except asyncio.TimeoutError: self._process.kill() self.closed = True + self.lost = self.lost or f"the connection to {self.host} was closed" self._live.clear() _report_live() for future in self._pending.values(): diff --git a/priv/python/albedo_ssh.py b/priv/python/albedo_ssh.py index f966c40f..99805adf 100644 --- a/priv/python/albedo_ssh.py +++ b/priv/python/albedo_ssh.py @@ -171,13 +171,25 @@ def in_login_shell(script: str) -> str: user's tools. Profiles often print (a motd, a greeting), which would corrupt a framed stdout and fill the daemon's log on every command, so the login shell starts with both on /dev/null and the real ones kept on - fds 3 and 4, and `script` gets them back. The outer `/bin/sh` makes the - redirections work whatever the user's shell is. + fds 3 and 4, and `script` gets them back. + + `$SHELL` is not always a POSIX shell (`nu`, `fish` and `tcsh` are not), + so it never sees the script: `/bin/sh` writes it to a temporary file and + the login shell is asked for one quote-free command, `/bin/sh `. + Both the redirections and the script stay POSIX, and stdin is never + touched, which the bridge's frames travel on. """ - login = f"exec 1>&3 2>&4 3>&- 4>&-; {script}" + # The script deletes its own file before running: an unlinked script keeps + # reading from its open handle, and the exec below leaves no process behind + # whose command line still holds it. + login = f'rm -f "$0"; exec 1>&3 2>&4 3>&- 4>&-; {script}' outer = ( - "exec 3>&1 4>&2 1>/dev/null 2>&1; " - f'exec "${{SHELL:-/bin/sh}}" -l -c {shlex.quote(login)}' + "f=$(mktemp /tmp/albedo-remote-XXXXXX 2>/dev/null) || exit 1\n" + "cat > \"$f\" <<'ALBEDO-REMOTE' || exit 1\n" + f"{login}\n" + "ALBEDO-REMOTE\n" + "exec 3>&1 4>&2 1>/dev/null 2>&1\n" + 'exec "${SHELL:-/bin/sh}" -l -c "/bin/sh $f"\n' ) return f"/bin/sh -c {shlex.quote(outer)}" diff --git a/robot-docs/kernel.md b/robot-docs/kernel.md index 3d297622..345f3184 100644 --- a/robot-docs/kernel.md +++ b/robot-docs/kernel.md @@ -512,12 +512,19 @@ with backoff and `resume_kernels` treat an ssh drop as a bridge that exited. own ssh, colmena and git in `run` jobs: one sign-in (one hardware-key touch) per host. with none configured it is `/tmp/albedo-ssh-cm/%C` for 600 s. -- **clean stdout**: every remote command is `/bin/sh -c 'exec 3>&1 1>&2; - exec "$SHELL" -l -c "exec 1>&3 3>&-; "'`, quoted with shlex in - `albedo_ssh.in_login_shell` only. whatever the login profile prints (a - motd, a greeting) goes to stderr, and the command gets the real stdout - back, so the bridge's frames, the remote plugin's kernel and the probe's - lines stay clean whatever the user's shell is. +- **clean stdout**: every remote command is `/bin/sh -c ''`, quoted + with shlex in `albedo_ssh.in_login_shell` only. `` writes the + command to a temporary file, keeps the real stdout and stderr on fds 3 and + 4, sends the login shell's own output to /dev/null, and asks `$SHELL -l -c + "/bin/sh "`: whatever the login profile prints (a motd, a greeting) + is dropped, the command gets the real fds back, and the bridge's frames, + the remote plugin's kernel and the probe's lines stay clean. the shell is + only ever asked for that one quote-free command, so a login shell that is + not POSIX (`nu`, `fish`, `tcsh`) still runs it; the redirections and the + command itself stay POSIX, and stdin is never redirected (the bridge's + frames travel on it). the script unlinks itself before it runs and the + shell `exec`s the login shell, so no process is left whose command line + holds it (a test finds a host's bridge by its command line). - **fixed command lines**: the probe (and, with no network, `albedo_ssh.py commands `) answers the complete remote commands the daemon runs: `bridge` (`albedo_bridge.py --frame`), `signal` @@ -583,7 +590,13 @@ with backoff and `resume_kernels` treat an ssh drop as a bridge that exited. call still in flight (resolved in order), `{"__bytes__": b64}` for byte data, decoded remotely by `albedo_values`, so `rem.run(..., stdin=b"...")` and byte-sized results work. `rem.write` takes text or bytes; either travels - on the stdin of one ssh command, never in the command line. + on the stdin of one ssh command, never in the command line. a connection + that dies stops the jobs it was mirroring as far as this side can tell: its + live jobs leave the `/jobs` list, and every one no reference here awaited or + read wakes the session once, naming the job and "connection lost, exit status + unknown" (a busy session is retried, as for a local wake). a reference on a + dead connection raises `RemoteLost` from `poll()`/`exit_code` instead of + reporting `None`, which reads as a job that never ends. - **the model** gets a context line (python extension, `place.gleam`): "your python kernel and run jobs execute on chernobog (Linux aarch64)…", with the remote home. diff --git a/test/harness/remote_plugin_test.py b/test/harness/remote_plugin_test.py index b465b39f..1a570cc0 100644 --- a/test/harness/remote_plugin_test.py +++ b/test/harness/remote_plugin_test.py @@ -550,6 +550,31 @@ class RemotePluginTest(unittest.IsolatedAsyncioTestCase): self.assertEqual([e["live"] for e in job_events], [1, 0]) self.assertEqual(job_events[-1]["items"], []) self.assertIsNone(await remote.stop_remote_job("ghost-job")) + for _ in range(100): + if notices: + break + await asyncio.sleep(0.01) + self.assertEqual( + len(notices), 1, "a job lost with its connection must wake the session" + ) + [fact] = notices[0]["jobs"] + self.assertEqual(fact["id"], "ghost-job") + self.assertEqual(fact["summary"], "connection lost, exit status unknown") + self.assertEqual(fact["host"], "loopback") + self.assertIn("ghost-job", notices[0]["text"]) + + async def test_a_reference_on_a_lost_connection_says_so_instead_of_running(self): + connection, stream, call = self.reader_connection() + reference = remote.RemoteRef(connection, handle="h") + connection._mirrors["h"] = { + "handle": "h", + "job": "ghost-job", + "exit_code": None, + } + stream.feed_eof() + await self.assert_reader_lost(connection, call) + with self.assertRaises(remote.RemoteLost): + reference.poll() async def test_stop_remote_job_fallback_job_stops_locally(self): rem = await self.connect(modules=["definitely-not-a-module"]) diff --git a/test/harness/ssh_login_shell_test.py b/test/harness/ssh_login_shell_test.py new file mode 100644 index 00000000..174578ca --- /dev/null +++ b/test/harness/ssh_login_shell_test.py @@ -0,0 +1,66 @@ +"""A remote login shell that is not POSIX (`nu`, `fish`, `tcsh`) must still run +the command albedo sends it. Real hosts with those shells are not in the test +environment, so every remote command would fail there as it did on +`regent@astaroth`: `-c` quoted POSIX redirections into `$SHELL`, and nu cannot +parse `exec 1>&3 2>&4`, so a login shell exits 1 with no output at all.""" + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "priv" / "python")) +import albedo_ssh # noqa: E402 + +SCRIPT = "cat; echo to-stdout; echo to-stderr >&2; exit 3" + + +def stand_in_shell(directory: Path, *, motd: str = "") -> str: + """A shell that takes -l -c like nu and refuses POSIX redirection in it.""" + path = directory / "nu-stand-in" + lines = ["#!/bin/sh", "shift", "shift"] + if motd: + lines.append(f'echo "{motd}"') + lines.append('case "$1" in *">"*) echo "nu: refused: $1" >&2; exit 91 ;; esac') + lines.append('exec /bin/sh -c "$1"') + path.write_text("\n".join(lines) + "\n") + path.chmod(0o755) + return str(path) + + +class LoginShellTest(unittest.TestCase): + def run_command(self, shell, command): + return subprocess.run( + ["/bin/sh", "-c", command], + input="from-stdin\n", + text=True, + capture_output=True, + env={**os.environ, "SHELL": shell}, + timeout=30, + ) + + def test_a_login_shell_that_is_not_posix_still_runs_the_command(self): + with tempfile.TemporaryDirectory() as scratch: + shell = stand_in_shell(Path(scratch), motd="Welcome to astaroth") + done = self.run_command(shell, albedo_ssh.in_login_shell(SCRIPT)) + self.assertEqual(done.returncode, 3, done.stderr) + self.assertEqual(done.stdout, "from-stdin\nto-stdout\n") + self.assertEqual(done.stderr, "to-stderr\n") + + def test_a_posix_login_shell_still_passes_stdin_through(self): + done = self.run_command("/bin/sh", albedo_ssh.in_login_shell(SCRIPT)) + self.assertEqual(done.returncode, 3) + self.assertEqual(done.stdout, "from-stdin\nto-stdout\n") + self.assertEqual(done.stderr, "to-stderr\n") + + def test_the_script_leaves_no_temporary_file(self): + command = albedo_ssh.in_login_shell("true") + before = set(Path("/tmp").glob("albedo-remote-*")) + self.run_command("/bin/sh", command) + self.assertEqual(set(Path("/tmp").glob("albedo-remote-*")), before) + + +if __name__ == "__main__": + unittest.main()