"""A python tree that fails at startup never leaves a session without a kernel. In a checkout the daemon boots new kernels straight from the working tree, so a half-saved edit there used to break every fresh kernel in every session, with only "python exited at startup" to go on. Each tree a fresh kernel starts cleanly from is saved under the home; a fresh boot that fails starts from that copy instead, and the model is told why, with the traceback's last lines. Once the tree is fixed, the next fresh kernel runs it again. Saved trees beyond the newest few are pruned, but never one a live kernel still runs from: a kernel that fell back keeps importing from its saved copy however many trees boot after it. """ import json import os from pathlib import Path import time import unittest import harness from harness import Albedo, Provider, exclusive, python, text from kernel_support import ModelCells, editable_daemon BROKEN = '\nraise TypeError("broken on purpose")\n' class Fallback(ModelCells, unittest.TestCase): bundle: Path def setUp(self): self.cells = [] def script(request): if request["input"][-1].get("role") == "user" and self.cells: return python(self.cells.pop(0)) return text("ok") def prepare(app): launcher, self.bundle = editable_daemon(app.root) app.env["ALBEDO_DAEMON"] = str(launcher) self.provider = Provider(script) self.addCleanup(self.provider.close) self.app = Albedo(self.provider, protocol="responses", prepare=prepare) self.app.__enter__() self.addCleanup(self.app.__exit__, None, None, None) def fresh(self, code): """A cell in a new session, so its kernel is a fresh boot.""" self.session = self.app.session() return self.cell(code) @exclusive def test_a_tree_that_fails_at_startup_boots_the_last_one_that_started(self): api = self.bundle / "albedo_api.py" good = self.app.home / "kernel-bundles" / "good" first, prompt = self.fresh("import albedo_api\nhasattr(albedo_api, 'BROKEN')") self.assertEqual(first.get("status"), "ok", first) self.assertNotIn("started cleanly", prompt) harness.wait_until(good.exists, 10, "the tree that booted was never saved") original = api.read_text() api.write_text(original + BROKEN) fallen, prompt = self.fresh("import albedo_api\nalbedo_api.__file__") self.assertEqual(fallen.get("status"), "ok", fallen) # It runs the saved copy, not the broken tree. self.assertIn("kernel-bundles", fallen["value"]) self.assertIn("runs the last albedo python code that started cleanly", prompt) self.assertIn("TypeError: broken on purpose", prompt) with self.app.api(f"/sessions/{self.session}?tail=0") as response: self.assertFalse(json.load(response)["kernel"]["stale"]) api.write_text(original) fixed, prompt = self.fresh("import albedo_api\nalbedo_api.__file__") self.assertEqual(fixed.get("status"), "ok", fixed) self.assertNotIn("kernel-bundles", fixed["value"]) self.assertNotIn("started cleanly", prompt) def saved(self): """The saved tree `good` names now.""" return (self.app.home / "kernel-bundles" / "good").read_text().strip() def boot_saved(self, before): """A fresh kernel on the tree as it reads now, once it is saved.""" booted, _ = self.fresh("import albedo_api\nalbedo_api.__file__") self.assertEqual(booted.get("status"), "ok", booted) harness.wait_until( lambda: self.saved() != before, 10, "the tree that booted was never saved" ) return self.saved() @exclusive def test_pruning_spares_the_tree_a_fallen_back_kernel_runs_from(self): api = self.bundle / "albedo_api.py" bundles = self.app.home / "kernel-bundles" first, _ = self.fresh("import albedo_api\nalbedo_api.__file__") self.assertEqual(first.get("status"), "ok", first) harness.wait_until( (bundles / "good").exists, 10, "the tree that booted was never saved" ) held = self.saved() # Oldest by far, so pruning reaches it first. old = time.time() - 3600 os.utime(bundles / held, (old, old)) original = api.read_text() api.write_text(original + BROKEN) # The plugin modules this kernel has not imported yet, which it # imports from its tree only when something first needs them. fallen, _ = self.fresh( "import albedo_api, albedo_plugins, pkgutil, sys\n" "lazy = sorted(f'albedo_plugins.{m.name}' " "for m in pkgutil.iter_modules(albedo_plugins.__path__) " "if f'albedo_plugins.{m.name}' not in sys.modules)\n" "(albedo_api.__file__, len(lazy))" ) self.assertEqual(fallen.get("status"), "ok", fallen) self.assertIn(held, fallen["value"]) self.assertNotRegex(fallen["value"], r", 0\)$", fallen) fallen_session = self.session # More trees boot than pruning keeps besides the newest. newest = held for variant in range(5): api.write_text(original + f"\nVARIANT = {variant}\n") newest = self.boot_saved(newest) self.session = fallen_session late, _ = self.cell( "import importlib, os\n" "(os.path.isfile(albedo_api.__file__), " "importlib.import_module(lazy[0]).__name__ == lazy[0])" ) self.assertEqual(late.get("status"), "ok", late) self.assertEqual(late["value"], "(True, True)", late) # Once no kernel runs from it, the next save prunes it. with self.app.api(f"/sessions/{self.session}/kernel/restart", {}) as response: response.read() restarted, _ = self.fresh("1") self.assertEqual(restarted.get("status"), "ok", restarted) harness.wait_until( lambda: not (bundles / held).exists(), 10, "a tree no kernel runs from was never pruned", ) if __name__ == "__main__": unittest.main()