"""What the daemon's folder browser and project readers need from a remote host. albedo_gather.py reads one JSON request line on stdin and prints one JSON snapshot. The daemon interprets the snapshot with the same code it uses on its own disk (albedo/daemon/folders.gleam, albedo/harness/vcs.gleam), so this side only collects: directory entries, which repository markers exist, the vcs commands the daemon planned, file sizes, and project files. Everything comes back in one ssh round trip per question. Request: `{route, dir, files, plans, tracked, deadline, counted}`, `dir` absolute and normalised. `route` is list, repo, preview or project, or exists, which answers `directory` alone. A list runs its plans only with `files`. The file route takes `{route, path, limit}` instead and answers `{size, modified, data}` (base64, left out past `limit`) or `{error}`. The run route takes `{route, root, commands, reads, cap, head_limit, head_budget, deadline}`: it runs each command (an argv) at `root` side by side and answers `{outputs, heads}`, an output `{status, stdout, truncated}` (stdout base64, at most `cap` bytes) or `{timed_out}` or `{missing}`, a head `{size, lines, data}` (as daemon/changes.gleam reads it) or null. `tracked` names, per kind, the planned command that lists a preview's tracked files and the separator its output uses. Snapshot: `{directory, entries: {dir: [[name, dir, modified, symlink, size]]}, exists: [path], outputs: {key: stdout}, sizes: {path: bytes}, files: {relative: base64}}`. An output's key is its program and arguments joined by NUL; only commands that exit 0 within the deadline answer. """ from __future__ import annotations import base64 import json import os import stat import subprocess import sys import threading from concurrent.futures import ThreadPoolExecutor MARKERS = (".jj", ".git") SUBDIRECTORIES = 256 # child directories whose entries a preview lists PROJECT_FILES = 512 PROJECT_BYTES = 8 * 1024 * 1024 FILE_BYTES = 1024 * 1024 + 1 # one past the instruction limit, so it still warns def child(directory: str, name: str) -> str: return "/" + name if directory == "/" else f"{directory}/{name}" def shown(name: str) -> bool: """Names the daemon can show and send back: valid UTF-8.""" try: name.encode() return True except UnicodeEncodeError: return False def entries(directory: str) -> list[list[object]]: try: names = os.listdir(directory) except OSError: return [] listed: list[list[object]] = [] for name in filter(shown, names): path = child(directory, name) symlink = os.path.islink(path) try: info = os.stat(path) listed.append( [ name, stat.S_ISDIR(info.st_mode), int(info.st_mtime), symlink, info.st_size, ] ) except OSError: listed.append([name, False, 0, symlink, 0]) return listed def marks(directory: str, exists: list[str]) -> str | None: """Record the markers at `directory`; the kind it roots, .jj first.""" found = None for marker in MARKERS: path = child(directory, marker) if os.path.lexists(path) and (os.path.isfile(path) or os.path.isdir(path)): exists.append(path) found = found or marker return found def parent(directory: str) -> str | None: return None if directory == "/" else os.path.dirname(directory) or "/" def checkout(directory: str, exists: list[str]) -> tuple[str, str] | None: """The nearest enclosing repository: (kind, root).""" current: str | None = directory while current is not None: found = marks(current, exists) if found is not None: return found[1:], current current = parent(current) return None def key(program: str, args: list[str]) -> str: return "\0".join([program, *args]) def run( command: list, root: str, directory: str, deadline: float ) -> tuple[str, str | None]: program, args, at = command try: done = subprocess.run( [program, *args], cwd=root if at == "root" else directory, stdin=subprocess.DEVNULL, capture_output=True, timeout=deadline, ) except (OSError, subprocess.TimeoutExpired): return key(program, args), None output = done.stdout.decode(errors="replace") if done.returncode == 0 else None return key(program, args), output def project(directory: str) -> dict[str, str]: """Project instruction files and skills, relative to `directory`.""" picked: list[str] = [] for relative in ("", ".agents", ".albedo"): base = child(directory, relative) if relative else directory try: names = sorted(filter(shown, os.listdir(base))) except OSError: continue for name in names: if name.lower().endswith(".md") and os.path.isfile(child(base, name)): picked.append(f"{relative}/{name}" if relative else name) for relative in (".agents/skills", ".albedo/skills"): for root, directories, names in os.walk(child(directory, relative)): directories.sort() for name in sorted(filter(shown, names)): path = os.path.join(root, name) if os.path.isfile(path) and len(picked) < PROJECT_FILES: picked.append(os.path.relpath(path, directory)) files: dict[str, str] = {} budget = PROJECT_BYTES for relative in picked: try: with open(child(directory, relative), "rb") as handle: data = handle.read(min(FILE_BYTES, budget)) except OSError: continue budget -= len(data) files[relative] = base64.b64encode(data).decode() if budget <= 0: break return files def read(path: str, limit: int) -> dict[str, object]: """A regular file's size, mtime and bytes: at most one byte past `limit`, so a file that grew since the stat still reads as too large.""" try: info = os.stat(path) except (FileNotFoundError, NotADirectoryError): return {"error": "does not exist"} except OSError: return {"error": "cannot be read"} if stat.S_ISDIR(info.st_mode): return {"error": "is a directory"} if not stat.S_ISREG(info.st_mode): return {"error": "is not a regular file"} found: dict[str, object] = {"size": info.st_size, "modified": int(info.st_mtime)} if info.st_size > limit: return found try: with open(path, "rb") as handle: data = handle.read(limit + 1) except OSError: return {"error": "cannot be read"} found["data"] = base64.b64encode(data).decode() return found SNIFF = 8000 # bytes that tell a binary file COUNTED = 64 * 1024 * 1024 # lines are counted in files up to this size def ran(command: list[str], root: str, deadline: float, cap: int) -> dict: try: done = subprocess.run( command, cwd=root, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=deadline, ) except subprocess.TimeoutExpired: return {"timed_out": True} except OSError: return {"missing": True} return { "status": done.returncode, "stdout": base64.b64encode(done.stdout[:cap]).decode(), "truncated": len(done.stdout) > cap, } def head(path: str, limit: int, budget: int) -> tuple[dict | None, int]: """A file's size, line count and first bytes: all of them while it fits `limit` and `budget`, else enough to tell binary content.""" try: info = os.stat(path) except OSError: return None, budget if not stat.S_ISREG(info.st_mode): return None, budget size = info.st_size whole = size <= limit and size <= budget want = size if whole else min(size, SNIFF) lines, kept, last = 0, b"", b"" try: with open(path, "rb") as handle: if size > COUNTED: kept = handle.read(want) else: while chunk := handle.read(1 << 20): if len(kept) < want: kept += chunk[: want - len(kept)] lines += chunk.count(b"\n") last = chunk[-1:] if last not in (b"", b"\n"): lines += 1 except OSError: return None, budget found = {"size": size, "lines": lines, "data": base64.b64encode(kept).decode()} return found, budget - (size if whole else 0) def changes(request: dict) -> dict[str, object]: root, deadline = request["root"], float(request["deadline"]) commands = request["commands"] with ThreadPoolExecutor(max_workers=max(1, len(commands))) as pool: outputs = list( pool.map(lambda c: ran(c, root, deadline, int(request["cap"])), commands) ) heads = [] budget = int(request["head_budget"]) for relative in request["reads"]: found, budget = head(child(root, relative), int(request["head_limit"]), budget) heads.append(found) return {"outputs": outputs, "heads": heads} def file_paths(directory: str, anchored: bool = False) -> dict[str, object]: if directory.startswith("~/"): directory = os.path.expanduser("~") + "/" + directory[2:] try: process = subprocess.Popen( [ "rg", "--files", "--null", "--hidden", "--no-require-git", "-g", "!.git", "-g", "!.jj", ], cwd=directory, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, ) except OSError as error: return {"error": str(error)} timer = threading.Timer(2.0, process.kill) timer.start() try: assert process.stdout is not None output = process.stdout.read(2097153) truncated = len(output) > 2097152 if truncated: process.kill() code = process.wait() if anchored and code == 2 and output: truncated = True if not truncated and code not in (0, 1): return {"error": "workspace file listing failed or timed out"} paths = [] parts = output[:2097152].split(b"\0", 20000) truncated = truncated or len(parts) > 20000 and bool(parts[-1]) for path in parts[:-1]: if len(path) > 4096: continue try: paths.append(path.decode("utf-8")) except UnicodeDecodeError: continue return {"items": paths, "truncated": truncated} finally: timer.cancel() process.stdout.close() if process.poll() is None: process.kill() process.wait() def gather(request: dict) -> dict[str, object]: if request["route"] == "file-paths": return file_paths(request["dir"], request.get("anchored", False)) if request["route"] == "file": return read(request["path"], int(request["limit"])) if request["route"] == "run": return changes(request) route, directory = request["route"], request["dir"] snapshot: dict[str, object] = {"directory": os.path.isdir(directory)} if not snapshot["directory"] or route == "exists": return snapshot if route == "project": snapshot["files"] = project(directory) return snapshot exists: list[str] = [] listed = {directory: entries(directory)} snapshot.update(entries=listed, exists=exists) if route == "list": for name, is_directory, *_ in listed[directory]: if is_directory: marks(child(directory, str(name)), exists) if not request.get("files"): return snapshot found = checkout(directory, exists) if route == "preview": children = [ child(directory, str(name)) for name, is_directory, *_ in listed[directory] if is_directory and not str(name).startswith(".") ] for path in children[:SUBDIRECTORIES]: listed[path] = entries(path) if found is None: return snapshot kind, root = found deadline = float(request.get("deadline", 2.0)) plan = request.get("plans", {}).get(kind, []) with ThreadPoolExecutor(max_workers=max(1, len(plan))) as pool: answers = dict(pool.map(lambda c: run(c, root, directory, deadline), plan)) outputs = {k: v for k, v in answers.items() if v is not None} snapshot["outputs"] = outputs if route == "preview": relatives: list[str] = [] tracked = request.get("tracked", {}).get(kind) if tracked is not None: program, args, _ = tracked["command"] listing = outputs.get(key(program, args)) or "" relatives = listing.split(tracked["separator"]) sizes: dict[str, int] = {} for relative in relatives[: int(request.get("counted", 0))]: if not relative: continue path = child(directory, relative) try: info = os.stat(path) except OSError: continue if stat.S_ISREG(info.st_mode): sizes[path] = info.st_size snapshot["sizes"] = sizes return snapshot def main() -> int: request = json.loads(sys.stdin.readline()) print(json.dumps(gather(request))) return 0 if __name__ == "__main__": sys.exit(main())