"""Which processes belong to which session, and what they use. The daemon records each session's kernel and the process groups of its run jobs (`agents.resources`); this module reads the live process table and attributes processes to them. It can only see the machine it runs on, so a session whose workspace is on another host is left without processes. """ from __future__ import annotations import asyncio import ctypes import json import os import struct import subprocess import sys from dataclasses import dataclass import albedo_memory import albedo_proc PS = "ps" DARWIN_VNODE_PATH_INFO = 9 DARWIN_VNODE_PATH_INFO_SIZE = 2352 DARWIN_CWD_OFFSET = 152 # past the vnode_info that opens pvi_cdir @dataclass(frozen=True) class Row: pid: int ppid: int pgid: int rss_kib: int cpu: float etime: str name: str command: str def _ps(*columns: str) -> list[str]: result = subprocess.run( [PS, "-A", *(f"-o{column}=" for column in columns)], capture_output=True, text=True, timeout=5, check=True, ) return result.stdout.splitlines() def sample() -> list[Row]: """Every live, non-zombie process, as `ps` lists it (macOS and Linux).""" commands = {} for line in _ps("pid", "args"): pid, _, args = line.strip().partition(" ") if pid.isdigit(): commands[int(pid)] = args.strip() rows = [] for line in _ps("pid", "ppid", "pgid", "rss", "pcpu", "etime", "stat", "comm"): fields = line.split(None, 7) if len(fields) < 8 or fields[6].startswith("Z"): continue pid = int(fields[0]) if pid not in commands: continue name = os.path.basename(fields[7].strip()) or fields[7].strip() rows.append( Row( pid, int(fields[1]), int(fields[2]), int(fields[3]), float(fields[4]), fields[5], name, commands[pid], ) ) # The ps that sampled this is still in its own listing. me = os.getpid() return [row for row in rows if not (row.ppid == me and row.name == "ps")] def elapsed(etime: str) -> int: """Seconds in ps's `[[dd-]hh:]mm:ss`.""" days, _, clock = etime.rpartition("-") seconds = 0 for part in clock.split(":"): seconds = seconds * 60 + int(part) return seconds + (int(days) * 86400 if days else 0) def working_directory(pid: int) -> str | None: """A process's current directory, or None where it cannot be read.""" if sys.platform == "darwin": try: libproc = ctypes.CDLL("/usr/lib/libproc.dylib") except OSError: return None buffer = ctypes.create_string_buffer(DARWIN_VNODE_PATH_INFO_SIZE) if ( libproc.proc_pidinfo( pid, DARWIN_VNODE_PATH_INFO, 0, buffer, DARWIN_VNODE_PATH_INFO_SIZE ) <= 0 ): return None path = struct.unpack_from("1024s", buffer.raw, DARWIN_CWD_OFFSET)[0] return path.split(b"\0", 1)[0].decode(errors="replace") or None try: return os.readlink(f"/proc/{pid}/cwd") except OSError: return None def locality(cwd: str) -> str: """The host a workspace lives on: \"\" for this machine's paths, else the `user@host` of a remote `user@host:/path`.""" return "" if cwd.startswith("/") or ":" not in cwd else cwd.split(":", 1)[0] def _jobs(owner: dict, own_jobs: dict[str, int]) -> dict[str, dict]: """The job groups a session's kernel reported, by job id. The calling kernel's own live jobs win: the daemon may not have heard of one yet.""" kernel = owner.get("kernel") or {} try: groups = json.loads(kernel.get("groups") or "{}") except ValueError: groups = {} jobs = { job: group for job, group in groups.items() if isinstance(group, dict) and isinstance(group.get("pgid"), int) } for job, pgid in own_jobs.items(): jobs[job] = {**jobs.get(job, {}), "pgid": pgid} return jobs def _left_group(rows: list[Row], pgid: int) -> bool: """Whether a member of the group has a parent in another one.""" groups = {row.pid: row.pgid for row in rows} return any(row.pgid == pgid and groups.get(row.ppid, pgid) != pgid for row in rows) def attribute( owners: list[dict], rows: list[Row], own_id: str, own_jobs: dict[str, int], ) -> dict[str, list[dict]]: """Each session's live processes by session id. A process is a session's when it is in the group of one of its jobs or its kernel, or descends from one that is. A process that a job's member left the group still parents (ssh's ControlPersist proxy) is nobody's.""" own_locality = locality(next((o["cwd"] for o in owners if o["id"] == own_id), "")) owner_of: dict[int, tuple[str, str | None]] = {} by_session = {owner["id"]: owner for owner in owners} excluded: set[int] = set() for owner in owners: if locality(owner["cwd"]) != own_locality: continue kernel = owner.get("kernel") or {} for job, group in _jobs( owner, own_jobs if owner["id"] == own_id else {} ).items(): pgid = group["pgid"] if _left_group(rows, pgid): excluded |= albedo_proc.adopted( albedo_proc.Group(pgid, group.get("leader")) ) for row in rows: if row.pgid == pgid: owner_of.setdefault(row.pid, (owner["id"], job)) for row in rows: if row.pgid == kernel.get("pgid") or row.pid == kernel.get("pid"): owner_of.setdefault(row.pid, (owner["id"], None)) children: dict[int, list[Row]] = {} for row in rows: children.setdefault(row.ppid, []).append(row) for pid in excluded: owner_of.pop(pid, None) queue = list(owner_of) while queue: pid = queue.pop() for child in children.get(pid, []): if child.pid not in owner_of and child.pid not in excluded: owner_of[child.pid] = owner_of[pid] queue.append(child.pid) found: dict[str, list[dict]] = {} for row in rows: if row.pid in owner_of: session, job = owner_of[row.pid] found.setdefault(session, []).append( _process(row, session, job, by_session[session]["cwd"]) ) return found def _process(row: Row, session: str, job: str | None, fallback_cwd: str) -> dict: rss = row.rss_kib * 1024 memory = ( albedo_memory.darwin_footprint(row.pid) if sys.platform == "darwin" else None ) return { "pid": row.pid, "ppid": row.ppid, "pgid": row.pgid, "session": session, "job_id": job, "name": row.name, "command": row.command, "cwd": working_directory(row.pid) or (fallback_cwd if locality(fallback_cwd) == "" else ""), "cpu_percent": row.cpu, "rss_bytes": rss, "memory_bytes": memory or rss, "elapsed_seconds": elapsed(row.etime), } def totals(processes: list[dict]) -> dict[str, float | int]: return { "process_count": len(processes), "cpu_percent": round(sum(p["cpu_percent"] for p in processes), 1), "memory_bytes": sum(p["memory_bytes"] for p in processes), "rss_bytes": sum(p["rss_bytes"] for p in processes), } def session_record(owner: dict, processes: list[dict]) -> dict: total = totals(processes) return { "session": owner["id"], "name": owner["name"], "cwd": owner["cwd"], **total, "total": total, "processes": sorted(processes, key=lambda p: (p["job_id"] or "", p["pid"])), } async def collect( owners: list[dict], own_id: str, own_jobs: dict[str, int] ) -> dict[str, list[dict]]: rows = await asyncio.to_thread(sample) return attribute(owners, rows, own_id, own_jobs)