// The ops space: the service's own log, kept on the PDS in a private space of a dedicated account // (ops., made by the service on first use and held in state.json like the owner's). // // at:///space/ai.languid.space.ops/self. The ops account writes; the installation owner is a // read-only member, and /admin and /admin/logs read it. It holds what the operator needs to run the // service and nothing of anyone's agents: the sign-up funnel (src/funnel.ts), ops events, which // carry DIDs, methods and scrubbed errors, never prompts, messages or tool arguments, and, for owners // who share usage, daily visits (src/visits.ts). // // Events are buffered and written every 30 seconds; the same event repeating within five minutes // is one record with a count. Writing is best effort: an event that can't be written stays in stdout // (the caller's `log`) and is dropped. Events older than 90 days are deleted once a day. // // Before the ops account, the funnel lived in the installation owner's own ops space. When the new // space is first made, those records are copied over; the old space is left as it was. import { SpaceRepo, type Actor } from './atproto/space.ts' import { tid } from './atproto/tid.ts' import { XrpcError } from './atproto/xrpc.ts' import { languidVersion, scrub } from './telemetry.ts' import { now } from './records.ts' export const OPS_SPACE_TYPE = 'ai.languid.space.ops' export const FUNNEL_COLLECTION = 'ai.languid.ops.funnel' export const EVENT_COLLECTION = 'ai.languid.ops.event' export const opsSpaceRef = (did: string) => `at://${did}/space/${OPS_SPACE_TYPE}/self` export const LEVELS = ['error', 'warn', 'info'] as const export type Level = (typeof LEVELS)[number] /** What happened, broadly: a filter on /admin/logs. */ export const KINDS = ['boot', 'crash', 'bug', 'pds', 'purge', 'computer', 'report', 'auto_approval', 'compaction', 'client', 'other'] as const export type Kind = (typeof KINDS)[number] /** ai.languid.ops.event: one thing the service did or hit. */ export interface OpsEvent { level: Level kind: Kind message: string /** An agent's name or an owner's DID it concerns. Never content. */ subject?: string /** The XRPC method or page it happened in. */ method?: string /** Error class and stack frames, scrubbed. */ name?: string stack?: string /** A problem report's output and note, sent only when the person said yes (reportEvent). Scrubbed. */ detail?: string /** How many times it happened between createdAt and lastAt (one record for repeats). */ count?: number lastAt?: string /** The languid commit that was running. */ version?: string createdAt: string } const FLUSH_MS = 30_000 const REPEAT_MS = 5 * 60_000 export const RETENTION_DAYS = 90 const LIMITS = { message: 1000, stack: 4000, subject: 300, method: 200, name: 120, detail: 16_000 } /** Events kept in memory for /admin/logs when the PDS can't be reached. */ const RING = 500 export interface OpsSpaceOptions { /** Logs in as the ops account (making it if needed). */ account: () => Promise /** DIDs that may read the space: the installation owner. */ readers: () => string[] /** The installation owner, whose own ops space held the funnel before; copied once. */ legacy?: () => Promise log?: (line: string) => void } /** The ops space, opened (and made, and filled from the old one) once per process. */ export class OpsSpace { private repo?: Promise private opts: OpsSpaceOptions constructor(opts: OpsSpaceOptions) { this.opts = opts } open(): Promise { this.repo ??= this.make().catch((e) => { this.repo = undefined throw e }) return this.repo } private async make(): Promise { const actor = await this.opts.account() const space = opsSpaceRef(actor.did) let created = false try { await actor.call('com.atproto.space.getSpace', { params: { space } }) } catch (e) { if (!(e instanceof XrpcError && e.error === 'SpaceNotFound')) throw e try { await actor.call('com.atproto.space.createSpace', { body: { spaceType: OPS_SPACE_TYPE, skey: 'self', policy: { $type: 'com.atproto.memberspace.policy' } } }) created = true } catch (e) { if (!(e instanceof XrpcError && e.error === 'SpaceAlreadyExists')) throw e } } const readers = this.opts.readers().filter((d) => d && d !== actor.did) await actor.call('com.atproto.memberspace.putMembers', { body: { space, members: [{ did: actor.did, read: true, write: true }, ...readers.map((did) => ({ did, read: true, write: false }))] }, }) const repo = new SpaceRepo(actor, space) if (created) await this.copyLegacy(repo, actor.did) return repo } /** Copy the funnel from the installation owner's own ops space, once, when the new space is made. */ private async copyLegacy(repo: SpaceRepo, opsDid: string) { try { const old = await this.opts.legacy?.() if (!old || old.did === opsDid) return const from = new SpaceRepo(old, opsSpaceRef(old.did)) let records try { records = await from.list(FUNNEL_COLLECTION) } catch (e) { if (e instanceof XrpcError && e.error === 'SpaceNotFound') return throw e } for (const r of records) await repo.put(FUNNEL_COLLECTION, r.rkey, r.value) if (records.length) this.opts.log?.(`ops: copied ${records.length} funnel record(s) from ${opsSpaceRef(old.did)}`) } catch (e: any) { this.opts.log?.(`ops: couldn't copy the old funnel: ${e?.message ?? e}`) } } } /** * An ops event from a computer's report (src/computer-report.ts): its runner, tray or setup hit an * error. `owner` is the computer's owner's DID. Only the report's known fields are kept. */ export function computerEvent(report: any, owner: string | undefined, computer: string | undefined): Omit | undefined { const str = (v: unknown) => (typeof v === 'string' && v ? v : undefined) const source = str(report?.source) const message = str(report?.message) if (!message || !source || !['runner', 'tray', 'setup'].includes(source)) return const where = [str(report.platform), str(report.version)].filter(Boolean).join(' ') const ev: Omit = { level: report.level === 'warn' ? 'warn' : 'error', kind: 'computer', message: `${computer ?? 'a computer'}${where ? ` (${where})` : ''}: ${message}`, method: `computer ${source}`, } if (owner) ev.subject = owner if (str(report.name)) ev.name = report.name if (str(report.stack)) ev.stack = report.stack return ev } /** Where a problem report comes from: the installer's steps, sign-in, computer setup, or `languid report`. */ export const REPORT_STEPS = ['check', 'download', 'unpack', 'dependencies', 'wrapper', 'path', 'login', 'setup', 'tray', 'runner', 'report'] as const const INSTALL_STEPS = ['check', 'download', 'unpack', 'dependencies', 'wrapper', 'path'] /** * An ops event from ai.languid.ops.report: a person's computer hit a problem before (or without) a * runner token, like install.ps1 failing, or someone ran `languid report`. Anyone can send one, so * only known fields are kept and everything is scrubbed again by OpsLog.event. `owner` is the * signed-in caller's DID, when the report came with a valid token. `detail` (output and a note) is * only there when the person agreed to send it. */ export function reportEvent(input: any, id: string, owner?: string): Omit | undefined { const str = (v: unknown, max: number) => (typeof v === 'string' && v.trim() ? v.trim().slice(0, max) : undefined) const step = str(input?.step, 40) const message = str(input?.message, 1000) ?? (step === 'report' ? 'a problem report' : undefined) if (!step || !(REPORT_STEPS as readonly string[]).includes(step) || !message) return const where = ['platform', 'os', 'shell', 'node', 'version'].map((k) => str(input[k], 120)).filter(Boolean).join(', ') const level = input.level === 'warn' || input.level === 'info' ? input.level : step === 'report' ? 'info' : 'error' const ev: Omit = { level, kind: 'report', message: `${message}${where ? ` (${where})` : ''}`, method: `${step === 'report' ? 'languid report' : INSTALL_STEPS.includes(step) ? `install ${step}` : step} ยท ${id}`, } if (owner) ev.subject = owner const name = str(input.name, 120) if (name) ev.name = name const stack = str(input.stack, 4000) if (stack) ev.stack = stack const detail = [str(input.note, 2000) && `Note: ${input.note.trim().slice(0, 2000)}`, str(input.log, 14_000)].filter(Boolean).join('\n\n') if (detail) ev.detail = detail return ev } /** An ops event from an error: class, message and frames, scrubbed. */ export function errorEvent(level: Level, kind: Kind, error: unknown, extra: { subject?: string; method?: string; message?: string } = {}): Omit { const e = error as any const message = typeof e?.message === 'string' ? e.message : String(error) const ev: Omit = { level, kind, message: extra.message ? `${extra.message}: ${message}` : message } if (typeof e?.name === 'string') ev.name = e.name if (typeof e?.stack === 'string') { const frames = e.stack.split('\n').filter((l: string) => /^\s+at /.test(l)).join('\n') if (frames) ev.stack = frames } if (extra.subject) ev.subject = extra.subject if (extra.method) ev.method = extra.method return ev } /** The ops log: events buffered, deduplicated, and written to the ops space. */ export class OpsLog { private pending: OpsEvent[] = [] /** Recent events by dedupe key, so a repeat bumps a count instead of writing again. */ private recent = new Map() private ring: OpsEvent[] = [] private timer?: ReturnType private flushing: Promise = Promise.resolve() private lastPrune = 0 /** Others written with each batch (daily visits, src/visits.ts). */ private also: ((repo: SpaceRepo) => Promise)[] = [] private space: () => Promise private log: (line: string) => void private clock: () => number constructor( space: () => Promise, log: (line: string) => void = () => {}, clock: () => number = Date.now, ) { this.space = space this.log = log this.clock = clock } /** Start writing every 30 seconds (the service calls this; tests flush by hand). */ start() { this.timer ??= setInterval(() => void this.flush(), FLUSH_MS) this.timer.unref?.() } stop() { if (this.timer) clearInterval(this.timer) this.timer = undefined } /** Write `fn` with every batch, every 30 seconds and on shutdown, when the space can be reached. */ alsoFlush(fn: (repo: SpaceRepo) => Promise) { this.also.push(fn) } /** Note an event. Never throws. */ event(input: Omit) { const at = new Date(this.clock()).toISOString() const ev: OpsEvent = { ...input, message: scrub(input.message || '(no message)', LIMITS.message), createdAt: at } if (ev.stack) ev.stack = scrub(ev.stack, LIMITS.stack) if (ev.subject) ev.subject = scrub(ev.subject, LIMITS.subject) if (ev.method) ev.method = scrub(ev.method, LIMITS.method) if (ev.name) ev.name = scrub(ev.name, LIMITS.name) if (ev.detail) ev.detail = scrub(ev.detail, LIMITS.detail) const v = languidVersion() if (v) ev.version = v const key = `${ev.level}|${ev.kind}|${ev.subject ?? ''}|${ev.message}` const seen = this.recent.get(key) if (seen && this.clock() - Date.parse(seen.event.createdAt) < REPEAT_MS) { seen.event.count = (seen.event.count ?? 1) + 1 seen.event.lastAt = at // Already written: write it again under the same rkey with the new count. if (seen.rkey && !this.pending.includes(seen.event)) this.pending.push(seen.event) return } this.recent.set(key, { event: ev }) this.pending.push(ev) this.ring.push(ev) if (this.ring.length > RING) this.ring.shift() for (const [k, r] of this.recent) if (this.clock() - Date.parse(r.event.createdAt) >= REPEAT_MS) this.recent.delete(k) } /** Write what's pending. Resolves once written or dropped. */ flush(): Promise { const run = this.flushing.then(async () => { const batch = this.pending.splice(0) if (!batch.length && !this.also.length) return this.maybePrune() let repo: SpaceRepo try { repo = await this.space() } catch (e: any) { if (batch.length) this.log(`ops: ${batch.length} event(s) not written, space unavailable: ${e?.message ?? e}`) return } for (const ev of batch) { const entry = [...this.recent.values()].find((r) => r.event === ev) const rkey = entry?.rkey ?? tid() try { await repo.put(EVENT_COLLECTION, rkey, { ...ev }) if (entry) entry.rkey = rkey } catch (e: any) { this.log(`ops: event not written: ${e?.message ?? e}`) } } for (const fn of this.also) await fn(repo).catch((e: any) => this.log(`ops: ${e?.message ?? e}`)) await this.maybePrune(repo) }) this.flushing = run.catch(() => {}) return this.flushing } /** Events, newest first: from the PDS, or the in-memory ring when it can't be read. */ async list(): Promise<{ events: OpsEvent[]; fromMemory: boolean }> { try { const repo = await this.space() const records = await repo.list(EVENT_COLLECTION) return { events: records.map((r) => r.value).sort((a, b) => b.createdAt.localeCompare(a.createdAt)), fromMemory: false } } catch (e: any) { this.log(`ops: couldn't read events: ${e?.message ?? e}`) return { events: [...this.ring].reverse(), fromMemory: true } } } /** Once a day, delete events older than the retention window. */ private async maybePrune(repo?: SpaceRepo) { if (this.clock() - this.lastPrune < 24 * 3600_000) return try { repo ??= await this.space() this.lastPrune = this.clock() const cutoff = new Date(this.clock() - RETENTION_DAYS * 24 * 3600_000).toISOString() const old = (await repo.list(EVENT_COLLECTION)).filter((r) => (r.value.lastAt ?? r.value.createdAt) < cutoff) for (const r of old) await repo.delete(EVENT_COLLECTION, r.rkey) if (old.length) this.log(`ops: deleted ${old.length} event(s) older than ${RETENTION_DAYS} days`) } catch (e: any) { this.log(`ops: prune failed: ${e?.message ?? e}`) } } }