Something went wrong. Try again.
Monorepo for Tangled forked from tangled.org/core
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274import { encode } from "@atcute/cbor";import { CODEC_DCBOR, createSync, toString as cidString } from "@atcute/cid";import { execFile, spawn } from "node:child_process";import { realpath } from "node:fs/promises";import type { Socket } from "node:net";import { createInterface } from "node:readline";import { promisify } from "node:util";
import { config } from "./config";import { get, partsOf, until } from "./net";import { versionsOf, type Version } from "./versions";
const exec = promisify(execFile);
const GIT_IMAGE = process.env.E2E_GIT_IMAGE ?? "docker.io/alpine/git";
const installed = (program: string) => exec(program, ["--version"]).then( () => true, () => false );
type Knot = { scan: string } | { engine: string; container: string; scan: string };
const findKnot = async (): Promise<Knot> => { // outside a container the knot's repos sit on this disk, so E2E_KNOT_SCAN points straight at them const own = process.env.E2E_KNOT_SCAN; if (own) return { scan: own }; const engine = process.env.E2E_ENGINE ?? ((await installed("podman")) ? "podman" : "docker"); const root = await realpath(".."); const listed = await exec(engine, [ "ps", "-q", "--filter", "label=com.docker.compose.service=knot2", "--filter", `label=com.docker.compose.project.working_dir=${root}` ]); const container = process.env.E2E_KNOT_CONTAINER ?? listed.stdout.trim().split("\n")[0]; if (!container) throw new Error( `${engine} should be running a knot2 container from the compose project in ${root} but lists none, set E2E_KNOT_CONTAINER for a knot started some other way` ); const { stdout } = await exec(engine, [ "inspect", "--format", "{{range .Config.Env}}{{println .}}{{end}}", container ]); const scan = /^KNOT_SCAN_PATH=(.+)$/m.exec(stdout)?.[1]; if (!scan) throw new Error(`the knot container ${container} should set KNOT_SCAN_PATH but doesn't`); return { engine, container, scan };};
const shardOf = (did: string) => { const [, method, ...rest] = did.split(":"); const id = rest.join(":"); return `${method}/${id.slice(0, 2)}/${id.slice(2)}`;};
// one line in, "end" out, so one container serves every checkconst LOOP = `check() { set -eu cd "$1" tip=$(git rev-parse -q --verify "refs/cobs/$2/$3^{commit}") || { echo missing; return 0; } echo "tip $tip" for change in $(git rev-list --reverse --topo-order "$tip"); do head=$(git cat-file commit "$change" | sed '/^$/q') type=$(echo "$head" | sed -n 's/^cob-type //p') author=$(echo "$head" | sed -n 's/^cob-author //p') echo "change $change $type $author $(git ls-tree --name-only "$change" | tr '\\n' ,)" done if git cat-file -e "$tip:record" 2>/dev/null; then echo "record $(git cat-file blob "$tip:record" | base64 | tr -d '\\n')" fi echo "payload $(git cat-file blob "$3:payload" | base64 | tr -d '\\n')"}while read -r repo nsid root; do (check "$repo" "$nsid" "$root") 2>&1 || echo "failed $?" echo enddone`;
interface Asked { lines: string[]; done: (lines: string[]) => void; fail: (cause: Error) => void;}
// the knot's repos are owned by another user, so git refuses them without safe.directoryconst SAFE_DIRECTORY = { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "safe.directory", GIT_CONFIG_VALUE_0: "*"};
const spawnGit = (box: Knot) => "engine" in box ? spawn( box.engine, [ "run", "-i", "--rm", "--network=none", `--volumes-from=${box.container}:ro`, ...Object.entries(SAFE_DIRECTORY).map( ([key, value]) => `--env=${key}=${value}` ), "--entrypoint=sh", GIT_IMAGE, "-c", LOOP ], { stdio: ["pipe", "pipe", "pipe"] } ) : spawn("sh", ["-c", LOOP], { env: { ...process.env, ...SAFE_DIRECTORY }, stdio: ["pipe", "pipe", "pipe"] });
const startHelper = async () => { const box = await findKnot(); const child = spawnGit(box); // the loop only ends when stdin closes, and --rm cleans the container up then const answers = child.stdout as unknown as Socket; child.unref(); for (const pipe of [child.stdin, child.stdout, child.stderr]) (pipe as unknown as Socket).unref(); const waiting: Asked[] = []; let said = ""; child.stderr.on("data", (chunk) => (said = (said + chunk).slice(-2000))); createInterface({ input: child.stdout }).on("line", (line) => { const asked = waiting[0]; if (!asked) return; if (line !== "end") return void asked.lines.push(line); waiting.shift(); if (waiting.length === 0) answers.unref(); asked.done(asked.lines); }); const ended = new Promise<Error>((resolve) => { const gone = (why: string) => resolve(new Error(`the git helper ${why}: ${said.trim()}`)); child.on("error", (cause) => gone(cause.message)); child.on("exit", (code) => gone(`exited with ${code}`)); }); void ended.then((cause) => { helper = undefined; for (const asked of waiting.splice(0)) asked.fail(cause); }); return { scan: box.scan, ask: (line: string) => new Promise<string[]>((done, fail) => { answers.ref(); waiting.push({ lines: [], done, fail }); child.stdin.write(`${line}\n`); }) };};
let helper: ReturnType<typeof startHelper> | undefined;
interface Change { id: string; type: string; author: string; files: string[];}
interface CobInGit { tip: string; changes: Change[]; record?: Buffer; payload: Buffer;}
// ls-remote can't see refs/cobs and the knot hides them from reads, so read the diskconst cobInGit = async ( repoDid: string, nsid: string, root: string): Promise<CobInGit | undefined> => { const git = await (helper ??= startHelper()); const lines = await git.ask(`${git.scan}/${shardOf(repoDid)} ${nsid} ${root}`); if (lines[0] === "missing") return undefined; const field = (name: string) => lines .filter((line) => line.startsWith(`${name} `)) .map((line) => line.slice(name.length + 1)); if (field("payload").length === 0) throw new Error(`git couldn't read refs/cobs/${nsid}/${root}: ${lines.join(" ")}`); const record = field("record")[0]; return { tip: field("tip")[0], changes: field("change").map((line) => { const [id, type, author, files] = line.split(" "); return { id, type, author, files: files.split(",").filter(Boolean) }; }), record: record === undefined ? undefined : Buffer.from(record, "base64"), payload: Buffer.from(field("payload")[0], "base64") };};
const cidOf = (bytes: Buffer) => cidString(createSync(CODEC_DCBOR, bytes));
// the root payload holds address {rkey, collection} as dag-cbor, so match those bytesconst namesRecord = (payload: Buffer, collection: string, rkey: string) => payload.includes(Buffer.concat([encode("address"), encode({ rkey, collection })]));
const signingKey = async () => { const { body } = await get(`${config.knotUrl}/.well-known/did.json`); const methods: { id: string; publicKeyMultibase: string }[] = body?.verificationMethod ?? []; return methods.find((method) => method.id.endsWith("#atproto"))?.publicKeyMultibase;};
let knotKey: Promise<string | undefined> | undefined;
// a cid of undefined means the record was deleted, so the chain ends in a tombstoneconst gitDisagrees = async ( { uri, cid }: { uri: string; cid?: string }, versions: Version[]): Promise<string | undefined> => { const { repo, collection, rkey } = partsOf(uri); if (versions.length === 0) return "listCobVersions lists no versions to look up in git"; const root = versions[0].changeId; const ref = `refs/cobs/${collection}/${root}`; const [cob, key] = await Promise.all([ cobInGit(repo, collection, root), (knotKey ??= signingKey()) ]); if (!cob) return `there is no ${ref}`; const last = versions.at(-1)!; if (cob.tip !== last.changeId) return `${ref} points at ${cob.tip} but listCobVersions ends at ${last.changeId}`; const walked = cob.changes.map((change) => change.id); const listed = versions.map((version) => version.changeId); if (JSON.stringify([...walked].sort()) !== JSON.stringify([...listed].sort())) return `git walks ${walked.join(" ")} from ${ref} but listCobVersions lists ${listed.join(" ")}`; const stranger = cob.changes.find( (change) => change.type !== collection || change.author !== key ); if (stranger) return `change ${stranger.id} says cob-type ${stranger.type} by ${stranger.author}, it should be ${collection} by the knot's key ${key}`; if (!namesRecord(cob.payload, collection, rkey)) return `the root change of ${ref} doesn't name rkey ${rkey} in ${collection}`; const holds = cob.changes.at(-1)!.files; if (cid === undefined) return holds.includes("tombstone") && !cob.record ? undefined : `the tip of ${ref} should hold a tombstone but holds ${holds.join(", ")}`; if (!cob.record) return `the tip of ${ref} should hold a record but holds ${holds.join(", ")}`; const hashed = cidOf(cob.record); return hashed === cid ? undefined : `the record blob at the tip of ${ref} hashes to ${hashed} but the write answered ${cid}`;};
// versions are read again only when git disagrees, because a write can land between readsexport const checkGit = (written: { uri: string; cid?: string }, read: Version[]) => { const { collection } = partsOf(written.uri); let versions: Version[] | undefined = read; return until( `refs/cobs/${collection}/${read[0]?.changeId} should end at ${written.cid ?? "a tombstone"}`, 10_000, async () => { versions ??= (await versionsOf(written.uri)).versions; const problem = await gitDisagrees(written, versions); versions = undefined; return [!problem, problem ?? ""]; } );};