#!/usr/bin/env python3
"""sov: everything done from outside the server, for the instance repository you are in.

  sov status                         run the live checks now and show what fails
  sov drill                          run a full recovery drill now (a few minutes)
  sov secrets                        every secret: where it lives, how a new one is made
  sov secrets copy <secret>          put a secret into the clipboard (never shown)
  sov rotate <secret>... | --all     replace secrets (--all: every secret kept in git)
  sov secrets check                  prove every secret works
  sov keys                           where each owner key is registered
  sov keys sync                      every place holds exactly spec.owner.sshKeys (asks before removing)
  sov key rotate                     a new owner key, registered everywhere; the old one removed
  sov paper print [--out FILE]       a new paper recovery key and its printable sheet
  sov paper check                    type the paper key in: verified, yearly reminder reset
  sov add repo <name> "<desc>"       a repository on the knot (prints its spec entry)
  sov add zone <zone>                DNS secondaries at Hetzner for a zone of the spec
  sov add domain [--prices|--check] <domain>...
                                     move domains to INWX with the spec's nameservers
  sov install <user>@<ip>            wipe a VPS and install the instance onto it

Secrets never appear on the screen or on a command line: values are made or pasted into hidden
prompts and go through pipes. Changes to the repository are signed commits, pushed to the knot;
the server deploys them by itself.
"""
import getpass
import hashlib
import html
import json
import os
import re
import subprocess
import sys
import tempfile
import time
import urllib.parse
import urllib.request
from datetime import date, datetime, timezone
from http.cookiejar import CookieJar
from pathlib import Path

ROOT = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip())
SECRETS = ROOT / "secrets"
FILES = {"server": SECRETS / "server.yaml", "admin": SECRETS / "admin.yaml"}
HOST_KEY, HOST_PUB, RECOVERY_PUB = SECRETS / "host_key.sops", SECRETS / "host_key.pub", SECRETS / "recovery.pub"
_cache = {}


# ------------------------------------------------------------------ plumbing

def say(msg):
    print(msg, flush=True)


def fail(msg):
    sys.exit(f"sov: {msg}")


def run(*cmd, input=None, env=None, check=True):
    r = subprocess.run(cmd, input=input, capture_output=True, text=True, env=env)
    if check and r.returncode:
        fail(f"{cmd[0]} {cmd[1] if len(cmd) > 1 else ''} failed: {r.stderr.strip()[:500]}")
    return r.stdout


def flake(attr):
    if attr not in _cache:
        _cache[attr] = json.loads(run("nix", "eval", "--json", f"{ROOT}#{attr}"))
    return _cache[attr]


def spec(path=None):
    s = flake("spec")
    for part in (path.split(".") if path else []):
        s = s[part]
    return s


def ssh(cmd, input=None, check=True):
    """A command on the server as admin (sudo inside cmd as needed)."""
    return run("ssh", "-o", "BatchMode=yes", f"admin@{spec('server.ipv4')}", cmd, input=input, check=check)


def secret(store, key):
    return run("sops", "--decrypt", "--extract", json.dumps([key]), str(FILES[store])).strip()


def set_secret(store, key, value):
    run("sops", "set", "--value-stdin", str(FILES[store]), json.dumps([key]), input=json.dumps(value))


def http(method, url, token=None, body=None, query=None, headers=None):
    if query:
        url += "?" + urllib.parse.urlencode(query)
    req = urllib.request.Request(url, method=method, data=json.dumps(body).encode() if body is not None else None)
    req.add_header("content-type", "application/json")
    if token:
        req.add_header("authorization", f"Bearer {token}")
    for k, v in (headers or {}).items():
        req.add_header(k, v)
    try:
        with urllib.request.urlopen(req, timeout=60) as r:
            raw = r.read()
            return json.loads(raw) if raw else {}
    except urllib.error.HTTPError as e:
        fail(f"{method} {url.split('?')[0]} -> HTTP {e.code}: {e.read().decode(errors='replace')[:300]}")


def commit(message, paths):
    """A signed commit of exactly these paths, pushed to the knot. Returns the commit id."""
    run("git", "-C", str(ROOT), "add", "--", *map(str, paths))
    run("git", "-C", str(ROOT), "commit", "-S", "-q", "-m", message + "\n\n(made by sov)", "--", *map(str, paths))
    for attempt in range(12):  # a key just added to the knot may take a moment to count
        if subprocess.run(["git", "-C", str(ROOT), "push", "-q", "origin", "HEAD"], capture_output=True).returncode == 0:
            break
        time.sleep(15)
    else:
        fail("git push failed")
    return run("git", "-C", str(ROOT), "rev-parse", "HEAD").strip()


def wait_deployed(rev, minutes=30):
    say(f"  waiting for the server to deploy {rev[:8]} ...")
    end = time.time() + minutes * 60
    while time.time() < end:
        status = ssh("/run/current-system/sw/bin/comin status", check=False)
        if rev in status.split("Deployer", 1)[-1]:
            say("  deployed")
            return
        time.sleep(15)
    fail(f"the server did not deploy {rev[:8]} within {minutes} minutes (see: ssh admin@{spec('server.ipv4')} comin status)")


def unit_log(unit):
    """Start a oneshot unit on the server, wait for it, return (ok, its log)."""
    ok = ssh(f"sudo systemctl start {unit}", check=False) is not None and \
        ssh(f"systemctl is-failed {unit}", check=False).strip() != "failed"
    inv = ssh(f"systemctl show -p InvocationID --value {unit}").strip()
    return ok, ssh(f"sudo journalctl -o cat _SYSTEMD_INVOCATION_ID={inv}", check=False)


def ssh_to_age(pub):
    return run("ssh-to-age", input=pub).strip()


def pub_of(private_key_file):
    return run("ssh-keygen", "-y", "-f", str(private_key_file)).strip()


# ------------------------------------------------------------------ who can decrypt what

def write_sops_config(extra_server=()):
    """.sops.yaml, derived from the public keys: the owner's device keys (spec.owner.sshKeys), the
    paper recovery key (secrets/recovery.pub) and the server's host key (secrets/host_key.pub)."""
    owner = [" ".join(k.split()[:2]) for k in spec("owner.sshKeys")]
    recovery = [RECOVERY_PUB.read_text().strip()] if RECOVERY_PUB.exists() else []
    server = [ssh_to_age(HOST_PUB.read_text())] + [ssh_to_age(p) for p in extra_server]

    def rule(regex, recipients):
        return f"  - path_regex: {regex}\n    key_groups:\n      - age:\n" + "".join(f"          - {r}\n" for r in recipients)
    text = ("# Who can decrypt what. Generated by sov from spec.owner.sshKeys, secrets/recovery.pub and\n"
            "# secrets/host_key.pub: do not edit (sov key rotate, sov paper print, sov rotate host_key).\n"
            "creation_rules:\n"
            + rule("secrets/host_key\\.sops$", owner + recovery)
            + rule("secrets/admin\\.yaml$", owner + recovery)
            + rule("secrets/.*", owner + recovery + server))
    (ROOT / ".sops.yaml").write_text(text)
    for f in (FILES["server"], FILES["admin"], HOST_KEY):
        binary = ["--input-type", "binary"] if f == HOST_KEY else []
        run("sops", "updatekeys", "-y", *binary, str(f))
    return [ROOT / ".sops.yaml", FILES["server"], FILES["admin"], HOST_KEY]


# ------------------------------------------------------------------ status, drill

def status():
    ok, log = unit_log("invariants")
    lines = [ln for ln in log.splitlines() if ln.startswith(("ok", "FAIL"))]
    for ln in lines:
        if ln.startswith("FAIL"):
            say(ln)
    say(f"{sum(ln.startswith('ok') for ln in lines)} ok, {sum(ln.startswith('FAIL') for ln in lines)} failed")
    return ok


def drill():
    say("recovery drill: a temporary VM, installed and restored from the source bucket, compared ...")
    ok, log = unit_log("recovery-drill")
    for ln in log.splitlines():
        if ln.startswith(("ok", "FAIL", "recovery works")) or "check(s) failed" in ln:
            say(ln)
    return ok


# ------------------------------------------------------------------ secrets

def catalogue():
    return flake("sov.secrets")


def list_secrets():
    where = {"server": "git: secrets/server.yaml", "admin": "git: secrets/admin.yaml",
             "state": "server only (+ source bucket)", "host": "git: secrets/host_key.sops"}
    for name, s in sorted(catalogue().items()):
        how = ("never replaced (see below)" if s["fixed"] else "generated" if s["generate"] or s["store"] == "state"
               else f"pasted from {s['paste']['url']}" if s["paste"] else "own procedure")
        say(f"{name:24} {where[s['store']]:32} {how}\n  {s['description']}"
            + (f"\n  not replaceable: {s['fixed']}" if s["fixed"] else ""))


def copy_secret(name):
    s = catalogue().get(name) or fail(f"unknown secret {name} (see: sov secrets)")
    if s["store"] in ("server", "admin"):
        value = secret(s["store"], name)
    elif s["store"] == "state":
        value = ssh(f"sudo cat /var/lib/sov/secrets/{name}")
    else:
        fail(f"{name} is a key file, not a value to paste")
    for tool in (["clip.exe"], ["wl-copy"], ["xclip", "-selection", "clipboard"], ["pbcopy"]):
        try:
            subprocess.run(tool, input=value.strip(), text=True, check=True, capture_output=True)
            say(f"{name}: in the clipboard")
            return
        except (FileNotFoundError, subprocess.CalledProcessError):
            continue
    fail("no clipboard tool found (clip.exe, wl-copy, xclip, pbcopy)")


def github(*args, input=None):
    """gh with the owner's account; None if gh is missing or lacks the permission."""
    r = subprocess.run(["gh", "api", *args], input=input, capture_output=True, text=True)
    if r.returncode:
        say(f"  (gh api {args[0]}: {r.stderr.strip()[:200]})")
        return None
    return json.loads(r.stdout or "null")


def new_value(name, s):
    if s["generate"]:
        return run("sh", "-c", s["generate"]).strip()
    say(f"\n{name}: {s['description']}\n  {s['paste']['url']}\n  {s['paste']['steps']}")
    while True:
        v = getpass.getpass(f"  paste the new {name} (hidden): ").strip()
        if v:
            return v


def rotate(names):
    cat = catalogue()
    if names == ["--all"]:
        names = [n for n, s in cat.items() if s["store"] in ("server", "admin", "host")]
    unknown = [n for n in names if n not in cat]
    if unknown:
        fail(f"unknown secret(s): {', '.join(unknown)} (see: sov secrets)")
    host = "host_key" in names
    in_git = [n for n in names if cat[n]["store"] in ("server", "admin")]
    on_server = [n for n in names if cat[n]["store"] == "state"]

    if in_git:
        for n in in_git:
            s = cat[n]
            value = new_value(n, s)
            if s["apply"] == "restic-password":  # the bucket accepts the new password before the server uses it
                ssh("sudo sov-restic key add --user sov --host rotation --new-password-file /dev/stdin -q", input=value)
            set_secret(s["store"], n, value)
            say(f"  {n}: new value stored")
        rev = commit(f"Rotate {', '.join(in_git)}", list(FILES.values()))
        wait_deployed(rev)
        for n in in_git:
            apply_after(n, cat[n])

    for n in on_server:
        s = cat[n]
        if s["fixed"]:
            say(f"  {n}: not replaceable: {s['fixed']}")
            continue
        old_pub = ssh(f"sudo ssh-keygen -y -f /var/lib/sov/secrets/{n}", check=False).strip() if s["apply"] == "mirror-key" else ""
        ssh(f"sudo rm -f /var/lib/sov/secrets/{n} && sudo systemctl restart sov-secrets")
        if s["apply"] == "mirror-key":
            mirror_key(n, old_pub)
        for unit in s["restart"]:
            ssh(f"sudo systemctl restart {unit}", check=False)
        say(f"  {n}: made anew on the server")

    if host:
        rotate_host_key()
    say("\nchecking ...")
    check_secrets(names if names != ["--all"] else None)
    status()


def mirror_key(name, old_pub):
    """Register the server's new mirror key as the GitHub mirror's deploy key; remove the old one."""
    m = spec(f"repos.{name.removeprefix('mirror_key_')}.mirror")
    pub = ssh(f"sudo ssh-keygen -y -f /var/lib/sov/secrets/{name}").strip()
    path = f"repos/{m['owner']}/{m['name']}/keys"
    keys = github(path)
    if keys is None:
        say(f"\n{name}: register this as the deploy key (allow write access) of the mirror, replacing the old one:\n"
            f"  https://github.com/{m['owner']}/{m['name']}/settings/keys\n  {pub}")
        input("  press Enter once it is registered ")
        return
    if not any(k["key"].split()[:2] == pub.split()[:2] for k in keys):
        github(path, "-f", "title=sov mirror", "-f", f"key={pub}", "-F", "read_only=false")
    for k in keys:  # a mirror has exactly one deploy key: the server's current one
        if k["key"].split()[:2] != pub.split()[:2]:
            github(f"{path}/{k['id']}", "-X", "DELETE")
    say(f"  {name}: registered on github.com/{m['owner']}/{m['name']}")


def check_one(name, s):
    """Prove that the current value of a secret works. Returns (ok, what was shown)."""
    c = s["check"]

    def unit(u):
        return unit_log(u)[0]
    if c == "bucket":
        return ssh("sudo sov-restic cat config >/dev/null && echo ok", check=False).strip() == "ok", "opens the source bucket"
    if c == "backup":
        return unit("restic-backups-box"), "the server reaches the source bucket and backs up"
    if c == "secondaries":
        ssh("sudo knotc -s /run/knot/knot.sock zone-notify >/dev/null", check=False)
        time.sleep(20)
        log = ssh("sudo journalctl -u knot --since -1min -o cat", check=False)
        bad = [ln for ln in log.splitlines() if "notify" in ln and ("failed" in ln or "BADKEY" in ln or "NOTAUTH" in ln)]
        return not bad and "notify, outgoing" in log, "the secondaries accept signed change messages"
    if c == "heartbeats":
        ssh("sudo systemctl restart heartbeats-sync", check=False)
        return ssh("systemctl is-failed heartbeats-sync", check=False).strip() != "failed", "healthchecks.io accepts it"
    if c == "hetzner":
        zones = http("GET", "https://api.hetzner.cloud/v1/zones", secret("server", name))["zones"]
        return bool(zones), f"Hetzner's API answers ({len(zones)} zones)"
    if c == "atproto":
        atproto()
        return True, "Bluesky accepts it"
    if c == "mirror":
        return unit("sov-mirror"), "the server pushes to the mirrors"
    if c == "stats":
        return ssh("systemctl is-active goatcounter", check=False).strip() == "active", "the statistics took it at start"
    if c == "knot":
        return ssh("curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:5555/xrpc/_health", check=False) == "200", \
            "the knot opens its sealed keys"
    return True, "nothing to check"


def check_secrets(names=None):
    cat = catalogue()
    good = True
    for name in sorted(names or cat):
        try:
            ok, what = check_one(name, cat[name])
        except SystemExit as e:
            ok, what = False, str(e)
        good &= ok
        say(f"{'ok  ' if ok else 'FAIL'}  {name}: {what}")
    return good


def apply_after(name, s):
    if s["apply"] == "restic-password":
        keys = json.loads(ssh("sudo sov-restic key list --json"))
        for k in keys:
            if not k.get("current") and k.get("userName") != "recovery":
                ssh(f"sudo sov-restic key remove {k['id']} -q")
        say("  restic_password: the old password no longer opens the source bucket")
    elif s["apply"] == "dns-secondaries":
        token, tsig = secret("server", "hetzner_api_token"), secret("server", "dns_secondaries_secret")
        server = spec("server")
        for zone in spec("dns.records"):
            z = http("GET", "https://api.hetzner.cloud/v1/zones", token, query={"name": zone})["zones"]
            if not z:
                say(f"  dns_secondaries_secret: {zone} has no Hetzner secondary (sov add zone {zone})")
                continue
            http("POST", f"https://api.hetzner.cloud/v1/zones/{z[0]['id']}/actions/change_primary_nameservers", token,
                 body={"primary_nameservers": [{"address": a, "tsig_algorithm": "hmac-sha256", "tsig_key": tsig}
                                               for a in (server["ipv4"], server["ipv6"])]})
        say("  dns_secondaries_secret: the secondaries use the new secret")


def box_keys(edit):
    """Rewrite the source bucket's authorized_keys with edit(lines) -> lines (sftp; the owner's key)."""
    box = spec("backup.box")
    target = f"{box['user']}@{box['host']}"
    with tempfile.TemporaryDirectory() as d:
        local = Path(d) / "authorized_keys"
        run("sftp", "-q", "-P", "23", "-b", "-", target, input=f"get .ssh/authorized_keys {local}\n")
        lines = [ln for ln in local.read_text().splitlines() if ln.strip()]
        local.write_text("\n".join(edit(lines)) + "\n")
        run("sftp", "-q", "-P", "23", "-b", "-", target, input=f"put {local} .ssh/authorized_keys\n")


def rotate_host_key():
    say("\nhost_key: a new identity for the server")
    old_pub = HOST_PUB.read_text().strip()
    ip = spec("server.ipv4")
    with tempfile.TemporaryDirectory() as d:
        key = Path(d) / "host_key"
        run("ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", f"{spec('server.hostname')} host key", "-f", str(key))
        new_pub = " ".join((Path(d) / "host_key.pub").read_text().split()[:2])
        # 1. the server can decrypt with either key
        rev = commit("host key: the next key may decrypt server secrets", write_sops_config(extra_server=[new_pub]))
        wait_deployed(rev)
        # 2. the source bucket lets either key in (same forced command)
        old_line = next(ln for ln in box_keys_lines() if old_pub.split()[1] in ln)
        box_keys(lambda lines: lines + [old_line.replace(old_pub.split()[1], new_pub.split()[1])])
        # 3. the server switches to the new key
        ssh("sudo install -m 600 /dev/stdin /etc/ssh/ssh_host_ed25519_key.next", input=key.read_text())
        ssh("sudo install -m 644 /dev/stdin /etc/ssh/ssh_host_ed25519_key.pub.next", input=new_pub + "\n")
        ssh("sudo mv /etc/ssh/ssh_host_ed25519_key.next /etc/ssh/ssh_host_ed25519_key && "
            "sudo mv /etc/ssh/ssh_host_ed25519_key.pub.next /etc/ssh/ssh_host_ed25519_key.pub && "
            "sudo systemctl reload sshd")
        known = Path.home() / ".ssh" / "known_hosts"
        run("ssh-keygen", "-R", ip, check=False)
        with known.open("a") as f:
            f.write(f"{ip} {new_pub}\n")
        # 4. the repository records the new key; only it decrypts server secrets
        enc = run("sops", "--encrypt", "--filename-override", str(HOST_KEY), "--input-type", "binary",
                  "--output-type", "binary", "/dev/stdin", input=key.read_text())
        HOST_KEY.write_text(enc)
        HOST_PUB.write_text(new_pub + "\n")
        rev = commit("host key rotated", write_sops_config() + [HOST_PUB])
        wait_deployed(rev)
    # 5. the old key no longer opens the bucket
    box_keys(lambda lines: [ln for ln in lines if old_pub.split()[1] not in ln])
    ok, _ = unit_log("restic-backups-box")
    say(f"  host_key: rotated; backup with the new key {'works' if ok else 'FAILED'}")


def box_keys_lines():
    box = spec("backup.box")
    with tempfile.TemporaryDirectory() as d:
        local = Path(d) / "authorized_keys"
        run("sftp", "-q", "-P", "23", "-b", "-", f"{box['user']}@{box['host']}", input=f"get .ssh/authorized_keys {local}\n")
        return [ln for ln in local.read_text().splitlines() if ln.strip()]


# ------------------------------------------------------------------ the owner's key

def atproto():
    owner = spec("owner")
    doc = http("GET", f"https://plc.directory/{owner['did']}")
    pds = next(s["serviceEndpoint"] for s in doc["service"] if s["id"] == "#atproto_pds")
    jwt = http("POST", f"{pds}/xrpc/com.atproto.server.createSession",
               body={"identifier": owner["handle"], "password": secret("admin", "bsky_app_password")})["accessJwt"]
    return owner["did"], pds, jwt


def tangled_keys(add=None, remove=None):
    """The owner's SSH keys on Tangled (collection org.tangled.key.sshKey): the knot lets them push."""
    did, pds, jwt = atproto()
    coll = "org.tangled.key.sshKey"
    recs = http("GET", f"{pds}/xrpc/com.atproto.repo.listRecords", jwt,
                query={"repo": did, "collection": coll, "limit": 100})["records"]
    for r in recs:
        if remove and r["value"].get("publicKey", "").split()[:2] == remove.split()[:2]:
            http("POST", f"{pds}/xrpc/com.atproto.repo.deleteRecord", jwt,
                 body={"repo": did, "collection": coll, "rkey": r["uri"].rsplit("/", 1)[1]})
    if add and not any(r["value"].get("publicKey", "").split()[:2] == add.split()[:2] for r in recs):
        http("POST", f"{pds}/xrpc/com.atproto.repo.createRecord", jwt, body={
            "repo": did, "collection": coll,
            "record": {"$type": coll, "publicKey": add, "name": "sov owner key", "createdAt": now()}})


def now():
    return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")


def keys(sync=False):
    """The owner's keys in every place that accepts them, compared with spec.owner.sshKeys."""
    want = {k.split()[1]: k for k in spec("owner.sshKeys")}
    host = HOST_PUB.read_text().split()[1]
    places = {}
    places["server login"] = {k.split()[1]: k for k in ssh("cat /etc/ssh/authorized_keys.d/admin").splitlines() if k.strip()}
    box = box_keys_lines()
    places["source bucket"] = {ln.split("ssh-")[1].split()[1]: ln for ln in box if "ssh-" in ln and host not in ln}
    did, pds, jwt = atproto()
    recs = http("GET", f"{pds}/xrpc/com.atproto.repo.listRecords", jwt,
                query={"repo": did, "collection": "org.tangled.key.sshKey", "limit": 100})["records"]
    places["Tangled"] = {r["value"]["publicKey"].split()[1]: r["value"]["publicKey"] for r in recs}
    gh = github("user/keys")
    if gh is not None:
        places["GitHub"] = {k["key"].split()[1]: f"{k['key']} ({k['title']})" for k in gh}
    clean = True
    for place, have in places.items():
        missing = [k for k in want if k not in have]
        extra = [k for k in have if k not in want]
        clean &= not missing and not extra
        say(f"{place:14} {'ok' if not missing and not extra else ''}")
        for k in missing:
            say(f"  missing: {want[k][:60]}")
        for k in extra:
            say(f"  not in the spec: {have[k][:80]}")
        if sync and (missing or extra):
            sync_place(place, [want[k] for k in missing], [have[k] for k in extra])
    if gh is None:
        say("GitHub         (not checked: gh lacks admin:public_key; gh auth refresh -h github.com -s admin:public_key)")
    return clean


def sync_place(place, add, remove):
    if remove and input(f"  remove {len(remove)} key(s) from {place}? [y/N] ").strip().lower() != "y":
        remove = []
    if place == "server login":
        say("  (the server's login keys follow the spec at every deploy)")
    elif place == "source bucket":
        gone = {k.split("ssh-")[1].split()[1] for k in remove}
        box_keys(lambda lines: [ln for ln in lines if not ("ssh-" in ln and ln.split("ssh-")[1].split()[1] in gone)]
                 + [k for k in add])
    elif place == "Tangled":
        for k in add:
            tangled_keys(add=k)
        for k in remove:
            tangled_keys(remove=k)
    elif place == "GitHub":
        for k in add:
            github_keys(add=k)
        for k in remove:
            github_keys(remove=k.split(" (")[0])
    say(f"  {place}: in line with the spec")


def key_rotate():
    keyfile = Path(run("git", "-C", str(ROOT), "config", "user.signingkey").strip()).expanduser()
    real = keyfile.resolve()
    old_pub = pub_of(keyfile)
    spec_file = ROOT / "spec" / "default.nix"
    text = spec_file.read_text()
    m = re.search(r'"' + re.escape(old_pub) + r'[^"]*"', text)
    if not m:
        fail(f"the current key ({keyfile}) is not in spec.owner.sshKeys")
    box = spec("backup.box")
    with tempfile.TemporaryDirectory() as d:
        new, old = Path(d) / "new", Path(d) / "old"
        run("ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "sov owner key", "-f", str(new))
        new_pub = (Path(d) / "new.pub").read_text().strip()
        old.write_bytes(real.read_bytes())
        old.chmod(0o600)
        say("owner key: registering the new key next to the old one ...")
        run("ssh-copy-id", "-s", "-p", "23", "-f", "-i", f"{new}.pub", "-o", f"IdentityFile={old}",
            "-o", "IdentitiesOnly=yes", f"{box['user']}@{box['host']}")
        tangled_keys(add=new_pub)
        spec_file.write_text(text.replace(m.group(0), f'{m.group(0)} "{new_pub}"'))
        _cache.clear()
        rev = commit("owner key: add the next key", write_sops_config() + [spec_file])
        wait_deployed(rev)

        say("owner key: switching to the new key ...")
        real.write_bytes(new.read_bytes())
        allowed = ROOT / ".git" / "allowed_signers"
        if allowed.exists():
            allowed.chmod(0o644)
            allowed.write_text(f'{spec("owner.email")} namespaces="git" {" ".join(new_pub.split()[:2])}\n')
        spec_file.write_text(spec_file.read_text().replace(f'{m.group(0)} ', ""))
        _cache.clear()
        rev = commit("owner key: the old key is retired", write_sops_config() + [spec_file])
        wait_deployed(rev)
        box_keys(lambda lines: [ln for ln in lines if old_pub.split()[1] not in ln])
        tangled_keys(remove=old_pub)
        github_ok = github_keys(add=new_pub, remove=old_pub)
        fp = run("ssh-keygen", "-l", "-f", str(old)).split()[1]
    say(f"owner key: rotated. {keyfile} is the new key; the server, the source bucket, Tangled"
        + (" and GitHub accept only it." if github_ok else
           f" accept only it.\nOne step on GitHub (gh lacks the permission admin:public_key):\n"
           f"https://github.com/settings/keys: delete the key {fp}, add:\n  {new_pub}"))


def github_keys(add=None, remove=None):
    """The owner's SSH keys on GitHub (login and pushing over SSH). False if gh may not change them."""
    keys = github("user/keys")
    if keys is None:
        return False
    for k in keys:
        if remove and k["key"].split()[:2] == remove.split()[:2]:
            github(f"user/keys/{k['id']}", "-X", "DELETE")
    if add and not any(k["key"].split()[:2] == add.split()[:2] for k in keys):
        github("user/keys", "-f", "title=sov owner key", "-f", f"key={add}")
    return True


# ------------------------------------------------------------------ the paper recovery key

LINE = 16


def checksum(i, chunk):
    return hashlib.sha256(f"{i}:{chunk}".encode()).hexdigest()[:4]


def paper_print(out):
    identity = [ln for ln in run("age-keygen").splitlines() if ln.startswith("AGE-SECRET-KEY-")][0]
    recipient = run("age-keygen", "-y", input=identity).strip()
    say("paper: a new recovery key ...")
    before = {k["id"] for k in json.loads(ssh("sudo sov-restic key list --json"))}
    ssh("sudo sov-restic key add --user recovery --host paper --new-password-file /dev/stdin -q", input=identity)
    for k in json.loads(ssh("sudo sov-restic key list --json")):  # the previous paper key stops working
        if k.get("userName") == "recovery" and k["id"] in before:
            ssh(f"sudo sov-restic key remove {k['id']} -q")
    RECOVERY_PUB.write_text(recipient + "\n")
    commit("paper: a new recovery key", write_sops_config() + [RECOVERY_PUB])

    chunks = [identity[i:i + LINE] for i in range(0, len(identity), LINE)]
    rows = "".join(f"<tr><td>{i + 1}</td><td><code>{c}</code></td><td><code>{checksum(i + 1, c)}</code></td></tr>"
                   for i, c in enumerate(chunks))
    qr = run("qrencode", "-t", "SVG", "-l", "M", "-m", "2", "-o", "-", input=identity)
    s, box = spec(), spec("backup.box")
    mirrors = "".join(f"<li>{n}: https://{s['services']['knot']['hostname']}/{r['did']}"
                      + (f" or github.com/{r['mirror']['owner']}/{r['mirror']['name']}" if r.get("mirror") else "")
                      + "</li>" for n, r in s["repos"].items() if n in (s["specRepo"], "sov"))
    page = f"""<!doctype html><meta charset=utf-8><title>{html.escape(s['server']['hostname'])}: paper recovery key</title>
<style>body{{font:11pt/1.45 system-ui,sans-serif;max-width:180mm;margin:12mm auto;color:#000;background:#fff}}
h1{{font-size:16pt}}h2{{font-size:12pt;margin-top:7mm}}td{{padding:1mm 4mm 1mm 0}}code{{font:12pt ui-monospace,monospace}}
.k{{display:flex;gap:8mm;align-items:center}}svg{{width:50mm;height:auto}}li{{margin:1mm 0}}</style>
<h1>{html.escape(s['server']['hostname'])}: paper recovery key</h1>
<p>Printed {date.today().isoformat()}. This key alone decrypts every secret of the instance and opens its source
bucket. Keep it private and offline. Check it once a year: <code>sov paper check</code>.</p>
<div class=k>{qr}<table>{rows}</table></div>
<p>Each line ends with a check code, so a typo is caught at once when you type it back in.</p>
<h2>If everything else is lost</h2>
<ol>
<li>The repositories: {mirrors and f"<ul>{mirrors}</ul>"} Or, if those are gone too, from the source bucket (next step):
it holds the knot with all repositories.</li>
<li>The source bucket: Hetzner Storage Box <b>{html.escape(box['user'])}@{html.escape(box['host'])}</b>, folder
<b>{html.escape(box['dir'])}</b>, a restic repository. Log in to Hetzner's console to reach it (enable SSH or a password
for the box there), then <code>restic -r sftp:{html.escape(box['user'])}@{html.escape(box['host'])}:{html.escape(box['dir'])} restore latest</code>
with this key as the password.</li>
<li>In the instance repository: make the key available to sops
(<code>export SOPS_AGE_KEY=&lt;the key&gt;</code>), then <code>nix run .#install -- &lt;user&gt;@&lt;new server&gt;</code>.
The new server restores everything else by itself.</li>
<li>Then make a new owner key: <code>sov key rotate</code>, and a new paper key: <code>sov paper print</code>.</li>
</ol>"""
    Path(out).write_text(page)
    os.chmod(out, 0o600)
    shown = show_file(out)
    say(f"paper: the sheet is {shown}\n  it opens in your browser: print it, then delete the file")


def wsl():
    return "microsoft" in Path("/proc/version").read_text().lower() if Path("/proc/version").exists() else False


def downloads():
    """Where the user finds files: the Windows Downloads folder under WSL, else ~/Downloads or ~."""
    if wsl():
        home = run("cmd.exe", "/c", "echo %USERPROFILE%", check=False).strip()
        if home:
            d = Path(run("wslpath", home).strip()) / "Downloads"
            if d.is_dir():
                return d
    d = Path.home() / "Downloads"
    return d if d.is_dir() else Path.home()


def show_file(path):
    """Open a file in the user's default program; return its path as the user sees it."""
    if wsl():
        win = run("wslpath", "-w", str(path)).strip()
        subprocess.run(["cmd.exe", "/c", "start", "", win], capture_output=True)
        return win
    for opener in (["xdg-open"], ["open"]):
        try:
            subprocess.Popen([*opener, str(path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            break
        except FileNotFoundError:
            continue
    return str(path)


def paper_check():
    want = RECOVERY_PUB.read_text().strip() if RECOVERY_PUB.exists() else fail("no paper key yet (sov paper print)")
    say("Type each line of the paper key with its check code, e.g.  AGE-SECRET-KEY-1  9f3a  (input hidden)")
    identity, i = "", 1
    while len(identity) < 74:  # AGE-SECRET-KEY-1 + 58 characters
        line = getpass.getpass(f"  line {i}: ").split()
        if len(line) != 2 or checksum(i, line[0].upper()) != line[1].lower():
            say("  that line does not match its check code, again")
            continue
        identity += line[0].upper()
        i += 1
    if run("age-keygen", "-y", input=identity, check=False).strip() != want:
        fail("this is not the current paper key (secrets/recovery.pub)")
    with tempfile.TemporaryDirectory() as home:  # only the paper key, not the owner's key in ~/.ssh
        env = {k: v for k, v in os.environ.items() if not k.startswith("SOPS_AGE")}
        env.update(HOME=home, SOPS_AGE_KEY=identity)
        run("sops", "--decrypt", "--extract", '["restic_password"]', str(FILES["server"]), env=env)
    ssh("sudo sov-restic --password-file /dev/stdin cat config >/dev/null", input=identity)
    ssh("sudo systemctl start sov-paper-checked")
    say("paper: the key decrypts the secrets and opens the source bucket. Reminder reset for a year.")


# ------------------------------------------------------------------ once per new thing

def add_repo(name, description):
    owner, knot = spec("owner"), spec("services.knot.hostname")
    did, pds, jwt = atproto()
    have = http("GET", f"{pds}/xrpc/com.atproto.repo.listRecords", jwt,
                query={"repo": did, "collection": "org.tangled.repo.declaration", "limit": 100})
    for rec in have.get("records", []):
        if rec["uri"].rsplit("/", 1)[1] == name:
            say(f'{name} exists already:  repos."{name}" = {{ did = "{rec["value"]["repo"]}"; description = "{description}"; }};')
            return
    nsid = "org.tangled.repo.createRepo"
    auth = http("GET", f"{pds}/xrpc/com.atproto.server.getServiceAuth", jwt,
                query={"aud": f"did:web:{knot}", "lxm": nsid, "exp": int(time.time()) + 60})["token"]
    repo = http("POST", f"https://{knot}/xrpc/{nsid}", auth,
                body={"name": name, "defaultBranch": "main", "description": description})["did"]
    http("POST", f"{pds}/xrpc/com.atproto.repo.putRecord", jwt, body={
        "repo": owner["did"], "collection": "org.tangled.repo.declaration", "rkey": name,
        "record": {"$type": "org.tangled.repo.declaration", "repo": repo, "createdAt": now()}})
    say(f'created. Add to the spec:  repos."{name}" = {{ did = "{repo}"; description = "{description}"; }};')


def add_zone(zone):
    if zone not in spec("dns.records"):
        fail(f"{zone} is not in spec.dns.records")
    server = spec("server")
    token, tsig = secret("server", "hetzner_api_token"), secret("server", "dns_secondaries_secret")
    api = "https://api.hetzner.cloud/v1/zones"
    if http("GET", api, token, query={"name": zone})["zones"]:
        say(f"{zone}: Hetzner secondary exists already")
        return
    z = http("POST", api, token, body={"name": zone, "mode": "secondary", "primary_nameservers": [
        {"address": a, "tsig_algorithm": "hmac-sha256", "tsig_key": tsig} for a in (server["ipv4"], server["ipv6"])]})["zone"]
    say(f"{zone}: Hetzner secondary created ({z['status']}); nameservers: {' '.join(z['authoritative_nameservers']['assigned'])}")


def add_domain(args):
    prices, check = "--prices" in args, "--check" in args
    domains = [a for a in args if not a.startswith("--")]
    ns = spec("dns.nameservers")
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(CookieJar()))  # INWX session

    def call(method, **params):
        req = urllib.request.Request("https://api.domrobot.com/jsonrpc/", headers={"content-type": "application/json"},
                                     data=json.dumps({"method": method, "params": params}).encode())
        with opener.open(req, timeout=60) as r:
            res = json.loads(r.read())
        if res.get("code", 0) >= 2000:
            raise RuntimeError(f"{method}: {res.get('code')} {res.get('msg')} {res.get('reason', '')}".strip())
        return res.get("resData", {})

    login = call("account.login", user=input("INWX username: ").strip(), **{"pass": getpass.getpass("INWX password: ")})
    if str(login.get("tfa", "0")) != "0":
        call("account.unlock", tan=input("INWX 2FA code: ").strip())
    if prices:
        entries = {e.get("tld", "").lstrip("."): e for e in (lambda r: r.get("price") or r.get("pricelist") or [])(
            call("domain.getPrices", tld=sorted({d.split(".", 1)[1] for d in domains})))}
        for d in domains:
            e = entries.get(d.split(".", 1)[1], {})
            say(f"{d:22} transfer {float(e.get('transferPrice') or 0):8.2f}  renewal {float(e.get('renewalPrice') or 0):8.2f} {e.get('currency', '')}")
        return
    contacts = call("contact.list").get("contact", [])
    if not contacts:
        fail("no contact in the INWX account yet: add your address in INWX once, then run again")
    c = contacts[0]["roId"]
    for d in domains:
        params = dict(domain=d, authCode=getpass.getpass(f"auth code for {d}: ").strip(), ns=ns,
                      registrant=c, admin=c, tech=c, billing=c)
        try:
            res = call("domain.transfer", testing=check, **params)
            say(f"  {'ok  ' if check else 'sent'}  {d}: {res.get('status', 'transfer started') if not check else 'would work'}")
        except RuntimeError as e:
            say(f"  FAIL  {d}: {e}")
    call("account.logout")


# ------------------------------------------------------------------ main

def main(a):
    cmd = " ".join(a[:2])
    if a[:1] == ["status"]:
        sys.exit(0 if status() else 1)
    if a[:1] == ["drill"]:
        sys.exit(0 if drill() else 1)
    if cmd == "secrets check":
        sys.exit(0 if check_secrets() else 1)
    if a == ["keys"]:
        sys.exit(0 if keys() else 1)
    if cmd == "keys sync":
        sys.exit(0 if keys(sync=True) else 1)
    if cmd == "secrets copy" and len(a) == 3:
        return copy_secret(a[2])
    if a[:1] == ["secrets"]:
        return list_secrets()
    if a[:1] == ["rotate"] and len(a) > 1:
        return rotate(a[1:])
    if cmd == "key rotate":
        return key_rotate()
    if cmd == "paper print":
        out = a[a.index("--out") + 1] if "--out" in a else str(downloads() / f"sov-paper-{spec('server.hostname')}.html")
        return paper_print(out)
    if cmd == "paper check":
        return paper_check()
    if cmd == "add repo" and len(a) == 4:
        return add_repo(a[2], a[3])
    if cmd == "add zone" and len(a) == 3:
        return add_zone(a[2])
    if cmd == "add domain" and len(a) > 2:
        return add_domain(a[2:])
    if a[:1] == ["install"] and len(a) == 2:
        os.execvp("nix", ["nix", "run", f"{ROOT}#install", "--", a[1]])
    say(__doc__)
    sys.exit(0 if a[:1] in ([], ["help"], ["--help"], ["-h"]) else 2)


if __name__ == "__main__":
    main(sys.argv[1:])