#!/usr/bin/env bash # Checks a deploy of a didbot server against a baseline taken before it. # # scripts/verify-deploy.sh before [options] # scripts/verify-deploy.sh after [options] [--revision ] [--post-as ] # # --zone the server's zone (default pds.did.bot) # --state where `before` saves the baseline and `after` reads it # (default ~/.local/state/didbot/verify-deploy-.json) # --name the instance's Name tag (default didbot-pds) # --baseline-hours before: hours of journal the warning baseline covers (default 24) # --revision after: the commit the server should run (default HEAD) # --outage-budget after: the longest restart allowed, in seconds (default 2) # --max-rss-mib after: the most anonymous memory the process may hold (default 512) # --post-as after: also write one scrobble as with `vibescrobble post`, # then read it back from the server # --post-wait after: how long to wait for the post to read back (default 15) # # `before` records the time, the revision served, the process's memory, and # the kinds of warning the journal logged over the baseline window. `after` # checks, since that time: # # - /xrpc/_health reports ; # - each restart's outage, from `no longer accepting` to `listening (tls)`, # is within the budget, and no process started without a clean stop; # - the journal logged no warning kind the baseline lacks; # - the process's anonymous memory is under the ceiling. # # A warning's kind is its level, target and message, without its fields. # Everything is read-only except --post-as, which publishes a public record. # The journal and memory are read over SSM with the caller's own AWS # credentials (AWS_PROFILE, AWS_REGION), through scripts/ssm-run.py. set -euo pipefail cd "$(dirname "$0")/.." usage() { echo "usage: scripts/verify-deploy.sh before|after [--zone Z] [--state FILE] [--name TAG] [--baseline-hours H] [--revision REV] [--outage-budget S] [--max-rss-mib MIB] [--post-as DID] [--post-wait S]" } unit=didbot-pds container=didbot-pds mode="${1:-}" case "$mode" in before | after) shift ;; -h | --help) usage exit 0 ;; *) usage >&2 exit 2 ;; esac zone=pds.did.bot state="" name=didbot-pds baseline_hours=24 rev=HEAD outage_budget=2 max_rss_mib=512 post_as="" post_wait=15 while [ $# -gt 0 ]; do [ $# -ge 2 ] || { usage >&2 exit 2 } case "$1" in --zone) zone="$2" ;; --state) state="$2" ;; --name) name="$2" ;; --baseline-hours) baseline_hours="$2" ;; --revision) rev="$2" ;; --outage-budget) outage_budget="$2" ;; --max-rss-mib) max_rss_mib="$2" ;; --post-as) post_as="$2" ;; --post-wait) post_wait="$2" ;; *) usage >&2 exit 2 ;; esac shift 2 done state="${state:-${XDG_STATE_HOME:-$HOME/.local/state}/didbot/verify-deploy-$zone.json}" for tool in curl jq uv; do command -v "$tool" >/dev/null || { echo "verify-deploy: $tool is not on PATH" >&2 exit 2 } done ssm() { scripts/ssm-run.py --name "$name" --timeout 150 "$1" } # Shell run on the instance. `since` is prepended by the caller. It prints # `key value` lines, then sections headed `== `. read -r -d '' remote <<'EOF' || true kinds() { sed -E 's/\x1b\[3m.*//; s/\x1b\[[0-9;]*m//g' | grep -E '^ *[0-9.]+s +(WARN|ERROR) ' | sed -E 's/^ *[0-9.]+s +//; s/ +$//; s/[0-9a-f]{8,}/X/g; s/did:[a-z]+:[A-Za-z0-9.:%-]+/DID/g; s/[0-9]+/N/g' | cut -c1-160 | sort | uniq -c } echo "now $(date +%s)" pid="$(docker inspect --format '{{.State.Pid}}' "$container")" awk '/^(VmRSS|RssAnon):/ { print $1, $2 }' "/proc/$pid/status" | tr -d : echo "== restarts" journalctl -u "$unit" -o short-unix --since "@$since" | grep -E 'no longer accepting|listening \(tls\)' | awk '{ print $1, (/no longer accepting/ ? "stop" : "listen") }' echo "== warnings" journalctl -u "$unit" -o cat --since "@$since" ${until:+--until "@$until"} | kinds EOF # The `key value` lines, or one section, of the remote's output. field() { awk -v k="$1" '/^== /{exit} $1==k {print $2}' <<<"$2"; } section() { awk -v s="== $1" '/^== /{on=($0==s); next} on' <<<"$2"; } kinds_json() { jq -R -s 'split("\n") | map(capture("^ *(?[0-9]+) (?.+)$")? // empty) | map({(.k): (.n | tonumber)}) | add // {}' } mkdir -p "$(dirname "$state")" if [ "$mode" = before ]; then now="$(date +%s)" since=$((now - baseline_hours * 3600)) out="$(ssm "since=$since; until=$now; unit=$unit; container=$container $remote")" serving="$(curl -fsS --max-time 10 "https://$zone/xrpc/_health" | jq -r '.version // empty')" jq -n \ --arg zone "$zone" \ --argjson time "$(field now "$out")" \ --arg revision "$serving" \ --argjson rss_anon_kib "$(field RssAnon "$out")" \ --argjson vm_rss_kib "$(field VmRSS "$out")" \ --argjson baseline_hours "$baseline_hours" \ --argjson warnings "$(section warnings "$out" | kinds_json)" \ '{zone: $zone, time: $time, revision: $revision, memory: {rss_anon_kib: $rss_anon_kib, vm_rss_kib: $vm_rss_kib}, baseline_hours: $baseline_hours, warnings: $warnings}' >"$state" jq -r '"verify-deploy: baseline for \(.zone) at \(.time | todate)", " revision \(.revision), RssAnon \(.memory.rss_anon_kib / 1024 | floor) MiB", " \(.warnings | length) warning kinds over \(.baseline_hours) h:", (.warnings | to_entries[] | " \(.value)\t\(.key)")' "$state" echo " saved to $state" exit 0 fi [ -f "$state" ] || { echo "verify-deploy: no baseline at $state; run \`scripts/verify-deploy.sh before\` first" >&2 exit 2 } [ "$(jq -r .zone "$state")" = "$zone" ] || { echo "verify-deploy: $state is a baseline for $(jq -r .zone "$state"), not $zone" >&2 exit 2 } failures=() ok() { echo "ok $1"; } fail() { echo "FAIL $1" failures+=("$1") } # Revision. _health reports a short sha; compare on the shorter of the two. want="$(git rev-parse --verify --quiet "$rev^{commit}" 2>/dev/null || echo "$rev")" serving="$(curl -fsS --max-time 10 "https://$zone/xrpc/_health" | jq -r '.version // empty' || true)" if [ -n "$serving" ] && { [[ $want == "$serving"* ]] || [[ $serving == "$want"* ]]; }; then ok "revision: _health reports $serving (was $(jq -r .revision "$state"))" else fail "revision: _health reports ${serving:-nothing}, want ${want:0:12}" fi since="$(jq -r .time "$state")" out="$(ssm "since=$since; unit=$unit; container=$container $remote")" # Restarts: pair each `stop` with the `listen` after it. restarts="$(section restarts "$out" | awk -v budget="$outage_budget" ' $2 == "stop" { stop = $1; next } $2 == "listen" && stop != "" { gap = $1 - stop printf "%s %.3f %d\n", (gap > budget ? "over" : "within"), gap, stop stop = ""; next } $2 == "listen" { printf "unclean 0 %d\n", $1 } END { if (stop != "") printf "down 0 %d\n", stop }')" if [ -z "$restarts" ]; then ok "outage: no restart since $(date -u -d "@$since" +%T)" fi while read -r verdict gap at; do [ -n "$verdict" ] || continue at="$(date -u -d "@$at" +%T)" case "$verdict" in within) ok "outage: ${gap} s at $at UTC (budget ${outage_budget} s)" ;; over) fail "outage: ${gap} s at $at UTC, over the ${outage_budget} s budget" ;; unclean) fail "outage: a process started at $at UTC without a clean stop before it" ;; down) fail "outage: stopped accepting at $at UTC and has not logged \`listening (tls)\` since" ;; esac done <<<"$restarts" # Warning kinds the baseline lacks. new="$(section warnings "$out" | kinds_json | jq -r --slurpfile base "$state" \ 'to_entries[] | select($base[0].warnings[.key] == null) | " \(.value)\t\(.key)"')" total="$(section warnings "$out" | kinds_json | jq 'add // 0')" if [ -z "$new" ]; then ok "warnings: $total since the baseline, all of kinds it holds" else fail "warnings: kinds the baseline lacks:" echo "$new" fi # Memory. rss="$(field RssAnon "$out")" before_rss="$(jq .memory.rss_anon_kib "$state")" if [ -z "$rss" ]; then fail "memory: could not read the process's RssAnon" elif [ $((rss / 1024)) -lt "$max_rss_mib" ]; then ok "memory: RssAnon $((rss / 1024)) MiB (was $((before_rss / 1024)) MiB, ceiling $max_rss_mib MiB)" else fail "memory: RssAnon $((rss / 1024)) MiB, over the $max_rss_mib MiB ceiling" fi # A write through a signed-in session, read back from the server. if [ -n "$post_as" ]; then uri="$(vibescrobble post --as "$post_as" --emoji '🩺' \ --text "post-deploy check: $zone serves ${serving:-?}" 2>&1 | grep -o 'at://[^ ]*' | tail -1 || true)" if [ -z "$uri" ]; then fail "post: vibescrobble post as $post_as returned no record" else IFS=/ read -r _ _ repo collection rkey <<<"$uri" got="" for _ in $(seq "$post_wait"); do got="$(curl -fs --max-time 10 \ "https://$zone/xrpc/com.atproto.repo.getRecord?repo=$repo&collection=$collection&rkey=$rkey" | jq -r '.uri // empty' || true)" [ "$got" != "$uri" ] || break sleep 1 done if [ "$got" = "$uri" ]; then ok "post: wrote and read back $uri" else fail "post: wrote $uri, but $zone did not return it within $post_wait s" fi fi fi if [ ${#failures[@]} -gt 0 ]; then echo "verify-deploy: ${#failures[@]} of the checks on $zone failed" >&2 exit 1 fi echo "verify-deploy: $zone passed every check"