Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260#!/usr/bin/env bash# Checks a deploy of a didbot server against a baseline taken before it.## scripts/verify-deploy.sh before [options]# scripts/verify-deploy.sh after [options] [--revision <rev>] [--post-as <did>]## --zone <zone> the server's zone (default pds.did.bot)# --state <file> where `before` saves the baseline and `after` reads it# (default ~/.local/state/didbot/verify-deploy-<zone>.json)# --name <tag> the instance's Name tag (default didbot-pds)# --baseline-hours <h> before: hours of journal the warning baseline covers (default 24)# --revision <rev> after: the commit the server should run (default HEAD)# --outage-budget <s> after: the longest restart allowed, in seconds (default 2)# --max-rss-mib <mib> after: the most anonymous memory the process may hold (default 512)# --post-as <did> after: also write one scrobble as <did> with `vibescrobble post`,# then read it back from the server# --post-wait <s> after: how long to wait for the post to read back (default 15)## `before` records the time, the revision served, the process's memory, and# the kinds of warning the journal logged over the baseline window. `after`# checks, since that time:## - /xrpc/_health reports <rev>;# - each restart's outage, from `no longer accepting` to `listening (tls)`,# is within the budget, and no process started without a clean stop;# - the journal logged no warning kind the baseline lacks;# - the process's anonymous memory is under the ceiling.## A warning's kind is its level, target and message, without its fields.# Everything is read-only except --post-as, which publishes a public record.# The journal and memory are read over SSM with the caller's own AWS# credentials (AWS_PROFILE, AWS_REGION), through scripts/ssm-run.py.set -euo pipefail
cd "$(dirname "$0")/.."
usage() { echo "usage: scripts/verify-deploy.sh before|after [--zone Z] [--state FILE] [--name TAG] [--baseline-hours H] [--revision REV] [--outage-budget S] [--max-rss-mib MIB] [--post-as DID] [--post-wait S]"}
unit=didbot-pdscontainer=didbot-pds
mode="${1:-}"case "$mode" in before | after) shift ;; -h | --help) usage exit 0 ;; *) usage >&2 exit 2 ;;esac
zone=pds.did.botstate=""name=didbot-pdsbaseline_hours=24rev=HEADoutage_budget=2max_rss_mib=512post_as=""post_wait=15while [ $# -gt 0 ]; do [ $# -ge 2 ] || { usage >&2 exit 2 } case "$1" in --zone) zone="$2" ;; --state) state="$2" ;; --name) name="$2" ;; --baseline-hours) baseline_hours="$2" ;; --revision) rev="$2" ;; --outage-budget) outage_budget="$2" ;; --max-rss-mib) max_rss_mib="$2" ;; --post-as) post_as="$2" ;; --post-wait) post_wait="$2" ;; *) usage >&2 exit 2 ;; esac shift 2donestate="${state:-${XDG_STATE_HOME:-$HOME/.local/state}/didbot/verify-deploy-$zone.json}"
for tool in curl jq uv; do command -v "$tool" >/dev/null || { echo "verify-deploy: $tool is not on PATH" >&2 exit 2 }done
ssm() { scripts/ssm-run.py --name "$name" --timeout 150 "$1"}
# Shell run on the instance. `since` is prepended by the caller. It prints# `key value` lines, then sections headed `== <name>`.read -r -d '' remote <<'EOF' || truekinds() { sed -E 's/\x1b\[3m.*//; s/\x1b\[[0-9;]*m//g' | grep -E '^ *[0-9.]+s +(WARN|ERROR) ' | sed -E 's/^ *[0-9.]+s +//; s/ +$//; s/[0-9a-f]{8,}/X/g; s/did:[a-z]+:[A-Za-z0-9.:%-]+/DID/g; s/[0-9]+/N/g' | cut -c1-160 | sort | uniq -c}echo "now $(date +%s)"pid="$(docker inspect --format '{{.State.Pid}}' "$container")"awk '/^(VmRSS|RssAnon):/ { print $1, $2 }' "/proc/$pid/status" | tr -d :echo "== restarts"journalctl -u "$unit" -o short-unix --since "@$since" | grep -E 'no longer accepting|listening \(tls\)' | awk '{ print $1, (/no longer accepting/ ? "stop" : "listen") }'echo "== warnings"journalctl -u "$unit" -o cat --since "@$since" ${until:+--until "@$until"} | kindsEOF
# The `key value` lines, or one section, of the remote's output.field() { awk -v k="$1" '/^== /{exit} $1==k {print $2}' <<<"$2"; }section() { awk -v s="== $1" '/^== /{on=($0==s); next} on' <<<"$2"; }kinds_json() { jq -R -s 'split("\n") | map(capture("^ *(?<n>[0-9]+) (?<k>.+)$")? // empty) | map({(.k): (.n | tonumber)}) | add // {}'}
mkdir -p "$(dirname "$state")"
if [ "$mode" = before ]; then now="$(date +%s)" since=$((now - baseline_hours * 3600)) out="$(ssm "since=$since; until=$now; unit=$unit; container=$container$remote")" serving="$(curl -fsS --max-time 10 "https://$zone/xrpc/_health" | jq -r '.version // empty')" jq -n \ --arg zone "$zone" \ --argjson time "$(field now "$out")" \ --arg revision "$serving" \ --argjson rss_anon_kib "$(field RssAnon "$out")" \ --argjson vm_rss_kib "$(field VmRSS "$out")" \ --argjson baseline_hours "$baseline_hours" \ --argjson warnings "$(section warnings "$out" | kinds_json)" \ '{zone: $zone, time: $time, revision: $revision, memory: {rss_anon_kib: $rss_anon_kib, vm_rss_kib: $vm_rss_kib}, baseline_hours: $baseline_hours, warnings: $warnings}' >"$state" jq -r '"verify-deploy: baseline for \(.zone) at \(.time | todate)", " revision \(.revision), RssAnon \(.memory.rss_anon_kib / 1024 | floor) MiB", " \(.warnings | length) warning kinds over \(.baseline_hours) h:", (.warnings | to_entries[] | " \(.value)\t\(.key)")' "$state" echo " saved to $state" exit 0fi
[ -f "$state" ] || { echo "verify-deploy: no baseline at $state; run \`scripts/verify-deploy.sh before\` first" >&2 exit 2}[ "$(jq -r .zone "$state")" = "$zone" ] || { echo "verify-deploy: $state is a baseline for $(jq -r .zone "$state"), not $zone" >&2 exit 2}
failures=()ok() { echo "ok $1"; }fail() { echo "FAIL $1" failures+=("$1")}
# Revision. _health reports a short sha; compare on the shorter of the two.want="$(git rev-parse --verify --quiet "$rev^{commit}" 2>/dev/null || echo "$rev")"serving="$(curl -fsS --max-time 10 "https://$zone/xrpc/_health" | jq -r '.version // empty' || true)"if [ -n "$serving" ] && { [[ $want == "$serving"* ]] || [[ $serving == "$want"* ]]; }; then ok "revision: _health reports $serving (was $(jq -r .revision "$state"))"else fail "revision: _health reports ${serving:-nothing}, want ${want:0:12}"fi
since="$(jq -r .time "$state")"out="$(ssm "since=$since; unit=$unit; container=$container$remote")"
# Restarts: pair each `stop` with the `listen` after it.restarts="$(section restarts "$out" | awk -v budget="$outage_budget" ' $2 == "stop" { stop = $1; next } $2 == "listen" && stop != "" { gap = $1 - stop printf "%s %.3f %d\n", (gap > budget ? "over" : "within"), gap, stop stop = ""; next } $2 == "listen" { printf "unclean 0 %d\n", $1 } END { if (stop != "") printf "down 0 %d\n", stop }')"if [ -z "$restarts" ]; then ok "outage: no restart since $(date -u -d "@$since" +%T)"fiwhile read -r verdict gap at; do [ -n "$verdict" ] || continue at="$(date -u -d "@$at" +%T)" case "$verdict" in within) ok "outage: ${gap} s at $at UTC (budget ${outage_budget} s)" ;; over) fail "outage: ${gap} s at $at UTC, over the ${outage_budget} s budget" ;; unclean) fail "outage: a process started at $at UTC without a clean stop before it" ;; down) fail "outage: stopped accepting at $at UTC and has not logged \`listening (tls)\` since" ;; esacdone <<<"$restarts"
# Warning kinds the baseline lacks.new="$(section warnings "$out" | kinds_json | jq -r --slurpfile base "$state" \ 'to_entries[] | select($base[0].warnings[.key] == null) | " \(.value)\t\(.key)"')"total="$(section warnings "$out" | kinds_json | jq 'add // 0')"if [ -z "$new" ]; then ok "warnings: $total since the baseline, all of kinds it holds"else fail "warnings: kinds the baseline lacks:" echo "$new"fi
# Memory.rss="$(field RssAnon "$out")"before_rss="$(jq .memory.rss_anon_kib "$state")"if [ -z "$rss" ]; then fail "memory: could not read the process's RssAnon"elif [ $((rss / 1024)) -lt "$max_rss_mib" ]; then ok "memory: RssAnon $((rss / 1024)) MiB (was $((before_rss / 1024)) MiB, ceiling $max_rss_mib MiB)"else fail "memory: RssAnon $((rss / 1024)) MiB, over the $max_rss_mib MiB ceiling"fi
# A write through a signed-in session, read back from the server.if [ -n "$post_as" ]; then uri="$(vibescrobble post --as "$post_as" --emoji '🩺' \ --text "post-deploy check: $zone serves ${serving:-?}" 2>&1 | grep -o 'at://[^ ]*' | tail -1 || true)" if [ -z "$uri" ]; then fail "post: vibescrobble post as $post_as returned no record" else IFS=/ read -r _ _ repo collection rkey <<<"$uri" got="" for _ in $(seq "$post_wait"); do got="$(curl -fs --max-time 10 \ "https://$zone/xrpc/com.atproto.repo.getRecord?repo=$repo&collection=$collection&rkey=$rkey" | jq -r '.uri // empty' || true)" [ "$got" != "$uri" ] || break sleep 1 done if [ "$got" = "$uri" ]; then ok "post: wrote and read back $uri" else fail "post: wrote $uri, but $zone did not return it within $post_wait s" fi fifi
if [ ${#failures[@]} -gt 0 ]; then echo "verify-deploy: ${#failures[@]} of the checks on $zone failed" >&2 exit 1fiecho "verify-deploy: $zone passed every check"