#!/usr/bin/env bash # Make vibescrobble.com serve a commit of this repository. # # scripts/deploy.sh # HEAD # scripts/deploy.sh v0.2.0 # any commit-ish: a sha, a tag, a branch # scripts/deploy.sh --dry-run # plan only; nothing is built, pushed or applied # scripts/deploy.sh --yes # apply without asking, for an unattended run # # A deploy is an apply. The commit is resolved to its short sha and passed to # tofu as `release_sha`; the apply writes it to the parameter the instance # polls, and the instance converges — pulls that image, unpacks that canvas, # restarts what changed. State records what is live, so `tofu -chdir=infra # output release_sha` answers it, and a rollback is this script with an # earlier commit. What the apply names has to exist first, which is # scripts/release.sh's job: this calls it, and it skips what is already built. # # The plan is saved and read before it is applied, so what runs is what was # on screen; the one confirmation is this script's, since applying a plan file # never prompts. Credentials are the caller's — whatever already works, as for # a hand-run apply — and have to be the did.bot account's; the profile is # jmm-did-bot-admin, the same one didbot deploys with. # # The first run stops once, on purpose, for the registrar: a zone is not a # delegation. It says what to do; run it again after. set -euo pipefail cd "$(dirname "$0")/.." stack=infra account=226086920337 profile=jmm-did-bot-admin domain=vibescrobble.com usage() { echo "usage: scripts/deploy.sh [--dry-run] [--yes] []" } dry_run="" assume_yes="" rev="" for arg in "$@"; do case "$arg" in -n | --dry-run) dry_run=1 ;; -y | --yes) assume_yes=1 ;; -h | --help) usage exit 0 ;; -*) usage >&2 exit 2 ;; *) if [ -n "$rev" ]; then usage >&2 exit 2 fi rev="$arg" ;; esac done rev="${rev:-HEAD}" say() { printf '\033[1m==> %s\033[0m\n' "$*"; } warn() { printf '\033[1;33m %s\033[0m\n' "$*" >&2; } for tool in tofu aws git jq dig curl; do command -v "$tool" >/dev/null || { echo "deploy: $tool is required and was not found on PATH" >&2 exit 1 } done # Fail before anything is built or applied, and in the right account: an # expired session and a different profile both stop here. read -r caller arn _ < <(aws sts get-caller-identity --query '[Account,Arn]' --output text 2>/dev/null) || { echo "deploy: no working AWS credentials; aws sso login --profile $profile" >&2 exit 1 } if [ "$caller" != "$account" ]; then echo "deploy: signed in to account $caller, not $account (did.bot); AWS_PROFILE=$profile" >&2 exit 1 fi say "deploying as $arn" sha="$(git rev-parse --verify --quiet --short=12 "$rev^{commit}")" || { echo "deploy: $rev is not a commit" >&2 exit 1 } if [ "$rev" = HEAD ] && [ -n "$(git status --porcelain --untracked-files=no)" ]; then warn "the working tree has uncommitted changes; deploying HEAD ($sha), which does not include them" fi # ----------------------------------------------------------------- the plan say "initialising $stack" tofu -chdir="$stack" init -input=false >/dev/null # What is live, or nothing at all before the first apply. `output -raw` cannot # tell the two apart: against a state with no outputs it exits zero and puts a # "No outputs found" warning on stdout, which reads as a release by that name. live="$(tofu -chdir="$stack" output -json | jq -r '.release_sha.value // empty')" # Artifacts before the pointer that names them, so the instance never sees a # release it cannot fetch. On the first run there is no bucket to put them in # until the apply has made one, so the order flips — see below. if [ -n "$live" ] && [ -z "$dry_run" ]; then scripts/release.sh "$sha" fi # A saved plan, so what is applied is what was read. It embeds resolved # values, so it lives outside the repository and not past this run. plan="$(mktemp -t vibescrobble-XXXXXX.tfplan)" trap 'rm -f "$plan"' EXIT # The upstream refs are the commit's too, read the way scripts/release.sh # reads them, so the apply names the images that release built. A rollback # takes its relay and Jetstream back with it. indigo_ref="$(scripts/pin.sh "$sha" indigo_ref)" jetstream_ref="$(scripts/pin.sh "$sha" jetstream_ref)" say "planning release $sha${live:+ (live: $live)}" rc=0 tofu -chdir="$stack" plan -input=false -detailed-exitcode -out="$plan" \ -var "release_sha=$sha" \ -var "indigo_ref=$indigo_ref" \ -var "jetstream_ref=$jetstream_ref" || rc=$? case "$rc" in 0) say "no changes: the stack already names $sha" ;; 2) ;; *) exit "$rc" ;; esac if [ -n "$dry_run" ]; then say "dry run: nothing built, nothing applied" exit 0 fi if [ "$rc" -eq 2 ]; then if [ -z "$assume_yes" ]; then printf '\napply this plan? [y/N] ' # Nothing on stdin — piped, or a cron that forgot --yes — is a no. read -r reply || reply="" case "$reply" in y | Y | yes | Yes) ;; *) echo "nothing applied" exit 1 ;; esac fi tofu -chdir="$stack" apply -input=false "$plan" fi if [ -z "$live" ]; then scripts/release.sh "$sha" fi # ------------------------------------------------------------------ the DNS # Creating a zone is not delegating to it, and nothing here can do that part. # Everything downstream — the certificate above all — fails in a way that # looks like a bug if this is skipped, so it is checked rather than assumed: # the registrar has to answer with one of this zone's own name servers. expected="$(tofu -chdir="$stack" output -json nameservers | jq -r '.[]' | sed 's/\.$//')" delegated="$(dig +short NS "$domain" @1.1.1.1 | sed 's/\.$//')" if ! grep -qxF -f <(printf '%s\n' "$expected") <<<"$delegated"; then say "$domain is not delegated to this zone yet" echo echo " Point the registrar for $domain at:" printf '%s\n' "$expected" | sed 's/^/ /' echo warn "then run this again. Nothing can get a certificate until it resolves." exit 0 fi # ------------------------------------------------------------- the converge instance="$(tofu -chdir="$stack" output -raw instance_id)" # The instance's poller would get there within thirty seconds on its own. # Asking directly is faster and, more to the point, brings its output back # here. The instance takes a minute or two to register with Session Manager # after its first boot, so the ask is retried until it is accepted; and the # converge script itself does not exist until user data has finished, so the # command waits for that too. say "converging the instance on $sha" command_id="" for _ in $(seq 20); do command_id="$(aws ssm send-command \ --instance-ids "$instance" \ --document-name AWS-RunShellScript \ --comment "vibescrobble deploy $sha" \ --parameters '{"commands":["until [ -x /usr/local/bin/vibescrobble-converge ]; do sleep 5; done","/usr/local/bin/vibescrobble-converge"]}' \ --query Command.CommandId --output text 2>/dev/null || true)" [ -n "$command_id" ] && break printf '.' sleep 15 done echo if [ -z "$command_id" ]; then warn "the instance is not reachable over Session Manager yet; leaving it to the poller" else while :; do status="$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance" \ --query Status --output text 2>/dev/null || echo Pending)" case "$status" in Success) break ;; Failed | Cancelled | TimedOut) aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance" \ --query '[StandardOutputContent, StandardErrorContent]' --output text | tail -40 warn "converge failed: $status. The poller retries every thirty seconds; to watch it:" echo " aws ssm start-session --target $instance" echo " sudo journalctl -u vibescrobble-converge -f" exit 1 ;; *) printf '.' sleep 10 ;; esac done echo aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance" \ --query StandardOutputContent --output text | sed 's/^/ /' fi # ------------------------------------------------------------------ the check # release.txt is written into the canvas tree and the tree goes live last, so # the site answering with this sha means the whole activation succeeded. say "waiting for https://$domain to serve $sha" serving="" for _ in $(seq 60); do serving="$(curl -sf --max-time 10 "https://$domain/release.txt" 2>/dev/null || true)" [ "$serving" = "$sha" ] && break printf '.' sleep 10 done echo if [ "$serving" != "$sha" ]; then warn "https://$domain is serving ${serving:-nothing} rather than $sha." warn "Certificates can take a few minutes, and the poller keeps trying. To watch it:" echo " aws ssm start-session --target $instance" echo " sudo journalctl -u vibescrobble-converge -f" exit 1 fi say "up: $sha" echo " canvas https://$domain" echo " relay https://relay.$domain" health="$(curl -sf --max-time 10 "https://$domain/health" || true)" echo " health ${health:-no answer}" case "$health" in *'"connected":true'*) ;; *) warn "the query service is not connected to the index yet; the not-indexing alarm watches for this" ;; esac echo echo " Nothing is indexed until a server asks to be crawled:" echo " curl -X POST https://relay.$domain/xrpc/com.atproto.sync.requestCrawl \\" echo " -H 'content-type: application/json' -d '{\"hostname\":\"pds.example\"}'" echo if [ "$(aws sns list-subscriptions-by-topic \ --topic-arn "$(tofu -chdir="$stack" output -raw alarms_topic)" \ --query "length(Subscriptions[?SubscriptionArn!='PendingConfirmation'])" \ --output text 2>/dev/null || echo 0)" = "0" ]; then warn "the alarm email is not confirmed yet: AWS sent a link, and until it is" warn "clicked no alarm will reach you." echo fi echo " A shell, without SSH:" echo " aws ssm start-session --target $instance"