dww.sh at main · zio.sh/didweb.watch

zio.sh / didweb.watch
🦋🤖 Bluesky bot to track did:web usage — https://bsky.app/profile/didweb.watch
didweb.watch / dww.sh
at main 11 kB view raw
  1#!/usr/bin/env bash
  2
  3# Sourcing
  4
  5unset _atfile_path
  6
  7#if [[ -d "$(dirname "$(realpath "$0")")/../.git" ]]; then
  8#    _atfile_path="$(dirname "$(realpath "$0")")/../atfile.sh"
  9#else
 10    _atfile_path="$(which atfile)"
 11#    [[ $? != 0 ]] && unset _atfile_path
 12#fi
 13
 14if [[ ! -f "$_atfile_path" ]]; then
 15    echo -e "\033[1;31mError: ATFile not found (download: https://zio.sh/atfile)\033[0m"
 16    exit 0
 17fi
 18
 19source "$_atfile_path"
 20
 21# Die
 22
 23function dww.die() {
 24    message="$1"
 25    atfile.say.die "$message"
 26    exit 255
 27}
 28
 29# Utils
 30
 31function dww.util.get_at_stats_json() {
 32    stats="$(atfile.http.get "https://raw.githubusercontent.com/mary-ext/atproto-scraping/refs/heads/trunk/state.json")"
 33    [[ $? == 0 ]] && echo "$stats" | jq
 34}
 35
 36function dww.util.get_bsky_user_count() {
 37    stats="$(atfile.http.get "https://bsky-stats.lut.li")"
 38    [[ $? == 0 ]] && echo "$stats" | jq '.total_users'
 39}
 40
 41function dww.util.get_change_phrase() {
 42    type="$1"
 43
 44    case $type in
 45        "high-score") echo "🏆" ;;
 46        "upward") echo "↗️" ;;
 47        "steady") echo "➡️" ;;
 48        "downward") echo "↘️" ;;
 49        *) echo "🤔" ;;
 50    esac
 51}
 52
 53function dww.util.get_dw_stats() {
 54    didwebs="$(dww.util.get_at_stats_json | jq -c '.firehose.didWebs | to_entries[]')"
 55    updatedAt="$(dww.util.get_at_stats_json | jq -c '.firehose.cursor')"
 56    
 57    active_count=0
 58    error_count=0
 59    pds_count=0
 60    pds_list=()
 61    unset pds_top
 62
 63    while IFS=$"\n" read -r a; do
 64        error_at="$(echo "$a" | jq -r ".value.errorAt")"
 65        pds="$(echo "$a" | jq -r ".value.pds")"
 66        
 67        if [[ $error_at != "null" ]]; then
 68            ((error_count++))
 69        else
 70            ((active_count++))
 71        fi
 72
 73        [[ $pds != "null" ]] && pds_list+=("$pds")
 74    done <<< "$didwebs"
 75
 76    pds_list_string="$(printf '%s\n' "${pds_list[@]}" | sort | uniq -c | sort -k1,1nr -k2)"
 77    pds_count="$(echo -e "$pds_list_string" | wc -l)"
 78
 79    if [[ -n $pds_list ]]; then
 80        while IFS=$"\n" read -r a; do
 81            pds_top+="$(echo $a | cut -d " " -f 2);"
 82        done <<< "$(printf '%s\n' "${pds_list[@]}" | sort | uniq -c | sort -k1,1nr -k2 | head -n3)"
 83    fi
 84
 85    echo "$active_count|$error_count|$pds_top|$pds_count|${updatedAt::-6}"
 86}
 87
 88function dww.util.get_dw_stats_current_json() {
 89    stats_record="$(com.atproto.repo.getRecord "$_username" "$_stats_record_nsid" "self")"
 90    [[ -z "$(atfile.util.get_xrpc_error $? "$stats_record")" ]] && echo "$stats_record"
 91}
 92
 93function dww.util.fmt_int() {
 94    printf "%'d\n" "$1"
 95}
 96
 97# Main Functions
 98
 99function dww.auth() {
100    [[ -z "$_atf_username" ]] && dww.die "\$DWW_USERNAME not set"
101    [[ -z "$_atf_password" ]] && dww.die "\$DWW_PASSWORD not set"
102    atfile.auth "$_atf_username" "$_atf_password"
103}
104
105function dww.bot() {
106    echo "👀 Checking for update..."
107
108    bsky_users="$(dww.util.get_bsky_user_count)"
109    dw_stats="$(dww.util.get_dw_stats)"
110    dw_stats_current="$(dww.util.get_dw_stats_current_json)"
111
112    change_phrase="$(dww.util.get_change_phrase "unknown")"
113    dw_cursor="$(echo $dw_stats | cut -d "|" -f 5)"
114    dw_nodes_top="$(echo $dw_stats | cut -d "|" -f 3)"
115    dw_nodes_total="$(echo $dw_stats | cut -d "|" -f 4)"
116    dw_users_active="$(echo $dw_stats | cut -d "|" -f 1)"
117    dw_users_active_prev=0
118    dw_users_errors="$(echo $dw_stats | cut -d "|" -f 2)"
119    dw_users_max=0
120    is_error=0
121    is_update=1
122
123    if [[ $_force_users_active != 0 || $_force_nodes_total != 0 ]]; then
124        dw_cursor="$(date +%s)"
125
126        [[ $_force_users_active != 0 ]] && dw_users_active=$_force_users_active
127        [[ $_force_nodes_total != 0 ]] && dw_nodes_total=$_force_nodes_total
128    fi
129
130    dw_cursor_pretty="$(date -d @$dw_cursor +"%d-%b-%Y %H:%M:%S %:z")"
131    dw_users_total=$(( $dw_users_active + $dw_users_errors ))
132    dw_dist="$(echo "scale=7; ($dw_users_total / $bsky_users) * 100" | bc)"
133    IFS=';' read -r -a dw_nodes_top_array <<< "$dw_nodes_top"
134
135    [[ -z "$bsky_users" || "$bsky_users" == 0 || "$bsky_users" == "null" ]] && is_error=1
136    [[ -z "$dw_stats" ]] && is_error=1
137    [[ $dw_nodes_total == 0 ]] && is_error=1
138    [[ $dw_users_active == 0 ]] && is_error=1
139
140    if [[ -n $dw_stats_current ]]; then
141        dw_users_max="$(echo "$dw_stats_current" | jq -r ".value.users.max")"
142        dw_users_active_prev="$(echo "$dw_stats_current" | jq -r ".value.users.active")"
143        cursor_current="$(echo "$dw_stats_current" | jq -r ".value.cursor")"
144
145        if (( $dw_cursor <= $cursor_current )); then
146            is_update=0
147        fi
148    fi
149
150    if (( $dw_users_total > $dw_users_max )); then
151        change_phrase="$(dww.util.get_change_phrase "high-score")"
152    elif (( $dw_users_active == $dw_users_active_prev )); then
153        change_phrase="$(dww.util.get_change_phrase "steady")"
154    elif (( $dw_users_active == $dw_users_active_prev )); then
155        change_phrase="$(dww.util.get_change_phrase "steady")"
156    elif (( $dw_users_active > $dw_users_active_prev )); then
157        change_phrase="$(dww.util.get_change_phrase "upward")"
158    elif (( $dw_users_active < $dw_users_active_prev )); then
159        change_phrase="$(dww.util.get_change_phrase "downward")"
160    fi
161
162    (( $dw_users_total > $dw_users_max )) && dw_users_max=$dw_users_total
163    [[ $dw_dist == "."* ]] && dw_dist="0$dw_dist"
164
165    stats_record="{
166        \"cursor\": \"$dw_cursor\",
167        \"nodes\": {
168            \"top_hosts\": [],
169            \"total\": $dw_nodes_total
170        },
171        \"users\": {
172            \"active\": $dw_users_active,
173            \"errors\": $dw_users_errors,
174            \"max\": $dw_users_max
175        }
176    }"
177
178    unset post_message_facets
179    post_message="did:web Stats $change_phrase\n—\n👥  Users: $(dww.util.fmt_int $dw_users_active) Active · $(dww.util.fmt_int $dw_users_errors) Errors · $(dww.util.fmt_int $dw_users_max) Max\n🖥️  Nodes: $(dww.util.fmt_int $dw_nodes_total) Total\n ↳ Top: "
180
181    if [[ -n $dw_nodes_top ]]; then
182        for a in "${dw_nodes_top_array[@]}"; do
183            facet_byte_start=$(( $(echo -e "$post_message" | wc -c) - 1))
184
185            host="$(atfile.util.get_uri_segment "$a" host)"
186            post_message+="$host, "
187
188            facet_byte_end=$(( $(echo -e "$post_message" | wc -c) - 3 ))
189
190            post_message_facets+="{
191        \"index\": {
192            \"byteStart\": $facet_byte_start,
193            \"byteEnd\": $facet_byte_end
194        },
195        \"features\": [
196            {
197                \"\$type\": \"app.bsky.richtext.facet#link\",
198                \"uri\": \"https://pdsls.dev/$host\"
199            }
200        ]
201    },"
202        done
203    else
204        post_message+="(None)"
205    fi
206
207    post_message="${post_message::-2}"
208    post_message+="\n↔️  Distrib.: $dw_dist% ($(dww.util.fmt_int $bsky_users) Total)\n—\n📅  Updated: $dw_cursor_pretty"
209    post_message_facets="${post_message_facets::-1}"
210
211    post_record="{
212        \"createdAt\": \"$(atfile.util.get_date)\",
213        \"langs\": [\"en\"],
214        \"facets\": [
215            {
216                \"index\": {
217                    \"byteStart\": 0,
218                    \"byteEnd\": 13
219                },
220                \"features\": [
221                    {
222                        \"\$type\": \"app.bsky.richtext.facet#tag\",
223                        \"tag\": \"blueskystats\"
224                    }
225                ]
226            },
227            $post_message_facets
228        ],
229        \"text\": \"$post_message\"
230    }"
231
232    echo "---
233ℹ️  Update: $dw_cursor_pretty
234   Phrase: $change_phrase
235   Nodes: $(dww.util.fmt_int $dw_nodes_total)
236   ↳ Top: $(echo "${dw_nodes_top::-1}" | sed -e 's/;/, /g' -e 's/\///g' -e 's/https://g')
237   Users: $(dww.util.fmt_int $dw_users_total)
238   ↳ Active: $(dww.util.fmt_int $dw_users_active)
239   ↳ Errors: $(dww.util.fmt_int $dw_users_errors)
240   ↳ Max: $(dww.util.fmt_int $dw_users_max)
241   Dist: $dw_dist%
242   ↳ Total: $(dww.util.fmt_int $bsky_users)
243---"
244
245    if [[ $is_error == 0 ]] && [[ $_force_update == 1 ]] || [[ $is_update == 1 ]]; then
246        echo -e "✅ Posting update for $dw_cursor_pretty...\n---"
247
248        if [[ $_dry_run == 0 ]]; then
249            com.atproto.repo.createRecord "$_username" "app.bsky.feed.post" "$post_record"
250            [[ $? != 0 ]] && is_error=1
251
252            com.atproto.repo.putRecord "$_username" "$_stats_record_nsid" "self" "$stats_record"
253            [[ $? != 0 ]] && is_error=1
254        else
255            echo -e "(Dry Run)"
256        fi
257        
258        echo "---"
259    else
260        echo -e "❌ Skipping update (outdated?)"
261    fi
262
263    if [[ $is_error == 1 ]]; then
264        dww.bot_error
265        exit 0
266    fi
267}
268
269function dww.bot_error() {
270    echo -e "⚠️  Sending error message...\n---"
271
272    error_post_record="{
273        \"createdAt\": \"$(atfile.util.get_date)\",
274        \"langs\": [\"en\"],
275        \"facets\": [
276            {
277                \"index\": {
278                    \"byteStart\": 0,
279                    \"byteEnd\": 6
280                },
281                \"features\": [
282                    {
283                        \"\$type\": \"app.bsky.richtext.facet#mention\",
284                        \"did\": \"did:web:zio.sh\"
285                    }
286                ]
287            }
288        ],
289        \"text\": \"⚠️ Something went wrong during an update. Service has been paused.\"
290    }"
291
292    com.atproto.repo.createRecord "$_username" "app.bsky.feed.post" "$error_post_record"
293}
294
295function dww.reset() {
296    echo -e "🗑️ Deleting stats...\n---"
297    com.atproto.repo.deleteRecord "$_username" "$_stats_record_nsid" "self"
298    echo "---"
299}
300
301# Main
302
303_sleep_default=43200
304
305_atf_password="$(atfile.util.get_envvar "DWW_PASSWORD")"
306_atf_username="$(atfile.util.get_envvar "DWW_USERNAME")"
307_delete_stats="$(atfile.util.get_envvar "DWW_DELETE_STATS" 0)"
308_dry_run="$(atfile.util.get_envvar "DWW_DRY_RUN" 0)"
309_force_users_active="$(atfile.util.get_envvar "DWW_FORCE_USERS_ACTIVE" 0)"
310_force_nodes_total="$(atfile.util.get_envvar "DWW_FORCE_NODES_TOTAL" 0)"
311_force_update="$(atfile.util.get_envvar "DWW_FORCE_UPDATE" 0)"
312_sleep="$(atfile.util.get_envvar "DWW_SLEEP" 43200)"
313_stats_record_nsid="self.dww.stats"
314
315if [[ $_command == "help" || $_command == "h" || $_command == "--help" || $_command == "-h" ]]; then
316    echo -e "did:web:watch
317    Keeping a watchful eye over Bluesky's minority
318    
319Usage
320    $_prog
321    
322Environment Variables
323    DWW_USERNAME <string> (required)
324    DWW_PASSWORD <string> (required)
325    DWW_DELETE_STATS <bool> (default: 0)
326    DWW_DRY_RUN <bool> (default: 0)
327    DWW_FORCE_USERS_ACTIVE <int> (default: 0)
328    DWW_FORCE_NODES_TOTAL <int> (default: 0)
329    DWW_FORCE_UPDATE <bool> (default: 0)
330    DWW_SLEEP <int> (default: $_sleep_default)
331    "
332 
333    exit 0
334fi
335
336dww.auth
337
338if [[ $_delete_stats == 1 ]]; then
339    dww.reset
340fi
341
342while true; do
343    dww.bot
344    sleep $_sleep
345done