learn and share notes on atproto (wip) 馃
malfestio.stormlightlabs.org/
desertthunder.dev
readability
solid
axum
atproto
srs
1#!/bin/bash
2mkdir -p crates/readability/tests/data
3
4fetch_if_missing() {
5 local url="$1"
6 local output="$2"
7
8 if [ -f "$output" ]; then
9 echo "Cached: $output"
10 else
11 echo "Fetching $url..."
12 curl -L -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" -o "$output" "$url"
13 fi
14}
15
16fetch_if_missing "https://arxiv.org/abs/2009.03017" "crates/readability/tests/data/arxiv.html"
17fetch_if_missing "https://en.wikipedia.org/wiki/Rust_(programming_language)" "crates/readability/tests/data/wikipedia.html"
18fetch_if_missing "https://dougshapiro.medium.com/how-will-the-disruption-of-hollywood-play-out-42f724c921e1" "crates/readability/tests/data/medium.html"
19fetch_if_missing "https://taibbi.substack.com/p/glenn-greenwald-on-his-resignation" "crates/readability/tests/data/substack.html"
20fetch_if_missing "https://www.theonion.com/theresa-may-narrowly-manages-to-survive-parliamentary-f-1831077604" "crates/readability/tests/data/theonion.html"
21fetch_if_missing "http://docs.readthedocs.io/en/latest/getting_started.html" "crates/readability/tests/data/readthedocs.html"
22fetch_if_missing "https://www.rust-lang.org/" "crates/readability/tests/data/generic.html"
23
24echo "Done."