tangled
alpha
login
or
join now
cherry.computer
/
website
My personal site
cherry.computer
htmx
tailwind
axum
askama
0
fork
atom
overview
issues
pulls
pipelines
feat: loop over scraped media in template
cherry.computer
3 months ago
73cb3a3c
beb83603
verified
This commit was signed with the committer's
known signature
.
cherry.computer
SSH Key Fingerprint:
SHA256:SIA77Ll0IpMb8Xd3RtaGT+PBIGIePhJJg5W2r6Td7cc=
+124
-151
6 changed files
expand all
collapse all
unified
split
server
src
index.rs
scrapers
apple_music.rs
backloggd.rs
letterboxd.rs
scrapers.rs
templates
index.html
+7
-7
server/src/index.rs
···
1
1
use crate::scrapers::{
2
2
-
apple_music::{self, AppleMusic, AppleMusicClient},
3
3
-
backloggd::{self, Backloggd},
4
4
-
letterboxd::{self, Letterboxd},
2
2
+
Media,
3
3
+
apple_music::{self, AppleMusicClient},
4
4
+
backloggd, letterboxd,
5
5
};
6
6
7
7
use askama::Template;
···
9
9
#[derive(Template, Debug, Clone)]
10
10
#[template(path = "index.html")]
11
11
pub struct RootTemplate {
12
12
-
game: Option<Backloggd>,
13
13
-
movie: Option<Letterboxd>,
14
14
-
song: Option<AppleMusic>,
12
12
+
media: Vec<Media>,
15
13
}
16
14
17
15
impl RootTemplate {
···
21
19
letterboxd::cached_fetch(),
22
20
apple_music::cached_fetch(apple_music_client)
23
21
);
24
24
-
RootTemplate { game, movie, song }
22
22
+
let media = [game, movie, song].into_iter().flatten().collect();
23
23
+
24
24
+
RootTemplate { media }
25
25
}
26
26
}
+6
server/src/scrapers.rs
···
1
1
pub mod apple_music;
2
2
pub mod backloggd;
3
3
pub mod letterboxd;
4
4
+
5
5
+
#[derive(Debug, Clone)]
6
6
+
pub struct Media {
7
7
+
pub name: String,
8
8
+
pub image: String,
9
9
+
}
+5
-9
server/src/scrapers/apple_music.rs
···
6
6
use reqwest::Client;
7
7
use serde::{Deserialize, Serialize};
8
8
9
9
-
#[derive(Debug, Clone)]
10
10
-
pub struct AppleMusic {
11
11
-
pub name: String,
12
12
-
pub art: String,
13
13
-
}
9
9
+
use super::Media;
14
10
15
11
#[derive(Serialize, Debug, Clone)]
16
12
struct Claims {
···
79
75
})
80
76
}
81
77
82
82
-
pub async fn fetch(&self) -> anyhow::Result<AppleMusic> {
78
78
+
pub async fn fetch(&self) -> anyhow::Result<Media> {
83
79
let jwt = self.build_developer_token()?;
84
80
85
81
let response: AppleMusicResponse = self
···
102
98
.replace("{w}", dimensions)
103
99
.replace("{h}", dimensions);
104
100
105
105
-
Ok(AppleMusic {
101
101
+
Ok(Media {
106
102
name: track.attributes.name.clone(),
107
107
-
art: artwork_url,
103
103
+
image: artwork_url,
108
104
})
109
105
}
110
106
···
119
115
}
120
116
121
117
#[once(time = 30, option = false)]
122
122
-
pub async fn cached_fetch(this: &AppleMusicClient) -> Option<AppleMusic> {
118
118
+
pub async fn cached_fetch(this: &AppleMusicClient) -> Option<Media> {
123
119
this.fetch()
124
120
.await
125
121
.map_err(|error| tracing::warn!(?error, "failed to call Apple Music"))
+36
-43
server/src/scrapers/backloggd.rs
···
4
4
use cached::proc_macro::once;
5
5
use scraper::{Html, Selector};
6
6
7
7
-
#[derive(Debug, Clone)]
8
8
-
pub struct Backloggd {
9
9
-
pub name: String,
10
10
-
pub image: String,
11
11
-
}
7
7
+
use super::Media;
12
8
13
13
-
impl Backloggd {
14
14
-
pub async fn fetch() -> anyhow::Result<Self> {
15
15
-
static FIRST_ENTRY_SEL: LazyLock<Selector> =
16
16
-
LazyLock::new(|| Selector::parse(".journal_entry:first-child").unwrap());
17
17
-
static NAME_SEL: LazyLock<Selector> =
18
18
-
LazyLock::new(|| Selector::parse(".game-name a").unwrap());
19
19
-
static IMAGE_SEL: LazyLock<Selector> =
20
20
-
LazyLock::new(|| Selector::parse(".card-img").unwrap());
9
9
+
pub async fn fetch() -> anyhow::Result<Media> {
10
10
+
static FIRST_ENTRY_SEL: LazyLock<Selector> =
11
11
+
LazyLock::new(|| Selector::parse(".journal_entry:first-child").unwrap());
12
12
+
static NAME_SEL: LazyLock<Selector> =
13
13
+
LazyLock::new(|| Selector::parse(".game-name a").unwrap());
14
14
+
static IMAGE_SEL: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".card-img").unwrap());
21
15
22
22
-
let html = reqwest::get("https://backloggd.com/u/cherryfunk/journal")
23
23
-
.await
24
24
-
.context("failed to fetch Backloggd page")?
25
25
-
.text()
26
26
-
.await
27
27
-
.context("failed to get HTML text")?;
28
28
-
let document = Html::parse_document(&html);
16
16
+
let html = reqwest::get("https://backloggd.com/u/cherryfunk/journal")
17
17
+
.await
18
18
+
.context("failed to fetch Backloggd page")?
19
19
+
.text()
20
20
+
.await
21
21
+
.context("failed to get HTML text")?;
22
22
+
let document = Html::parse_document(&html);
29
23
30
30
-
let first_entry = document
31
31
-
.select(&FIRST_ENTRY_SEL)
32
32
-
.next()
33
33
-
.context("couldn't find any journal entries")?;
34
34
-
let name = first_entry
35
35
-
.select(&NAME_SEL)
36
36
-
.next()
37
37
-
.context("couldn't find name element")?
38
38
-
.text()
39
39
-
.next()
40
40
-
.context("name element didn't have any text")?
41
41
-
.to_owned();
42
42
-
let image = first_entry
43
43
-
.select(&IMAGE_SEL)
44
44
-
.next()
45
45
-
.context("couldn't find image element")?
46
46
-
.attr("src")
47
47
-
.context("image element didn't have src attribute")?
48
48
-
.to_owned();
24
24
+
let first_entry = document
25
25
+
.select(&FIRST_ENTRY_SEL)
26
26
+
.next()
27
27
+
.context("couldn't find any journal entries")?;
28
28
+
let name = first_entry
29
29
+
.select(&NAME_SEL)
30
30
+
.next()
31
31
+
.context("couldn't find name element")?
32
32
+
.text()
33
33
+
.next()
34
34
+
.context("name element didn't have any text")?
35
35
+
.to_owned();
36
36
+
let image = first_entry
37
37
+
.select(&IMAGE_SEL)
38
38
+
.next()
39
39
+
.context("couldn't find image element")?
40
40
+
.attr("src")
41
41
+
.context("image element didn't have src attribute")?
42
42
+
.to_owned();
49
43
50
50
-
Ok(Self { name, image })
51
51
-
}
44
44
+
Ok(Media { name, image })
52
45
}
53
46
54
47
#[once(time = 300, option = false)]
55
55
-
pub async fn cached_fetch() -> Option<Backloggd> {
56
56
-
Backloggd::fetch()
48
48
+
pub async fn cached_fetch() -> Option<Media> {
49
49
+
fetch()
57
50
.await
58
51
.map_err(|error| tracing::warn!(?error, "failed to scrape Backloggd"))
59
52
.ok()
+66
-75
server/src/scrapers/letterboxd.rs
···
6
6
use scraper::{ElementRef, Html, Selector};
7
7
use serde::Deserialize;
8
8
9
9
-
#[derive(Debug, Clone)]
10
10
-
pub struct Letterboxd {
11
11
-
pub name: String,
12
12
-
pub poster: String,
13
13
-
}
9
9
+
use super::Media;
14
10
15
11
#[derive(Deserialize, Debug, Clone)]
16
12
pub struct ImageUrlMetadata {
···
22
18
image_url: Url,
23
19
}
24
20
25
25
-
impl Letterboxd {
26
26
-
pub async fn fetch() -> anyhow::Result<Self> {
27
27
-
let client = Client::new();
28
28
-
let html = client
29
29
-
.get("https://letterboxd.com/ivom/films/diary/")
30
30
-
.send()
31
31
-
.await
32
32
-
.context("failed to fetch Letterboxd page")?
33
33
-
.text()
34
34
-
.await
35
35
-
.context("failed to get HTML text")?;
36
36
-
let Extracted { name, image_url } = Self::parse_html(&html)?;
21
21
+
pub async fn fetch() -> anyhow::Result<Media> {
22
22
+
let client = Client::new();
23
23
+
let html = client
24
24
+
.get("https://letterboxd.com/ivom/films/diary/")
25
25
+
.send()
26
26
+
.await
27
27
+
.context("failed to fetch Letterboxd page")?
28
28
+
.text()
29
29
+
.await
30
30
+
.context("failed to get HTML text")?;
31
31
+
let Extracted { name, image_url } = parse_html(&html)?;
32
32
+
33
33
+
let image_url_data: ImageUrlMetadata = client
34
34
+
.get(image_url.clone())
35
35
+
.send()
36
36
+
.await
37
37
+
.with_context(|| format!("failed to fetch image metadata from URL {}", image_url))?
38
38
+
.json()
39
39
+
.await
40
40
+
.context("failed to parse image metadata")?;
37
41
38
38
-
let image_url_data: ImageUrlMetadata = client
39
39
-
.get(image_url.clone())
40
40
-
.send()
41
41
-
.await
42
42
-
.with_context(|| format!("failed to fetch image metadata from URL {}", image_url))?
43
43
-
.json()
44
44
-
.await
45
45
-
.context("failed to parse image metadata")?;
42
42
+
Ok(Media {
43
43
+
name,
44
44
+
image: image_url_data.url,
45
45
+
})
46
46
+
}
46
47
47
47
-
Ok(Self {
48
48
-
name,
49
49
-
poster: image_url_data.url,
50
50
-
})
51
51
-
}
48
48
+
fn parse_html(html: &str) -> anyhow::Result<Extracted> {
49
49
+
static FIRST_ENTRY_SEL: LazyLock<Selector> =
50
50
+
LazyLock::new(|| Selector::parse(".diary-entry-row:first-child").unwrap());
51
51
+
static NAME_SEL: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".name").unwrap());
52
52
+
static POSTER_COMPONENT_SEL: LazyLock<Selector> =
53
53
+
LazyLock::new(|| Selector::parse(".react-component:has(> .poster)").unwrap());
52
54
53
53
-
fn parse_html(html: &str) -> anyhow::Result<Extracted> {
54
54
-
static FIRST_ENTRY_SEL: LazyLock<Selector> =
55
55
-
LazyLock::new(|| Selector::parse(".diary-entry-row:first-child").unwrap());
56
56
-
static NAME_SEL: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".name").unwrap());
57
57
-
static POSTER_COMPONENT_SEL: LazyLock<Selector> =
58
58
-
LazyLock::new(|| Selector::parse(".react-component:has(> .poster)").unwrap());
55
55
+
let document = Html::parse_document(html);
59
56
60
60
-
let document = Html::parse_document(html);
57
57
+
let first_entry = document
58
58
+
.select(&FIRST_ENTRY_SEL)
59
59
+
.next()
60
60
+
.context("couldn't find any journal entries")?;
61
61
+
let name = first_entry
62
62
+
.select(&NAME_SEL)
63
63
+
.next()
64
64
+
.context("couldn't find name element")?
65
65
+
.text()
66
66
+
.next()
67
67
+
.context("name element didn't have any text")?
68
68
+
.to_owned();
69
69
+
let poster_component = first_entry
70
70
+
.select(&POSTER_COMPONENT_SEL)
71
71
+
.next()
72
72
+
.context("couldn't find post component")?;
61
73
62
62
-
let first_entry = document
63
63
-
.select(&FIRST_ENTRY_SEL)
64
64
-
.next()
65
65
-
.context("couldn't find any journal entries")?;
66
66
-
let name = first_entry
67
67
-
.select(&NAME_SEL)
68
68
-
.next()
69
69
-
.context("couldn't find name element")?
70
70
-
.text()
71
71
-
.next()
72
72
-
.context("name element didn't have any text")?
73
73
-
.to_owned();
74
74
-
let poster_component = first_entry
75
75
-
.select(&POSTER_COMPONENT_SEL)
76
76
-
.next()
77
77
-
.context("couldn't find post component")?;
74
74
+
let image_url = build_image_url(poster_component)?;
78
75
79
79
-
let image_url = Self::build_image_url(poster_component)?;
76
76
+
Ok(Extracted { name, image_url })
77
77
+
}
80
78
81
81
-
Ok(Extracted { name, image_url })
79
79
+
fn build_image_url(poster_component: ElementRef) -> anyhow::Result<Url> {
80
80
+
let film_path = poster_component
81
81
+
.attr("data-item-link")
82
82
+
.context("poster component didn't have an image URL path")?;
83
83
+
let cache_key = poster_component.attr("data-cache-busting-key");
84
84
+
let image_size = 230;
85
85
+
let image_url = format!("https://letterboxd.com{film_path}/poster/std/{image_size}/",);
86
86
+
let mut image_url =
87
87
+
Url::parse(&image_url).with_context(|| format!("failed to parse URL {image_url}"))?;
88
88
+
if let Some(cache_key) = cache_key {
89
89
+
image_url.query_pairs_mut().append_pair("k", cache_key);
82
90
}
83
91
84
84
-
fn build_image_url(poster_component: ElementRef) -> anyhow::Result<Url> {
85
85
-
let film_path = poster_component
86
86
-
.attr("data-item-link")
87
87
-
.context("poster component didn't have an image URL path")?;
88
88
-
let cache_key = poster_component.attr("data-cache-busting-key");
89
89
-
let image_size = 230;
90
90
-
let image_url = format!(
91
91
-
"https://letterboxd.com{}/poster/std/{}/",
92
92
-
film_path, image_size
93
93
-
);
94
94
-
let mut image_url =
95
95
-
Url::parse(&image_url).with_context(|| format!("failed to parse URL {}", image_url))?;
96
96
-
if let Some(cache_key) = cache_key {
97
97
-
image_url.query_pairs_mut().append_pair("k", cache_key);
98
98
-
}
99
99
-
100
100
-
Ok(image_url)
101
101
-
}
92
92
+
Ok(image_url)
102
93
}
103
94
104
95
#[once(time = 1800, option = false)]
105
105
-
pub async fn cached_fetch() -> Option<Letterboxd> {
106
106
-
Letterboxd::fetch()
96
96
+
pub async fn cached_fetch() -> Option<Media> {
97
97
+
fetch()
107
98
.await
108
99
.map_err(|error| tracing::warn!(?error, "failed to scrape Letterboxd"))
109
100
.ok()
+4
-17
server/templates/index.html
···
18
18
</h1>
19
19
</div>
20
20
<div class="flex justify-center">
21
21
-
{% if let Some(game) = game -%}
21
21
+
{% for media in media -%}
22
22
<img
23
23
class="p-3"
24
24
-
src="{{ game.image }}"
25
25
-
alt="Cover art for {{ game.name }}"
24
24
+
src="{{ media.image }}"
25
25
+
alt="Cover art for {{ media.name }}"
26
26
/>
27
27
-
{%- endif %} {% if let Some(movie) = movie -%}
28
28
-
<img
29
29
-
class="p-3"
30
30
-
src="{{ movie.poster }}"
31
31
-
alt="Poster for {{ movie.name }}"
32
32
-
/>
33
33
-
{%- endif %} {% if let Some(song) = song -%}
34
34
-
<img
35
35
-
class="p-3"
36
36
-
src="{{ song.art }}"
37
37
-
alt="Album art for {{ song.name }}"
38
38
-
/>
39
39
-
{%- endif %}
40
40
-
27
27
+
{%- endfor %}
41
28
</div>
42
29
</div>
43
30
</body>