tangled
alpha
login
or
join now
cherry.computer
/
website
My personal site
cherry.computer
htmx
tailwind
axum
askama
0
fork
atom
overview
issues
pulls
pipelines
feat: loop over scraped media in template
cherry.computer
3 months ago
73cb3a3c
beb83603
verified
This commit was signed with the committer's
known signature
.
cherry.computer
SSH Key Fingerprint:
SHA256:SIA77Ll0IpMb8Xd3RtaGT+PBIGIePhJJg5W2r6Td7cc=
+124
-151
6 changed files
expand all
collapse all
unified
split
server
src
index.rs
scrapers
apple_music.rs
backloggd.rs
letterboxd.rs
scrapers.rs
templates
index.html
+7
-7
server/src/index.rs
···
1
use crate::scrapers::{
2
-
apple_music::{self, AppleMusic, AppleMusicClient},
3
-
backloggd::{self, Backloggd},
4
-
letterboxd::{self, Letterboxd},
5
};
6
7
use askama::Template;
···
9
#[derive(Template, Debug, Clone)]
10
#[template(path = "index.html")]
11
pub struct RootTemplate {
12
-
game: Option<Backloggd>,
13
-
movie: Option<Letterboxd>,
14
-
song: Option<AppleMusic>,
15
}
16
17
impl RootTemplate {
···
21
letterboxd::cached_fetch(),
22
apple_music::cached_fetch(apple_music_client)
23
);
24
-
RootTemplate { game, movie, song }
0
0
25
}
26
}
···
1
use crate::scrapers::{
2
+
Media,
3
+
apple_music::{self, AppleMusicClient},
4
+
backloggd, letterboxd,
5
};
6
7
use askama::Template;
···
9
#[derive(Template, Debug, Clone)]
10
#[template(path = "index.html")]
11
pub struct RootTemplate {
12
+
media: Vec<Media>,
0
0
13
}
14
15
impl RootTemplate {
···
19
letterboxd::cached_fetch(),
20
apple_music::cached_fetch(apple_music_client)
21
);
22
+
let media = [game, movie, song].into_iter().flatten().collect();
23
+
24
+
RootTemplate { media }
25
}
26
}
+6
server/src/scrapers.rs
···
1
pub mod apple_music;
2
pub mod backloggd;
3
pub mod letterboxd;
0
0
0
0
0
0
···
1
pub mod apple_music;
2
pub mod backloggd;
3
pub mod letterboxd;
4
+
5
+
#[derive(Debug, Clone)]
6
+
pub struct Media {
7
+
pub name: String,
8
+
pub image: String,
9
+
}
+5
-9
server/src/scrapers/apple_music.rs
···
6
use reqwest::Client;
7
use serde::{Deserialize, Serialize};
8
9
-
#[derive(Debug, Clone)]
10
-
pub struct AppleMusic {
11
-
pub name: String,
12
-
pub art: String,
13
-
}
14
15
#[derive(Serialize, Debug, Clone)]
16
struct Claims {
···
79
})
80
}
81
82
-
pub async fn fetch(&self) -> anyhow::Result<AppleMusic> {
83
let jwt = self.build_developer_token()?;
84
85
let response: AppleMusicResponse = self
···
102
.replace("{w}", dimensions)
103
.replace("{h}", dimensions);
104
105
-
Ok(AppleMusic {
106
name: track.attributes.name.clone(),
107
-
art: artwork_url,
108
})
109
}
110
···
119
}
120
121
#[once(time = 30, option = false)]
122
-
pub async fn cached_fetch(this: &AppleMusicClient) -> Option<AppleMusic> {
123
this.fetch()
124
.await
125
.map_err(|error| tracing::warn!(?error, "failed to call Apple Music"))
···
6
use reqwest::Client;
7
use serde::{Deserialize, Serialize};
8
9
+
use super::Media;
0
0
0
0
10
11
#[derive(Serialize, Debug, Clone)]
12
struct Claims {
···
75
})
76
}
77
78
+
pub async fn fetch(&self) -> anyhow::Result<Media> {
79
let jwt = self.build_developer_token()?;
80
81
let response: AppleMusicResponse = self
···
98
.replace("{w}", dimensions)
99
.replace("{h}", dimensions);
100
101
+
Ok(Media {
102
name: track.attributes.name.clone(),
103
+
image: artwork_url,
104
})
105
}
106
···
115
}
116
117
#[once(time = 30, option = false)]
118
+
pub async fn cached_fetch(this: &AppleMusicClient) -> Option<Media> {
119
this.fetch()
120
.await
121
.map_err(|error| tracing::warn!(?error, "failed to call Apple Music"))
+36
-43
server/src/scrapers/backloggd.rs
···
4
use cached::proc_macro::once;
5
use scraper::{Html, Selector};
6
7
-
#[derive(Debug, Clone)]
8
-
pub struct Backloggd {
9
-
pub name: String,
10
-
pub image: String,
11
-
}
12
13
-
impl Backloggd {
14
-
pub async fn fetch() -> anyhow::Result<Self> {
15
-
static FIRST_ENTRY_SEL: LazyLock<Selector> =
16
-
LazyLock::new(|| Selector::parse(".journal_entry:first-child").unwrap());
17
-
static NAME_SEL: LazyLock<Selector> =
18
-
LazyLock::new(|| Selector::parse(".game-name a").unwrap());
19
-
static IMAGE_SEL: LazyLock<Selector> =
20
-
LazyLock::new(|| Selector::parse(".card-img").unwrap());
21
22
-
let html = reqwest::get("https://backloggd.com/u/cherryfunk/journal")
23
-
.await
24
-
.context("failed to fetch Backloggd page")?
25
-
.text()
26
-
.await
27
-
.context("failed to get HTML text")?;
28
-
let document = Html::parse_document(&html);
29
30
-
let first_entry = document
31
-
.select(&FIRST_ENTRY_SEL)
32
-
.next()
33
-
.context("couldn't find any journal entries")?;
34
-
let name = first_entry
35
-
.select(&NAME_SEL)
36
-
.next()
37
-
.context("couldn't find name element")?
38
-
.text()
39
-
.next()
40
-
.context("name element didn't have any text")?
41
-
.to_owned();
42
-
let image = first_entry
43
-
.select(&IMAGE_SEL)
44
-
.next()
45
-
.context("couldn't find image element")?
46
-
.attr("src")
47
-
.context("image element didn't have src attribute")?
48
-
.to_owned();
49
50
-
Ok(Self { name, image })
51
-
}
52
}
53
54
#[once(time = 300, option = false)]
55
-
pub async fn cached_fetch() -> Option<Backloggd> {
56
-
Backloggd::fetch()
57
.await
58
.map_err(|error| tracing::warn!(?error, "failed to scrape Backloggd"))
59
.ok()
···
4
use cached::proc_macro::once;
5
use scraper::{Html, Selector};
6
7
+
use super::Media;
0
0
0
0
8
9
+
pub async fn fetch() -> anyhow::Result<Media> {
10
+
static FIRST_ENTRY_SEL: LazyLock<Selector> =
11
+
LazyLock::new(|| Selector::parse(".journal_entry:first-child").unwrap());
12
+
static NAME_SEL: LazyLock<Selector> =
13
+
LazyLock::new(|| Selector::parse(".game-name a").unwrap());
14
+
static IMAGE_SEL: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".card-img").unwrap());
0
0
15
16
+
let html = reqwest::get("https://backloggd.com/u/cherryfunk/journal")
17
+
.await
18
+
.context("failed to fetch Backloggd page")?
19
+
.text()
20
+
.await
21
+
.context("failed to get HTML text")?;
22
+
let document = Html::parse_document(&html);
23
24
+
let first_entry = document
25
+
.select(&FIRST_ENTRY_SEL)
26
+
.next()
27
+
.context("couldn't find any journal entries")?;
28
+
let name = first_entry
29
+
.select(&NAME_SEL)
30
+
.next()
31
+
.context("couldn't find name element")?
32
+
.text()
33
+
.next()
34
+
.context("name element didn't have any text")?
35
+
.to_owned();
36
+
let image = first_entry
37
+
.select(&IMAGE_SEL)
38
+
.next()
39
+
.context("couldn't find image element")?
40
+
.attr("src")
41
+
.context("image element didn't have src attribute")?
42
+
.to_owned();
43
44
+
Ok(Media { name, image })
0
45
}
46
47
#[once(time = 300, option = false)]
48
+
pub async fn cached_fetch() -> Option<Media> {
49
+
fetch()
50
.await
51
.map_err(|error| tracing::warn!(?error, "failed to scrape Backloggd"))
52
.ok()
+66
-75
server/src/scrapers/letterboxd.rs
···
6
use scraper::{ElementRef, Html, Selector};
7
use serde::Deserialize;
8
9
-
#[derive(Debug, Clone)]
10
-
pub struct Letterboxd {
11
-
pub name: String,
12
-
pub poster: String,
13
-
}
14
15
#[derive(Deserialize, Debug, Clone)]
16
pub struct ImageUrlMetadata {
···
22
image_url: Url,
23
}
24
25
-
impl Letterboxd {
26
-
pub async fn fetch() -> anyhow::Result<Self> {
27
-
let client = Client::new();
28
-
let html = client
29
-
.get("https://letterboxd.com/ivom/films/diary/")
30
-
.send()
31
-
.await
32
-
.context("failed to fetch Letterboxd page")?
33
-
.text()
34
-
.await
35
-
.context("failed to get HTML text")?;
36
-
let Extracted { name, image_url } = Self::parse_html(&html)?;
0
0
0
0
0
0
0
0
37
38
-
let image_url_data: ImageUrlMetadata = client
39
-
.get(image_url.clone())
40
-
.send()
41
-
.await
42
-
.with_context(|| format!("failed to fetch image metadata from URL {}", image_url))?
43
-
.json()
44
-
.await
45
-
.context("failed to parse image metadata")?;
46
47
-
Ok(Self {
48
-
name,
49
-
poster: image_url_data.url,
50
-
})
51
-
}
0
52
53
-
fn parse_html(html: &str) -> anyhow::Result<Extracted> {
54
-
static FIRST_ENTRY_SEL: LazyLock<Selector> =
55
-
LazyLock::new(|| Selector::parse(".diary-entry-row:first-child").unwrap());
56
-
static NAME_SEL: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".name").unwrap());
57
-
static POSTER_COMPONENT_SEL: LazyLock<Selector> =
58
-
LazyLock::new(|| Selector::parse(".react-component:has(> .poster)").unwrap());
59
60
-
let document = Html::parse_document(html);
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
61
62
-
let first_entry = document
63
-
.select(&FIRST_ENTRY_SEL)
64
-
.next()
65
-
.context("couldn't find any journal entries")?;
66
-
let name = first_entry
67
-
.select(&NAME_SEL)
68
-
.next()
69
-
.context("couldn't find name element")?
70
-
.text()
71
-
.next()
72
-
.context("name element didn't have any text")?
73
-
.to_owned();
74
-
let poster_component = first_entry
75
-
.select(&POSTER_COMPONENT_SEL)
76
-
.next()
77
-
.context("couldn't find post component")?;
78
79
-
let image_url = Self::build_image_url(poster_component)?;
0
80
81
-
Ok(Extracted { name, image_url })
0
0
0
0
0
0
0
0
0
0
82
}
83
84
-
fn build_image_url(poster_component: ElementRef) -> anyhow::Result<Url> {
85
-
let film_path = poster_component
86
-
.attr("data-item-link")
87
-
.context("poster component didn't have an image URL path")?;
88
-
let cache_key = poster_component.attr("data-cache-busting-key");
89
-
let image_size = 230;
90
-
let image_url = format!(
91
-
"https://letterboxd.com{}/poster/std/{}/",
92
-
film_path, image_size
93
-
);
94
-
let mut image_url =
95
-
Url::parse(&image_url).with_context(|| format!("failed to parse URL {}", image_url))?;
96
-
if let Some(cache_key) = cache_key {
97
-
image_url.query_pairs_mut().append_pair("k", cache_key);
98
-
}
99
-
100
-
Ok(image_url)
101
-
}
102
}
103
104
#[once(time = 1800, option = false)]
105
-
pub async fn cached_fetch() -> Option<Letterboxd> {
106
-
Letterboxd::fetch()
107
.await
108
.map_err(|error| tracing::warn!(?error, "failed to scrape Letterboxd"))
109
.ok()
···
6
use scraper::{ElementRef, Html, Selector};
7
use serde::Deserialize;
8
9
+
use super::Media;
0
0
0
0
10
11
#[derive(Deserialize, Debug, Clone)]
12
pub struct ImageUrlMetadata {
···
18
image_url: Url,
19
}
20
21
+
pub async fn fetch() -> anyhow::Result<Media> {
22
+
let client = Client::new();
23
+
let html = client
24
+
.get("https://letterboxd.com/ivom/films/diary/")
25
+
.send()
26
+
.await
27
+
.context("failed to fetch Letterboxd page")?
28
+
.text()
29
+
.await
30
+
.context("failed to get HTML text")?;
31
+
let Extracted { name, image_url } = parse_html(&html)?;
32
+
33
+
let image_url_data: ImageUrlMetadata = client
34
+
.get(image_url.clone())
35
+
.send()
36
+
.await
37
+
.with_context(|| format!("failed to fetch image metadata from URL {}", image_url))?
38
+
.json()
39
+
.await
40
+
.context("failed to parse image metadata")?;
41
42
+
Ok(Media {
43
+
name,
44
+
image: image_url_data.url,
45
+
})
46
+
}
0
0
0
47
48
+
fn parse_html(html: &str) -> anyhow::Result<Extracted> {
49
+
static FIRST_ENTRY_SEL: LazyLock<Selector> =
50
+
LazyLock::new(|| Selector::parse(".diary-entry-row:first-child").unwrap());
51
+
static NAME_SEL: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".name").unwrap());
52
+
static POSTER_COMPONENT_SEL: LazyLock<Selector> =
53
+
LazyLock::new(|| Selector::parse(".react-component:has(> .poster)").unwrap());
54
55
+
let document = Html::parse_document(html);
0
0
0
0
0
56
57
+
let first_entry = document
58
+
.select(&FIRST_ENTRY_SEL)
59
+
.next()
60
+
.context("couldn't find any journal entries")?;
61
+
let name = first_entry
62
+
.select(&NAME_SEL)
63
+
.next()
64
+
.context("couldn't find name element")?
65
+
.text()
66
+
.next()
67
+
.context("name element didn't have any text")?
68
+
.to_owned();
69
+
let poster_component = first_entry
70
+
.select(&POSTER_COMPONENT_SEL)
71
+
.next()
72
+
.context("couldn't find post component")?;
73
74
+
let image_url = build_image_url(poster_component)?;
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
75
76
+
Ok(Extracted { name, image_url })
77
+
}
78
79
+
fn build_image_url(poster_component: ElementRef) -> anyhow::Result<Url> {
80
+
let film_path = poster_component
81
+
.attr("data-item-link")
82
+
.context("poster component didn't have an image URL path")?;
83
+
let cache_key = poster_component.attr("data-cache-busting-key");
84
+
let image_size = 230;
85
+
let image_url = format!("https://letterboxd.com{film_path}/poster/std/{image_size}/",);
86
+
let mut image_url =
87
+
Url::parse(&image_url).with_context(|| format!("failed to parse URL {image_url}"))?;
88
+
if let Some(cache_key) = cache_key {
89
+
image_url.query_pairs_mut().append_pair("k", cache_key);
90
}
91
92
+
Ok(image_url)
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
93
}
94
95
#[once(time = 1800, option = false)]
96
+
pub async fn cached_fetch() -> Option<Media> {
97
+
fetch()
98
.await
99
.map_err(|error| tracing::warn!(?error, "failed to scrape Letterboxd"))
100
.ok()
+4
-17
server/templates/index.html
···
18
</h1>
19
</div>
20
<div class="flex justify-center">
21
-
{% if let Some(game) = game -%}
22
<img
23
class="p-3"
24
-
src="{{ game.image }}"
25
-
alt="Cover art for {{ game.name }}"
26
/>
27
-
{%- endif %} {% if let Some(movie) = movie -%}
28
-
<img
29
-
class="p-3"
30
-
src="{{ movie.poster }}"
31
-
alt="Poster for {{ movie.name }}"
32
-
/>
33
-
{%- endif %} {% if let Some(song) = song -%}
34
-
<img
35
-
class="p-3"
36
-
src="{{ song.art }}"
37
-
alt="Album art for {{ song.name }}"
38
-
/>
39
-
{%- endif %}
40
-
41
</div>
42
</div>
43
</body>
···
18
</h1>
19
</div>
20
<div class="flex justify-center">
21
+
{% for media in media -%}
22
<img
23
class="p-3"
24
+
src="{{ media.image }}"
25
+
alt="Cover art for {{ media.name }}"
26
/>
27
+
{%- endfor %}
0
0
0
0
0
0
0
0
0
0
0
0
0
28
</div>
29
</div>
30
</body>