Add Spotify API retries and HTTP client timeouts

crates/scrobbler/src/scrobbler.rs

···
     xata::user::User,
 };
 
+const MAX_SPOTIFY_RETRIES: u32 = 3;
+const INITIAL_RETRY_DELAY_MS: u64 = 1000;
+
 fn parse_batch(form: &BTreeMap<String, String>) -> Result<Vec<Scrobble>, Error> {
     let mut result = vec![];
     let mut index = 0;
···
         let spotify_token = refresh_token(&spotify_token, &client_id, &client_secret).await?;
         let spotify_client = SpotifyClient::new(&spotify_token.access_token);
 
-        let result = spotify_client
-            .search(&format!(
-                r#"track:"{}" artist:"{}""#,
-                scrobble.track, scrobble.artist
-            ))
-            .await?;
+        let result = retry_spotify_call(
+            || async {
+                spotify_client
+                    .search(&format!(
+                        r#"track:"{}" artist:"{}""#,
+                        scrobble.track, scrobble.artist
+                    ))
+                    .await
+            },
+            "search",
+        )
+        .await?;
 
         if let Some(track) = result.tracks.items.first() {
             tracing::info!(artist = %scrobble.artist, track = %scrobble.track, "Spotify (track)");
             scrobble.album = Some(track.album.name.clone());
             let mut track = track.clone();
 
-            if let Some(album) = spotify_client.get_album(&track.album.id).await? {
+            if let Some(album) = retry_spotify_call(
+                || async { spotify_client.get_album(&track.album.id).await },
+                "get_album",
+            )
+            .await?
+            {
                 track.album = album;
             }
 
-            if let Some(artist) = spotify_client
-                .get_artist(&track.album.artists[0].id)
-                .await?
+            if let Some(artist) = retry_spotify_call(
+                || async { spotify_client.get_artist(&track.album.artists[0].id).await },
+                "get_artist",
+            )
+            .await?
             {
                 track.album.artists[0] = artist;
             }
···
     let spotify_token = refresh_token(&spotify_token, &client_id, &client_secret).await?;
     let spotify_client = SpotifyClient::new(&spotify_token.access_token);
 
-    let result = spotify_client
-        .search(&format!(
-            r#"track:"{}" artist:"{}""#,
-            scrobble.track, scrobble.artist
-        ))
-        .await?;
+    let result = retry_spotify_call(
+        || async {
+            spotify_client
+                .search(&format!(
+                    r#"track:"{}" artist:"{}""#,
+                    scrobble.track, scrobble.artist
+                ))
+                .await
+        },
+        "search",
+    )
+    .await?;
 
     if let Some(track) = result.tracks.items.first() {
         let normalize = |s: &str| -> String {
···
             scrobble.album = Some(track.album.name.clone());
             let mut track = track.clone();
 
-            if let Some(album) = spotify_client.get_album(&track.album.id).await? {
+            if let Some(album) = retry_spotify_call(
+                || async { spotify_client.get_album(&track.album.id).await },
+                "get_album",
+            )
+            .await?
+            {
                 track.album = album;
             }
 
-            if let Some(artist) = spotify_client
-                .get_artist(&track.album.artists[0].id)
-                .await?
+            if let Some(artist) = retry_spotify_call(
+                || async { spotify_client.get_artist(&track.album.artists[0].id).await },
+                "get_artist",
+            )
+            .await?
             {
                 track.album.artists[0] = artist;
             }
···
     let spotify_token = refresh_token(&spotify_token, &client_id, &client_secret).await?;
     let spotify_client = SpotifyClient::new(&spotify_token.access_token);
 
-    let result = spotify_client
-        .search(&format!(
-            r#"track:"{}" artist:"{}""#,
-            scrobble.track, scrobble.artist
-        ))
-        .await?;
+    let result = retry_spotify_call(
+        || async {
+            spotify_client
+                .search(&format!(
+                    r#"track:"{}" artist:"{}""#,
+                    scrobble.track, scrobble.artist
+                ))
+                .await
+        },
+        "search",
+    )
+    .await?;
 
     if let Some(track) = result.tracks.items.first() {
         let normalize = |s: &str| -> String {
···
             scrobble.album = Some(track.album.name.clone());
             let mut track = track.clone();
 
-            if let Some(album) = spotify_client.get_album(&track.album.id).await? {
+            if let Some(album) = retry_spotify_call(
+                || async { spotify_client.get_album(&track.album.id).await },
+                "get_album",
+            )
+            .await?
+            {
                 track.album = album;
             }
 
-            if let Some(artist) = spotify_client
-                .get_artist(&track.album.artists[0].id)
-                .await?
+            if let Some(artist) = retry_spotify_call(
+                || async { spotify_client.get_artist(&track.album.artists[0].id).await },
+                "get_artist",
+            )
+            .await?
             {
                 track.album.artists[0] = artist;
             }
···
 
     Ok(None)
 }
+
+async fn retry_spotify_call<F, Fut, T>(mut f: F, operation: &str) -> Result<T, Error>
+where
+    F: FnMut() -> Fut,
+    Fut: std::future::Future<Output = Result<T, Error>>,
+{
+    let mut last_error = None;
+
+    for attempt in 0..MAX_SPOTIFY_RETRIES {
+        match f().await {
+            Ok(result) => return Ok(result),
+            Err(e) => {
+                let is_timeout = e.to_string().contains("timed out")
+                    || e.to_string().contains("timeout")
+                    || e.to_string().contains("operation timed out");
+
+                if is_timeout && attempt < MAX_SPOTIFY_RETRIES - 1 {
+                    let delay = INITIAL_RETRY_DELAY_MS * 2_u64.pow(attempt);
+                    tracing::warn!(
+                        attempt = attempt + 1,
+                        max_attempts = MAX_SPOTIFY_RETRIES,
+                        delay_ms = delay,
+                        operation = operation,
+                        "Spotify API timeout, retrying..."
+                    );
+                    tokio::time::sleep(tokio::time::Duration::from_millis(delay)).await;
+                    last_error = Some(e);
+                } else {
+                    return Err(e);
+                }
+            }
+        }
+    }
+
+    Err(last_error.unwrap_or_else(|| Error::msg("Max retries exceeded")))
+}

crates/scrobbler/src/spotify/client.rs

···
 use super::types::{Album, Artist, SearchResponse};
 use anyhow::Error;
+use std::time::Duration;
 
 pub const BASE_URL: &str = "https://api.spotify.com/v1";
 
 pub struct SpotifyClient {
     token: String,
+    client: reqwest::Client,
 }
 
 impl SpotifyClient {
     pub fn new(token: &str) -> Self {
+        let client = reqwest::Client::builder()
+            .timeout(Duration::from_secs(30))
+            .connect_timeout(Duration::from_secs(10))
+            .build()
+            .expect("Failed to build HTTP client");
+
         SpotifyClient {
             token: token.to_string(),
+            client,
         }
     }
 
     pub async fn search(&self, query: &str) -> Result<SearchResponse, Error> {
         let url = format!("{}/search", BASE_URL);
-        let client = reqwest::Client::new();
-        let response = client
+        let response = self
+            .client
             .get(&url)
             .bearer_auth(&self.token)
             .query(&[("type", "track"), ("q", query)])
···
 
     pub async fn get_album(&self, id: &str) -> Result<Option<Album>, Error> {
         let url = format!("{}/albums/{}", BASE_URL, id);
-        let client = reqwest::Client::new();
-        let response = client.get(&url).bearer_auth(&self.token).send().await?;
+        let response = self
+            .client
+            .get(&url)
+            .bearer_auth(&self.token)
+            .send()
+            .await?;
 
         let headers = response.headers().clone();
         let data = response.text().await?;
···
 
     pub async fn get_artist(&self, id: &str) -> Result<Option<Artist>, Error> {
         let url = format!("{}/artists/{}", BASE_URL, id);
-        let client = reqwest::Client::new();
-        let response = client.get(&url).bearer_auth(&self.token).send().await?;
+        let response = self
+            .client
+            .get(&url)
+            .bearer_auth(&self.token)
+            .send()
+            .await?;
 
         let headers = response.headers().clone();
         let data = response.text().await?;

crates/webscrobbler/src/musicbrainz/client.rs

···
         let redis = client.get_multiplexed_tokio_connection().await?;
         let http = reqwest::Client::builder()
             .user_agent(USER_AGENT)
+            .timeout(Duration::from_secs(30))
+            .connect_timeout(Duration::from_secs(10))
             .build()
             .context("build http client")?;
         let me = MusicbrainzClient {

crates/webscrobbler/src/scrobbler.rs

···
 use rand::Rng;
 use sqlx::{Pool, Postgres};
 
+const MAX_SPOTIFY_RETRIES: u32 = 3;
+const INITIAL_RETRY_DELAY_MS: u64 = 1000;
+
 pub async fn scrobble(
     pool: &Pool<Postgres>,
     cache: &Cache,
···
 
     tracing::info!(query = %query, "Searching on Spotify");
 
-    let result = spotify_client.search(&query).await?;
+    let result =
+        retry_spotify_call(|| async { spotify_client.search(&query).await }, "search").await?;
 
     tracing::info!(total = %result.tracks.total, "Spotify search results");
 
···
             tracing::info!("Spotify (track)");
             let mut track = track.clone();
 
-            if let Some(album) = spotify_client.get_album(&track.album.id).await? {
+            if let Some(album) = retry_spotify_call(
+                || async { spotify_client.get_album(&track.album.id).await },
+                "get_album",
+            )
+            .await?
+            {
                 track.album = album;
             }
 
-            if let Some(artist) = spotify_client
-                .get_artist(&track.album.artists[0].id)
-                .await?
+            if let Some(artist) = retry_spotify_call(
+                || async { spotify_client.get_artist(&track.album.artists[0].id).await },
+                "get_artist",
+            )
+            .await?
             {
                 track.album.artists[0] = artist;
             }
···
 
     Ok(None)
 }
+
+async fn retry_spotify_call<F, Fut, T>(mut f: F, operation: &str) -> Result<T, Error>
+where
+    F: FnMut() -> Fut,
+    Fut: std::future::Future<Output = Result<T, Error>>,
+{
+    let mut last_error = None;
+
+    for attempt in 0..MAX_SPOTIFY_RETRIES {
+        match f().await {
+            Ok(result) => return Ok(result),
+            Err(e) => {
+                let is_timeout = e.to_string().contains("timed out")
+                    || e.to_string().contains("timeout")
+                    || e.to_string().contains("operation timed out");
+
+                if is_timeout && attempt < MAX_SPOTIFY_RETRIES - 1 {
+                    let delay = INITIAL_RETRY_DELAY_MS * 2_u64.pow(attempt);
+                    tracing::warn!(
+                        attempt = attempt + 1,
+                        max_attempts = MAX_SPOTIFY_RETRIES,
+                        delay_ms = delay,
+                        operation = operation,
+                        "Spotify API timeout, retrying..."
+                    );
+                    tokio::time::sleep(tokio::time::Duration::from_millis(delay)).await;
+                    last_error = Some(e);
+                } else {
+                    return Err(e);
+                }
+            }
+        }
+    }
+
+    Err(last_error.unwrap_or_else(|| Error::msg("Max retries exceeded")))
+}

crates/webscrobbler/src/spotify/client.rs

···
 use super::types::{Album, Artist, SearchResponse};
 use anyhow::Error;
+use std::time::Duration;
 
 pub const BASE_URL: &str = "https://api.spotify.com/v1";
 
 pub struct SpotifyClient {
     token: String,
+    client: reqwest::Client,
 }
 
 impl SpotifyClient {
     pub fn new(token: &str) -> Self {
+        let client = reqwest::Client::builder()
+            .timeout(Duration::from_secs(30))
+            .connect_timeout(Duration::from_secs(10))
+            .build()
+            .expect("Failed to build HTTP client");
+
         SpotifyClient {
             token: token.to_string(),
+            client,
         }
     }
 
     pub async fn search(&self, query: &str) -> Result<SearchResponse, Error> {
         let url = format!("{}/search", BASE_URL);
-        let client = reqwest::Client::new();
-        let response = client
+        let response = self
+            .client
             .get(&url)
             .bearer_auth(&self.token)
             .query(&[("type", "track"), ("q", query)])
···
 
     pub async fn get_album(&self, id: &str) -> Result<Option<Album>, Error> {
         let url = format!("{}/albums/{}", BASE_URL, id);
-        let client = reqwest::Client::new();
-        let response = client.get(&url).bearer_auth(&self.token).send().await?;
+        let response = self
+            .client
+            .get(&url)
+            .bearer_auth(&self.token)
+            .send()
+            .await?;
 
         let headers = response.headers().clone();
         let data = response.text().await?;
···
 
     pub async fn get_artist(&self, id: &str) -> Result<Option<Artist>, Error> {
         let url = format!("{}/artists/{}", BASE_URL, id);
-        let client = reqwest::Client::new();
-        let response = client.get(&url).bearer_auth(&self.token).send().await?;
+        let response = self
+            .client
+            .get(&url)
+            .bearer_auth(&self.token)
+            .send()
+            .await?;
 
         let headers = response.headers().clone();
         let data = response.text().await?;