back round 0 view raw

fix(m): wait for postgres connection #153

merged
opened by a.starrysky.fyi targeting main from private/minion/push-tuxqlnlmuxol

I woke up this morning to a dead menu instance and the following backtrace:

Jan 05 04:46:24 teal menu-start[1351]: thread 'main' (1351) panicked at src/main.rs:173:6:
Jan 05 04:46:24 teal menu-start[1351]: Failed to connect to database defined in $DATABASE_URL: Io(Os { code: 2, kind: NotFound, message: "No such file or directory" })
Jan 05 04:46:24 teal menu-start[1351]: note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

It seems like this is because, when teal restarted, postgresql hadn't created a socket by the time menu connected. Restarting menu after the fact let it run Just Fine. Adding some retries should stop this problem from coming up...

...a proper solution is probably to fix whatever decides that postgresql is ready in packetmix, but this is much easier to locate and implement

options
unified split
Changed files
+39 -20
menu
src
main.rs
+39 -20
menu/src/main.rs
··· 13 use serde::Deserialize; 14 use sqlx::{Connection, PgConnection}; 15 use std::{collections::HashMap, env, ops::DerefMut, sync::OnceLock}; 16 - use tokio::sync::Mutex; 17 use tower_http::normalize_path::NormalizePathLayer; 18 use tower_layer::Layer; 19 ··· 50 } 51 52 async fn get_redirect(default_location: &str, go: &str) -> Redirect { 53 - let redirect = sqlx::query!(r#"SELECT ("to") FROM direct WHERE "from" = $1 LIMIT 1"#, go.to_lowercase()) 54 - .fetch_one( 55 - STATE 56 - .get() 57 - .expect("Server must be initialized before processing connections") 58 - .sqlx_connection 59 - .lock() 60 - .await 61 - .deref_mut(), 62 - ) 63 - .await; 64 65 if let Ok(record) = redirect { 66 Redirect::temporary(&record.to) ··· 162 163 #[tokio::main] 164 async fn main() { 165 - let mut connection = PgConnection::connect( 166 - env::var("DATABASE_URL") 167 - .expect( 168 - "Please ensure you set your database URL in the $DATABASE_URL environment variable", 169 ) 170 - .as_str(), 171 - ) 172 - .await 173 - .expect("Failed to connect to database defined in $DATABASE_URL"); 174 175 sqlx::migrate!() 176 .run(&mut connection)
··· 13 use serde::Deserialize; 14 use sqlx::{Connection, PgConnection}; 15 use std::{collections::HashMap, env, ops::DerefMut, sync::OnceLock}; 16 + use tokio::{ 17 + sync::Mutex, 18 + time::{Duration, sleep}, 19 + }; 20 use tower_http::normalize_path::NormalizePathLayer; 21 use tower_layer::Layer; 22 ··· 53 } 54 55 async fn get_redirect(default_location: &str, go: &str) -> Redirect { 56 + let redirect = sqlx::query!( 57 + r#"SELECT ("to") FROM direct WHERE "from" = $1 LIMIT 1"#, 58 + go.to_lowercase() 59 + ) 60 + .fetch_one( 61 + STATE 62 + .get() 63 + .expect("Server must be initialized before processing connections") 64 + .sqlx_connection 65 + .lock() 66 + .await 67 + .deref_mut(), 68 + ) 69 + .await; 70 71 if let Ok(record) = redirect { 72 Redirect::temporary(&record.to) ··· 168 169 #[tokio::main] 170 async fn main() { 171 + let mut connection = { 172 + let mut maybe_connection; 173 + for _ in 0..3 { 174 + maybe_connection = PgConnection::connect( 175 + env::var("DATABASE_URL") 176 + .expect( 177 + "Please ensure you set your database URL in the $DATABASE_URL environment variable", 178 + ) 179 + .as_str(), 180 ) 181 + .await; 182 + 183 + if maybe_connection.is_ok() { 184 + break; 185 + } 186 + 187 + sleep(Duration::from_secs(5)).await; 188 + } 189 + 190 + maybe_connection 191 + .expect("Failed to connect to database defined in $DATABASE_URL after 3 retries") 192 + }; 193 194 sqlx::migrate!() 195 .run(&mut connection)