From c87ef3b6287bd8b1477a3a3622118b0805f68c9c Mon Sep 17 00:00:00 2001
From: Skyler Grey <minion@freshlybakedca.ke>
Date: Mon, 5 Jan 2026 07:44:35 +0000
Subject: [PATCH] fix(m): wait for postgres connection
Change-Id: tuxqlnlmuxolwuxluuoxvqnvnrsyqlsm

I woke up this morning to a dead menu instance and the following
backtrace:

    Jan 05 04:46:24 teal menu-start[1351]: thread 'main' (1351) panicked at src/main.rs:173:6:
    Jan 05 04:46:24 teal menu-start[1351]: Failed to connect to database defined in $DATABASE_URL: Io(Os { code: 2, kind: NotFound, message: "No such file or directory" })
    Jan 05 04:46:24 teal menu-start[1351]: note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

It seems like this is because, when teal restarted, postgresql hadn't
created a socket by the time menu connected. Restarting menu after the
fact let it run Just Fine. Adding some retries should stop this problem
from coming up...

...a proper solution is probably to fix whatever decides that postgresql
is ready in packetmix, but this is much easier to locate and implement
---
 menu/src/main.rs | 62 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 42 insertions(+), 20 deletions(-)

diff --git a/menu/src/main.rs b/menu/src/main.rs
index 0b5d70c3..a72a7377 100644
--- a/menu/src/main.rs
+++ b/menu/src/main.rs
@@ -13,7 +13,10 @@ use axum::{
 use serde::Deserialize;
 use sqlx::{Connection, PgConnection};
 use std::{collections::HashMap, env, ops::DerefMut, sync::OnceLock};
-use tokio::sync::Mutex;
+use tokio::{
+    sync::Mutex,
+    time::{Duration, sleep},
+};
 use tower_http::normalize_path::NormalizePathLayer;
 use tower_layer::Layer;
 
@@ -50,17 +53,20 @@ fn clean_host(provided_host: &str) -> &str {
 }
 
 async fn get_redirect(default_location: &str, go: &str) -> Redirect {
-    let redirect = sqlx::query!(r#"SELECT ("to") FROM direct WHERE "from" = $1 LIMIT 1"#, go.to_lowercase())
-        .fetch_one(
-            STATE
-                .get()
-                .expect("Server must be initialized before processing connections")
-                .sqlx_connection
-                .lock()
-                .await
-                .deref_mut(),
-        )
-        .await;
+    let redirect = sqlx::query!(
+        r#"SELECT ("to") FROM direct WHERE "from" = $1 LIMIT 1"#,
+        go.to_lowercase()
+    )
+    .fetch_one(
+        STATE
+            .get()
+            .expect("Server must be initialized before processing connections")
+            .sqlx_connection
+            .lock()
+            .await
+            .deref_mut(),
+    )
+    .await;
 
     if let Ok(record) = redirect {
         Redirect::temporary(&record.to)
@@ -162,15 +168,31 @@ async fn handle_404() -> impl IntoResponse {
 
 #[tokio::main]
 async fn main() {
-    let mut connection = PgConnection::connect(
-        env::var("DATABASE_URL")
-            .expect(
-                "Please ensure you set your database URL in the $DATABASE_URL environment variable",
+    let mut connection = {
+        let mut maybe_connection;
+        let mut tries = 3;
+        loop {
+            // We can't use a for loop here as rust doesn't know it will run at least once...
+            tries -= 1;
+            maybe_connection = PgConnection::connect(
+                env::var("DATABASE_URL")
+                    .expect(
+                        "Please ensure you set your database URL in the $DATABASE_URL environment variable",
+                    )
+                    .as_str(),
             )
-            .as_str(),
-    )
-    .await
-    .expect("Failed to connect to database defined in $DATABASE_URL");
+            .await;
+
+            if maybe_connection.is_ok() || tries == 0 {
+                break;
+            }
+
+            sleep(Duration::from_secs(5)).await;
+        }
+
+        maybe_connection
+            .expect("Failed to connect to database defined in $DATABASE_URL after 3 retries")
+    };
 
     sqlx::migrate!()
         .run(&mut connection)
-- 
2.43.0