expo backoff, fix concurrency

apps/hosting-service/src/index.ts

···
 
 const PORT = process.env.PORT ? parseInt(process.env.PORT) : 3001;
 const CACHE_DIR = process.env.CACHE_DIR || './cache/sites';
+const BACKFILL_CONCURRENCY = process.env.BACKFILL_CONCURRENCY
+  ? parseInt(process.env.BACKFILL_CONCURRENCY)
+  : undefined; // Let backfill.ts default (10) apply
 
 // Parse CLI arguments
 const args = process.argv.slice(2);
···
   console.log('🔄 Backfill requested, starting cache backfill...');
   backfillCache({
     skipExisting: true,
-    concurrency: 3,
+    concurrency: BACKFILL_CONCURRENCY,
   }).then((stats) => {
     console.log('✅ Cache backfill completed');
   }).catch((err) => {
···
 Cache:        ${CACHE_DIR}
 Firehose:     Connected to Firehose
 Cache-Only:   ${CACHE_ONLY_MODE ? 'ENABLED (no DB writes)' : 'DISABLED'}
+Backfill:     ${backfillOnStartup ? `ENABLED (concurrency: ${BACKFILL_CONCURRENCY || 10})` : 'DISABLED'}
 `);
 
 // Graceful shutdown

apps/hosting-service/src/lib/utils.ts

···
 export async function fetchSiteRecord(did: string, rkey: string): Promise<{ record: WispFsRecord; cid: string } | null> {
   try {
     const pdsEndpoint = await getPdsForDid(did);
-    if (!pdsEndpoint) return null;
+    if (!pdsEndpoint) {
+      console.error('[hosting-service] Failed to get PDS endpoint for DID', { did, rkey });
+      return null;
+    }
 
     const url = `${pdsEndpoint}/xrpc/com.atproto.repo.getRecord?repo=${encodeURIComponent(did)}&collection=place.wisp.fs&rkey=${encodeURIComponent(rkey)}`;
     const data = await safeFetchJson(url);
···
       cid: data.cid || ''
     };
   } catch (err) {
-    console.error('Failed to fetch site record', did, rkey, err);
+    const errorCode = (err as any)?.code;
+    const errorMsg = err instanceof Error ? err.message : String(err);
+
+    // Better error logging to distinguish between network errors and 404s
+    if (errorMsg.includes('HTTP 404') || errorMsg.includes('Not Found')) {
+      console.log('[hosting-service] Site record not found', { did, rkey });
+    } else if (errorCode && ['ECONNRESET', 'ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR', 'ETIMEDOUT'].includes(errorCode)) {
+      console.error('[hosting-service] Network/SSL error fetching site record (after retries)', {
+        did,
+        rkey,
+        error: errorMsg,
+        code: errorCode
+      });
+    } else {
+      console.error('[hosting-service] Failed to fetch site record', {
+        did,
+        rkey,
+        error: errorMsg,
+        code: errorCode
+      });
+    }
+
     return null;
   }
 }

packages/@wisp/safe-fetch/src/index.ts

···
 const MAX_BLOB_SIZE = 500 * 1024 * 1024; // 500MB
 const MAX_REDIRECTS = 10;
 
+// Retry configuration
+const MAX_RETRIES = 3;
+const INITIAL_RETRY_DELAY = 1000; // 1 second
+const MAX_RETRY_DELAY = 10000; // 10 seconds
+
 function isBlockedHost(hostname: string): boolean {
 	const lowerHost = hostname.toLowerCase();
 
···
 	return false;
 }
 
+/**
+ * Check if an error is retryable (network/SSL errors, not HTTP errors)
+ */
+function isRetryableError(err: unknown): boolean {
+	if (!(err instanceof Error)) return false;
+
+	// Network errors (ECONNRESET, ENOTFOUND, etc.)
+	const errorCode = (err as any).code;
+	if (errorCode) {
+		const retryableCodes = [
+			'ECONNRESET',
+			'ECONNREFUSED',
+			'ETIMEDOUT',
+			'ENOTFOUND',
+			'ENETUNREACH',
+			'EAI_AGAIN',
+			'EPIPE',
+			'ERR_SSL_TLSV1_ALERT_INTERNAL_ERROR', // SSL/TLS handshake failures
+			'ERR_SSL_WRONG_VERSION_NUMBER',
+			'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
+		];
+		if (retryableCodes.includes(errorCode)) {
+			return true;
+		}
+	}
+
+	// Timeout errors
+	if (err.name === 'AbortError' || err.message.includes('timeout')) {
+		return true;
+	}
+
+	// Fetch failures (generic network errors)
+	if (err.message.includes('fetch failed')) {
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * Sleep for a given number of milliseconds
+ */
+function sleep(ms: number): Promise<void> {
+	return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Retry a function with exponential backoff
+ */
+async function withRetry<T>(
+	fn: () => Promise<T>,
+	options: { maxRetries?: number; initialDelay?: number; maxDelay?: number; context?: string } = {}
+): Promise<T> {
+	const maxRetries = options.maxRetries ?? MAX_RETRIES;
+	const initialDelay = options.initialDelay ?? INITIAL_RETRY_DELAY;
+	const maxDelay = options.maxDelay ?? MAX_RETRY_DELAY;
+	const context = options.context ?? 'Request';
+
+	let lastError: unknown;
+
+	for (let attempt = 0; attempt <= maxRetries; attempt++) {
+		try {
+			return await fn();
+		} catch (err) {
+			lastError = err;
+
+			// Don't retry if this is the last attempt or error is not retryable
+			if (attempt === maxRetries || !isRetryableError(err)) {
+				throw err;
+			}
+
+			// Calculate delay with exponential backoff
+			const delay = Math.min(initialDelay * Math.pow(2, attempt), maxDelay);
+
+			const errorCode = (err as any)?.code;
+			const errorMsg = err instanceof Error ? err.message : String(err);
+			console.warn(
+				`${context} failed (attempt ${attempt + 1}/${maxRetries + 1}): ${errorMsg}${errorCode ? ` [${errorCode}]` : ''} - retrying in ${delay}ms`
+			);
+
+			await sleep(delay);
+		}
+	}
+
+	throw lastError;
+}
+
 export async function safeFetch(
 	url: string,
-	options?: RequestInit & { maxSize?: number; timeout?: number }
+	options?: RequestInit & { maxSize?: number; timeout?: number; retry?: boolean }
 ): Promise<Response> {
+	const shouldRetry = options?.retry !== false; // Default to true
 	const timeoutMs = options?.timeout ?? FETCH_TIMEOUT;
 	const maxSize = options?.maxSize ?? MAX_RESPONSE_SIZE;
 
-	// Parse and validate URL
+	// Parse and validate URL (done once, outside retry loop)
 	let parsedUrl: URL;
 	try {
 		parsedUrl = new URL(url);
···
 		throw new Error(`Blocked host: ${hostname}`);
 	}
 
-	const controller = new AbortController();
-	const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+	const fetchFn = async () => {
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+		try {
+			const response = await fetch(url, {
+				...options,
+				signal: controller.signal,
+				redirect: 'follow',
+				headers: {
+					'User-Agent': 'wisp-place hosting-service',
+					...(options?.headers || {}),
+				},
+			});
 
-	try {
-		const response = await fetch(url, {
-			...options,
-			signal: controller.signal,
-			redirect: 'follow',
-			headers: {
-				'User-Agent': 'wisp-place hosting-service',
-				...(options?.headers || {}),
-			},
-		});
+			const contentLength = response.headers.get('content-length');
+			if (contentLength && parseInt(contentLength, 10) > maxSize) {
+				throw new Error(`Response too large: ${contentLength} bytes`);
+			}
 
-		const contentLength = response.headers.get('content-length');
-		if (contentLength && parseInt(contentLength, 10) > maxSize) {
-			throw new Error(`Response too large: ${contentLength} bytes`);
+			return response;
+		} catch (err) {
+			if (err instanceof Error && err.name === 'AbortError') {
+				throw new Error(`Request timeout after ${timeoutMs}ms`);
+			}
+			throw err;
+		} finally {
+			clearTimeout(timeoutId);
 		}
+	};
 
-		return response;
-	} catch (err) {
-		if (err instanceof Error && err.name === 'AbortError') {
-			throw new Error(`Request timeout after ${timeoutMs}ms`);
-		}
-		throw err;
-	} finally {
-		clearTimeout(timeoutId);
+	if (shouldRetry) {
+		return withRetry(fetchFn, { context: `Fetch ${parsedUrl.hostname}` });
+	} else {
+		return fetchFn();
 	}
 }
 
 export async function safeFetchJson<T = any>(
 	url: string,
-	options?: RequestInit & { maxSize?: number; timeout?: number }
+	options?: RequestInit & { maxSize?: number; timeout?: number; retry?: boolean }
 ): Promise<T> {
 	const maxJsonSize = options?.maxSize ?? MAX_JSON_SIZE;
 	const response = await safeFetch(url, { ...options, maxSize: maxJsonSize });
···
 
 export async function safeFetchBlob(
 	url: string,
-	options?: RequestInit & { maxSize?: number; timeout?: number }
+	options?: RequestInit & { maxSize?: number; timeout?: number; retry?: boolean }
 ): Promise<Uint8Array> {
 	const maxBlobSize = options?.maxSize ?? MAX_BLOB_SIZE;
 	const timeoutMs = options?.timeout ?? FETCH_TIMEOUT_BLOB;