dont serve content while caching new site

hosting-service/src/lib/backfill.ts

···
 import { getAllSites } from './db';
 import { fetchSiteRecord, getPdsForDid, downloadAndCacheSite, isCached } from './utils';
 import { logger } from './observability';
+import { markSiteAsBeingCached, unmarkSiteAsBeingCached } from './cache';
 
 export interface BackfillOptions {
   skipExisting?: boolean; // Skip sites already in cache
···
               return;
             }
 
-            // Download and cache site
-            await downloadAndCacheSite(site.did, site.rkey, siteData.record, pdsEndpoint, siteData.cid);
-            stats.cached++;
-            processed++;
-            logger.info('Successfully cached site during backfill', { did: site.did, rkey: site.rkey });
-            console.log(`✅ [${processed}/${sites.length}] Cached: ${site.display_name || site.rkey}`);
+            // Mark site as being cached to prevent serving stale content during update
+            markSiteAsBeingCached(site.did, site.rkey);
+
+            try {
+              // Download and cache site
+              await downloadAndCacheSite(site.did, site.rkey, siteData.record, pdsEndpoint, siteData.cid);
+              stats.cached++;
+              processed++;
+              logger.info('Successfully cached site during backfill', { did: site.did, rkey: site.rkey });
+              console.log(`✅ [${processed}/${sites.length}] Cached: ${site.display_name || site.rkey}`);
+            } finally {
+              // Always unmark, even if caching fails
+              unmarkSiteAsBeingCached(site.did, site.rkey);
+            }
           } catch (err) {
             stats.failed++;
             processed++;

hosting-service/src/lib/cache.ts

···
   console.log(`[Cache] Invalidated site ${did}:${rkey} - ${fileCount} files, ${metaCount} metadata, ${htmlCount} HTML`);
 }
 
+// Track sites currently being cached (to prevent serving stale cache during updates)
+const sitesBeingCached = new Set<string>();
+
+export function markSiteAsBeingCached(did: string, rkey: string): void {
+  const key = `${did}:${rkey}`;
+  sitesBeingCached.add(key);
+}
+
+export function unmarkSiteAsBeingCached(did: string, rkey: string): void {
+  const key = `${did}:${rkey}`;
+  sitesBeingCached.delete(key);
+}
+
+export function isSiteBeingCached(did: string, rkey: string): boolean {
+  const key = `${did}:${rkey}`;
+  return sitesBeingCached.has(key);
+}
+
 // Get overall cache statistics
 export function getCacheStats() {
   return {
···
     metadataHitRate: metadataCache.getHitRate(),
     rewrittenHtml: rewrittenHtmlCache.getStats(),
     rewrittenHtmlHitRate: rewrittenHtmlCache.getHitRate(),
+    sitesBeingCached: sitesBeingCached.size,
   };
 }

hosting-service/src/lib/firehose.ts

···
 import { isRecord, validateRecord } from '../lexicon/types/place/wisp/fs'
 import { Firehose } from '@atproto/sync'
 import { IdResolver } from '@atproto/identity'
-import { invalidateSiteCache } from './cache'
+import { invalidateSiteCache, markSiteAsBeingCached, unmarkSiteAsBeingCached } from './cache'
 
 const CACHE_DIR = './cache/sites'
 
···
 		// Invalidate in-memory caches before updating
 		invalidateSiteCache(did, site)
 
-		// Cache the record with verified CID (uses atomic swap internally)
-		// All instances cache locally for edge serving
-		await downloadAndCacheSite(
-			did,
-			site,
-			fsRecord,
-			pdsEndpoint,
-			verifiedCid
-		)
-
-		// Acquire distributed lock only for database write to prevent duplicate writes
-		// Note: upsertSite will check cache-only mode internally and skip if needed
-		const lockKey = `db:upsert:${did}:${site}`
-		const lockAcquired = await tryAcquireLock(lockKey)
-
-		if (!lockAcquired) {
-			this.log('Another instance is writing to DB, skipping upsert', {
-				did,
-				site
-			})
-			this.log('Successfully processed create/update (cached locally)', {
-				did,
-				site
-			})
-			return
-		}
+		// Mark site as being cached to prevent serving stale content during update
+		markSiteAsBeingCached(did, site)
 
 		try {
-			// Upsert site to database (only one instance does this)
-			// In cache-only mode, this will be a no-op
-			await upsertSite(did, site, fsRecord.site)
-			this.log(
-				'Successfully processed create/update (cached + DB updated)',
-				{ did, site }
+			// Cache the record with verified CID (uses atomic swap internally)
+			// All instances cache locally for edge serving
+			await downloadAndCacheSite(
+				did,
+				site,
+				fsRecord,
+				pdsEndpoint,
+				verifiedCid
 			)
+
+			// Acquire distributed lock only for database write to prevent duplicate writes
+			// Note: upsertSite will check cache-only mode internally and skip if needed
+			const lockKey = `db:upsert:${did}:${site}`
+			const lockAcquired = await tryAcquireLock(lockKey)
+
+			if (!lockAcquired) {
+				this.log('Another instance is writing to DB, skipping upsert', {
+					did,
+					site
+				})
+				this.log('Successfully processed create/update (cached locally)', {
+					did,
+					site
+				})
+				return
+			}
+
+			try {
+				// Upsert site to database (only one instance does this)
+				// In cache-only mode, this will be a no-op
+				await upsertSite(did, site, fsRecord.site)
+				this.log(
+					'Successfully processed create/update (cached + DB updated)',
+					{ did, site }
+				)
+			} finally {
+				// Always release lock, even if DB write fails
+				await releaseLock(lockKey)
+			}
 		} finally {
-			// Always release lock, even if DB write fails
-			await releaseLock(lockKey)
+			// Always unmark, even if caching fails
+			unmarkSiteAsBeingCached(did, site)
 		}
 	}
 

hosting-service/src/server.ts

···
 import { readFile, access } from 'fs/promises';
 import { lookup } from 'mime-types';
 import { logger, observabilityMiddleware, observabilityErrorHandler, logCollector, errorTracker, metricsCollector } from './lib/observability';
-import { fileCache, metadataCache, rewrittenHtmlCache, getCacheKey, type FileMetadata } from './lib/cache';
+import { fileCache, metadataCache, rewrittenHtmlCache, getCacheKey, type FileMetadata, markSiteAsBeingCached, unmarkSiteAsBeingCached, isSiteBeingCached } from './lib/cache';
 import { loadRedirectRules, matchRedirectRule, parseCookies, parseQueryString, type RedirectRule } from './lib/redirects';
 
 const BASE_HOST = process.env.BASE_HOST || 'wisp.place';
···
   }
 }
 
+/**
+ * Return a response indicating the site is being updated
+ */
+function siteUpdatingResponse(): Response {
+  const html = `<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Site Updating</title>
+  <style>
+    @media (prefers-color-scheme: light) {
+      :root {
+        --background: oklch(0.90 0.012 35);
+        --foreground: oklch(0.18 0.01 30);
+        --primary: oklch(0.35 0.02 35);
+        --accent: oklch(0.78 0.15 345);
+      }
+    }
+    @media (prefers-color-scheme: dark) {
+      :root {
+        --background: oklch(0.23 0.015 285);
+        --foreground: oklch(0.90 0.005 285);
+        --primary: oklch(0.70 0.10 295);
+        --accent: oklch(0.85 0.08 5);
+      }
+    }
+    body {
+      font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      min-height: 100vh;
+      margin: 0;
+      background: var(--background);
+      color: var(--foreground);
+    }
+    .container {
+      text-align: center;
+      padding: 2rem;
+      max-width: 500px;
+    }
+    h1 {
+      font-size: 2.5rem;
+      margin-bottom: 1rem;
+      font-weight: 600;
+      color: var(--primary);
+    }
+    p {
+      font-size: 1.25rem;
+      opacity: 0.8;
+      margin-bottom: 2rem;
+      color: var(--foreground);
+    }
+    .spinner {
+      border: 4px solid var(--accent);
+      border-radius: 50%;
+      border-top: 4px solid var(--primary);
+      width: 40px;
+      height: 40px;
+      animation: spin 1s linear infinite;
+      margin: 0 auto;
+    }
+    @keyframes spin {
+      0% { transform: rotate(0deg); }
+      100% { transform: rotate(360deg); }
+    }
+  </style>
+  <meta http-equiv="refresh" content="3">
+</head>
+<body>
+  <div class="container">
+    <h1>Site Updating</h1>
+    <p>This site is undergoing an update right now. Check back in a moment...</p>
+    <div class="spinner"></div>
+  </div>
+</body>
+</html>`;
+
+  return new Response(html, {
+    status: 503,
+    headers: {
+      'Content-Type': 'text/html; charset=utf-8',
+      'Cache-Control': 'no-store, no-cache, must-revalidate',
+      'Retry-After': '3',
+    },
+  });
+}
+
 // Cache for redirect rules (per site)
 const redirectRulesCache = new Map<string, RedirectRule[]>();
 
···
 
 // Internal function to serve a file (used by both normal serving and rewrites)
 async function serveFileInternal(did: string, rkey: string, filePath: string) {
+  // Check if site is currently being cached - if so, return updating response
+  if (isSiteBeingCached(did, rkey)) {
+    return siteUpdatingResponse();
+  }
+
   // Default to first index file if path is empty
   let requestPath = filePath || INDEX_FILES[0];
 
···
 
 // Internal function to serve a file with rewriting
 async function serveFileInternalWithRewrite(did: string, rkey: string, filePath: string, basePath: string) {
+  // Check if site is currently being cached - if so, return updating response
+  if (isSiteBeingCached(did, rkey)) {
+    return siteUpdatingResponse();
+  }
+
   // Default to first index file if path is empty
   let requestPath = filePath || INDEX_FILES[0];
 
···
     return false;
   }
 
+  // Mark site as being cached to prevent serving stale content during update
+  markSiteAsBeingCached(did, rkey);
+
   try {
     await downloadAndCacheSite(did, rkey, siteData.record, pdsEndpoint, siteData.cid);
     // Clear redirect rules cache since the site was updated
···
   } catch (err) {
     logger.error('Failed to cache site', err, { did, rkey });
     return false;
+  } finally {
+    // Always unmark, even if caching fails
+    unmarkSiteAsBeingCached(did, rkey);
   }
 }
 
···
   const rawPath = url.pathname.replace(/^\//, '');
   const path = sanitizePath(rawPath);
 
-  // Check if this is sites.wisp.place subdomain
-  if (hostname === `sites.${BASE_HOST}` || hostname === `sites.${BASE_HOST}:${process.env.PORT || 3000}`) {
+  // Check if this is sites.wisp.place subdomain (strip port for comparison)
+  const hostnameWithoutPort = hostname.split(':')[0];
+  if (hostnameWithoutPort === `sites.${BASE_HOST}`) {
     // Sanitize the path FIRST to prevent path traversal
     const sanitizedFullPath = sanitizePath(rawPath);
 
···
     const did = await resolveDid(identifier);
     if (!did) {
       return c.text('Invalid identifier', 400);
+    }
+
+    // Check if site is currently being cached - return updating response early
+    if (isSiteBeingCached(did, site)) {
+      return siteUpdatingResponse();
     }
 
     // Ensure site is cached
···
       return c.text('Invalid site configuration', 500);
     }
 
+    // Check if site is currently being cached - return updating response early
+    if (isSiteBeingCached(customDomain.did, rkey)) {
+      return siteUpdatingResponse();
+    }
+
     const cached = await ensureSiteCached(customDomain.did, rkey);
     if (!cached) {
       return c.text('Site not found', 404);
···
       return c.text('Invalid site configuration', 500);
     }
 
+    // Check if site is currently being cached - return updating response early
+    if (isSiteBeingCached(domainInfo.did, rkey)) {
+      return siteUpdatingResponse();
+    }
+
     const cached = await ensureSiteCached(domainInfo.did, rkey);
     if (!cached) {
       return c.text('Site not found', 404);
···
   const rkey = customDomain.rkey;
   if (!isValidRkey(rkey)) {
     return c.text('Invalid site configuration', 500);
+  }
+
+  // Check if site is currently being cached - return updating response early
+  if (isSiteBeingCached(customDomain.did, rkey)) {
+    return siteUpdatingResponse();
   }
 
   const cached = await ensureSiteCached(customDomain.did, rkey);

src/lib/oauth-client.ts

···
         // Loopback client for local development
         // For loopback, scopes and redirect_uri must be in client_id query string
         const redirectUri = 'http://127.0.0.1:8000/api/auth/callback';
-        const scope = 'atproto repo:place.wisp.fs repo:place.wisp.domain repo:place.wisp.subfs blob:*/* blob?maxSize=100000000 rpc:app.bsky.actor.getProfile?aud=*';
+        const scope = 'atproto repo:place.wisp.fs repo:place.wisp.domain repo:place.wisp.subfs blob:*/* blob?maxSize=100000000 rpc:*?aud=did:web:api.bsky.app#bsky_appview';
         const params = new URLSearchParams();
         params.append('redirect_uri', redirectUri);
         params.append('scope', scope);
···
         application_type: 'web',
         token_endpoint_auth_method: 'private_key_jwt',
         token_endpoint_auth_signing_alg: "ES256",
-        scope: "atproto repo:place.wisp.fs repo:place.wisp.domain repo:place.wisp.subfs blob:*/* blob?maxSize=100000000 rpc:app.bsky.actor.getProfile?aud=*",
+        scope: "atproto repo:place.wisp.fs repo:place.wisp.domain repo:place.wisp.subfs blob:*/* blob?maxSize=100000000 rpc:*?aud=did:web:api.bsky.app#bsky_appview",
         dpop_bound_access_tokens: true,
         jwks_uri: `${config.domain}/jwks.json`,
         subject_type: 'public',