fix fallback behavior in hosting service

hosting-service/src/lib/firehose.ts

···
 		// Invalidate in-memory caches (includes metadata which stores settings)
 		invalidateSiteCache(did, rkey)
 
-		// Update on-disk metadata with new settings
+		// Check if site is already cached
+		const cacheDir = `${CACHE_DIR}/${did}/${rkey}`
+		const isCached = existsSync(cacheDir)
+
+		if (!isCached) {
+			this.log('Site not cached yet, checking if fs record exists', { did, rkey })
+
+			// If site exists on PDS, cache it (which will include the new settings)
+			try {
+				const siteRecord = await fetchSiteRecord(did, rkey)
+
+				if (siteRecord) {
+					this.log('Site record found, triggering full cache with settings', { did, rkey })
+					const pdsEndpoint = await getPdsForDid(did)
+
+					if (pdsEndpoint) {
+						// Mark as being cached
+						markSiteAsBeingCached(did, rkey)
+
+						try {
+							await downloadAndCacheSite(did, rkey, siteRecord.record, pdsEndpoint, siteRecord.cid)
+							this.log('Successfully cached site with new settings', { did, rkey })
+						} finally {
+							unmarkSiteAsBeingCached(did, rkey)
+						}
+					} else {
+						this.log('Could not resolve PDS for DID', { did })
+					}
+				} else {
+					this.log('No fs record found for site, skipping cache', { did, rkey })
+				}
+			} catch (err) {
+				this.log('Failed to cache site after settings change', {
+					did,
+					rkey,
+					error: err instanceof Error ? err.message : String(err)
+				})
+			}
+
+			this.log('Successfully processed settings change (new cache)', { did, rkey })
+			return
+		}
+
+		// Site is already cached, just update the settings in metadata
 		try {
 			const { fetchSiteSettings, updateCacheMetadataSettings } = await import('./utils')
 			const settings = await fetchSiteSettings(did, rkey)

hosting-service/src/lib/utils.ts

···
 
 export async function getCachedSettings(did: string, rkey: string): Promise<WispSettings | null> {
   const metadata = await getCacheMetadata(did, rkey);
-  return metadata?.settings || null;
+
+  // If metadata has settings, return them
+  if (metadata?.settings) {
+    return metadata.settings;
+  }
+
+  // If metadata exists but has no settings, try to fetch from PDS and update cache
+  if (metadata) {
+    console.log('[Cache] Metadata missing settings, fetching from PDS', { did, rkey });
+    try {
+      const settings = await fetchSiteSettings(did, rkey);
+      if (settings) {
+        // Update the cached metadata with the fetched settings
+        await updateCacheMetadataSettings(did, rkey, settings);
+        console.log('[Cache] Updated metadata with fetched settings', { did, rkey });
+        return settings;
+      }
+    } catch (err) {
+      console.error('[Cache] Failed to fetch/update settings', { did, rkey, err });
+    }
+  }
+
+  return null;
 }
 
 export async function updateCacheMetadataSettings(did: string, rkey: string, settings: WispSettings | null): Promise<void> {

hosting-service/src/server.ts

···
     }
   }
 
+  // Directory listing fallback: if enabled, show root directory listing on 404
+  if (settings?.directoryListing) {
+    const rootPath = getCachedFilePath(did, rkey, '');
+    if (await fileExists(rootPath)) {
+      const { stat, readdir } = await import('fs/promises');
+      try {
+        const stats = await stat(rootPath);
+        if (stats.isDirectory()) {
+          const entries = await readdir(rootPath);
+          // Filter out .meta files and metadata
+          const visibleEntries = entries.filter(entry =>
+            !entry.endsWith('.meta') && entry !== '.metadata.json'
+          );
+
+          // Check which entries are directories
+          const entriesWithType = await Promise.all(
+            visibleEntries.map(async (name) => {
+              try {
+                const entryPath = `${rootPath}/${name}`;
+                const entryStats = await stat(entryPath);
+                return { name, isDirectory: entryStats.isDirectory() };
+              } catch {
+                return { name, isDirectory: false };
+              }
+            })
+          );
+
+          const html = generateDirectoryListing('', entriesWithType);
+          return new Response(html, {
+            status: 404,
+            headers: {
+              'Content-Type': 'text/html; charset=utf-8',
+              'Cache-Control': 'public, max-age=300',
+            },
+          });
+        }
+      } catch (err) {
+        // If directory listing fails, fall through to 404
+      }
+    }
+  }
+
   // Default styled 404 page
   const html = generate404Page();
   return new Response(html, {
···
         status: 404,
         headers: response.headers,
       });
+    }
+  }
+
+  // Directory listing fallback: if enabled, show root directory listing on 404
+  if (settings?.directoryListing) {
+    const rootPath = getCachedFilePath(did, rkey, '');
+    if (await fileExists(rootPath)) {
+      const { stat, readdir } = await import('fs/promises');
+      try {
+        const stats = await stat(rootPath);
+        if (stats.isDirectory()) {
+          const entries = await readdir(rootPath);
+          // Filter out .meta files and metadata
+          const visibleEntries = entries.filter(entry =>
+            !entry.endsWith('.meta') && entry !== '.metadata.json'
+          );
+
+          // Check which entries are directories
+          const entriesWithType = await Promise.all(
+            visibleEntries.map(async (name) => {
+              try {
+                const entryPath = `${rootPath}/${name}`;
+                const entryStats = await stat(entryPath);
+                return { name, isDirectory: entryStats.isDirectory() };
+              } catch {
+                return { name, isDirectory: false };
+              }
+            })
+          );
+
+          const html = generateDirectoryListing('', entriesWithType);
+          return new Response(html, {
+            status: 404,
+            headers: {
+              'Content-Type': 'text/html; charset=utf-8',
+              'Cache-Control': 'public, max-age=300',
+            },
+          });
+        }
+      } catch (err) {
+        // If directory listing fails, fall through to 404
+      }
     }
   }