add flat: boolean to subfs definition in place.wisp.fs to define if to merge flat or as a subdirectory, handle this logic better across backend and hosting service

hosting-service/src/lexicon/lexicons.ts

···
             type: 'string',
             format: 'at-uri',
             description:
-              'AT-URI pointing to a place.wisp.subfs record containing this subtree',
+              'AT-URI pointing to a place.wisp.subfs record containing this subtree.',
+          },
+          flat: {
+            type: 'boolean',
+            description:
+              "If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure.",
           },
         },
       },
···
       main: {
         type: 'record',
         description:
-          'Virtual filesystem manifest within a place.wisp.fs record',
+          'Virtual filesystem subtree referenced by place.wisp.fs records. When a subfs entry is expanded, its root entries are merged (flattened) into the parent directory, allowing large directories to be split across multiple records while maintaining a flat structure.',
         record: {
           type: 'object',
           required: ['root', 'createdAt'],
···
             type: 'string',
             format: 'at-uri',
             description:
-              'AT-URI pointing to another place.wisp.subfs record for nested subtrees',
+              "AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures.",
           },
         },
       },

hosting-service/src/lexicon/types/place/wisp/fs.ts

···
 export interface Subfs {
   $type?: 'place.wisp.fs#subfs'
   type: 'subfs'
-  /** AT-URI pointing to a place.wisp.subfs record containing this subtree */
+  /** AT-URI pointing to a place.wisp.subfs record containing this subtree. */
   subject: string
+  /** If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure. */
+  flat?: boolean
 }
 
 const hashSubfs = 'subfs'

hosting-service/src/lexicon/types/place/wisp/subfs.ts

···
 export interface Subfs {
   $type?: 'place.wisp.subfs#subfs'
   type: 'subfs'
-  /** AT-URI pointing to another place.wisp.subfs record for nested subtrees */
+  /** AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures. */
   subject: string
 }
 

hosting-service/src/lib/firehose.ts

···
 								evt.cid?.toString()
 							)
 						} catch (err) {
+							console.error('Full error details:', err);
 							this.log('Error handling event', {
 								did: evt.did,
 								event: evt.event,

hosting-service/src/lib/utils.ts

···
       const node = entry.node;
 
       if ('type' in node && node.type === 'subfs') {
-        // Merge subfs entries into parent directory
+        // Check if this is a flat merge or subdirectory merge (default to flat if not specified)
+        const subfsNode = node as any;
+        const isFlat = subfsNode.flat !== false; // Default to true
         const subfsEntries = subfsMap.get(fullPath);
+
         if (subfsEntries) {
-          console.log(`Merging subfs node at ${fullPath} (${subfsEntries.length} entries)`);
-          // Recursively process the merged entries in case they contain nested subfs
-          const processedEntries = replaceSubfsInEntries(subfsEntries, currentPath);
-          result.push(...processedEntries);
+          console.log(`Merging subfs node at ${fullPath} (${subfsEntries.length} entries, flat: ${isFlat})`);
+
+          if (isFlat) {
+            // Flat merge: hoist entries directly into parent directory
+            const processedEntries = replaceSubfsInEntries(subfsEntries, currentPath);
+            result.push(...processedEntries);
+          } else {
+            // Subdirectory merge: create a directory with the subfs node's name
+            const processedEntries = replaceSubfsInEntries(subfsEntries, fullPath);
+            result.push({
+              name: entry.name,
+              node: {
+                type: 'directory',
+                entries: processedEntries
+              }
+            });
+          }
         } else {
           // If fetch failed, skip this entry
           console.warn(`Failed to fetch subfs at ${fullPath}, skipping`);
···
 
   // Download new/changed files concurrently - increased from 3 to 20 for much better performance
   const downloadLimit = 20;
+  let successCount = 0;
+  let failureCount = 0;
+
   for (let i = 0; i < downloadTasks.length; i += downloadLimit) {
     const batch = downloadTasks.slice(i, i + downloadLimit);
-    await Promise.all(batch.map(task => task()));
+    const results = await Promise.allSettled(batch.map(task => task()));
+
+    // Count successes and failures
+    results.forEach((result, index) => {
+      if (result.status === 'fulfilled') {
+        successCount++;
+      } else {
+        failureCount++;
+        console.error(`[Cache] Failed to download file (continuing with others):`, result.reason);
+      }
+    });
+
     if (downloadTasks.length > downloadLimit) {
-      console.log(`[Cache Progress] Downloaded ${Math.min(i + downloadLimit, downloadTasks.length)}/${downloadTasks.length} files`);
+      console.log(`[Cache Progress] Downloaded ${Math.min(i + downloadLimit, downloadTasks.length)}/${downloadTasks.length} files (${failureCount} failed)`);
     }
+  }
+
+  if (failureCount > 0) {
+    console.warn(`[Cache] Completed with ${successCount} successful and ${failureCount} failed file downloads`);
   }
 }
 
···
   }
 
   const blobUrl = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`;
+
+  console.log(`[Cache] Fetching blob for file: ${filePath}, CID: ${cid}`);
 
   // Allow up to 500MB per file blob, with 5 minute timeout
   let content = await safeFetchBlob(blobUrl, { maxSize: 500 * 1024 * 1024, timeout: 300000 });

lexicons/fs.json

···
       "required": ["type", "subject"],
       "properties": {
         "type": { "type": "string", "const": "subfs" },
-        "subject": { "type": "string", "format": "at-uri", "description": "AT-URI pointing to a place.wisp.subfs record containing this subtree. When expanded, the subfs record's root entries are merged (flattened) into the parent directory - the subfs entry itself is removed and replaced by all entries from the referenced record's root. This allows splitting large directories across multiple records while maintaining a flat structure." }
+        "subject": { "type": "string", "format": "at-uri", "description": "AT-URI pointing to a place.wisp.subfs record containing this subtree." },
+        "flat": { "type": "boolean", "description": "If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure." }
       }
     }
   }

src/lexicons/lexicons.ts

···
             type: 'string',
             format: 'at-uri',
             description:
-              'AT-URI pointing to a place.wisp.subfs record containing this subtree',
+              'AT-URI pointing to a place.wisp.subfs record containing this subtree.',
+          },
+          flat: {
+            type: 'boolean',
+            description:
+              "If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure.",
           },
         },
       },
···
       main: {
         type: 'record',
         description:
-          'Virtual filesystem manifest within a place.wisp.fs record',
+          'Virtual filesystem subtree referenced by place.wisp.fs records. When a subfs entry is expanded, its root entries are merged (flattened) into the parent directory, allowing large directories to be split across multiple records while maintaining a flat structure.',
         record: {
           type: 'object',
           required: ['root', 'createdAt'],
···
             type: 'string',
             format: 'at-uri',
             description:
-              'AT-URI pointing to another place.wisp.subfs record for nested subtrees',
+              "AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures.",
           },
         },
       },

src/lexicons/types/place/wisp/fs.ts

···
 export interface Subfs {
   $type?: 'place.wisp.fs#subfs'
   type: 'subfs'
-  /** AT-URI pointing to a place.wisp.subfs record containing this subtree */
+  /** AT-URI pointing to a place.wisp.subfs record containing this subtree. */
   subject: string
+  /** If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure. */
+  flat?: boolean
 }
 
 const hashSubfs = 'subfs'

src/lexicons/types/place/wisp/subfs.ts

···
 export interface Subfs {
   $type?: 'place.wisp.subfs#subfs'
   type: 'subfs'
-  /** AT-URI pointing to another place.wisp.subfs record for nested subtrees */
+  /** AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures. */
   subject: string
 }
 

src/lib/wisp-utils.ts

···
 					node: {
 						$type: 'place.wisp.fs#subfs' as const,
 						type: 'subfs' as const,
-						subject: subfsUri
+						subject: subfsUri,
+						flat: false  // Preserve directory structure
 					}
 				};
 			}

src/routes/wisp.ts

···
                         node: {
                             $type: 'place.wisp.fs#subfs' as const,
                             type: 'subfs' as const,
-                            subject: subfsUri
+                            subject: subfsUri,
+                            flat: true  // Merge entries directly into parent (default, but explicit for clarity)
                         }
                     });