fixes to cli not handling subfs correctly, fix hosting service storing metadata as object Object I LOVE JAVASCRIPT

apps/main-app/src/routes/wisp.ts

···
     updateFileBlobs,
     findLargeDirectories,
     replaceDirectoryWithSubfs,
-    estimateDirectorySize
+    estimateDirectorySize,
+    splitDirectoryIntoChunks,
+    countFilesInDirectory
 } from '@wispplace/fs-utils'
 import {
     shouldCompressFile,
···
         const MAX_MANIFEST_SIZE = 140 * 1024; // 140KB to be safe (PDS limit is 150KB)
         const FILE_COUNT_THRESHOLD = 250; // Start splitting at this many files
         const TARGET_FILE_COUNT = 200; // Try to keep main manifest under this many files
+        const MAX_SUBFS_SIZE = 75 * 1024; // 75KB per subfs record
         const subfsRecords: Array<{ uri: string; path: string }> = [];
         let workingDirectory = updatedDirectory;
         let currentFileCount = actualFileCount;
···
                     const largestDir = directories[0];
                     console.log(`  Split #${attempts}: ${largestDir.path} (${largestDir.fileCount} files, ${(largestDir.size / 1024).toFixed(1)}KB)`);
 
-                    // Create a subfs record for this directory
-                    const subfsRkey = TID.nextStr();
-                    const subfsManifest = {
-                        $type: 'place.wisp.subfs' as const,
-                        root: largestDir.directory,
-                        fileCount: largestDir.fileCount,
-                        createdAt: new Date().toISOString()
-                    };
+                    let subfsUri: string;
+
+                    if (largestDir.size > MAX_SUBFS_SIZE) {
+                        // Directory too large for a single subfs — split into chunks
+                        console.log(`    → Directory too large (${(largestDir.size / 1024).toFixed(1)}KB), splitting into chunks...`);
+                        const chunks = splitDirectoryIntoChunks(largestDir.directory, MAX_SUBFS_SIZE);
+                        console.log(`    → Created ${chunks.length} chunks`);
+
+                        const chunkUris: string[] = [];
+                        for (let i = 0; i < chunks.length; i++) {
+                            const chunk = chunks[i]!;
+                            const chunkRkey = TID.nextStr();
+                            const chunkFileCount = countFilesInDirectory(chunk);
+                            console.log(`    → Uploading chunk ${i + 1}/${chunks.length} (${chunkFileCount} files)...`);
+
+                            const chunkRecord = await agent.com.atproto.repo.putRecord({
+                                repo: did,
+                                collection: 'place.wisp.subfs',
+                                rkey: chunkRkey,
+                                record: {
+                                    $type: 'place.wisp.subfs' as const,
+                                    root: chunk,
+                                    fileCount: chunkFileCount,
+                                    createdAt: new Date().toISOString()
+                                }
+                            });
+
+                            chunkUris.push(chunkRecord.data.uri);
+                        }
+
+                        // Create parent subfs referencing all chunks with flat: true
+                        console.log(`    → Creating parent subfs with ${chunkUris.length} chunk references...`);
+                        const parentDirectory: Directory = {
+                            $type: 'place.wisp.fs#directory' as const,
+                            type: 'directory' as const,
+                            entries: chunkUris.map((uri, i) => ({
+                                name: `chunk${i}`,
+                                node: {
+                                    $type: 'place.wisp.fs#subfs' as const,
+                                    type: 'subfs' as const,
+                                    subject: uri,
+                                    flat: true
+                                }
+                            }))
+                        };
+
+                        const parentRkey = TID.nextStr();
+                        const parentRecord = await agent.com.atproto.repo.putRecord({
+                            repo: did,
+                            collection: 'place.wisp.subfs',
+                            rkey: parentRkey,
+                            record: {
+                                $type: 'place.wisp.subfs' as const,
+                                root: parentDirectory,
+                                fileCount: largestDir.fileCount,
+                                createdAt: new Date().toISOString()
+                            }
+                        });
 
-                    // Validate subfs record
-                    // const subfsValidation = validateSubfsRecord(subfsManifest);
-                    // if (!subfsValidation.success) {
-                    //     throw new Error(`Invalid subfs manifest: ${subfsValidation.error?.message || 'Validation failed'}`);
-                    // }
+                        subfsUri = parentRecord.data.uri;
+                        console.log(`    ✓ Created parent subfs with ${chunks.length} chunks`);
+                        logger.info(`Created chunked subfs for ${largestDir.path}: ${subfsUri} (${chunks.length} chunks)`);
+                    } else {
+                        // Directory fits in a single subfs record
+                        const subfsRkey = TID.nextStr();
+                        const subfsRecord = await agent.com.atproto.repo.putRecord({
+                            repo: did,
+                            collection: 'place.wisp.subfs',
+                            rkey: subfsRkey,
+                            record: {
+                                $type: 'place.wisp.subfs' as const,
+                                root: largestDir.directory,
+                                fileCount: largestDir.fileCount,
+                                createdAt: new Date().toISOString()
+                            }
+                        });
 
-                    // Upload subfs record to PDS
-                    const subfsRecord = await agent.com.atproto.repo.putRecord({
-                        repo: did,
-                        collection: 'place.wisp.subfs',
-                        rkey: subfsRkey,
-                        record: subfsManifest
-                    });
+                        subfsUri = subfsRecord.data.uri;
+                        console.log(`  ✅ Created subfs: ${subfsUri}`);
+                        logger.info(`Created subfs record for ${largestDir.path}: ${subfsUri}`);
+                    }
 
-                    const subfsUri = subfsRecord.data.uri;
                     subfsRecords.push({ uri: subfsUri, path: largestDir.path });
-                    console.log(`  ✅ Created subfs: ${subfsUri}`);
-                    logger.info(`Created subfs record for ${largestDir.path}: ${subfsUri}`);
 
                     // Replace directory with subfs node in the main tree
                     workingDirectory = replaceDirectoryWithSubfs(workingDirectory, largestDir.path, subfsUri);

cli/commands/deploy.ts

···
 // Constants for manifest splitting
 const MAX_MANIFEST_SIZE = 140 * 1024; // 140KB (PDS limit is 150KB)
 const FILE_COUNT_THRESHOLD = 250;
+const TARGET_FILE_COUNT = 200;
 const MAX_SUBFS_SIZE = 75 * 1024; // 75KB per subfs
-const MAX_CONCURRENT_UPLOADS = 5;
+const DEFAULT_CONCURRENT_UPLOADS = 3;
+const RATELIMITED_CONCURRENT_UPLOADS = 2;
 
 export interface DeployOptions {
   path: string;
···
   directory?: boolean;
   spa?: boolean;
   yes?: boolean;
+  concurrency?: number;
 }
 
 interface FileInfo {
···
   agent: Agent,
   content: Buffer,
   mimeType: string,
-  retries = 3
+  retries = 3,
+  onRateLimit?: () => void
 ): Promise<BlobRef> {
   for (let attempt = 0; attempt < retries; attempt++) {
     try {
···
 
       // Handle rate limits
       if (err?.status === 429) {
+        onRateLimit?.();
         const delay = Math.pow(2, attempt) * 2000;
         await new Promise(r => setTimeout(r, delay));
       } else {
···
 async function processAndUploadFiles(
   agent: Agent,
   files: FileInfo[],
-  existingBlobMap: Map<string, { blobRef: BlobRef; cid: string }>
+  existingBlobMap: Map<string, { blobRef: BlobRef; cid: string }>,
+  concurrency: number
 ): Promise<{ uploadedFiles: UploadedFile[]; uploadResults: FileUploadResult[]; filePaths: string[] }> {
   const spinner = createSpinner(`Processing ${files.length} files...`).start();
 
···
 
   let uploaded = 0;
   let reused = 0;
+  let currentConcurrency = concurrency;
+
+  const onRateLimit = () => {
+    const reduced = Math.min(currentConcurrency, RATELIMITED_CONCURRENT_UPLOADS);
+    if (reduced < currentConcurrency) {
+      currentConcurrency = reduced;
+      spinner.text = `Rate limited — reducing concurrency to ${currentConcurrency}`;
+    }
+  };
 
   // Process in batches for concurrency
-  for (let i = 0; i < files.length; i += MAX_CONCURRENT_UPLOADS) {
-    const batch = files.slice(i, i + MAX_CONCURRENT_UPLOADS);
+  for (let i = 0; i < files.length; i += currentConcurrency) {
+    const batch = files.slice(i, i + currentConcurrency);
 
     await Promise.all(batch.map(async (file) => {
       const content = readFileSync(file.path);
···
         reused++;
       } else {
         // Upload new blob
-        blobRef = await uploadBlob(agent, processedContent, 'application/octet-stream');
+        blobRef = await uploadBlob(agent, processedContent, 'application/octet-stream', 3, onRateLimit);
         uploaded++;
       }
 
···
   const spinner = createSpinner('Splitting large site into subfs records...').start();
   const subfsRkeys: string[] = [];
   let currentDir = directory;
+  let currentFileCount = countFilesInDirectory(currentDir);
   let iteration = 0;
   let chunkCounter = 0;
 
-  while (
-    (estimateDirectorySize(currentDir) > MAX_MANIFEST_SIZE ||
-      countFilesInDirectory(currentDir) > FILE_COUNT_THRESHOLD) &&
-    iteration < 50
-  ) {
+  let manifest = createManifest(siteRkey, currentDir, currentFileCount);
+  let manifestSize = JSON.stringify(manifest).length;
+
+  while ((manifestSize > MAX_MANIFEST_SIZE || currentFileCount > TARGET_FILE_COUNT) && iteration < 100) {
     iteration++;
 
-    // Find largest directories
-    const largeDirs = findLargeDirectories(currentDir)
-      .filter(d => d.size > 1000) // Only consider dirs with meaningful size
-      .sort((a, b) => b.size - a.size);
+    // Find all directories sorted by size (largest first)
+    const directories = findLargeDirectories(currentDir);
+    directories.sort((a, b) => b.size - a.size);
 
-    if (largeDirs.length === 0) break;
+    if (directories.length > 0) {
+      const largest = directories[0]!;
+      spinner.text = `Split #${iteration}: ${largest.path} (${largest.fileCount} files, ${formatBytes(largest.size)})`;
 
-    const largest = largeDirs[0]!;
-    spinner.text = `Creating subfs ${iteration} for ${largest.path} (${formatBytes(largest.size)})`;
+      let subfsUri: string;
 
-    let subfsUri: string;
+      // Check if directory is too large for a single subfs record
+      if (largest.size > MAX_SUBFS_SIZE) {
+        // Split into chunks
+        console.log(pc.dim(`\n    → Directory too large (${formatBytes(largest.size)}), splitting into chunks...`));
+        const chunks = splitDirectoryIntoChunks(largest.directory, MAX_SUBFS_SIZE);
+        console.log(pc.dim(`    → Created ${chunks.length} chunks`));
 
-    // Check if directory is too large for a single subfs record
-    if (largest.size > MAX_SUBFS_SIZE) {
-      // Split into chunks
-      console.log(pc.dim(`\n    → Directory too large (${formatBytes(largest.size)}), splitting into chunks...`));
-      const chunks = splitDirectoryIntoChunks(largest.directory, MAX_SUBFS_SIZE);
-      console.log(pc.dim(`    → Created ${chunks.length} chunks`));
+        // Upload each chunk as a subfs record
+        const chunkUris: string[] = [];
+        for (let i = 0; i < chunks.length; i++) {
+          const chunk = chunks[i]!;
+          const chunkRkey = `${siteRkey}-chunk-${chunkCounter++}`;
+          const chunkSize = estimateDirectorySize(chunk);
+          const chunkFileCount = countFilesInDirectory(chunk);
+
+          console.log(pc.dim(`    → Uploading chunk ${i + 1}/${chunks.length} (${chunkFileCount} files, ${formatBytes(chunkSize)})...`));
 
-      // Upload each chunk as a subfs record
-      const chunkUris: string[] = [];
-      for (let i = 0; i < chunks.length; i++) {
-        const chunk = chunks[i]!;
-        const chunkRkey = `${siteRkey}-chunk-${chunkCounter++}`;
-        const chunkSize = estimateDirectorySize(chunk);
-        const chunkFileCount = countFilesInDirectory(chunk);
+          const chunkUri = await createSubfsRecord(agent, did, chunk, chunkRkey);
+          chunkUris.push(chunkUri);
+          subfsRkeys.push(chunkRkey);
+        }
 
-        console.log(pc.dim(`    → Uploading chunk ${i + 1}/${chunks.length} (${chunkFileCount} files, ${formatBytes(chunkSize)})...`));
+        // Create parent subfs that references all chunks with flat: true
+        console.log(pc.dim(`    → Creating parent subfs with ${chunkUris.length} chunk references...`));
 
-        const chunkUri = await createSubfsRecord(agent, did, chunk, chunkRkey);
-        chunkUris.push(chunkUri);
-        subfsRkeys.push(chunkRkey);
+        const parentEntries = chunkUris.map((uri, i) => ({
+          name: `chunk${i}`,
+          node: {
+            $type: 'place.wisp.fs#subfs' as const,
+            type: 'subfs' as const,
+            subject: uri,
+            flat: true
+          }
+        }));
+
+        const parentDirectory: Directory = {
+          $type: 'place.wisp.fs#directory' as const,
+          type: 'directory' as const,
+          entries: parentEntries
+        };
+
+        const parentRkey = `${siteRkey}-subfs-${iteration}`;
+        subfsUri = await createSubfsRecord(agent, did, parentDirectory, parentRkey);
+        subfsRkeys.push(parentRkey);
+
+        console.log(pc.green(`    ✓ Created parent subfs with ${chunks.length} chunks`));
+      } else {
+        // Directory fits in a single subfs record
+        const subfsRkey = `${siteRkey}-subfs-${iteration}`;
+        subfsUri = await createSubfsRecord(agent, did, largest.directory, subfsRkey);
+        subfsRkeys.push(subfsRkey);
       }
 
-      // Create parent subfs that references all chunks with flat: true
-      console.log(pc.dim(`    → Creating parent subfs with ${chunkUris.length} chunk references...`));
+      // Replace directory with subfs reference
+      currentDir = replaceDirectoryWithSubfs(currentDir, largest.path, subfsUri);
+      currentFileCount -= largest.fileCount;
+    } else {
+      // No subdirectories — split flat files at root level
+      const rootFiles = currentDir.entries.filter(e => 'type' in e.node && e.node.type === 'file');
+
+      if (rootFiles.length === 0) {
+        spinner.fail(`Cannot split further — no files or directories available (${currentFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB)`);
+        break;
+      }
+
+      // Take a chunk of ~100 files
+      const CHUNK_SIZE = 100;
+      const chunkFiles = rootFiles.slice(0, Math.min(CHUNK_SIZE, rootFiles.length));
+      spinner.text = `Split #${iteration}: flat root (${chunkFiles.length} files)`;
 
-      const parentEntries = chunkUris.map((uri, i) => ({
-        name: `chunk${i}`,
+      // Create a directory with just these files
+      const chunkDirectory: Directory = {
+        $type: 'place.wisp.fs#directory' as const,
+        type: 'directory' as const,
+        entries: chunkFiles
+      };
+
+      const subfsRkey = `${siteRkey}-subfs-${iteration}`;
+      const subfsUri = await createSubfsRecord(agent, did, chunkDirectory, subfsRkey);
+      subfsRkeys.push(subfsRkey);
+
+      // Remove chunked files and add a flat subfs entry
+      const remainingEntries = currentDir.entries.filter(
+        e => !chunkFiles.some(cf => cf.name === e.name)
+      );
+
+      remainingEntries.push({
+        name: `__subfs_${iteration}`,
         node: {
           $type: 'place.wisp.fs#subfs' as const,
           type: 'subfs' as const,
-          subject: uri,
-          flat: true // Merge chunk contents into parent
+          subject: subfsUri,
+          flat: true
         }
-      }));
+      });
 
-      const parentDirectory: Directory = {
+      currentDir = {
         $type: 'place.wisp.fs#directory' as const,
         type: 'directory' as const,
-        entries: parentEntries
+        entries: remainingEntries
       };
 
-      const parentRkey = `${siteRkey}-subfs-${iteration}`;
-      subfsUri = await createSubfsRecord(agent, did, parentDirectory, parentRkey);
-      subfsRkeys.push(parentRkey);
-
-      console.log(pc.green(`    ✓ Created parent subfs with ${chunks.length} chunks`));
-    } else {
-      // Directory fits in a single subfs record
-      const subfsRkey = `${siteRkey}-subfs-${iteration}`;
-      subfsUri = await createSubfsRecord(agent, did, largest.directory, subfsRkey);
-      subfsRkeys.push(subfsRkey);
+      currentFileCount -= chunkFiles.length;
     }
 
-    // Replace directory with subfs reference
-    currentDir = replaceDirectoryWithSubfs(currentDir, largest.path, subfsUri);
+    // Recreate manifest and check new size
+    manifest = createManifest(siteRkey, currentDir, currentFileCount);
+    manifestSize = JSON.stringify(manifest).length;
   }
 
   spinner.succeed(`Created ${subfsRkeys.length} subfs records`);
···
   const { uploadedFiles, uploadResults, filePaths } = await processAndUploadFiles(
     agent,
     files,
-    existingBlobMap
+    existingBlobMap,
+    options.concurrency ?? DEFAULT_CONCURRENT_UPLOADS
   );
 
   // 5. Build directory structure
···
   let finalDirectory = directory;
   let subfsRkeys: string[] = [];
 
+  const initialManifest = createManifest(siteName, directory, fileCount);
   if (
-    estimateDirectorySize(directory) > MAX_MANIFEST_SIZE ||
-    fileCount > FILE_COUNT_THRESHOLD
+    fileCount >= FILE_COUNT_THRESHOLD ||
+    JSON.stringify(initialManifest).length > MAX_MANIFEST_SIZE
   ) {
     const result = await splitIntoSubfs(agent, did, directory, siteName);
     finalDirectory = result.directory;

cli/index.ts

···
   .option('-s, --site <name>', 'Site name (defaults to directory name)')
   .option('--directory', 'Enable directory listing')
   .option('--spa', 'Enable SPA mode (serve index.html for all routes)')
+  .option('-c, --concurrency <n>', 'Number of concurrent uploads (backs off to 2 on rate limit)', '3')
   .option('--password <password>', 'App password for headless authentication')
   .option('--store <path>', 'OAuth session store path')
   .option('-y, --yes', 'Skip confirmation prompts')
···
         site: resolvedSite,
         directory: options.directory,
         spa: options.spa,
-        yes: options.yes
+        yes: options.yes,
+        concurrency: parseInt(options.concurrency, 10)
       });
 
       console.log();

packages/@wispplace/fs-utils/src/tree.ts

···
 			collectFileCidsFromEntries(node.entries, currentPath, fileCids);
 		} else if ('type' in node && node.type === 'file' && 'blob' in node) {
 			const fileNode = node as File;
-			// Extract CID from blob ref
 			if (fileNode.blob && fileNode.blob.ref) {
-				const cid = fileNode.blob.ref.toString();
+				const ref = fileNode.blob.ref;
+			// Handle both CID instances (CBOR/firehose) and IPLD link objects { $link } (JSON API)
+			const cid = (typeof ref === 'object' && '$link' in ref) ? (ref as { $link: string }).$link : ref.toString();
 				fileCids[currentPath] = cid;
 			}
 		}