Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol. wisp.place

better validations during file uploading

Changed files
+399 -58
hosting-service
public
editor
src
+10 -4
hosting-service/src/lib/utils.ts
··· 26 */ 27 export function shouldCompressMimeType(mimeType: string | undefined): boolean { 28 if (!mimeType) return false; 29 - 30 const mime = mimeType.toLowerCase(); 31 - 32 - // Text-based web assets that benefit from compression 33 const compressibleTypes = [ 34 'text/html', 35 'text/css', ··· 41 'application/json', 42 'text/plain', 43 'image/svg+xml', 44 ]; 45 - 46 if (compressibleTypes.some(type => mime === type || mime.startsWith(type))) { 47 return true; 48 }
··· 26 */ 27 export function shouldCompressMimeType(mimeType: string | undefined): boolean { 28 if (!mimeType) return false; 29 + 30 const mime = mimeType.toLowerCase(); 31 + 32 + // Text-based web assets and uncompressed audio that benefit from compression 33 const compressibleTypes = [ 34 'text/html', 35 'text/css', ··· 41 'application/json', 42 'text/plain', 43 'image/svg+xml', 44 + // Uncompressed audio formats 45 + 'audio/wav', 46 + 'audio/wave', 47 + 'audio/x-wav', 48 + 'audio/aiff', 49 + 'audio/x-aiff', 50 ]; 51 + 52 if (compressibleTypes.some(type => mime === type || mime.startsWith(type))) { 53 return true; 54 }
+43 -12
hosting-service/src/server.ts
··· 166 const shouldServeCompressed = shouldCompressMimeType(meta.mimeType); 167 168 if (!shouldServeCompressed) { 169 - const { gunzipSync } = await import('zlib'); 170 - const decompressed = gunzipSync(content); 171 - headers['Content-Type'] = meta.mimeType; 172 - headers['Cache-Control'] = 'public, max-age=31536000, immutable'; 173 - return new Response(decompressed, { headers }); 174 } 175 176 headers['Content-Type'] = meta.mimeType; ··· 368 if (isHtmlContent(requestPath, mimeType)) { 369 let htmlContent: string; 370 if (isGzipped) { 371 - const { gunzipSync } = await import('zlib'); 372 - htmlContent = gunzipSync(content).toString('utf-8'); 373 } else { 374 htmlContent = content.toString('utf-8'); 375 } ··· 400 if (isGzipped) { 401 const shouldServeCompressed = shouldCompressMimeType(mimeType); 402 if (!shouldServeCompressed) { 403 - const { gunzipSync } = await import('zlib'); 404 - const decompressed = gunzipSync(content); 405 - return new Response(decompressed, { headers }); 406 } 407 headers['Content-Encoding'] = 'gzip'; 408 } ··· 449 450 let htmlContent: string; 451 if (isGzipped) { 452 - const { gunzipSync } = await import('zlib'); 453 - htmlContent = gunzipSync(indexContent).toString('utf-8'); 454 } else { 455 htmlContent = indexContent.toString('utf-8'); 456 }
··· 166 const shouldServeCompressed = shouldCompressMimeType(meta.mimeType); 167 168 if (!shouldServeCompressed) { 169 + // Verify content is actually gzipped before attempting decompression 170 + const isGzipped = content.length >= 2 && content[0] === 0x1f && content[1] === 0x8b; 171 + if (isGzipped) { 172 + const { gunzipSync } = await import('zlib'); 173 + const decompressed = gunzipSync(content); 174 + headers['Content-Type'] = meta.mimeType; 175 + headers['Cache-Control'] = 'public, max-age=31536000, immutable'; 176 + return new Response(decompressed, { headers }); 177 + } else { 178 + // Meta says gzipped but content isn't - serve as-is 179 + console.warn(`File ${filePath} has gzip encoding in meta but content lacks gzip magic bytes`); 180 + headers['Content-Type'] = meta.mimeType; 181 + headers['Cache-Control'] = 'public, max-age=31536000, immutable'; 182 + return new Response(content, { headers }); 183 + } 184 } 185 186 headers['Content-Type'] = meta.mimeType; ··· 378 if (isHtmlContent(requestPath, mimeType)) { 379 let htmlContent: string; 380 if (isGzipped) { 381 + // Verify content is actually gzipped 382 + const hasGzipMagic = content.length >= 2 && content[0] === 0x1f && content[1] === 0x8b; 383 + if (hasGzipMagic) { 384 + const { gunzipSync } = await import('zlib'); 385 + htmlContent = gunzipSync(content).toString('utf-8'); 386 + } else { 387 + console.warn(`File ${requestPath} marked as gzipped but lacks magic bytes, serving as-is`); 388 + htmlContent = content.toString('utf-8'); 389 + } 390 } else { 391 htmlContent = content.toString('utf-8'); 392 } ··· 417 if (isGzipped) { 418 const shouldServeCompressed = shouldCompressMimeType(mimeType); 419 if (!shouldServeCompressed) { 420 + // Verify content is actually gzipped 421 + const hasGzipMagic = content.length >= 2 && content[0] === 0x1f && content[1] === 0x8b; 422 + if (hasGzipMagic) { 423 + const { gunzipSync } = await import('zlib'); 424 + const decompressed = gunzipSync(content); 425 + return new Response(decompressed, { headers }); 426 + } else { 427 + console.warn(`File ${requestPath} marked as gzipped but lacks magic bytes, serving as-is`); 428 + return new Response(content, { headers }); 429 + } 430 } 431 headers['Content-Encoding'] = 'gzip'; 432 } ··· 473 474 let htmlContent: string; 475 if (isGzipped) { 476 + // Verify content is actually gzipped 477 + const hasGzipMagic = indexContent.length >= 2 && indexContent[0] === 0x1f && indexContent[1] === 0x8b; 478 + if (hasGzipMagic) { 479 + const { gunzipSync } = await import('zlib'); 480 + htmlContent = gunzipSync(indexContent).toString('utf-8'); 481 + } else { 482 + console.warn(`Index file marked as gzipped but lacks magic bytes, serving as-is`); 483 + htmlContent = indexContent.toString('utf-8'); 484 + } 485 } else { 486 htmlContent = indexContent.toString('utf-8'); 487 }
+162 -8
public/editor/tabs/UploadTab.tsx
··· 15 Globe, 16 Upload, 17 AlertCircle, 18 - Loader2 19 } from 'lucide-react' 20 import type { SiteWithDomains } from '../hooks/useSiteData' 21 22 interface UploadTabProps { 23 sites: SiteWithDomains[] ··· 38 const [isUploading, setIsUploading] = useState(false) 39 const [uploadProgress, setUploadProgress] = useState('') 40 const [skippedFiles, setSkippedFiles] = useState<Array<{ name: string; reason: string }>>([]) 41 const [uploadedCount, setUploadedCount] = useState(0) 42 43 // Keep SSE connection alive across tab switches 44 const eventSourceRef = useRef<EventSource | null>(null) ··· 79 const progressData = JSON.parse(event.data) 80 const { progress, status } = progressData 81 82 // Update progress message based on phase 83 let message = 'Processing...' 84 if (progress.phase === 'validating') { ··· 110 eventSourceRef.current = null 111 currentJobIdRef.current = null 112 113 - setUploadProgress('Upload complete!') 114 setSkippedFiles(result.skippedFiles || []) 115 setUploadedCount(result.uploadedCount || result.fileCount || 0) 116 setSelectedSiteRkey('') 117 setNewSiteName('') ··· 120 // Refresh sites list 121 onUploadComplete() 122 123 - // Reset form 124 - const resetDelay = result.skippedFiles && result.skippedFiles.length > 0 ? 4000 : 1500 125 setTimeout(() => { 126 setUploadProgress('') 127 setSkippedFiles([]) 128 setUploadedCount(0) 129 setIsUploading(false) 130 }, resetDelay) 131 }) ··· 376 </div> 377 </div> 378 379 - {skippedFiles.length > 0 && ( 380 - <div className="p-4 bg-yellow-500/10 border border-yellow-500/20 rounded-lg"> 381 - <div className="flex items-start gap-2 text-yellow-600 dark:text-yellow-400 mb-2"> 382 <AlertCircle className="w-4 h-4 mt-0.5 shrink-0" /> 383 <div className="flex-1"> 384 <span className="font-medium"> 385 - {skippedFiles.length} file{skippedFiles.length > 1 ? 's' : ''} skipped 386 </span> 387 {uploadedCount > 0 && ( 388 <span className="text-sm ml-2"> 389 ({uploadedCount} uploaded successfully) 390 </span> 391 )} 392 </div> 393 </div> 394 <div className="ml-6 space-y-1 max-h-32 overflow-y-auto">
··· 15 Globe, 16 Upload, 17 AlertCircle, 18 + Loader2, 19 + ChevronDown, 20 + ChevronUp, 21 + CheckCircle2, 22 + XCircle, 23 + RefreshCw 24 } from 'lucide-react' 25 import type { SiteWithDomains } from '../hooks/useSiteData' 26 + 27 + type FileStatus = 'pending' | 'checking' | 'uploading' | 'uploaded' | 'reused' | 'failed' 28 + 29 + interface FileProgress { 30 + name: string 31 + status: FileStatus 32 + error?: string 33 + } 34 35 interface UploadTabProps { 36 sites: SiteWithDomains[] ··· 51 const [isUploading, setIsUploading] = useState(false) 52 const [uploadProgress, setUploadProgress] = useState('') 53 const [skippedFiles, setSkippedFiles] = useState<Array<{ name: string; reason: string }>>([]) 54 + const [failedFiles, setFailedFiles] = useState<Array<{ name: string; index: number; error: string; size: number }>>([]) 55 const [uploadedCount, setUploadedCount] = useState(0) 56 + const [fileProgressList, setFileProgressList] = useState<FileProgress[]>([]) 57 + const [showFileProgress, setShowFileProgress] = useState(false) 58 59 // Keep SSE connection alive across tab switches 60 const eventSourceRef = useRef<EventSource | null>(null) ··· 95 const progressData = JSON.parse(event.data) 96 const { progress, status } = progressData 97 98 + // Update file progress list if we have current file info 99 + if (progress.currentFile && progress.currentFileStatus) { 100 + setFileProgressList(prev => { 101 + const existing = prev.find(f => f.name === progress.currentFile) 102 + if (existing) { 103 + // Update existing file status 104 + return prev.map(f => 105 + f.name === progress.currentFile 106 + ? { ...f, status: progress.currentFileStatus as FileStatus } 107 + : f 108 + ) 109 + } else { 110 + // Add new file 111 + return [...prev, { 112 + name: progress.currentFile, 113 + status: progress.currentFileStatus as FileStatus 114 + }] 115 + } 116 + }) 117 + } 118 + 119 // Update progress message based on phase 120 let message = 'Processing...' 121 if (progress.phase === 'validating') { ··· 147 eventSourceRef.current = null 148 currentJobIdRef.current = null 149 150 + const hasIssues = (result.skippedFiles && result.skippedFiles.length > 0) || 151 + (result.failedFiles && result.failedFiles.length > 0) 152 + 153 + // Update file progress list with failed files 154 + if (result.failedFiles && result.failedFiles.length > 0) { 155 + setFileProgressList(prev => { 156 + const updated = [...prev] 157 + result.failedFiles.forEach((failedFile: any) => { 158 + const existing = updated.find(f => f.name === failedFile.name) 159 + if (existing) { 160 + existing.status = 'failed' 161 + existing.error = failedFile.error 162 + } else { 163 + updated.push({ 164 + name: failedFile.name, 165 + status: 'failed', 166 + error: failedFile.error 167 + }) 168 + } 169 + }) 170 + return updated 171 + }) 172 + } 173 + 174 + setUploadProgress(hasIssues ? 'Upload completed with issues' : 'Upload complete!') 175 setSkippedFiles(result.skippedFiles || []) 176 + setFailedFiles(result.failedFiles || []) 177 setUploadedCount(result.uploadedCount || result.fileCount || 0) 178 setSelectedSiteRkey('') 179 setNewSiteName('') ··· 182 // Refresh sites list 183 onUploadComplete() 184 185 + // Reset form (wait longer if there are issues to show) 186 + const resetDelay = hasIssues ? 6000 : 1500 187 setTimeout(() => { 188 setUploadProgress('') 189 setSkippedFiles([]) 190 + setFailedFiles([]) 191 setUploadedCount(0) 192 + setFileProgressList([]) 193 setIsUploading(false) 194 }, resetDelay) 195 }) ··· 440 </div> 441 </div> 442 443 + {fileProgressList.length > 0 && ( 444 + <div className="border rounded-lg overflow-hidden"> 445 + <button 446 + onClick={() => setShowFileProgress(!showFileProgress)} 447 + className="w-full p-3 bg-muted/50 hover:bg-muted transition-colors flex items-center justify-between text-sm font-medium" 448 + > 449 + <span> 450 + Processing files ({fileProgressList.filter(f => f.status === 'uploaded' || f.status === 'reused').length}/{fileProgressList.length}) 451 + </span> 452 + {showFileProgress ? ( 453 + <ChevronUp className="w-4 h-4" /> 454 + ) : ( 455 + <ChevronDown className="w-4 h-4" /> 456 + )} 457 + </button> 458 + {showFileProgress && ( 459 + <div className="max-h-64 overflow-y-auto p-3 space-y-1 bg-background"> 460 + {fileProgressList.map((file, idx) => ( 461 + <div 462 + key={idx} 463 + className="flex items-start gap-2 text-xs p-2 rounded hover:bg-muted/50 transition-colors" 464 + > 465 + {file.status === 'checking' && ( 466 + <Loader2 className="w-3 h-3 mt-0.5 animate-spin text-blue-500 shrink-0" /> 467 + )} 468 + {file.status === 'uploading' && ( 469 + <Loader2 className="w-3 h-3 mt-0.5 animate-spin text-purple-500 shrink-0" /> 470 + )} 471 + {file.status === 'uploaded' && ( 472 + <CheckCircle2 className="w-3 h-3 mt-0.5 text-green-500 shrink-0" /> 473 + )} 474 + {file.status === 'reused' && ( 475 + <RefreshCw className="w-3 h-3 mt-0.5 text-cyan-500 shrink-0" /> 476 + )} 477 + {file.status === 'failed' && ( 478 + <XCircle className="w-3 h-3 mt-0.5 text-red-500 shrink-0" /> 479 + )} 480 + <div className="flex-1 min-w-0"> 481 + <div className="font-mono truncate">{file.name}</div> 482 + {file.error && ( 483 + <div className="text-red-500 mt-0.5"> 484 + {file.error} 485 + </div> 486 + )} 487 + {file.status === 'checking' && ( 488 + <div className="text-muted-foreground">Checking for changes...</div> 489 + )} 490 + {file.status === 'uploading' && ( 491 + <div className="text-muted-foreground">Uploading to PDS...</div> 492 + )} 493 + {file.status === 'reused' && ( 494 + <div className="text-muted-foreground">Reused (unchanged)</div> 495 + )} 496 + </div> 497 + </div> 498 + ))} 499 + </div> 500 + )} 501 + </div> 502 + )} 503 + 504 + {failedFiles.length > 0 && ( 505 + <div className="p-4 bg-red-500/10 border border-red-500/20 rounded-lg"> 506 + <div className="flex items-start gap-2 text-red-600 dark:text-red-400 mb-2"> 507 <AlertCircle className="w-4 h-4 mt-0.5 shrink-0" /> 508 <div className="flex-1"> 509 <span className="font-medium"> 510 + {failedFiles.length} file{failedFiles.length > 1 ? 's' : ''} failed to upload 511 </span> 512 {uploadedCount > 0 && ( 513 <span className="text-sm ml-2"> 514 ({uploadedCount} uploaded successfully) 515 </span> 516 )} 517 + </div> 518 + </div> 519 + <div className="ml-6 space-y-1 max-h-40 overflow-y-auto"> 520 + {failedFiles.slice(0, 10).map((file, idx) => ( 521 + <div key={idx} className="text-xs"> 522 + <div className="font-mono font-semibold">{file.name}</div> 523 + <div className="text-muted-foreground ml-2"> 524 + Error: {file.error} 525 + {file.size > 0 && ` (${(file.size / 1024).toFixed(1)} KB)`} 526 + </div> 527 + </div> 528 + ))} 529 + {failedFiles.length > 10 && ( 530 + <div className="text-xs text-muted-foreground"> 531 + ...and {failedFiles.length - 10} more 532 + </div> 533 + )} 534 + </div> 535 + </div> 536 + )} 537 + 538 + {skippedFiles.length > 0 && ( 539 + <div className="p-4 bg-yellow-500/10 border border-yellow-500/20 rounded-lg"> 540 + <div className="flex items-start gap-2 text-yellow-600 dark:text-yellow-400 mb-2"> 541 + <AlertCircle className="w-4 h-4 mt-0.5 shrink-0" /> 542 + <div className="flex-1"> 543 + <span className="font-medium"> 544 + {skippedFiles.length} file{skippedFiles.length > 1 ? 's' : ''} skipped 545 + </span> 546 </div> 547 </div> 548 <div className="ml-6 space-y-1 max-h-32 overflow-y-auto">
+3
src/lib/upload-jobs.ts
··· 8 filesUploaded: number; 9 filesReused: number; 10 currentFile?: string; 11 phase: 'validating' | 'compressing' | 'uploading' | 'creating_manifest' | 'finalizing' | 'done'; 12 } 13 ··· 24 fileCount?: number; 25 siteName?: string; 26 skippedFiles?: Array<{ name: string; reason: string }>; 27 uploadedCount?: number; 28 }; 29 error?: string; 30 createdAt: number;
··· 8 filesUploaded: number; 9 filesReused: number; 10 currentFile?: string; 11 + currentFileStatus?: 'checking' | 'uploading' | 'uploaded' | 'reused' | 'failed'; 12 phase: 'validating' | 'compressing' | 'uploading' | 'creating_manifest' | 'finalizing' | 'done'; 13 } 14 ··· 25 fileCount?: number; 26 siteName?: string; 27 skippedFiles?: Array<{ name: string; reason: string }>; 28 + failedFiles?: Array<{ name: string; index: number; error: string; size: number }>; 29 uploadedCount?: number; 30 + hasFailures?: boolean; 31 }; 32 error?: string; 33 createdAt: number;
+9 -2
src/lib/wisp-utils.ts
··· 14 mimeType: string; 15 size: number; 16 compressed?: boolean; 17 originalMimeType?: string; 18 } 19 ··· 34 * Determine if a file should be gzip compressed based on its MIME type 35 */ 36 export function shouldCompressFile(mimeType: string): boolean { 37 - // Compress text-based files 38 const compressibleTypes = [ 39 'text/html', 40 'text/css', ··· 45 'text/xml', 46 'application/xml', 47 'text/plain', 48 - 'application/x-javascript' 49 ]; 50 51 // Check if mime type starts with any compressible type
··· 14 mimeType: string; 15 size: number; 16 compressed?: boolean; 17 + base64Encoded?: boolean; 18 originalMimeType?: string; 19 } 20 ··· 35 * Determine if a file should be gzip compressed based on its MIME type 36 */ 37 export function shouldCompressFile(mimeType: string): boolean { 38 + // Compress text-based files and uncompressed audio formats 39 const compressibleTypes = [ 40 'text/html', 41 'text/css', ··· 46 'text/xml', 47 'application/xml', 48 'text/plain', 49 + 'application/x-javascript', 50 + // Uncompressed audio formats (WAV, AIFF, etc.) 51 + 'audio/wav', 52 + 'audio/wave', 53 + 'audio/x-wav', 54 + 'audio/aiff', 55 + 'audio/x-aiff' 56 ]; 57 58 // Check if mime type starts with any compressible type
+172 -32
src/routes/wisp.ts
··· 149 150 for (let i = 0; i < fileArray.length; i++) { 151 const file = fileArray[i]; 152 console.log(`Processing file ${i + 1}/${fileArray.length}:`, file.name, file.size, 'bytes'); 153 updateJobProgress(jobId, { 154 filesProcessed: i + 1, ··· 180 const originalContent = Buffer.from(arrayBuffer); 181 const originalMimeType = file.type || 'application/octet-stream'; 182 183 - // Compress and base64 encode ALL files 184 - const compressedContent = compressFile(originalContent); 185 - const base64Content = Buffer.from(compressedContent.toString('base64'), 'binary'); 186 - const compressionRatio = (compressedContent.length / originalContent.length * 100).toFixed(1); 187 - console.log(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`); 188 - logger.info(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`); 189 190 uploadedFiles.push({ 191 name: file.name, 192 - content: base64Content, 193 mimeType: originalMimeType, 194 - size: base64Content.length, 195 - compressed: true, 196 originalMimeType 197 }); 198 } ··· 275 console.log('Starting blob upload/reuse phase...'); 276 updateJobProgress(jobId, { phase: 'uploading' }); 277 278 - // Helper function to upload blob with exponential backoff retry 279 const uploadBlobWithRetry = async ( 280 agent: Agent, 281 content: Buffer, 282 mimeType: string, 283 fileName: string, 284 - maxRetries = 3 285 ) => { 286 for (let attempt = 0; attempt < maxRetries; attempt++) { 287 try { 288 - return await agent.com.atproto.repo.uploadBlob(content, { encoding: mimeType }); 289 } catch (error: any) { 290 const isDPoPNonceError = 291 error?.message?.toLowerCase().includes('nonce') || 292 error?.message?.toLowerCase().includes('dpop') || 293 error?.status === 409; 294 295 - if (isDPoPNonceError && attempt < maxRetries - 1) { 296 - const backoffMs = 100 * Math.pow(2, attempt); // 100ms, 200ms, 400ms 297 - logger.info(`[File Upload] 🔄 DPoP nonce conflict for ${fileName}, retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})`); 298 await new Promise(resolve => setTimeout(resolve, backoffMs)); 299 continue; 300 } 301 throw error; 302 } 303 } ··· 305 }; 306 307 // Use sliding window concurrency for maximum throughput 308 - const CONCURRENCY_LIMIT = 50; // Maximum concurrent uploads with retry logic 309 const uploadedBlobs: Array<{ 310 result: FileUploadResult; 311 filePath: string; ··· 313 returnedMimeType: string; 314 reused: boolean; 315 }> = []; 316 317 // Process file with sliding window concurrency 318 const processFile = async (file: UploadedFile, index: number) => { ··· 327 328 if (existingBlob && existingBlob.cid === fileCID) { 329 logger.info(`[File Upload] ♻️ Reused: ${file.name} (unchanged, CID: ${fileCID})`); 330 - updateJobProgress(jobId, { filesReused: (getUploadJob(jobId)?.progress.filesReused || 0) + 1 }); 331 332 return { 333 result: { ··· 336 ...(file.compressed && { 337 encoding: 'gzip' as const, 338 mimeType: file.originalMimeType || file.mimeType, 339 - base64: true 340 }) 341 }, 342 filePath: file.name, ··· 362 ); 363 364 const returnedBlobRef = uploadResult.data.blob; 365 - updateJobProgress(jobId, { filesUploaded: (getUploadJob(jobId)?.progress.filesUploaded || 0) + 1 }); 366 logger.info(`[File Upload] ✅ Uploaded: ${file.name} (CID: ${fileCID})`); 367 368 return { ··· 372 ...(file.compressed && { 373 encoding: 'gzip' as const, 374 mimeType: file.originalMimeType || file.mimeType, 375 - base64: true 376 }) 377 }, 378 filePath: file.name, ··· 381 reused: false 382 }; 383 } catch (uploadError) { 384 - logger.error('Upload failed for file', uploadError); 385 - throw uploadError; 386 } 387 }; 388 ··· 390 const processWithConcurrency = async () => { 391 const results: any[] = []; 392 let fileIndex = 0; 393 - const executing = new Set<Promise<void>>(); 394 395 for (const file of validUploadedFiles) { 396 const currentIndex = fileIndex++; ··· 398 const promise = processFile(file, currentIndex) 399 .then(result => { 400 results[currentIndex] = result; 401 }) 402 .catch(error => { 403 - logger.error(`Failed to process file at index ${currentIndex}`, error); 404 - throw error; // Re-throw to fail the entire upload 405 }) 406 .finally(() => { 407 executing.delete(promise); 408 }); 409 410 - executing.add(promise); 411 412 if (executing.size >= CONCURRENCY_LIMIT) { 413 - await Promise.race(executing); 414 } 415 } 416 417 // Wait for remaining uploads 418 - await Promise.all(executing); 419 - return results.filter(r => r !== undefined); // Filter out any undefined entries 420 }; 421 422 const allResults = await processWithConcurrency(); ··· 424 425 const currentReused = uploadedBlobs.filter(b => b.reused).length; 426 const currentUploaded = uploadedBlobs.filter(b => !b.reused).length; 427 - logger.info(`[File Upload] 🎉 Upload complete → ${uploadedBlobs.length}/${validUploadedFiles.length} files (${currentUploaded} uploaded, ${currentReused} reused)`); 428 429 const reusedCount = uploadedBlobs.filter(b => b.reused).length; 430 const uploadedCount = uploadedBlobs.filter(b => !b.reused).length; 431 - logger.info(`[File Upload] 🎉 Upload phase complete! Total: ${uploadedBlobs.length} files (${uploadedCount} uploaded, ${reusedCount} reused)`); 432 433 const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result); 434 const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath); ··· 594 fileCount, 595 siteName, 596 skippedFiles, 597 - uploadedCount: validUploadedFiles.length 598 }); 599 600 console.log('=== UPLOAD FILES COMPLETE ===');
··· 149 150 for (let i = 0; i < fileArray.length; i++) { 151 const file = fileArray[i]; 152 + 153 + // Skip undefined/null files 154 + if (!file || !file.name) { 155 + console.log(`Skipping undefined file at index ${i}`); 156 + skippedFiles.push({ 157 + name: `[undefined file at index ${i}]`, 158 + reason: 'Invalid file object' 159 + }); 160 + continue; 161 + } 162 + 163 console.log(`Processing file ${i + 1}/${fileArray.length}:`, file.name, file.size, 'bytes'); 164 updateJobProgress(jobId, { 165 filesProcessed: i + 1, ··· 191 const originalContent = Buffer.from(arrayBuffer); 192 const originalMimeType = file.type || 'application/octet-stream'; 193 194 + // Determine if file should be compressed 195 + const shouldCompress = shouldCompressFile(originalMimeType); 196 + 197 + // Text files (HTML/CSS/JS) need base64 encoding to prevent PDS content sniffing 198 + // Audio files just need compression without base64 199 + const needsBase64 = originalMimeType.startsWith('text/') || 200 + originalMimeType.includes('html') || 201 + originalMimeType.includes('javascript') || 202 + originalMimeType.includes('css') || 203 + originalMimeType.includes('json') || 204 + originalMimeType.includes('xml') || 205 + originalMimeType.includes('svg'); 206 + 207 + let finalContent: Buffer; 208 + let compressed = false; 209 + let base64Encoded = false; 210 + 211 + if (shouldCompress) { 212 + const compressedContent = compressFile(originalContent); 213 + compressed = true; 214 + 215 + if (needsBase64) { 216 + // Text files: compress AND base64 encode 217 + finalContent = Buffer.from(compressedContent.toString('base64'), 'binary'); 218 + base64Encoded = true; 219 + const compressionRatio = (compressedContent.length / originalContent.length * 100).toFixed(1); 220 + console.log(`Compressing+base64 ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${finalContent.length} bytes`); 221 + logger.info(`Compressing+base64 ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${finalContent.length} bytes`); 222 + } else { 223 + // Audio files: just compress, no base64 224 + finalContent = compressedContent; 225 + const compressionRatio = (compressedContent.length / originalContent.length * 100).toFixed(1); 226 + console.log(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%)`); 227 + logger.info(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%)`); 228 + } 229 + } else { 230 + // Binary files: upload directly 231 + finalContent = originalContent; 232 + console.log(`Uploading ${file.name} directly: ${originalContent.length} bytes (no compression)`); 233 + logger.info(`Uploading ${file.name} directly: ${originalContent.length} bytes (binary)`); 234 + } 235 236 uploadedFiles.push({ 237 name: file.name, 238 + content: finalContent, 239 mimeType: originalMimeType, 240 + size: finalContent.length, 241 + compressed, 242 + base64Encoded, 243 originalMimeType 244 }); 245 } ··· 322 console.log('Starting blob upload/reuse phase...'); 323 updateJobProgress(jobId, { phase: 'uploading' }); 324 325 + // Helper function to upload blob with exponential backoff retry and timeout 326 const uploadBlobWithRetry = async ( 327 agent: Agent, 328 content: Buffer, 329 mimeType: string, 330 fileName: string, 331 + maxRetries = 5 332 ) => { 333 for (let attempt = 0; attempt < maxRetries; attempt++) { 334 try { 335 + console.log(`[File Upload] Starting upload attempt ${attempt + 1}/${maxRetries} for ${fileName} (${content.length} bytes, ${mimeType})`); 336 + 337 + // Add timeout wrapper to prevent hanging requests 338 + const uploadPromise = agent.com.atproto.repo.uploadBlob(content, { encoding: mimeType }); 339 + const timeoutMs = 300000; // 5 minute timeout per upload 340 + 341 + const timeoutPromise = new Promise((_, reject) => { 342 + setTimeout(() => reject(new Error('Upload timeout')), timeoutMs); 343 + }); 344 + 345 + const result = await Promise.race([uploadPromise, timeoutPromise]) as any; 346 + console.log(`[File Upload] ✅ Successfully uploaded ${fileName} on attempt ${attempt + 1}`); 347 + return result; 348 } catch (error: any) { 349 const isDPoPNonceError = 350 error?.message?.toLowerCase().includes('nonce') || 351 error?.message?.toLowerCase().includes('dpop') || 352 error?.status === 409; 353 354 + const isTimeout = error?.message === 'Upload timeout'; 355 + const isRateLimited = error?.status === 429 || error?.message?.toLowerCase().includes('rate'); 356 + 357 + // Retry on DPoP nonce conflicts, timeouts, or rate limits 358 + if ((isDPoPNonceError || isTimeout || isRateLimited) && attempt < maxRetries - 1) { 359 + let backoffMs: number; 360 + if (isRateLimited) { 361 + backoffMs = 2000 * Math.pow(2, attempt); // 2s, 4s, 8s, 16s for rate limits 362 + } else if (isTimeout) { 363 + backoffMs = 1000 * Math.pow(2, attempt); // 1s, 2s, 4s, 8s for timeouts 364 + } else { 365 + backoffMs = 100 * Math.pow(2, attempt); // 100ms, 200ms, 400ms for DPoP 366 + } 367 + 368 + const reason = isDPoPNonceError ? 'DPoP nonce conflict' : isTimeout ? 'timeout' : 'rate limit'; 369 + logger.info(`[File Upload] 🔄 ${reason} for ${fileName}, retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})`); 370 + console.log(`[File Upload] 🔄 ${reason} for ${fileName}, retrying in ${backoffMs}ms`); 371 await new Promise(resolve => setTimeout(resolve, backoffMs)); 372 continue; 373 } 374 + 375 + // Log detailed error information before throwing 376 + logger.error(`[File Upload] ❌ Upload failed for ${fileName} (size: ${content.length} bytes, mimeType: ${mimeType}, attempt: ${attempt + 1}/${maxRetries})`, { 377 + error: error?.error || error?.message || 'Unknown error', 378 + status: error?.status, 379 + headers: error?.headers, 380 + success: error?.success 381 + }); 382 + console.error(`[File Upload] ❌ Upload failed for ${fileName}:`, { 383 + error: error?.error || error?.message || 'Unknown error', 384 + status: error?.status, 385 + size: content.length, 386 + mimeType, 387 + attempt: attempt + 1 388 + }); 389 throw error; 390 } 391 } ··· 393 }; 394 395 // Use sliding window concurrency for maximum throughput 396 + const CONCURRENCY_LIMIT = 20; // Maximum concurrent uploads 397 const uploadedBlobs: Array<{ 398 result: FileUploadResult; 399 filePath: string; ··· 401 returnedMimeType: string; 402 reused: boolean; 403 }> = []; 404 + const failedFiles: Array<{ 405 + name: string; 406 + index: number; 407 + error: string; 408 + size: number; 409 + }> = []; 410 411 // Process file with sliding window concurrency 412 const processFile = async (file: UploadedFile, index: number) => { ··· 421 422 if (existingBlob && existingBlob.cid === fileCID) { 423 logger.info(`[File Upload] ♻️ Reused: ${file.name} (unchanged, CID: ${fileCID})`); 424 + updateJobProgress(jobId, { 425 + filesReused: (getUploadJob(jobId)?.progress.filesReused || 0) + 1 426 + }); 427 428 return { 429 result: { ··· 432 ...(file.compressed && { 433 encoding: 'gzip' as const, 434 mimeType: file.originalMimeType || file.mimeType, 435 + base64: file.base64Encoded || false 436 }) 437 }, 438 filePath: file.name, ··· 458 ); 459 460 const returnedBlobRef = uploadResult.data.blob; 461 + updateJobProgress(jobId, { 462 + filesUploaded: (getUploadJob(jobId)?.progress.filesUploaded || 0) + 1 463 + }); 464 logger.info(`[File Upload] ✅ Uploaded: ${file.name} (CID: ${fileCID})`); 465 466 return { ··· 470 ...(file.compressed && { 471 encoding: 'gzip' as const, 472 mimeType: file.originalMimeType || file.mimeType, 473 + base64: file.base64Encoded || false 474 }) 475 }, 476 filePath: file.name, ··· 479 reused: false 480 }; 481 } catch (uploadError) { 482 + const fileName = file?.name || 'unknown'; 483 + const fileSize = file?.size || 0; 484 + const errorMessage = uploadError instanceof Error ? uploadError.message : 'Unknown error'; 485 + const errorDetails = { 486 + fileName, 487 + fileSize, 488 + index, 489 + error: errorMessage, 490 + stack: uploadError instanceof Error ? uploadError.stack : undefined 491 + }; 492 + logger.error(`Upload failed for file: ${fileName} (${fileSize} bytes) at index ${index}`, errorDetails); 493 + console.error(`Upload failed for file: ${fileName} (${fileSize} bytes) at index ${index}`, errorDetails); 494 + 495 + // Track failed file but don't throw - continue with other files 496 + failedFiles.push({ 497 + name: fileName, 498 + index, 499 + error: errorMessage, 500 + size: fileSize 501 + }); 502 + 503 + return null; // Return null to indicate failure 504 } 505 }; 506 ··· 508 const processWithConcurrency = async () => { 509 const results: any[] = []; 510 let fileIndex = 0; 511 + const executing = new Map<Promise<void>, { index: number; name: string }>(); 512 513 for (const file of validUploadedFiles) { 514 const currentIndex = fileIndex++; ··· 516 const promise = processFile(file, currentIndex) 517 .then(result => { 518 results[currentIndex] = result; 519 + console.log(`[Concurrency] File ${currentIndex} (${file.name}) completed successfully`); 520 }) 521 .catch(error => { 522 + // This shouldn't happen since processFile catches errors, but just in case 523 + logger.error(`Unexpected error processing file at index ${currentIndex}`, error); 524 + console.error(`[Concurrency] File ${currentIndex} (${file.name}) had unexpected error:`, error); 525 + results[currentIndex] = null; 526 }) 527 .finally(() => { 528 executing.delete(promise); 529 + const remaining = Array.from(executing.values()).map(f => `${f.index}:${f.name}`); 530 + console.log(`[Concurrency] File ${currentIndex} (${file.name}) removed. Remaining ${executing.size}: [${remaining.join(', ')}]`); 531 }); 532 533 + executing.set(promise, { index: currentIndex, name: file.name }); 534 + const current = Array.from(executing.values()).map(f => `${f.index}:${f.name}`); 535 + console.log(`[Concurrency] Added file ${currentIndex} (${file.name}). Total ${executing.size}: [${current.join(', ')}]`); 536 537 if (executing.size >= CONCURRENCY_LIMIT) { 538 + console.log(`[Concurrency] Hit limit (${CONCURRENCY_LIMIT}), waiting for one to complete...`); 539 + await Promise.race(executing.keys()); 540 + console.log(`[Concurrency] One completed, continuing. Remaining: ${executing.size}`); 541 } 542 } 543 544 // Wait for remaining uploads 545 + const remaining = Array.from(executing.values()).map(f => `${f.index}:${f.name}`); 546 + console.log(`[Concurrency] Waiting for ${executing.size} remaining uploads: [${remaining.join(', ')}]`); 547 + await Promise.all(executing.keys()); 548 + console.log(`[Concurrency] All uploads complete!`); 549 + return results.filter(r => r !== undefined && r !== null); // Filter out null (failed) and undefined entries 550 }; 551 552 const allResults = await processWithConcurrency(); ··· 554 555 const currentReused = uploadedBlobs.filter(b => b.reused).length; 556 const currentUploaded = uploadedBlobs.filter(b => !b.reused).length; 557 + const successfulCount = uploadedBlobs.length; 558 + const failedCount = failedFiles.length; 559 + 560 + logger.info(`[File Upload] 🎉 Upload complete → ${successfulCount}/${validUploadedFiles.length} files succeeded (${currentUploaded} uploaded, ${currentReused} reused), ${failedCount} failed`); 561 + 562 + if (failedCount > 0) { 563 + logger.warn(`[File Upload] ⚠️ Failed files:`, failedFiles); 564 + console.warn(`[File Upload] ⚠️ ${failedCount} files failed to upload:`, failedFiles.map(f => f.name).join(', ')); 565 + } 566 567 const reusedCount = uploadedBlobs.filter(b => b.reused).length; 568 const uploadedCount = uploadedBlobs.filter(b => !b.reused).length; 569 + logger.info(`[File Upload] 🎉 Upload phase complete! Total: ${successfulCount} files (${uploadedCount} uploaded, ${reusedCount} reused)`); 570 571 const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result); 572 const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath); ··· 732 fileCount, 733 siteName, 734 skippedFiles, 735 + failedFiles, 736 + uploadedCount: validUploadedFiles.length - failedFiles.length, 737 + hasFailures: failedFiles.length > 0 738 }); 739 740 console.log('=== UPLOAD FILES COMPLETE ===');