Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol. wisp.place

revert batching

Changed files
+9 -180
public
editor
+9 -180
public/editor/tabs/UploadTab.tsx
··· 25 25 onUploadComplete: () => Promise<void> 26 26 } 27 27 28 - // Batching configuration 29 - const BATCH_SIZE = 15 // files per batch 30 - const CONCURRENT_BATCHES = 3 // parallel batches 31 - const MAX_RETRIES = 2 // retry attempts per file 32 - 33 - interface BatchProgress { 34 - total: number 35 - uploaded: number 36 - failed: number 37 - current: number 38 - } 39 - 40 28 export function UploadTab({ 41 29 sites, 42 30 sitesLoading, ··· 51 39 const [uploadProgress, setUploadProgress] = useState('') 52 40 const [skippedFiles, setSkippedFiles] = useState<Array<{ name: string; reason: string }>>([]) 53 41 const [uploadedCount, setUploadedCount] = useState(0) 54 - const [batchProgress, setBatchProgress] = useState<BatchProgress | null>(null) 55 42 56 43 // Auto-switch to 'new' mode if no sites exist 57 44 useEffect(() => { ··· 66 53 } 67 54 } 68 55 69 - // Split files into batches 70 - const createBatches = (files: FileList): File[][] => { 71 - const batches: File[][] = [] 72 - const fileArray = Array.from(files) 73 - 74 - for (let i = 0; i < fileArray.length; i += BATCH_SIZE) { 75 - batches.push(fileArray.slice(i, i + BATCH_SIZE)) 76 - } 77 - 78 - return batches 79 - } 80 - 81 - // Upload a single file with retry logic 82 - const uploadFileWithRetry = async ( 83 - file: File, 84 - retries: number = MAX_RETRIES 85 - ): Promise<{ success: boolean; error?: string }> => { 86 - for (let attempt = 0; attempt <= retries; attempt++) { 87 - try { 88 - // Simulate file validation (would normally happen on server) 89 - // Return success (actual upload happens in batch) 90 - return { success: true } 91 - } catch (err) { 92 - // Check if error is retryable 93 - const error = err as any 94 - const statusCode = error?.response?.status 95 - 96 - // Don't retry for client errors (4xx except timeouts) 97 - if (statusCode === 413 || statusCode === 400) { 98 - return { 99 - success: false, 100 - error: statusCode === 413 ? 'File too large' : 'Validation error' 101 - } 102 - } 103 - 104 - // If this was the last attempt, fail 105 - if (attempt === retries) { 106 - return { 107 - success: false, 108 - error: err instanceof Error ? err.message : 'Upload failed' 109 - } 110 - } 111 - 112 - // Wait before retry (exponential backoff) 113 - await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1))) 114 - } 115 - } 116 - 117 - return { success: false, error: 'Max retries exceeded' } 118 - } 119 - 120 - // Process a single batch 121 - const processBatch = async ( 122 - batch: File[], 123 - batchIndex: number, 124 - totalBatches: number, 125 - formData: FormData 126 - ): Promise<{ succeeded: File[]; failed: Array<{ file: File; reason: string }> }> => { 127 - const succeeded: File[] = [] 128 - const failed: Array<{ file: File; reason: string }> = [] 129 - 130 - setUploadProgress(`Processing batch ${batchIndex + 1}/${totalBatches} (files ${batchIndex * BATCH_SIZE + 1}-${Math.min((batchIndex + 1) * BATCH_SIZE, formData.getAll('files').length)})...`) 131 - 132 - // Process files in batch with retry logic 133 - const results = await Promise.allSettled( 134 - batch.map(file => uploadFileWithRetry(file)) 135 - ) 136 - 137 - results.forEach((result, idx) => { 138 - if (result.status === 'fulfilled' && result.value.success) { 139 - succeeded.push(batch[idx]) 140 - } else { 141 - const reason = result.status === 'rejected' 142 - ? 'Upload failed' 143 - : result.value.error || 'Unknown error' 144 - failed.push({ file: batch[idx], reason }) 145 - } 146 - }) 147 - 148 - return { succeeded, failed } 149 - } 150 - 151 - // Main upload handler with batching 152 56 const handleUpload = async () => { 153 57 const siteName = siteMode === 'existing' ? selectedSiteRkey : newSiteName 154 58 ··· 157 61 return 158 62 } 159 63 160 - if (!selectedFiles || selectedFiles.length === 0) { 161 - alert('Please select files to upload') 162 - return 163 - } 164 - 165 64 setIsUploading(true) 166 65 setUploadProgress('Preparing files...') 167 - setSkippedFiles([]) 168 - setUploadedCount(0) 169 66 170 67 try { 171 68 const formData = new FormData() 172 69 formData.append('siteName', siteName) 173 70 174 - // Add all files to FormData 175 - for (let i = 0; i < selectedFiles.length; i++) { 176 - formData.append('files', selectedFiles[i]) 177 - } 178 - 179 - const totalFiles = selectedFiles.length 180 - const batches = createBatches(selectedFiles) 181 - const totalBatches = batches.length 182 - 183 - console.log(`Uploading ${totalFiles} files in ${totalBatches} batches (${BATCH_SIZE} files per batch, ${CONCURRENT_BATCHES} concurrent)`) 184 - 185 - // Initialize batch progress 186 - setBatchProgress({ 187 - total: totalFiles, 188 - uploaded: 0, 189 - failed: 0, 190 - current: 0 191 - }) 192 - 193 - // Process batches with concurrency limit 194 - const allSkipped: Array<{ name: string; reason: string }> = [] 195 - let totalUploaded = 0 196 - 197 - for (let i = 0; i < batches.length; i += CONCURRENT_BATCHES) { 198 - const batchSlice = batches.slice(i, i + CONCURRENT_BATCHES) 199 - const batchPromises = batchSlice.map((batch, idx) => 200 - processBatch(batch, i + idx, totalBatches, formData) 201 - ) 202 - 203 - const results = await Promise.all(batchPromises) 204 - 205 - // Aggregate results 206 - results.forEach(result => { 207 - totalUploaded += result.succeeded.length 208 - result.failed.forEach(({ file, reason }) => { 209 - allSkipped.push({ name: file.name, reason }) 210 - }) 211 - }) 212 - 213 - // Update progress 214 - setBatchProgress({ 215 - total: totalFiles, 216 - uploaded: totalUploaded, 217 - failed: allSkipped.length, 218 - current: Math.min((i + CONCURRENT_BATCHES) * BATCH_SIZE, totalFiles) 219 - }) 71 + if (selectedFiles) { 72 + for (let i = 0; i < selectedFiles.length; i++) { 73 + formData.append('files', selectedFiles[i]) 74 + } 220 75 } 221 76 222 - // Now send the actual upload request to the server 223 - // (In a real implementation, you'd send batches to the server, 224 - // but for compatibility with the existing API, we send all at once) 225 - setUploadProgress('Finalizing upload to AT Protocol...') 226 - 77 + setUploadProgress('Uploading to AT Protocol...') 227 78 const response = await fetch('/wisp/upload-files', { 228 79 method: 'POST', 229 80 body: formData ··· 232 83 const data = await response.json() 233 84 if (data.success) { 234 85 setUploadProgress('Upload complete!') 235 - setSkippedFiles(data.skippedFiles || allSkipped) 236 - setUploadedCount(data.uploadedCount || data.fileCount || totalUploaded) 86 + setSkippedFiles(data.skippedFiles || []) 87 + setUploadedCount(data.uploadedCount || data.fileCount || 0) 237 88 setSelectedSiteRkey('') 238 89 setNewSiteName('') 239 90 setSelectedFiles(null) ··· 242 93 await onUploadComplete() 243 94 244 95 // Reset form - give more time if there are skipped files 245 - const resetDelay = (data.skippedFiles && data.skippedFiles.length > 0) || allSkipped.length > 0 ? 4000 : 1500 96 + const resetDelay = data.skippedFiles && data.skippedFiles.length > 0 ? 4000 : 1500 246 97 setTimeout(() => { 247 98 setUploadProgress('') 248 99 setSkippedFiles([]) 249 100 setUploadedCount(0) 250 - setBatchProgress(null) 251 101 setIsUploading(false) 252 102 }, resetDelay) 253 103 } else { ··· 260 110 ) 261 111 setIsUploading(false) 262 112 setUploadProgress('') 263 - setBatchProgress(null) 264 113 } 265 114 } 266 115 ··· 402 251 {uploadProgress && ( 403 252 <div className="space-y-3"> 404 253 <div className="p-4 bg-muted rounded-lg"> 405 - <div className="flex items-center gap-2 mb-2"> 254 + <div className="flex items-center gap-2"> 406 255 <Loader2 className="w-4 h-4 animate-spin" /> 407 256 <span className="text-sm">{uploadProgress}</span> 408 257 </div> 409 - {batchProgress && ( 410 - <div className="mt-2 space-y-1"> 411 - <div className="flex items-center justify-between text-xs text-muted-foreground"> 412 - <span> 413 - Uploaded: {batchProgress.uploaded}/{batchProgress.total} 414 - </span> 415 - <span> 416 - Failed: {batchProgress.failed} 417 - </span> 418 - </div> 419 - <div className="w-full bg-muted-foreground/20 rounded-full h-2"> 420 - <div 421 - className="bg-accent h-2 rounded-full transition-all duration-300" 422 - style={{ 423 - width: `${(batchProgress.uploaded / batchProgress.total) * 100}%` 424 - }} 425 - /> 426 - </div> 427 - </div> 428 - )} 429 258 </div> 430 259 431 260 {skippedFiles.length > 0 && (