Implement chunked blob upload for large podcast episodes

Problem:
- AT Protocol PDS servers have blob size limits (~5-10 MB)
- Users were getting PayloadTooLarge (413) errors when uploading large episodes

Solution:
- Implement automatic file chunking for files >5MB
- Upload each chunk as a separate blob to AT Protocol
- Store chunk manifest in episode record with audioChunks array
- Reconstruct full audio file when streaming by concatenating chunks

Changes:
- Upload: Split large files into 5MB chunks and upload sequentially
- Episode schema: Add audioChunks[], totalSize, and isChunked fields
- Streaming: Detect chunked episodes and reassemble chunks on-the-fly
- Frontend: Update file size limits (500 MB max) with chunking indicator
- Error handling: Better messages for file size issues

Benefits:
- Supports episode files up to 500 MB (from ~5-10 MB)
- Transparent to users - chunking happens automatically
- Backwards compatible with existing single-blob episodes
- Maintains audio quality (no compression/transcoding)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[e-mail]>

client/src/components/UploadModal.jsx

···
   const [error, setError] = useState('');
   const [success, setSuccess] = useState('');
 
+  const formatFileSize = (bytes) => {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return Math.round((bytes / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i];
+  };
+
   const handleSubmit = async (e) => {
     e.preventDefault();
 
···
               onChange={(e) => setFile(e.target.files[0])}
               required
             />
-            <small>Supported formats: MP3, M4A, WAV, etc.</small>
+            {file && (
+              <small style={{ display: 'block', marginTop: '0.5rem', color: file.size > 500 * 1024 * 1024 ? '#ef4444' : 'inherit' }}>
+                Selected: {file.name} ({formatFileSize(file.size)})
+                {file.size > 500 * 1024 * 1024 && ' - File exceeds 500 MB limit'}
+                {file.size > 5 * 1024 * 1024 && file.size <= 500 * 1024 * 1024 && ' - Will be uploaded in chunks'}
+              </small>
+            )}
+            <small>Supported formats: MP3, M4A, WAV, etc. Large files will be automatically chunked (max 500 MB)</small>
           </div>
 
           <div className="form-group">

src/index.js

···
   exposedHeaders: ['Set-Cookie']
 }));
 
-// Parse JSON bodies
-app.use(express.json());
+// Parse JSON bodies with increased limit for large payloads
+app.use(express.json({ limit: '50mb' }));
+app.use(express.urlencoded({ limit: '50mb', extended: true }));
 
 // Session middleware
 app.use(session({

src/routes/media.js

···
 /**
  * Stream media from AT Protocol blob storage
  * GET /api/media/stream/:did/:cid
+ * Supports both single blob and chunked blob episodes
  */
 router.get('/stream/:did/:cid', async (req, res) => {
   try {
     const { did, cid } = req.params;
-    
+
     console.log('Stream request:', { did, cid });
-    
-    // Fetch blob from AT Protocol
+
+    // Check if this is a chunked episode by trying to get the episode record
+    // The CID in the URL will be from the first chunk for chunked episodes
+    try {
+      // Try to find the episode record that matches this CID
+      const rpc = new Client({
+        handler: (pathname, init) => {
+          return fetch(pathname, init);
+        }
+      });
+
+      const records = await ok(
+        rpc.get('com.atproto.repo.listRecords', {
+          params: {
+            repo: did,
+            collection: 'app.podcast.episode'
+          }
+        })
+      );
+
+      // Find the episode with this CID in its audio field
+      const episode = records.records?.find(record => {
+        const audioCid = record.value.audio?.ref?.$link || record.value.audio?.ref?.toString();
+        return audioCid === cid;
+      });
+
+      // If we found a chunked episode, stream all chunks
+      if (episode?.value?.isChunked && episode.value.audioChunks) {
+        console.log(`Streaming chunked episode with ${episode.value.audioChunks.length} chunks`);
+
+        const chunks = [];
+        for (let i = 0; i < episode.value.audioChunks.length; i++) {
+          const chunk = episode.value.audioChunks[i];
+          const chunkCid = chunk.ref.$link || chunk.ref.toString();
+
+          console.log(`Fetching chunk ${i + 1}/${episode.value.audioChunks.length}: ${chunkCid}`);
+
+          const blobResponse = await atprotoClient.getBlob(did, chunkCid);
+
+          if (!blobResponse.ok) {
+            console.error('Chunk not found:', { did, chunkCid, chunkIndex: i });
+            throw new Error(`Chunk ${i + 1} not found`);
+          }
+
+          const buffer = await blobResponse.arrayBuffer();
+          chunks.push(Buffer.from(buffer));
+        }
+
+        // Concatenate all chunks
+        const fullAudio = Buffer.concat(chunks);
+        const contentType = episode.value.audioChunks[0].mimeType || 'audio/mpeg';
+
+        res.setHeader('Content-Type', contentType);
+        res.setHeader('Content-Length', fullAudio.length);
+        res.setHeader('Accept-Ranges', 'bytes');
+
+        console.log(`Streaming complete file: ${fullAudio.length} bytes`);
+        return res.send(fullAudio);
+      }
+    } catch (recordError) {
+      // If we can't find the record, fall back to single blob streaming
+      console.log('Could not check for chunked episode, falling back to single blob:', recordError.message);
+    }
+
+    // Fall back to single blob streaming
     const blobResponse = await atprotoClient.getBlob(did, cid);
-    
+
     if (!blobResponse.ok) {
       console.error('Blob not found:', { did, cid, status: blobResponse.status });
       return res.status(404).json({ error: 'Media not found' });

src/routes/upload.js

···
 import { storePodcastMetadata } from '../storage/metadata.js';
 
 const router = express.Router();
-const upload = multer({ dest: 'public/uploads/' });
+
+// AT Protocol PDS typically has a 10MB limit for blobs
+// We'll chunk files larger than this into smaller pieces
+const PDS_BLOB_LIMIT = 5 * 1024 * 1024; // 5MB per chunk (conservative estimate)
+const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB total file limit
+
+const upload = multer({
+  dest: 'public/uploads/',
+  limits: {
+    fileSize: MAX_FILE_SIZE
+  }
+});
+
+/**
+ * Split buffer into chunks for multi-blob upload
+ */
+function splitIntoChunks(buffer, chunkSize) {
+  const chunks = [];
+  for (let i = 0; i < buffer.length; i += chunkSize) {
+    chunks.push(buffer.slice(i, i + chunkSize));
+  }
+  return chunks;
+}
 
 /**
  * Upload a podcast episode
···
  */
 router.post('/episode', requireAuth, upload.single('audio'), async (req, res) => {
   try {
+    console.log('Upload request received:', {
+      hasFile: !!req.file,
+      filename: req.file?.originalname,
+      size: req.file?.size,
+      mimetype: req.file?.mimetype,
+      title: req.body?.title,
+      userDid: req.oauthSession?.did
+    });
+
     if (!req.file) {
+      console.error('Upload failed: No audio file provided');
       return res.status(400).json({ error: 'No audio file provided' });
     }
 
     const { title, description, pubDate } = req.body;
-    
+
     if (!title) {
+      console.error('Upload failed: No title provided');
       return res.status(400).json({ error: 'Episode title is required' });
     }
 
+    console.log('Reading uploaded file from:', req.file.path);
     // Read the uploaded file
     const audioData = await fs.readFile(req.file.path);
+    console.log('File read successfully, size:', audioData.length, 'bytes');
 
     // Create atcute Client with OAuth session's authenticated fetch handler
     const rpc = new Client({
···
       }
     });
 
-    // Upload to AT Protocol as a blob
-    const uploadResponse = await ok(
-      rpc.post('com.atproto.repo.uploadBlob', {
-        input: audioData,
-        headers: {
-          'Content-Type': req.file.mimetype
+    // Determine if we need to chunk the file
+    const needsChunking = audioData.length > PDS_BLOB_LIMIT;
+    let audioBlobs = [];
+
+    if (needsChunking) {
+      console.log(`File is large (${(audioData.length / (1024 * 1024)).toFixed(2)} MB), splitting into chunks...`);
+
+      // Split file into chunks
+      const chunks = splitIntoChunks(audioData, PDS_BLOB_LIMIT);
+      console.log(`Split into ${chunks.length} chunks`);
+
+      // Upload each chunk
+      for (let i = 0; i < chunks.length; i++) {
+        console.log(`Uploading chunk ${i + 1}/${chunks.length} (${(chunks[i].length / (1024 * 1024)).toFixed(2)} MB)...`);
+
+        try {
+          const chunkResponse = await ok(
+            rpc.post('com.atproto.repo.uploadBlob', {
+              input: chunks[i],
+              headers: {
+                'Content-Type': req.file.mimetype
+              }
+            })
+          );
+
+          audioBlobs.push({
+            ...chunkResponse.blob,
+            chunkIndex: i,
+            chunkSize: chunks[i].length
+          });
+
+          console.log(`Chunk ${i + 1}/${chunks.length} uploaded successfully`);
+        } catch (uploadError) {
+          // Clean up temporary file before throwing
+          await fs.unlink(req.file.path).catch(() => {});
+
+          if (uploadError.status === 413 || uploadError.error === 'PayloadTooLarge') {
+            throw new Error(`Chunk ${i + 1} is too large for your PDS server. The PDS_BLOB_LIMIT may need to be reduced.`);
+          }
+          throw uploadError;
+        }
+      }
+
+      console.log('All chunks uploaded successfully');
+    } else {
+      // Upload as single blob
+      console.log('Uploading as single blob to AT Protocol...');
+      try {
+        const uploadResponse = await ok(
+          rpc.post('com.atproto.repo.uploadBlob', {
+            input: audioData,
+            headers: {
+              'Content-Type': req.file.mimetype
+            }
+          })
+        );
+
+        audioBlobs = [uploadResponse.blob];
+      } catch (uploadError) {
+        // Clean up temporary file before throwing
+        await fs.unlink(req.file.path).catch(() => {});
+
+        if (uploadError.status === 413 || uploadError.error === 'PayloadTooLarge') {
+          const sizeMB = (audioData.length / (1024 * 1024)).toFixed(2);
+          throw new Error(`File is too large (${sizeMB} MB). Your PDS server limits blob uploads. Try a smaller file or compress your audio.`);
         }
-      })
-    );
+        throw uploadError;
+      }
+    }
 
-    // The response contains a blob object
-    const blobData = uploadResponse.blob;
+    // Use the first blob for backwards compatibility in metadata
+    const blobData = audioBlobs[0];
 
     // Extract CID from blob reference
     const cid = blobData.ref.$link || blobData.ref.toString();
···
 
     // Use a simple app.podcast.episode collection (custom namespace)
     const rkey = `${Date.now()}`;
+
+    // Build the episode record
+    const episodeRecord = {
+      $type: 'app.podcast.episode',
+      title: title,
+      description: description || '',
+      publishedAt: episode.pubDate,
+      createdAt: episode.pubDate
+    };
+
+    // If chunked, store all chunks; otherwise store single blob
+    if (audioBlobs.length > 1) {
+      console.log(`Storing chunked episode with ${audioBlobs.length} chunks`);
+      episodeRecord.audioChunks = audioBlobs;
+      episodeRecord.audio = blobData; // Keep first chunk as primary for backwards compatibility
+      episodeRecord.totalSize = audioData.length;
+      episodeRecord.isChunked = true;
+    } else {
+      episodeRecord.audio = blobData;
+    }
+
     const record = await ok(
       rpc.post('com.atproto.repo.createRecord', {
         input: {
           repo: req.oauthSession.did,
           collection: 'app.podcast.episode',
           rkey: rkey,
-          record: {
-            $type: 'app.podcast.episode',
-            title: title,
-            description: description || '',
-            audio: blobData, // Reference the blob in the record
-            publishedAt: episode.pubDate,
-            createdAt: episode.pubDate
-          }
+          record: episodeRecord
         }
       })
     );
···
     });
   } catch (error) {
     console.error('Upload error:', error);
+
+    // Clean up temporary file if it exists
+    if (req.file?.path) {
+      await fs.unlink(req.file.path).catch(() => {});
+    }
+
     res.status(500).json({ error: 'Failed to upload episode', details: error.message });
   }
+});
+
+// Error handler for multer errors (file too large, etc.)
+router.use((error, req, res, next) => {
+  if (error instanceof multer.MulterError) {
+    if (error.code === 'LIMIT_FILE_SIZE') {
+      const maxSizeMB = (MAX_FILE_SIZE / (1024 * 1024)).toFixed(0);
+      return res.status(413).json({
+        error: 'File too large',
+        details: `Maximum file size is ${maxSizeMB} MB. Please compress your audio file or split it into multiple episodes.`
+      });
+    }
+    return res.status(400).json({
+      error: 'Upload error',
+      details: error.message
+    });
+  }
+  next(error);
 });
 
 export { router as uploadRouter };