Monorepo for Wisp.place. A static site hosting service built on top of the AT Protocol.

domain verfication worker, upload blobs as octet-stream

+36 -4
hosting-service/src/index.ts
··· 1 1 import { serve } from 'bun'; 2 2 import app from './server'; 3 3 import { FirehoseWorker } from './lib/firehose'; 4 + import { DNSVerificationWorker } from './lib/dns-verification-worker'; 4 5 import { mkdirSync, existsSync } from 'fs'; 5 6 6 7 const PORT = process.env.PORT || 3001; ··· 19 20 20 21 firehose.start(); 21 22 23 + // Start DNS verification worker (runs every hour) 24 + const dnsVerifier = new DNSVerificationWorker( 25 + 60 * 60 * 1000, // 1 hour 26 + (msg, data) => { 27 + console.log('[DNS Verifier]', msg, data || ''); 28 + } 29 + ); 30 + 31 + dnsVerifier.start(); 32 + 22 33 // Add health check endpoint 23 34 app.get('/health', (c) => { 24 35 const firehoseHealth = firehose.getHealth(); 36 + const dnsVerifierHealth = dnsVerifier.getHealth(); 25 37 return c.json({ 26 38 status: 'ok', 27 39 firehose: firehoseHealth, 40 + dnsVerifier: dnsVerifierHealth, 28 41 }); 29 42 }); 30 43 44 + // Add manual DNS verification trigger (for testing/admin) 45 + app.post('/admin/verify-dns', async (c) => { 46 + try { 47 + await dnsVerifier.trigger(); 48 + return c.json({ 49 + success: true, 50 + message: 'DNS verification triggered', 51 + }); 52 + } catch (error) { 53 + return c.json({ 54 + success: false, 55 + error: error instanceof Error ? error.message : String(error), 56 + }, 500); 57 + } 58 + }); 59 + 31 60 // Start HTTP server 32 61 const server = serve({ 33 62 port: PORT, ··· 37 66 console.log(` 38 67 Wisp Hosting Service 39 68 40 - Server: http://localhost:${PORT} 41 - Health: http://localhost:${PORT}/health 42 - Cache: ${CACHE_DIR} 43 - Firehose: Connected to Jetstream 69 + Server: http://localhost:${PORT} 70 + Health: http://localhost:${PORT}/health 71 + Cache: ${CACHE_DIR} 72 + Firehose: Connected to Jetstream 73 + DNS Verifier: Checking every hour 44 74 `); 45 75 46 76 // Graceful shutdown 47 77 process.on('SIGINT', () => { 48 78 console.log('\n🛑 Shutting down...'); 49 79 firehose.stop(); 80 + dnsVerifier.stop(); 50 81 server.stop(); 51 82 process.exit(0); 52 83 }); ··· 54 85 process.on('SIGTERM', () => { 55 86 console.log('\n🛑 Shutting down...'); 56 87 firehose.stop(); 88 + dnsVerifier.stop(); 57 89 server.stop(); 58 90 process.exit(0); 59 91 });
+170
hosting-service/src/lib/dns-verification-worker.ts
··· 1 + import { verifyCustomDomain } from '../../../src/lib/dns-verify'; 2 + import { db } from '../../../src/lib/db'; 3 + 4 + interface VerificationStats { 5 + totalChecked: number; 6 + verified: number; 7 + failed: number; 8 + errors: number; 9 + } 10 + 11 + export class DNSVerificationWorker { 12 + private interval: Timer | null = null; 13 + private isRunning = false; 14 + private lastRunTime: number | null = null; 15 + private stats: VerificationStats = { 16 + totalChecked: 0, 17 + verified: 0, 18 + failed: 0, 19 + errors: 0, 20 + }; 21 + 22 + constructor( 23 + private checkIntervalMs: number = 60 * 60 * 1000, // 1 hour default 24 + private onLog?: (message: string, data?: any) => void 25 + ) {} 26 + 27 + private log(message: string, data?: any) { 28 + if (this.onLog) { 29 + this.onLog(message, data); 30 + } 31 + } 32 + 33 + async start() { 34 + if (this.isRunning) { 35 + this.log('DNS verification worker already running'); 36 + return; 37 + } 38 + 39 + this.isRunning = true; 40 + this.log('Starting DNS verification worker', { 41 + intervalMinutes: this.checkIntervalMs / 60000, 42 + }); 43 + 44 + // Run immediately on start 45 + await this.verifyAllDomains(); 46 + 47 + // Then run on interval 48 + this.interval = setInterval(() => { 49 + this.verifyAllDomains(); 50 + }, this.checkIntervalMs); 51 + } 52 + 53 + stop() { 54 + if (this.interval) { 55 + clearInterval(this.interval); 56 + this.interval = null; 57 + } 58 + this.isRunning = false; 59 + this.log('DNS verification worker stopped'); 60 + } 61 + 62 + private async verifyAllDomains() { 63 + this.log('Starting DNS verification check'); 64 + const startTime = Date.now(); 65 + 66 + const runStats: VerificationStats = { 67 + totalChecked: 0, 68 + verified: 0, 69 + failed: 0, 70 + errors: 0, 71 + }; 72 + 73 + try { 74 + // Get all verified custom domains 75 + const domains = await db` 76 + SELECT id, domain, did FROM custom_domains WHERE verified = true 77 + `; 78 + 79 + if (!domains || domains.length === 0) { 80 + this.log('No verified custom domains to check'); 81 + this.lastRunTime = Date.now(); 82 + return; 83 + } 84 + 85 + this.log(`Checking ${domains.length} verified custom domains`); 86 + 87 + // Verify each domain 88 + for (const row of domains) { 89 + runStats.totalChecked++; 90 + const { id, domain, did } = row; 91 + 92 + try { 93 + // Extract hash from id (SHA256 of did:domain) 94 + const expectedHash = id.substring(0, 16); 95 + 96 + // Verify DNS records 97 + const result = await verifyCustomDomain(domain, did, expectedHash); 98 + 99 + if (result.verified) { 100 + // Update last_verified_at timestamp 101 + await db` 102 + UPDATE custom_domains 103 + SET last_verified_at = EXTRACT(EPOCH FROM NOW()) 104 + WHERE id = ${id} 105 + `; 106 + runStats.verified++; 107 + this.log(`Domain verified: ${domain}`, { did }); 108 + } else { 109 + // Mark domain as unverified 110 + await db` 111 + UPDATE custom_domains 112 + SET verified = false, 113 + last_verified_at = EXTRACT(EPOCH FROM NOW()) 114 + WHERE id = ${id} 115 + `; 116 + runStats.failed++; 117 + this.log(`Domain verification failed: ${domain}`, { 118 + did, 119 + error: result.error, 120 + found: result.found, 121 + }); 122 + } 123 + } catch (error) { 124 + runStats.errors++; 125 + this.log(`Error verifying domain: ${domain}`, { 126 + did, 127 + error: error instanceof Error ? error.message : String(error), 128 + }); 129 + } 130 + } 131 + 132 + // Update cumulative stats 133 + this.stats.totalChecked += runStats.totalChecked; 134 + this.stats.verified += runStats.verified; 135 + this.stats.failed += runStats.failed; 136 + this.stats.errors += runStats.errors; 137 + 138 + const duration = Date.now() - startTime; 139 + this.lastRunTime = Date.now(); 140 + 141 + this.log('DNS verification check completed', { 142 + duration: `${duration}ms`, 143 + ...runStats, 144 + }); 145 + } catch (error) { 146 + this.log('Fatal error in DNS verification worker', { 147 + error: error instanceof Error ? error.message : String(error), 148 + }); 149 + } 150 + } 151 + 152 + getHealth() { 153 + return { 154 + isRunning: this.isRunning, 155 + lastRunTime: this.lastRunTime, 156 + intervalMs: this.checkIntervalMs, 157 + stats: this.stats, 158 + healthy: this.isRunning && ( 159 + this.lastRunTime === null || 160 + Date.now() - this.lastRunTime < this.checkIntervalMs * 2 161 + ), 162 + }; 163 + } 164 + 165 + // Manual trigger for testing 166 + async trigger() { 167 + this.log('Manual DNS verification triggered'); 168 + await this.verifyAllDomains(); 169 + } 170 + }
+20 -20
hosting-service/src/lib/html-rewriter.test.ts
··· 8 8 9 9 test('rewriteHtmlPaths - rewrites absolute paths in src attributes', () => { 10 10 const html = '<img src="/logo.png">'; 11 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 12 - expect(result).toBe('<img src="/s/did:plc:123/mysite/logo.png">'); 11 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 12 + expect(result).toBe('<img src="/did:plc:123/mysite/logo.png">'); 13 13 }); 14 14 15 15 test('rewriteHtmlPaths - rewrites absolute paths in href attributes', () => { 16 16 const html = '<link rel="stylesheet" href="/style.css">'; 17 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 18 - expect(result).toBe('<link rel="stylesheet" href="/s/did:plc:123/mysite/style.css">'); 17 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 18 + expect(result).toBe('<link rel="stylesheet" href="/did:plc:123/mysite/style.css">'); 19 19 }); 20 20 21 21 test('rewriteHtmlPaths - preserves external URLs', () => { 22 22 const html = '<img src="https://example.com/logo.png">'; 23 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 23 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 24 24 expect(result).toBe('<img src="https://example.com/logo.png">'); 25 25 }); 26 26 27 27 test('rewriteHtmlPaths - preserves protocol-relative URLs', () => { 28 28 const html = '<script src="//cdn.example.com/script.js"></script>'; 29 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 29 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 30 30 expect(result).toBe('<script src="//cdn.example.com/script.js"></script>'); 31 31 }); 32 32 33 33 test('rewriteHtmlPaths - preserves data URIs', () => { 34 34 const html = '<img src="data:image/png;base64,abc123">'; 35 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 35 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 36 36 expect(result).toBe('<img src="data:image/png;base64,abc123">'); 37 37 }); 38 38 39 39 test('rewriteHtmlPaths - preserves anchors', () => { 40 40 const html = '<a href="/#section">Jump</a>'; 41 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 41 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 42 42 expect(result).toBe('<a href="/#section">Jump</a>'); 43 43 }); 44 44 45 45 test('rewriteHtmlPaths - preserves relative paths', () => { 46 46 const html = '<img src="./logo.png">'; 47 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 47 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 48 48 expect(result).toBe('<img src="./logo.png">'); 49 49 }); 50 50 51 51 test('rewriteHtmlPaths - handles single quotes', () => { 52 52 const html = "<img src='/logo.png'>"; 53 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 54 - expect(result).toBe("<img src='/s/did:plc:123/mysite/logo.png'>"); 53 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 54 + expect(result).toBe("<img src='/did:plc:123/mysite/logo.png'>"); 55 55 }); 56 56 57 57 test('rewriteHtmlPaths - handles srcset', () => { 58 58 const html = '<img srcset="/logo.png 1x, /logo@2x.png 2x">'; 59 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 60 - expect(result).toBe('<img srcset="/s/did:plc:123/mysite/logo.png 1x, /s/did:plc:123/mysite/logo@2x.png 2x">'); 59 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 60 + expect(result).toBe('<img srcset="/did:plc:123/mysite/logo.png 1x, /did:plc:123/mysite/logo@2x.png 2x">'); 61 61 }); 62 62 63 63 test('rewriteHtmlPaths - handles form actions', () => { 64 64 const html = '<form action="/submit"></form>'; 65 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 66 - expect(result).toBe('<form action="/s/did:plc:123/mysite/submit"></form>'); 65 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 66 + expect(result).toBe('<form action="/did:plc:123/mysite/submit"></form>'); 67 67 }); 68 68 69 69 test('rewriteHtmlPaths - handles complex HTML', () => { ··· 83 83 </html> 84 84 `.trim(); 85 85 86 - const result = rewriteHtmlPaths(html, '/s/did:plc:123/mysite/'); 86 + const result = rewriteHtmlPaths(html, '/did:plc:123/mysite/'); 87 87 88 - expect(result).toContain('href="/s/did:plc:123/mysite/style.css"'); 89 - expect(result).toContain('src="/s/did:plc:123/mysite/app.js"'); 90 - expect(result).toContain('src="/s/did:plc:123/mysite/images/logo.png"'); 91 - expect(result).toContain('href="/s/did:plc:123/mysite/about"'); 88 + expect(result).toContain('href="/did:plc:123/mysite/style.css"'); 89 + expect(result).toContain('src="/did:plc:123/mysite/app.js"'); 90 + expect(result).toContain('src="/did:plc:123/mysite/images/logo.png"'); 91 + expect(result).toContain('href="/did:plc:123/mysite/about"'); 92 92 expect(result).toContain('href="https://example.com"'); // External preserved 93 93 expect(result).toContain('href="#section"'); // Anchor preserved 94 94 });
+49 -9
hosting-service/src/lib/utils.ts
··· 3 3 import { existsSync, mkdirSync } from 'fs'; 4 4 import { writeFile } from 'fs/promises'; 5 5 import { safeFetchJson, safeFetchBlob } from './safe-fetch'; 6 + import { CID } from 'multiformats/cid'; 6 7 7 8 const CACHE_DIR = './cache/sites'; 9 + 10 + // Type guards for different blob reference formats 11 + interface IpldLink { 12 + $link: string; 13 + } 14 + 15 + interface TypedBlobRef { 16 + ref: CID | IpldLink; 17 + } 18 + 19 + interface UntypedBlobRef { 20 + cid: string; 21 + } 22 + 23 + function isIpldLink(obj: unknown): obj is IpldLink { 24 + return typeof obj === 'object' && obj !== null && '$link' in obj && typeof (obj as IpldLink).$link === 'string'; 25 + } 26 + 27 + function isTypedBlobRef(obj: unknown): obj is TypedBlobRef { 28 + return typeof obj === 'object' && obj !== null && 'ref' in obj; 29 + } 30 + 31 + function isUntypedBlobRef(obj: unknown): obj is UntypedBlobRef { 32 + return typeof obj === 'object' && obj !== null && 'cid' in obj && typeof (obj as UntypedBlobRef).cid === 'string'; 33 + } 8 34 9 35 export async function resolveDid(identifier: string): Promise<string | null> { 10 36 try { ··· 85 111 } 86 112 } 87 113 88 - export function extractBlobCid(blobRef: any): string | null { 89 - if (typeof blobRef === 'object' && blobRef !== null) { 90 - if ('ref' in blobRef && blobRef.ref?.$link) { 91 - return blobRef.ref.$link; 92 - } 93 - if ('cid' in blobRef && typeof blobRef.cid === 'string') { 94 - return blobRef.cid; 114 + export function extractBlobCid(blobRef: unknown): string | null { 115 + // Check if it's a direct IPLD link 116 + if (isIpldLink(blobRef)) { 117 + return blobRef.$link; 118 + } 119 + 120 + // Check if it's a typed blob ref with a ref property 121 + if (isTypedBlobRef(blobRef)) { 122 + const ref = blobRef.ref; 123 + 124 + // Check if ref is a CID object 125 + if (CID.isCID(ref)) { 126 + return ref.toString(); 95 127 } 96 - if ('$link' in blobRef && typeof blobRef.$link === 'string') { 97 - return blobRef.$link; 128 + 129 + // Check if ref is an IPLD link object 130 + if (isIpldLink(ref)) { 131 + return ref.$link; 98 132 } 99 133 } 134 + 135 + // Check if it's an untyped blob ref with a cid string 136 + if (isUntypedBlobRef(blobRef)) { 137 + return blobRef.cid; 138 + } 139 + 100 140 return null; 101 141 } 102 142
+55 -35
hosting-service/src/server.ts
··· 34 34 35 35 if (existsSync(cachedFile)) { 36 36 const file = Bun.file(cachedFile); 37 - return new Response(file); 37 + return new Response(file, { 38 + headers: { 39 + 'Content-Type': file.type || 'application/octet-stream', 40 + }, 41 + }); 38 42 } 39 43 40 44 // Try index.html for directory-like paths ··· 42 46 const indexFile = getCachedFilePath(did, rkey, `${requestPath}/index.html`); 43 47 if (existsSync(indexFile)) { 44 48 const file = Bun.file(indexFile); 45 - return new Response(file); 49 + return new Response(file, { 50 + headers: { 51 + 'Content-Type': 'text/html; charset=utf-8', 52 + }, 53 + }); 46 54 } 47 55 } 48 56 49 57 return new Response('Not Found', { status: 404 }); 50 58 } 51 59 52 - // Helper to serve files from cache with HTML path rewriting for /s/ routes 60 + // Helper to serve files from cache with HTML path rewriting for sites.wisp.place routes 53 61 async function serveFromCacheWithRewrite( 54 62 did: string, 55 63 rkey: string, ··· 78 86 }); 79 87 } 80 88 81 - // Non-HTML files served as-is 82 - return new Response(file); 89 + // Non-HTML files served with proper MIME type 90 + return new Response(file, { 91 + headers: { 92 + 'Content-Type': file.type || 'application/octet-stream', 93 + }, 94 + }); 83 95 } 84 96 85 97 // Try index.html for directory-like paths ··· 128 140 } 129 141 } 130 142 131 - // Route 4: Direct file serving (no DB) - /s.wisp.place/:identifier/:site/* 132 - app.get('/s/:identifier/:site/*', async (c) => { 133 - const identifier = c.req.param('identifier'); 134 - const site = c.req.param('site'); 135 - const rawPath = c.req.path.replace(`/s/${identifier}/${site}/`, ''); 136 - const filePath = sanitizePath(rawPath); 137 - 138 - console.log('[Direct] Serving', { identifier, site, filePath }); 139 - 140 - // Validate site name (rkey) 141 - if (!isValidRkey(site)) { 142 - return c.text('Invalid site name', 400); 143 - } 144 - 145 - // Resolve identifier to DID 146 - const did = await resolveDid(identifier); 147 - if (!did) { 148 - return c.text('Invalid identifier', 400); 149 - } 150 - 151 - // Ensure site is cached 152 - const cached = await ensureSiteCached(did, site); 153 - if (!cached) { 154 - return c.text('Site not found', 404); 155 - } 156 - 157 - // Serve with HTML path rewriting to handle absolute paths 158 - const basePath = `/s/${identifier}/${site}/`; 159 - return serveFromCacheWithRewrite(did, site, filePath, basePath); 160 - }); 143 + // Route 4: Direct file serving (no DB) - sites.wisp.place/:identifier/:site/* 144 + // This route is now handled in the catch-all route below 161 145 162 146 // Route 3: DNS routing for custom domains - /hash.dns.wisp.place/* 163 147 app.get('/*', async (c) => { ··· 166 150 const path = sanitizePath(rawPath); 167 151 168 152 console.log('[Request]', { hostname, path }); 153 + 154 + // Check if this is sites.wisp.place subdomain 155 + if (hostname === `sites.${BASE_HOST}` || hostname === `sites.${BASE_HOST}:${process.env.PORT || 3000}`) { 156 + // Extract identifier and site from path: /did:plc:123abc/sitename/file.html 157 + const pathParts = rawPath.split('/'); 158 + if (pathParts.length < 2) { 159 + return c.text('Invalid path format. Expected: /identifier/sitename/path', 400); 160 + } 161 + 162 + const identifier = pathParts[0]; 163 + const site = pathParts[1]; 164 + const filePath = sanitizePath(pathParts.slice(2).join('/')); 165 + 166 + console.log('[Sites] Serving', { identifier, site, filePath }); 167 + 168 + // Validate site name (rkey) 169 + if (!isValidRkey(site)) { 170 + return c.text('Invalid site name', 400); 171 + } 172 + 173 + // Resolve identifier to DID 174 + const did = await resolveDid(identifier); 175 + if (!did) { 176 + return c.text('Invalid identifier', 400); 177 + } 178 + 179 + // Ensure site is cached 180 + const cached = await ensureSiteCached(did, site); 181 + if (!cached) { 182 + return c.text('Site not found', 404); 183 + } 184 + 185 + // Serve with HTML path rewriting to handle absolute paths 186 + const basePath = `/${identifier}/${site}/`; 187 + return serveFromCacheWithRewrite(did, site, filePath, basePath); 188 + } 169 189 170 190 // Check if this is a DNS hash subdomain 171 191 const dnsMatch = hostname.match(/^([a-f0-9]{16})\.dns\.(.+)$/);
+2 -1
package.json
··· 30 30 "react-dom": "^19.2.0", 31 31 "tailwind-merge": "^3.3.1", 32 32 "tailwindcss": "4", 33 - "tw-animate-css": "^1.4.0" 33 + "tw-animate-css": "^1.4.0", 34 + "typescript": "^5.9.3" 34 35 }, 35 36 "devDependencies": { 36 37 "@types/react": "^19.2.2",
-34
src/lib/wisp-utils.ts
··· 145 145 filePaths: string[], 146 146 currentPath: string = '' 147 147 ): Directory { 148 - const mimeTypeMismatches: string[] = []; 149 - 150 148 const updatedEntries = directory.entries.map(entry => { 151 149 if ('type' in entry.node && entry.node.type === 'file') { 152 150 // Build the full path for this file ··· 162 160 163 161 if (fileIndex !== -1 && uploadResults[fileIndex]) { 164 162 const blobRef = uploadResults[fileIndex].blobRef; 165 - const uploadedPath = filePaths[fileIndex]; 166 - 167 - // Check if MIME types make sense for this file extension 168 - const expectedMime = getExpectedMimeType(entry.name); 169 - if (expectedMime && blobRef.mimeType !== expectedMime && !blobRef.mimeType.startsWith(expectedMime)) { 170 - mimeTypeMismatches.push(`${fullPath}: expected ${expectedMime}, got ${blobRef.mimeType} (from upload: ${uploadedPath})`); 171 - } 172 163 173 164 return { 174 165 ...entry, ··· 192 183 return entry; 193 184 }) as Entry[]; 194 185 195 - if (mimeTypeMismatches.length > 0) { 196 - console.error('\n⚠️ MIME TYPE MISMATCHES DETECTED IN MANIFEST:'); 197 - mimeTypeMismatches.forEach(m => console.error(` ${m}`)); 198 - console.error(''); 199 - } 200 - 201 186 const result = { 202 187 $type: 'place.wisp.fs#directory' as const, 203 188 type: 'directory' as const, ··· 206 191 207 192 return result; 208 193 } 209 - 210 - function getExpectedMimeType(filename: string): string | null { 211 - const ext = filename.toLowerCase().split('.').pop(); 212 - const mimeMap: Record<string, string> = { 213 - 'html': 'text/html', 214 - 'htm': 'text/html', 215 - 'css': 'text/css', 216 - 'js': 'text/javascript', 217 - 'mjs': 'text/javascript', 218 - 'json': 'application/json', 219 - 'jpg': 'image/jpeg', 220 - 'jpeg': 'image/jpeg', 221 - 'png': 'image/png', 222 - 'gif': 'image/gif', 223 - 'webp': 'image/webp', 224 - 'svg': 'image/svg+xml', 225 - }; 226 - return ext ? (mimeMap[ext] || null) : null; 227 - }
+4 -65
src/routes/wisp.ts
··· 159 159 uploadedFiles.push({ 160 160 name: file.name, 161 161 content: Buffer.from(arrayBuffer), 162 - mimeType: file.type || 'application/octet-stream', 162 + mimeType: 'application/octet-stream', 163 163 size: file.size 164 164 }); 165 165 } ··· 211 211 // Process files into directory structure 212 212 const { directory, fileCount } = processUploadedFiles(uploadedFiles); 213 213 214 - // Upload files as blobs in parallel 215 - const mimeTypeMismatches: Array<{file: string, sent: string, returned: string}> = []; 216 - 214 + // Upload files as blobs in parallel (always as octet-stream) 217 215 const uploadPromises = uploadedFiles.map(async (file, i) => { 218 216 try { 219 217 const uploadResult = await agent.com.atproto.repo.uploadBlob( 220 218 file.content, 221 219 { 222 - encoding: file.mimeType 220 + encoding: 'application/octet-stream' 223 221 } 224 222 ); 225 223 226 224 const sentMimeType = file.mimeType; 227 225 const returnedBlobRef = uploadResult.data.blob; 228 226 229 - // Track MIME type mismatches for summary 230 - if (sentMimeType !== returnedBlobRef.mimeType) { 231 - mimeTypeMismatches.push({ 232 - file: file.name, 233 - sent: sentMimeType, 234 - returned: returnedBlobRef.mimeType 235 - }); 236 - } 237 - 238 227 // Use the blob ref exactly as returned from PDS 239 228 return { 240 229 result: { 241 - hash: returnedBlobRef.ref.$link || returnedBlobRef.ref.toString(), 230 + hash: returnedBlobRef.ref.toString(), 242 231 blobRef: returnedBlobRef 243 232 }, 244 233 filePath: file.name, ··· 254 243 // Wait for all uploads to complete 255 244 const uploadedBlobs = await Promise.all(uploadPromises); 256 245 257 - // Show MIME type mismatch summary 258 - if (mimeTypeMismatches.length > 0) { 259 - console.warn(`\n⚠️ PDS changed MIME types for ${mimeTypeMismatches.length} files:`); 260 - mimeTypeMismatches.slice(0, 20).forEach(m => { 261 - console.warn(` ${m.file}: ${m.sent} → ${m.returned}`); 262 - }); 263 - if (mimeTypeMismatches.length > 20) { 264 - console.warn(` ... and ${mimeTypeMismatches.length - 20} more`); 265 - } 266 - console.warn(''); 267 - } 268 - 269 - // CRITICAL: Find files uploaded as application/octet-stream 270 - const octetStreamFiles = uploadedBlobs.filter(b => b.returnedMimeType === 'application/octet-stream'); 271 - if (octetStreamFiles.length > 0) { 272 - console.error(`\n🚨 FILES UPLOADED AS application/octet-stream (${octetStreamFiles.length}):`); 273 - octetStreamFiles.forEach(f => { 274 - console.error(` ${f.filePath}: sent=${f.sentMimeType}, returned=${f.returnedMimeType}`); 275 - }); 276 - console.error(''); 277 - } 278 - 279 246 // Extract results and file paths in correct order 280 247 const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result); 281 248 const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath); ··· 300 267 } catch (putRecordError: any) { 301 268 console.error('\n❌ Failed to create record on PDS'); 302 269 console.error('Error:', putRecordError.message); 303 - 304 - // Try to identify which file has the MIME type mismatch 305 - if (putRecordError.message?.includes('Mimetype') || putRecordError.message?.includes('mimeType')) { 306 - console.error('\n🔍 Analyzing manifest for MIME type issues...'); 307 - 308 - // Recursively check all blobs in manifest 309 - const checkBlobs = (node: any, path: string = '') => { 310 - if (node.type === 'file' && node.blob) { 311 - const mimeType = node.blob.mimeType; 312 - console.error(` File: ${path} - MIME: ${mimeType}`); 313 - } else if (node.type === 'directory' && node.entries) { 314 - for (const entry of node.entries) { 315 - const entryPath = path ? `${path}/${entry.name}` : entry.name; 316 - checkBlobs(entry.node, entryPath); 317 - } 318 - } 319 - }; 320 - 321 - checkBlobs(manifest.root, ''); 322 - 323 - console.error('\n📊 Blob upload summary:'); 324 - uploadedBlobs.slice(0, 20).forEach((b, i) => { 325 - console.error(` [${i}] ${b.filePath}: sent=${b.sentMimeType}, returned=${b.returnedMimeType}`); 326 - }); 327 - if (uploadedBlobs.length > 20) { 328 - console.error(` ... and ${uploadedBlobs.length - 20} more`); 329 - } 330 - } 331 270 332 271 throw putRecordError; 333 272 }