handle compression better

hosting-service/src/lib/utils.ts

···
   rkey: string;
 }
 
+/**
+ * Determines if a MIME type should benefit from gzip compression.
+ * Returns true for text-based web assets (HTML, CSS, JS, JSON, XML, SVG).
+ * Returns false for already-compressed formats (images, video, audio, PDFs).
+ * 
+ */
+export function shouldCompressMimeType(mimeType: string | undefined): boolean {
+  if (!mimeType) return false;
+  
+  const mime = mimeType.toLowerCase();
+  
+  // Text-based web assets that benefit from compression
+  const compressibleTypes = [
+    'text/html',
+    'text/css',
+    'text/javascript',
+    'application/javascript',
+    'application/x-javascript',
+    'text/xml',
+    'application/xml',
+    'application/json',
+    'text/plain',
+    'image/svg+xml',
+  ];
+  
+  if (compressibleTypes.some(type => mime === type || mime.startsWith(type))) {
+    return true;
+  }
+  
+  // Already-compressed formats that should NOT be double-compressed
+  const alreadyCompressedPrefixes = [
+    'video/',
+    'audio/',
+    'image/',
+    'application/pdf',
+    'application/zip',
+    'application/gzip',
+  ];
+  
+  if (alreadyCompressedPrefixes.some(prefix => mime.startsWith(prefix))) {
+    return false;
+  }
+  
+  // Default to not compressing for unknown types
+  return false;
+}
+
 interface IpldLink {
   $link: string;
 }
···
 
   console.log(`[DEBUG] ${filePath}: fetched ${content.length} bytes, base64=${base64}, encoding=${encoding}, mimeType=${mimeType}`);
 
-  // If content is base64-encoded, decode it back to binary (gzipped or not)
+  // If content is base64-encoded, decode it back to raw binary (gzipped or not)
   if (base64) {
     const originalSize = content.length;
-    // The content from the blob is base64 text, decode it directly to binary
-    const buffer = Buffer.from(content);
-    const base64String = buffer.toString('ascii');  // Use ascii for base64 text, not utf-8
-    console.log(`[DEBUG] ${filePath}: base64 string first 100 chars: ${base64String.substring(0, 100)}`);
+    // Decode base64 directly from raw bytes - no string conversion
+    // The blob contains base64-encoded text as raw bytes, decode it in-place
+    const textDecoder = new TextDecoder();
+    const base64String = textDecoder.decode(content);
     content = Buffer.from(base64String, 'base64');
-    console.log(`[DEBUG] ${filePath}: decoded from ${originalSize} bytes to ${content.length} bytes`);
+    console.log(`[DEBUG] ${filePath}: decoded base64 from ${originalSize} bytes to ${content.length} bytes`);
     
     // Check if it's actually gzipped by looking at magic bytes
     if (content.length >= 2) {
-      const magic = content[0] === 0x1f && content[1] === 0x8b;
-      const byte0 = content[0];
-      const byte1 = content[1];
-      console.log(`[DEBUG] ${filePath}: has gzip magic bytes: ${magic} (0x${byte0?.toString(16)}, 0x${byte1?.toString(16)})`);
+      const hasGzipMagic = content[0] === 0x1f && content[1] === 0x8b;
+      console.log(`[DEBUG] ${filePath}: has gzip magic bytes: ${hasGzipMagic}`);
     }
   }
 
···
     mkdirSync(fileDir, { recursive: true });
   }
 
-  // Determine if this is a web asset that should remain compressed
-  const webAssetTypes = [
-    'text/html', 'text/css', 'application/javascript', 'text/javascript',
-    'application/json', 'text/xml', 'application/xml'
-  ];
-  
-  const isWebAsset = mimeType && webAssetTypes.some(type => 
-    mimeType.toLowerCase().startsWith(type) || mimeType.toLowerCase() === type
-  );
+  // Use the shared function to determine if this should remain compressed
+  const shouldStayCompressed = shouldCompressMimeType(mimeType);
 
-  // Decompress non-web assets that are gzipped
-  if (encoding === 'gzip' && !isWebAsset && content.length >= 2 && 
+  // Decompress files that shouldn't be stored compressed
+  if (encoding === 'gzip' && !shouldStayCompressed && content.length >= 2 && 
       content[0] === 0x1f && content[1] === 0x8b) {
-    console.log(`[DEBUG] ${filePath}: decompressing non-web asset (${mimeType}) before caching`);
+    console.log(`[DEBUG] ${filePath}: decompressing non-compressible type (${mimeType}) before caching`);
     try {
       const { gunzipSync } = await import('zlib');
       const decompressed = gunzipSync(content);
···
       // Clear the encoding flag since we're storing decompressed
       encoding = undefined;
     } catch (error) {
-      console.log(`[DEBUG] ${filePath}: failed to decompress, storing original gzipped content`);
+      console.log(`[DEBUG] ${filePath}: failed to decompress, storing original gzipped content. Error:`, error);
     }
   }
 
   await writeFile(cacheFile, content);
 
-  // Store metadata only if file is still compressed (web assets)
+  // Store metadata only if file is still compressed
   if (encoding === 'gzip' && mimeType) {
     const metaFile = `${cacheFile}.meta`;
     await writeFile(metaFile, JSON.stringify({ encoding, mimeType }));
     console.log('Cached file', filePath, content.length, 'bytes (gzipped,', mimeType + ')');
   } else {
-    console.log('Cached file', filePath, content.length, 'bytes (decompressed)');
+    console.log('Cached file', filePath, content.length, 'bytes');
   }
 }
 

hosting-service/src/server.ts

···
 import { Hono } from 'hono';
 import { getWispDomain, getCustomDomain, getCustomDomainByHash } from './lib/db';
-import { resolveDid, getPdsForDid, fetchSiteRecord, downloadAndCacheSite, getCachedFilePath, isCached, sanitizePath } from './lib/utils';
+import { resolveDid, getPdsForDid, fetchSiteRecord, downloadAndCacheSite, getCachedFilePath, isCached, sanitizePath, shouldCompressMimeType } from './lib/utils';
 import { rewriteHtmlPaths, isHtmlContent } from './lib/html-rewriter';
 import { existsSync, readFileSync } from 'fs';
 import { lookup } from 'mime-types';
···
       // Check actual content for gzip magic bytes
       if (content.length >= 2) {
         const hasGzipMagic = content[0] === 0x1f && content[1] === 0x8b;
-        const byte0 = content[0];
-        const byte1 = content[1];
-        console.log(`[DEBUG SERVE] ${requestPath}: has gzip magic bytes=${hasGzipMagic} (0x${byte0?.toString(16)}, 0x${byte1?.toString(16)})`);
+        console.log(`[DEBUG SERVE] ${requestPath}: has gzip magic bytes=${hasGzipMagic}`);
       }
       
       if (meta.encoding === 'gzip' && meta.mimeType) {
-        // Don't serve already-compressed media formats with Content-Encoding: gzip
-        // These formats (video, audio, images) are already compressed and the browser
-        // can't decode them if we add another layer of compression
-        const alreadyCompressedTypes = [
-          'video/', 'audio/', 'image/jpeg', 'image/jpg', 'image/png', 
-          'image/gif', 'image/webp', 'application/pdf'
-        ];
+        // Use shared function to determine if this should be served compressed
+        const shouldServeCompressed = shouldCompressMimeType(meta.mimeType);
         
-        const isAlreadyCompressed = alreadyCompressedTypes.some(type => 
-          meta.mimeType.toLowerCase().startsWith(type)
-        );
-        
-        if (isAlreadyCompressed) {
-          // Decompress the file before serving
-          console.log(`[DEBUG SERVE] ${requestPath}: decompressing already-compressed media type`);
+        if (!shouldServeCompressed) {
+          // This shouldn't happen if caching is working correctly, but handle it gracefully
+          console.log(`[DEBUG SERVE] ${requestPath}: decompressing file that shouldn't be compressed (${meta.mimeType})`);
           const { gunzipSync } = await import('zlib');
           const decompressed = gunzipSync(content);
           console.log(`[DEBUG SERVE] ${requestPath}: decompressed from ${content.length} to ${decompressed.length} bytes`);
···
     }
 
     // Check if this is HTML content that needs rewriting
-    // Note: For gzipped HTML with path rewriting, we need to decompress, rewrite, and serve uncompressed
-    // This is a trade-off for the sites.wisp.place domain which needs path rewriting
+    // We decompress, rewrite paths, then recompress for efficient delivery
     if (isHtmlContent(requestPath, mimeType)) {
       let content: string;
       if (isGzipped) {
···
         content = readFileSync(cachedFile, 'utf-8');
       }
       const rewritten = rewriteHtmlPaths(content, basePath);
-      return new Response(rewritten, {
+      
+      // Recompress the HTML for efficient delivery
+      const { gzipSync } = await import('zlib');
+      const recompressed = gzipSync(Buffer.from(rewritten, 'utf-8'));
+      
+      return new Response(recompressed, {
         headers: {
           'Content-Type': 'text/html; charset=utf-8',
+          'Content-Encoding': 'gzip',
         },
       });
     }
···
     // Non-HTML files: serve gzipped content as-is with proper headers
     const content = readFileSync(cachedFile);
     if (isGzipped) {
-      // Don't serve already-compressed media formats with Content-Encoding: gzip
-      const alreadyCompressedTypes = [
-        'video/', 'audio/', 'image/jpeg', 'image/jpg', 'image/png', 
-        'image/gif', 'image/webp', 'application/pdf'
-      ];
-      
-      const isAlreadyCompressed = alreadyCompressedTypes.some(type => 
-        mimeType.toLowerCase().startsWith(type)
-      );
+      // Use shared function to determine if this should be served compressed
+      const shouldServeCompressed = shouldCompressMimeType(mimeType);
       
-      if (isAlreadyCompressed) {
-        // Decompress the file before serving
+      if (!shouldServeCompressed) {
+        // This shouldn't happen if caching is working correctly, but handle it gracefully
         const { gunzipSync } = await import('zlib');
         const decompressed = gunzipSync(content);
         return new Response(decompressed, {
···
         }
       }
 
-      // HTML needs path rewriting, so decompress if needed
+      // HTML needs path rewriting, decompress, rewrite, then recompress
       let content: string;
       if (isGzipped) {
         const { gunzipSync } = await import('zlib');
···
         content = readFileSync(indexFile, 'utf-8');
       }
       const rewritten = rewriteHtmlPaths(content, basePath);
-      return new Response(rewritten, {
+      
+      // Recompress the HTML for efficient delivery
+      const { gzipSync } = await import('zlib');
+      const recompressed = gzipSync(Buffer.from(rewritten, 'utf-8'));
+      
+      return new Response(recompressed, {
         headers: {
           'Content-Type': 'text/html; charset=utf-8',
+          'Content-Encoding': 'gzip',
         },
       });
     }