update

plcbundle.ts

···
 import path from 'path';
 import crypto from 'crypto';
 import { fileURLToPath } from 'url';
-import { init, compress, decompress } from '@bokuweb/zstd-wasm';
+import { init, compress } from '@bokuweb/zstd-wasm';
 import axios from 'axios';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
···
 };
 
 // ============================================================================
-// Bundle Loading
-// ============================================================================
-
-const loadBundle = async (dir: string, bundleNumber: number): Promise<PLCOperation[]> => {
-  const filename = `${String(bundleNumber).padStart(6, '0')}.jsonl.zst`;
-  const filepath = path.join(dir, filename);
-  
-  const compressed = await fs.readFile(filepath);
-  const decompressed = decompress(compressed);
-  const jsonl = Buffer.from(decompressed).toString('utf8');
-  
-  const lines = jsonl.trim().split('\n').filter(l => l);
-  return lines.map(line => {
-    const op = JSON.parse(line) as PLCOperation;
-    op._raw = line;
-    return op;
-  });
-};
-
-// ============================================================================
-// Boundary Handling
-// ============================================================================
-
-const getBoundaryCIDs = (operations: PLCOperation[]): Set<string> => {
-  if (operations.length === 0) return new Set();
-  
-  const lastOp = operations[operations.length - 1];
-  const boundaryTime = lastOp.createdAt;
-  const cidSet = new Set<string>();
-  
-  // Walk backwards from the end to find all operations with the same timestamp
-  for (let i = operations.length - 1; i >= 0; i--) {
-    if (operations[i].createdAt === boundaryTime) {
-      cidSet.add(operations[i].cid);
-    } else {
-      break;
-    }
-  }
-  
-  return cidSet;
-};
-
-const stripBoundaryDuplicates = (
-  operations: PLCOperation[],
-  prevBoundaryCIDs: Set<string>
-): PLCOperation[] => {
-  if (prevBoundaryCIDs.size === 0) return operations;
-  if (operations.length === 0) return operations;
-  
-  const boundaryTime = operations[0].createdAt;
-  let startIdx = 0;
-  
-  // Skip operations that are in the previous bundle's boundary
-  for (let i = 0; i < operations.length; i++) {
-    const op = operations[i];
-    
-    // Stop if we've moved past the boundary timestamp
-    if (op.createdAt > boundaryTime) {
-      break;
-    }
-    
-    // Skip if this CID was in the previous boundary
-    if (op.createdAt === boundaryTime && prevBoundaryCIDs.has(op.cid)) {
-      startIdx = i + 1;
-      continue;
-    }
-    
-    break;
-  }
-  
-  const stripped = operations.slice(startIdx);
-  if (startIdx > 0) {
-    console.log(`    Stripped ${startIdx} boundary duplicates`);
-  }
-  return stripped;
-};
-
-// ============================================================================
 // PLC Directory Client
 // ============================================================================
 
···
 // ============================================================================
 
 const serializeJSONL = (operations: PLCOperation[]): string => {
+  // Each operation followed by \n, but NO trailing newline at the end
+  // This matches the Go implementation exactly
   const lines = operations.map(op => {
     const json = op._raw || JSON.stringify(op);
     return json + '\n';
···
 const calculateChainHash = (parent: string, contentHash: string): string => {
   let data: string;
   if (!parent || parent === '') {
+    // Genesis bundle (first bundle)
     data = `plcbundle:genesis:${contentHash}`;
   } else {
+    // Subsequent bundles - chain parent hash with current content
     data = `${parent}:${contentHash}`;
   }
   return sha256(data);
···
   const filename = `${String(bundleNumber).padStart(6, '0')}.jsonl.zst`;
   const filepath = path.join(dir, filename);
   
+  // Serialize to JSONL (exact format: each line ends with \n, no trailing newline)
   const jsonl = serializeJSONL(operations);
   const uncompressedBuffer = Buffer.from(jsonl, 'utf8');
   
+  console.log(`    JSONL size: ${uncompressedBuffer.length} bytes`);
+  console.log(`    First 100 chars: ${jsonl.substring(0, 100)}`);
+  console.log(`    Last 100 chars: ${jsonl.substring(jsonl.length - 100)}`);
+  
+  // Calculate content hash
   const contentHash = sha256(uncompressedBuffer);
   const uncompressedSize = uncompressedBuffer.length;
   
+  // Calculate chain hash
   const chainHash = calculateChainHash(parentHash, contentHash);
   
+  // Compress with zstd level 3 (same as Go SpeedDefault)
   const compressed = compress(uncompressedBuffer, 3);
   const compressedBuffer = Buffer.from(compressed);
   const compressedHash = sha256(compressedBuffer);
   const compressedSize = compressedBuffer.length;
   
+  // Write file
   await fs.writeFile(filepath, compressedBuffer);
   
+  // Extract metadata
   const startTime = operations[0].createdAt;
   const endTime = operations[operations.length - 1].createdAt;
   const didCount = extractUniqueDIDs(operations);
···
     end_time: endTime,
     operation_count: operations.length,
     did_count: didCount,
-    hash: chainHash,
-    content_hash: contentHash,
-    parent: parentHash || '',
+    hash: chainHash,              // Chain hash (primary)
+    content_hash: contentHash,    // Content hash
+    parent: parentHash || '',     // Parent chain hash
     compressed_hash: compressedHash,
     compressed_size: compressedSize,
     uncompressed_size: uncompressedSize,
···
 // ============================================================================
 
 const run = async (): Promise<void> => {
-  const dir = process.argv[2] || './plc_bundles';
+  const dir = process.argv[2] || './bundles';
   
   console.log('PLC Bundle Fetcher');
   console.log('==================');
···
   
   await fs.mkdir(dir, { recursive: true });
   
+  // Load existing index
   const index = await loadIndex(dir);
   
   let currentBundle = index.last_bundle + 1;
   let cursor: string | null = null;
   let parentHash = '';
-  let prevBoundaryCIDs = new Set<string>();
   
+  // If resuming, get cursor and parent from last bundle
   if (index.bundles.length > 0) {
     const lastBundle = index.bundles[index.bundles.length - 1];
     cursor = lastBundle.end_time;
-    parentHash = lastBundle.hash;
-    
-    try {
-      const prevOps = await loadBundle(dir, lastBundle.bundle_number);
-      prevBoundaryCIDs = getBoundaryCIDs(prevOps);
-      console.log(`Loaded previous bundle boundary: ${prevBoundaryCIDs.size} CIDs`);
-    } catch (err) {
-      console.log(`Could not load previous bundle for boundary detection`);
-    }
-    
+    parentHash = lastBundle.hash; // Chain hash from previous bundle
     console.log(`Resuming from bundle ${currentBundle}`);
     console.log(`Last operation: ${cursor}`);
+    console.log(`Parent hash: ${parentHash}`);
   } else {
     console.log('Starting from the beginning (genesis)');
   }
···
   console.log();
   
   let mempool: PLCOperation[] = [];
-  const seenCIDs = new Set<string>(prevBoundaryCIDs);
   let totalFetched = 0;
   let totalBundles = 0;
   
   while (true) {
     try {
+      // Fetch operations
       console.log(`Fetching operations (cursor: ${cursor || 'start'})...`);
       const operations = await fetchOperations(cursor, 1000);
       
···
         break;
       }
       
-      // Deduplicate
-      const uniqueOps = operations.filter(op => {
-        if (seenCIDs.has(op.cid)) {
-          return false;
-        }
-        seenCIDs.add(op.cid);
-        return true;
-      });
+      console.log(`  Fetched ${operations.length} operations`);
+      totalFetched += operations.length;
       
-      console.log(`  Fetched ${operations.length} operations (${uniqueOps.length} unique)`);
-      totalFetched += uniqueOps.length;
+      // Add to mempool
+      mempool.push(...operations);
       
-      mempool.push(...uniqueOps);
+      // Update cursor
       cursor = operations[operations.length - 1].createdAt;
       
+      // Create bundles while we have enough operations
       while (mempool.length >= BUNDLE_SIZE) {
         const bundleOps = mempool.splice(0, BUNDLE_SIZE);
         
-        console.log(`\nCreating bundle ${String(currentBundle).padStart(6, '0')}...`);
+        console.log(`Creating bundle ${String(currentBundle).padStart(6, '0')}...`);
         
         const metadata = await saveBundle(dir, currentBundle, bundleOps, parentHash);
         
+        // Add to index
         index.bundles.push(metadata);
         index.last_bundle = currentBundle;
         index.total_size_bytes += metadata.compressed_size;
···
         console.log(`  ✓ Bundle ${String(currentBundle).padStart(6, '0')}: ${metadata.operation_count} ops, ${metadata.did_count} DIDs`);
         console.log(`    Chain Hash:   ${metadata.hash}`);
         console.log(`    Content Hash: ${metadata.content_hash}`);
-        console.log(`    Size: ${(metadata.compressed_size / 1024).toFixed(1)} KB`);
-        
-        // Get boundary CIDs for next bundle
-        prevBoundaryCIDs = getBoundaryCIDs(bundleOps);
-        console.log(`    Boundary CIDs: ${prevBoundaryCIDs.size}`);
+        if (metadata.parent) {
+          console.log(`    Parent Hash:  ${metadata.parent}`);
+        } else {
+          console.log(`    Parent Hash:  (genesis)`);
+        }
+        console.log(`    Compressed:   ${metadata.compressed_hash}`);
+        console.log(`    Size: ${(metadata.compressed_size / 1024).toFixed(1)} KB (${(metadata.compressed_size / metadata.uncompressed_size * 100).toFixed(1)}% of original)`);
         console.log();
         
+        // Update parent hash for next bundle
         parentHash = metadata.hash;
+        
         currentBundle++;
         totalBundles++;
       }
       
+      // Small delay to be nice to the server
       await new Promise(resolve => setTimeout(resolve, 100));
       
     } catch (err: any) {
···
     }
   }
   
+  // Save final index
   await saveIndex(dir, index);
   
   console.log();
···
   
   if (mempool.length > 0) {
     console.log();
-    console.log(`Note: ${mempool.length} operations in mempool`);
+    console.log(`Note: ${mempool.length} operations in mempool (need ${BUNDLE_SIZE - mempool.length} more for next bundle)`);
   }
 };