fist commit

.gitignore

···
+node_modules
+.DS_Store
+plc_bundles

package.json

···
+{
+  "name": "plcbundle-js",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@bokuweb/zstd-wasm": "^0.0.27",
+    "axios": "^1.13.0"
+  }
+}

plcbundle.ts

···
+#!/usr/bin/env node
+
+/**
+ * plcbundle.ts - Fetch from PLC Directory and create verifiable bundles
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
+import { fileURLToPath } from 'url';
+import { init, compress, decompress } from '@bokuweb/zstd-wasm';
+import axios from 'axios';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const BUNDLE_SIZE = 10000;
+const INDEX_FILE = 'plc_bundles.json';
+const PLC_URL = 'https://plc.directory';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+interface PLCOperation {
+  did: string;
+  cid: string;
+  createdAt: string;
+  operation: Record<string, any>;
+  nullified?: boolean | string;
+  _raw?: string;
+}
+
+interface BundleMetadata {
+  bundle_number: number;
+  start_time: string;
+  end_time: string;
+  operation_count: number;
+  did_count: number;
+  hash: string;
+  content_hash: string;
+  parent: string;
+  compressed_hash: string;
+  compressed_size: number;
+  uncompressed_size: number;
+  created_at: string;
+}
+
+interface Index {
+  version: string;
+  last_bundle: number;
+  updated_at: string;
+  total_size_bytes: number;
+  bundles: BundleMetadata[];
+}
+
+// Initialize zstd
+await init();
+
+// ============================================================================
+// Index Management
+// ============================================================================
+
+const loadIndex = async (dir: string): Promise<Index> => {
+  try {
+    const data = await fs.readFile(path.join(dir, INDEX_FILE), 'utf8');
+    return JSON.parse(data);
+  } catch (err) {
+    return {
+      version: '1.0',
+      last_bundle: 0,
+      updated_at: new Date().toISOString(),
+      total_size_bytes: 0,
+      bundles: []
+    };
+  }
+};
+
+const saveIndex = async (dir: string, index: Index): Promise<void> => {
+  index.updated_at = new Date().toISOString();
+  const indexPath = path.join(dir, INDEX_FILE);
+  const tempPath = indexPath + '.tmp';
+  await fs.writeFile(tempPath, JSON.stringify(index, null, 2));
+  await fs.rename(tempPath, indexPath);
+};
+
+// ============================================================================
+// Bundle Loading
+// ============================================================================
+
+const loadBundle = async (dir: string, bundleNumber: number): Promise<PLCOperation[]> => {
+  const filename = `${String(bundleNumber).padStart(6, '0')}.jsonl.zst`;
+  const filepath = path.join(dir, filename);
+  
+  const compressed = await fs.readFile(filepath);
+  const decompressed = decompress(compressed);
+  const jsonl = Buffer.from(decompressed).toString('utf8');
+  
+  const lines = jsonl.trim().split('\n').filter(l => l);
+  return lines.map(line => {
+    const op = JSON.parse(line) as PLCOperation;
+    op._raw = line;
+    return op;
+  });
+};
+
+// ============================================================================
+// Boundary Handling
+// ============================================================================
+
+const getBoundaryCIDs = (operations: PLCOperation[]): Set<string> => {
+  if (operations.length === 0) return new Set();
+  
+  const lastOp = operations[operations.length - 1];
+  const boundaryTime = lastOp.createdAt;
+  const cidSet = new Set<string>();
+  
+  // Walk backwards from the end to find all operations with the same timestamp
+  for (let i = operations.length - 1; i >= 0; i--) {
+    if (operations[i].createdAt === boundaryTime) {
+      cidSet.add(operations[i].cid);
+    } else {
+      break;
+    }
+  }
+  
+  return cidSet;
+};
+
+const stripBoundaryDuplicates = (
+  operations: PLCOperation[],
+  prevBoundaryCIDs: Set<string>
+): PLCOperation[] => {
+  if (prevBoundaryCIDs.size === 0) return operations;
+  if (operations.length === 0) return operations;
+  
+  const boundaryTime = operations[0].createdAt;
+  let startIdx = 0;
+  
+  // Skip operations that are in the previous bundle's boundary
+  for (let i = 0; i < operations.length; i++) {
+    const op = operations[i];
+    
+    // Stop if we've moved past the boundary timestamp
+    if (op.createdAt > boundaryTime) {
+      break;
+    }
+    
+    // Skip if this CID was in the previous boundary
+    if (op.createdAt === boundaryTime && prevBoundaryCIDs.has(op.cid)) {
+      startIdx = i + 1;
+      continue;
+    }
+    
+    break;
+  }
+  
+  const stripped = operations.slice(startIdx);
+  if (startIdx > 0) {
+    console.log(`    Stripped ${startIdx} boundary duplicates`);
+  }
+  return stripped;
+};
+
+// ============================================================================
+// PLC Directory Client
+// ============================================================================
+
+const fetchOperations = async (after: string | null, count: number = 1000): Promise<PLCOperation[]> => {
+  const params: Record<string, any> = { count };
+  if (after) {
+    params.after = after;
+  }
+  
+  const response = await axios.get<string>(`${PLC_URL}/export`, {
+    params,
+    responseType: 'text'
+  });
+  
+  const lines = response.data.trim().split('\n').filter(l => l);
+  
+  return lines.map(line => {
+    const op = JSON.parse(line) as PLCOperation;
+    op._raw = line; // Preserve exact JSON
+    return op;
+  });
+};
+
+// ============================================================================
+// Bundle Operations
+// ============================================================================
+
+const serializeJSONL = (operations: PLCOperation[]): string => {
+  const lines = operations.map(op => {
+    const json = op._raw || JSON.stringify(op);
+    return json + '\n';
+  });
+  return lines.join('');
+};
+
+const sha256 = (data: Buffer | string): string => {
+  return crypto.createHash('sha256').update(data).digest('hex');
+};
+
+const calculateChainHash = (parent: string, contentHash: string): string => {
+  let data: string;
+  if (!parent || parent === '') {
+    data = `plcbundle:genesis:${contentHash}`;
+  } else {
+    data = `${parent}:${contentHash}`;
+  }
+  return sha256(data);
+};
+
+const extractUniqueDIDs = (operations: PLCOperation[]): number => {
+  const dids = new Set<string>();
+  operations.forEach(op => dids.add(op.did));
+  return dids.size;
+};
+
+const saveBundle = async (
+  dir: string,
+  bundleNumber: number,
+  operations: PLCOperation[],
+  parentHash: string
+): Promise<BundleMetadata> => {
+  const filename = `${String(bundleNumber).padStart(6, '0')}.jsonl.zst`;
+  const filepath = path.join(dir, filename);
+  
+  const jsonl = serializeJSONL(operations);
+  const uncompressedBuffer = Buffer.from(jsonl, 'utf8');
+  
+  const contentHash = sha256(uncompressedBuffer);
+  const uncompressedSize = uncompressedBuffer.length;
+  
+  const chainHash = calculateChainHash(parentHash, contentHash);
+  
+  const compressed = compress(uncompressedBuffer, 3);
+  const compressedBuffer = Buffer.from(compressed);
+  const compressedHash = sha256(compressedBuffer);
+  const compressedSize = compressedBuffer.length;
+  
+  await fs.writeFile(filepath, compressedBuffer);
+  
+  const startTime = operations[0].createdAt;
+  const endTime = operations[operations.length - 1].createdAt;
+  const didCount = extractUniqueDIDs(operations);
+  
+  return {
+    bundle_number: bundleNumber,
+    start_time: startTime,
+    end_time: endTime,
+    operation_count: operations.length,
+    did_count: didCount,
+    hash: chainHash,
+    content_hash: contentHash,
+    parent: parentHash || '',
+    compressed_hash: compressedHash,
+    compressed_size: compressedSize,
+    uncompressed_size: uncompressedSize,
+    created_at: new Date().toISOString()
+  };
+};
+
+// ============================================================================
+// Main Logic
+// ============================================================================
+
+const run = async (): Promise<void> => {
+  const dir = process.argv[2] || './plc_bundles';
+  
+  console.log('PLC Bundle Fetcher');
+  console.log('==================');
+  console.log();
+  console.log(`Directory: ${dir}`);
+  console.log(`Source:    ${PLC_URL}`);
+  console.log();
+  
+  await fs.mkdir(dir, { recursive: true });
+  
+  const index = await loadIndex(dir);
+  
+  let currentBundle = index.last_bundle + 1;
+  let cursor: string | null = null;
+  let parentHash = '';
+  let prevBoundaryCIDs = new Set<string>();
+  
+  if (index.bundles.length > 0) {
+    const lastBundle = index.bundles[index.bundles.length - 1];
+    cursor = lastBundle.end_time;
+    parentHash = lastBundle.hash;
+    
+    try {
+      const prevOps = await loadBundle(dir, lastBundle.bundle_number);
+      prevBoundaryCIDs = getBoundaryCIDs(prevOps);
+      console.log(`Loaded previous bundle boundary: ${prevBoundaryCIDs.size} CIDs`);
+    } catch (err) {
+      console.log(`Could not load previous bundle for boundary detection`);
+    }
+    
+    console.log(`Resuming from bundle ${currentBundle}`);
+    console.log(`Last operation: ${cursor}`);
+  } else {
+    console.log('Starting from the beginning (genesis)');
+  }
+  
+  console.log();
+  
+  let mempool: PLCOperation[] = [];
+  const seenCIDs = new Set<string>(prevBoundaryCIDs);
+  let totalFetched = 0;
+  let totalBundles = 0;
+  
+  while (true) {
+    try {
+      console.log(`Fetching operations (cursor: ${cursor || 'start'})...`);
+      const operations = await fetchOperations(cursor, 1000);
+      
+      if (operations.length === 0) {
+        console.log('No more operations available');
+        break;
+      }
+      
+      // Deduplicate
+      const uniqueOps = operations.filter(op => {
+        if (seenCIDs.has(op.cid)) {
+          return false;
+        }
+        seenCIDs.add(op.cid);
+        return true;
+      });
+      
+      console.log(`  Fetched ${operations.length} operations (${uniqueOps.length} unique)`);
+      totalFetched += uniqueOps.length;
+      
+      mempool.push(...uniqueOps);
+      cursor = operations[operations.length - 1].createdAt;
+      
+      while (mempool.length >= BUNDLE_SIZE) {
+        const bundleOps = mempool.splice(0, BUNDLE_SIZE);
+        
+        console.log(`\nCreating bundle ${String(currentBundle).padStart(6, '0')}...`);
+        
+        const metadata = await saveBundle(dir, currentBundle, bundleOps, parentHash);
+        
+        index.bundles.push(metadata);
+        index.last_bundle = currentBundle;
+        index.total_size_bytes += metadata.compressed_size;
+        
+        await saveIndex(dir, index);
+        
+        console.log(`  ✓ Bundle ${String(currentBundle).padStart(6, '0')}: ${metadata.operation_count} ops, ${metadata.did_count} DIDs`);
+        console.log(`    Chain Hash:   ${metadata.hash}`);
+        console.log(`    Content Hash: ${metadata.content_hash}`);
+        console.log(`    Size: ${(metadata.compressed_size / 1024).toFixed(1)} KB`);
+        
+        // Get boundary CIDs for next bundle
+        prevBoundaryCIDs = getBoundaryCIDs(bundleOps);
+        console.log(`    Boundary CIDs: ${prevBoundaryCIDs.size}`);
+        console.log();
+        
+        parentHash = metadata.hash;
+        currentBundle++;
+        totalBundles++;
+      }
+      
+      await new Promise(resolve => setTimeout(resolve, 100));
+      
+    } catch (err: any) {
+      console.error(`Error: ${err.message}`);
+      
+      if (err.response) {
+        console.error(`HTTP Status: ${err.response.status}`);
+      }
+      
+      if (err.code === 'ECONNRESET' || err.code === 'ECONNABORTED') {
+        console.log('Connection error, retrying in 5 seconds...');
+        await new Promise(resolve => setTimeout(resolve, 5000));
+        continue;
+      }
+      
+      break;
+    }
+  }
+  
+  await saveIndex(dir, index);
+  
+  console.log();
+  console.log('================');
+  console.log('Complete!');
+  console.log('================');
+  console.log(`Total operations fetched: ${totalFetched}`);
+  console.log(`Bundles created: ${totalBundles}`);
+  console.log(`Total bundles: ${index.bundles.length}`);
+  console.log(`Mempool: ${mempool.length} operations`);
+  console.log(`Total size: ${(index.total_size_bytes / 1024 / 1024).toFixed(1)} MB`);
+  
+  if (mempool.length > 0) {
+    console.log();
+    console.log(`Note: ${mempool.length} operations in mempool`);
+  }
+};
+
+// ============================================================================
+// Entry Point
+// ============================================================================
+
+run().catch(err => {
+  console.error('Fatal error:', err.message);
+  console.error(err.stack);
+  process.exit(1);
+});

pnpm-lock.yaml

···
+lockfileVersion: '9.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+importers:
+
+  .:
+    dependencies:
+      '@bokuweb/zstd-wasm':
+        specifier: ^0.0.27
+        version: 0.0.27
+      axios:
+        specifier: ^1.13.0
+        version: 1.13.0
+
+packages:
+
+  '@bokuweb/zstd-wasm@0.0.27':
+    resolution: {integrity: sha512-GDm2uOTK3ESjnYmSeLQifJnBsRCWajKLvN32D2ZcQaaCIJI/Hse9s74f7APXjHit95S10UImsRGkTsbwHmrtmg==}
+
+  asynckit@0.4.0:
+    resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
+
+  axios@1.13.0:
+    resolution: {integrity: sha512-zt40Pz4zcRXra9CVV31KeyofwiNvAbJ5B6YPz9pMJ+yOSLikvPT4Yi5LjfgjRa9CawVYBaD1JQzIVcIvBejKeA==}
+
+  call-bind-apply-helpers@1.0.2:
+    resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==}
+    engines: {node: '>= 0.4'}
+
+  combined-stream@1.0.8:
+    resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
+    engines: {node: '>= 0.8'}
+
+  delayed-stream@1.0.0:
+    resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
+    engines: {node: '>=0.4.0'}
+
+  dunder-proto@1.0.1:
+    resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
+    engines: {node: '>= 0.4'}
+
+  es-define-property@1.0.1:
+    resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==}
+    engines: {node: '>= 0.4'}
+
+  es-errors@1.3.0:
+    resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==}
+    engines: {node: '>= 0.4'}
+
+  es-object-atoms@1.1.1:
+    resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==}
+    engines: {node: '>= 0.4'}
+
+  es-set-tostringtag@2.1.0:
+    resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==}
+    engines: {node: '>= 0.4'}
+
+  follow-redirects@1.15.11:
+    resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==}
+    engines: {node: '>=4.0'}
+    peerDependencies:
+      debug: '*'
+    peerDependenciesMeta:
+      debug:
+        optional: true
+
+  form-data@4.0.4:
+    resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==}
+    engines: {node: '>= 6'}
+
+  function-bind@1.1.2:
+    resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
+
+  get-intrinsic@1.3.0:
+    resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==}
+    engines: {node: '>= 0.4'}
+
+  get-proto@1.0.1:
+    resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==}
+    engines: {node: '>= 0.4'}
+
+  gopd@1.2.0:
+    resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==}
+    engines: {node: '>= 0.4'}
+
+  has-symbols@1.1.0:
+    resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==}
+    engines: {node: '>= 0.4'}
+
+  has-tostringtag@1.0.2:
+    resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}
+    engines: {node: '>= 0.4'}
+
+  hasown@2.0.2:
+    resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
+    engines: {node: '>= 0.4'}
+
+  math-intrinsics@1.1.0:
+    resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==}
+    engines: {node: '>= 0.4'}
+
+  mime-db@1.52.0:
+    resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
+    engines: {node: '>= 0.6'}
+
+  mime-types@2.1.35:
+    resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
+    engines: {node: '>= 0.6'}
+
+  proxy-from-env@1.1.0:
+    resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
+
+snapshots:
+
+  '@bokuweb/zstd-wasm@0.0.27': {}
+
+  asynckit@0.4.0: {}
+
+  axios@1.13.0:
+    dependencies:
+      follow-redirects: 1.15.11
+      form-data: 4.0.4
+      proxy-from-env: 1.1.0
+    transitivePeerDependencies:
+      - debug
+
+  call-bind-apply-helpers@1.0.2:
+    dependencies:
+      es-errors: 1.3.0
+      function-bind: 1.1.2
+
+  combined-stream@1.0.8:
+    dependencies:
+      delayed-stream: 1.0.0
+
+  delayed-stream@1.0.0: {}
+
+  dunder-proto@1.0.1:
+    dependencies:
+      call-bind-apply-helpers: 1.0.2
+      es-errors: 1.3.0
+      gopd: 1.2.0
+
+  es-define-property@1.0.1: {}
+
+  es-errors@1.3.0: {}
+
+  es-object-atoms@1.1.1:
+    dependencies:
+      es-errors: 1.3.0
+
+  es-set-tostringtag@2.1.0:
+    dependencies:
+      es-errors: 1.3.0
+      get-intrinsic: 1.3.0
+      has-tostringtag: 1.0.2
+      hasown: 2.0.2
+
+  follow-redirects@1.15.11: {}
+
+  form-data@4.0.4:
+    dependencies:
+      asynckit: 0.4.0
+      combined-stream: 1.0.8
+      es-set-tostringtag: 2.1.0
+      hasown: 2.0.2
+      mime-types: 2.1.35
+
+  function-bind@1.1.2: {}
+
+  get-intrinsic@1.3.0:
+    dependencies:
+      call-bind-apply-helpers: 1.0.2
+      es-define-property: 1.0.1
+      es-errors: 1.3.0
+      es-object-atoms: 1.1.1
+      function-bind: 1.1.2
+      get-proto: 1.0.1
+      gopd: 1.2.0
+      has-symbols: 1.1.0
+      hasown: 2.0.2
+      math-intrinsics: 1.1.0
+
+  get-proto@1.0.1:
+    dependencies:
+      dunder-proto: 1.0.1
+      es-object-atoms: 1.1.1
+
+  gopd@1.2.0: {}
+
+  has-symbols@1.1.0: {}
+
+  has-tostringtag@1.0.2:
+    dependencies:
+      has-symbols: 1.1.0
+
+  hasown@2.0.2:
+    dependencies:
+      function-bind: 1.1.2
+
+  math-intrinsics@1.1.0: {}
+
+  mime-db@1.52.0: {}
+
+  mime-types@2.1.35:
+    dependencies:
+      mime-db: 1.52.0
+
+  proxy-from-env@1.1.0: {}