core / objkt / kidlisp-extractor.mjs

#!/usr/bin/env node

// Utility to extract and fetch all $codes from KidLisp source
// Used by the Teia packer to create offline-capable packages

import { promises as fs } from "fs";
import https from "https";

// Extract all $code references from KidLisp source text
function extractCodes(source) {
  const codes = new Set();
  
  // Match $codes in various contexts:
  // - Standalone: $abc123XY
  // - Function calls: ($abc123XY)
  // - With parameters: ($abc123XY 128 128)
  // - In embed calls: (embed $abc123XY)
  // - In jump calls: (jump $abc123XY)
  const codePattern = /\$([0-9A-Za-z]{3,12})\b/g;
  
  let match;
  while ((match = codePattern.exec(source)) !== null) {
    codes.add(match[1]); // Add the code without the $ prefix
  }
  
  return Array.from(codes);
}

// Fetch a single code from the aesthetic.computer API
async function fetchCode(codeId) {
  const url = `https://aesthetic.computer/.netlify/functions/store-kidlisp?code=${codeId}`;
  
  return new Promise((resolve, reject) => {
    const options = { rejectUnauthorized: false }; // Allow self-signed certificates
    
    https.get(url, options, (res) => {
      let data = '';
      
      res.on('data', (chunk) => {
        data += chunk;
      });
      
      res.on('end', () => {
        try {
          const parsed = JSON.parse(data);
          if (parsed && parsed.source) {
            console.log(`✅ Fetched $${codeId} (${parsed.source.length} chars)`);
            resolve({ id: codeId, source: parsed.source });
          } else {
            console.error(`❌ No source found for $${codeId}`);
            resolve(null);
          }
        } catch (err) {
          console.error(`❌ Failed to parse response for $${codeId}:`, err);
          resolve(null);
        }
      });
    }).on('error', (err) => {
      console.error(`❌ Network error fetching $${codeId}:`, err);
      resolve(null);
    });
  });
}

// Recursively fetch all codes and their dependencies
async function fetchAllCodes(initialSource, maxDepth = 5) {
  const allCodes = new Map(); // codeId -> { source, dependencies }
  const toProcess = new Set();
  
  // Start with codes from the initial source
  const initialCodes = extractCodes(initialSource);
  console.log(`🔍 Found ${initialCodes.length} initial codes:`, initialCodes.map(c => `$${c}`).join(', '));
  
  for (const code of initialCodes) {
    toProcess.add(code);
  }
  
  let depth = 0;
  while (toProcess.size > 0 && depth < maxDepth) {
    depth++;
    console.log(`\n📚 Processing depth ${depth}...`);
    
    const currentBatch = Array.from(toProcess);
    toProcess.clear();
    
    // Fetch all codes in parallel
    const promises = currentBatch.map(async (codeId) => {
      if (allCodes.has(codeId)) return null; // Already processed
      
      const result = await fetchCode(codeId);
      if (result) {
        const dependencies = extractCodes(result.source);
        allCodes.set(codeId, {
          source: result.source,
          dependencies
        });
        
        // Add new dependencies to next batch
        for (const dep of dependencies) {
          if (!allCodes.has(dep)) {
            toProcess.add(dep);
          }
        }
        
        if (dependencies.length > 0) {
          console.log(`  $${codeId} depends on: ${dependencies.map(c => `$${c}`).join(', ')}`);
        }
      }
      return result;
    });
    
    await Promise.all(promises);
  }
  
  if (toProcess.size > 0) {
    console.log(`⚠️ Reached max depth ${maxDepth}, ${toProcess.size} codes not processed:`, 
                Array.from(toProcess).map(c => `$${c}`).join(', '));
  }
  
  return allCodes;
}

// Generate JavaScript code to pre-populate the cache
function generateCacheCode(codesMap) {
  const entries = Array.from(codesMap.entries()).map(([id, data]) => {
    return `  "${id}": ${JSON.stringify(data.source)}`;
  }).join(',\n');
  
  return `
// 🗂️ Pre-populated KidLisp code cache for PACK mode
// Generated by kidlisp-extractor.mjs

// Define OBJKT KidLisp codes for kidlisp.mjs to auto-load
// Set on multiple global objects for compatibility with main thread and worker contexts
const objktKidlispCodes = {
${entries}
};

// Set cache on all available global scopes
if (typeof window !== 'undefined') {
  window.objktKidlispCodes = objktKidlispCodes;
}
if (typeof globalThis !== 'undefined') {
  globalThis.objktKidlispCodes = objktKidlispCodes;
}
if (typeof global !== 'undefined') {
  global.objktKidlispCodes = objktKidlispCodes;
}
if (typeof self !== 'undefined') {
  self.objktKidlispCodes = objktKidlispCodes;
}

console.log('🎯 OBJKT cache defined with ${codesMap.size} codes');`;
}

// Main function
async function extractAndFetch(sourceFile, outputFile) {
  try {
    console.log(`📖 Reading source from: ${sourceFile}`);
    const source = await fs.readFile(sourceFile, 'utf8');
    
    console.log(`🔍 Extracting codes from source...`);
    const codesMap = await fetchAllCodes(source);
    
    console.log(`\n📦 Summary:`);
    console.log(`  Total codes fetched: ${codesMap.size}`);
    for (const [id, data] of codesMap) {
      console.log(`    $${id}: ${data.source.length} chars, ${data.dependencies.length} deps`);
    }
    
    const cacheCode = generateCacheCode(codesMap);
    await fs.writeFile(outputFile, cacheCode);
    
    console.log(`\n✅ Cache code written to: ${outputFile}`);
    return { codesMap, cacheCode };
    
  } catch (error) {
    console.error('❌ Error:', error);
    throw error;
  }
}

// CLI usage
if (import.meta.url === `file://${process.argv[1]}`) {
  const [sourceFile, outputFile] = process.argv.slice(2);
  
  if (!sourceFile || !outputFile) {
    console.log('Usage: node kidlisp-extractor.mjs <source-file> <output-file>');
    console.log('Example: node kidlisp-extractor.mjs cow-source.lisp cache-preload.js');
    process.exit(1);
  }
  
  extractAndFetch(sourceFile, outputFile).catch(console.error);
}

export { extractCodes, fetchCode, fetchAllCodes, generateCacheCode };