import { mkdir, writeFile } from 'node:fs/promises'; import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; const VERSION = '18.0.0'; const BASE = `https://www.unicode.org/Public/${VERSION}/ucd/`; const inputs = [ ['auxiliary/GraphemeBreakProperty.txt', 'gcb'], ['emoji/emoji-data.txt', 'emoji'], ['DerivedCoreProperties.txt', 'incb'], ['auxiliary/GraphemeBreakTest.txt', 'test'], ]; await mkdir('unicode', { recursive: true }); const data = {}; const hashes = []; for (const [path, key] of inputs) { const response = await fetch(BASE + path); if (!response.ok) throw new Error(`${response.status}: ${path}`); data[key] = await response.text(); await writeFile(`unicode/${key}.txt`, data[key]); hashes.push(`${createHash('sha256').update(data[key]).digest('hex')} ${key}.txt`); } const license = await fetch('https://www.unicode.org/license.txt'); if (!license.ok) throw new Error('Cannot read the Unicode license'); await writeFile('unicode/LICENSE.txt', await license.text()); await writeFile('unicode/SHA256SUMS', `${hashes.join('\n')}\n`); const properties = ['Other', 'CR', 'LF', 'Control', 'Extend', 'ZWJ', 'Regional_Indicator', 'Prepend', 'SpacingMark', 'L', 'V', 'T', 'LV', 'LVT']; function ranges(source, select) { return source.split('\n').flatMap(line => { const fields = line.split('#')[0].trim().split(';').map(value => value.trim()); if (fields.length < 2) return []; const value = select(fields); if (value === null) return []; const [low, high = low] = fields[0].split('..').map(value => parseInt(value, 16)); return [{ low, high, value }]; }).sort((a, b) => a.low - b.low); } const gcb = ranges(data.gcb, fields => properties.indexOf(fields[1])); const emoji = ranges(data.emoji, fields => fields[1] === 'Extended_Pictographic' ? 1 : null); const incb = ranges(data.incb, fields => fields[1] === 'InCB' ? ['None', 'Consonant', 'Extend', 'Linker'].indexOf(fields[2]) : null); const table = (name, entries) => `const ${name}: UnicodeRange[${entries.length}] = [\n${entries.map(({ low, high, value }) => ` UnicodeRange { ${low}, ${high}, ${value} },`).join('\n')}\n];\n`; await writeFile('unicode_tables.goose', `// Generated from Unicode ${VERSION}. See unicode/LICENSE.txt and tools/unicode.mjs.\nstruct UnicodeRange { low: i64, high: i64, value: i64 }\n${table('GRAPHEME_BREAK', gcb)}${table('EXTENDED_PICTOGRAPHIC', emoji)}${table('INDIC_CONJUNCT', incb)}`); const tests = []; for (const line of data.test.split('\n')) { const tokens = line.split('#')[0].trim().split(/\s+/); if (tokens[0] !== '÷') continue; let string = ''; const boundaries = []; for (const token of tokens) { if (token === '÷') boundaries.push(Buffer.byteLength(string)); else if (token !== '×') string += String.fromCodePoint(parseInt(token, 16)); } const bytes = [...Buffer.from(string)].map(value => `\\x${value.toString(16).padStart(2, '0')}`).join(''); tests.push(` cluster_case("${bytes}", [${boundaries.join(', ')}]);`); } await writeFile('grapheme_cases.goose', `// Generated Unicode ${VERSION} UAX #29 conformance cases. See unicode/LICENSE.txt.\nfn grapheme_cases() {\n${tests.join('\n')}\n}\n`); execFileSync('goose', ['format', 'unicode_tables.goose', 'grapheme_cases.goose']); console.log(`Generated Unicode ${VERSION} tables and ${tests.length} conformance cases`);