From de1969d3cd5b63d2353dfd54db24c8149bac5c52 Mon Sep 17 00:00:00 2001 From: Nate Moore Date: Wed, 11 Nov 2026 00:27:33 -0500 Subject: [PATCH] lex: replay tokens from a direct-mapped cache instead of re-lexing after lookahead Every lookahead ends with reseek(mark), so scan() ran about 1.5x per distinct token. scan() now checks a 256-entry table (4KiB at LEXC, below the frame stack; DATA_END moves to 7168) keyed by token begin and replays tb/ts/te/tt/tk/tnl. Only scan's own completed results are stored; rescans and needMore results are not, and init() clears the table for each new stream. ab.mjs: erase +22-34% on 3000 OSS files, downlevel +25-26%; 0 output diffs. erase.wasm +190 B raw / +117 B gz, downlevel.wasm +191 B / +123 B. --- lite/keywords.zena | 5 +++- lite/lex.zena | 43 +++++++++++++++++++++++++++++++++- lite/strip.zena | 3 ++- notes/speed-log.tsv | 1 + scripts/gen/downlevel-text.mjs | 4 ++-- scripts/gen/lite-keywords.mjs | 5 +++- scripts/gen/lite-layout.mjs | 6 ++++- 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/lite/keywords.zena b/lite/keywords.zena index a99cce9..78496b6 100644 --- a/lite/keywords.zena +++ b/lite/keywords.zena @@ -140,7 +140,10 @@ export let TS_OPTIONAL_FOLLOW: i64 = 0x1000000000182000; export let TS_MEMBER_NAME: i64 = 0x20e; export let TS_POSTFIX_UPDATE: i64 = 0x1a000000; // The static data below the frame stack: strip.zena starts the stack at DATA_END. -export let DATA_END: i32 = 3072; +export let DATA_END: i32 = 7168; +// The lexer's token replay cache: LEXC_MASK + 1 entries of 16 bytes at LEXC. +export let LEXC: i32 = 3072; +export let LEXC_MASK: i32 = 255; let KW_SLOT: i32 = 1024; let KW_OFF: i32 = 1280; let KW_TEXT: i32 = 1452; diff --git a/lite/lex.zena b/lite/lex.zena index 058a9a4..f0d7665 100644 --- a/lite/lex.zena +++ b/lite/lex.zena @@ -7,7 +7,7 @@ // (regex, template continuation, `>` compounds) because it knows which state // it is in. import { Memory } from 'zena:memory'; -import { keywordAt, tokenTypeIn, PUNCT, PUNCT_ENT } from './keywords.zena'; +import { keywordAt, tokenTypeIn, PUNCT, PUNCT_ENT, LEXC, LEXC_MASK } from './keywords.zena'; export var delta: i32 = 0; export var inEnd: i32 = 0; @@ -223,7 +223,48 @@ function finish(t: i32, e: i32, start: i32): void { pos = e; } +// Token replay cache. Lookahead ends by reseeking to a mark, so most tokens are lexed again; scan +// replays them from a direct-mapped table instead. Entry at LEXC + (tb & LEXC_MASK) * 16 holds +// [tb, ts, te, tt | tk << 8 | tnl << 16] for scan's own result at tb (never a rescan's, never a +// needMore). Scan is a pure function of the bytes at stream offsets, and a token it completes +// before `fin` never depends on bytes pushed later (it is at least 3 bytes short of the buffered +// end), so an entry stays valid for the whole stream: only init's new stream clears it. +export function lexReset(): void { + var i = 0; + while (i <= LEXC_MASK) { + Memory.default.setI32(LEXC + (i << 4), -1); + i = i + 1; + } +} + export function scan(): void { + let start = pos; + let c = LEXC + ((start & LEXC_MASK) << 4); + if (Memory.default.getI32(c) == start) { + tb = start; + ts = Memory.default.getI32(c + 4); + let e = Memory.default.getI32(c + 8); + let k = Memory.default.getI32(c + 12); + te = e; + pos = e; + tt = k & 255; + tk = (k >> 8) & 255; + tnl = k >> 16; + return; + } + let m = more; + more = 0; + lexToken(); + if (more == 0) { + Memory.default.setI32(c, start); + Memory.default.setI32(c + 4, ts); + Memory.default.setI32(c + 8, te); + Memory.default.setI32(c + 12, tt | (tk << 8) | (tnl << 16)); + more = m; + } +} + +function lexToken(): void { let start = pos; tb = start; var p = start; diff --git a/lite/strip.zena b/lite/strip.zena index a5121fc..669da35 100644 --- a/lite/strip.zena +++ b/lite/strip.zena @@ -9,7 +9,7 @@ // the lexer failing on the token after them leaves every frame consistent. import { Memory } from 'zena:memory'; import { - tokenIn, setInput, scan, seek, reseek, clearMore, more, pos, tb, ts, te, tt, tk, tnl, inEnd, + tokenIn, setInput, scan, lexReset, seek, reseek, clearMore, more, pos, tb, ts, te, tt, tk, tnl, inEnd, rescanRegex, rescanTemplate, rescanGreater, rescanLess, isBinop, isAssign, precOf, T_EOF, T_NAME, T_NUM, T_STR, T_TEMPLATE, T_TEMPLATE_HEAD, T_TEMPLATE_TAIL, T_REGEX, T_PRIVATE, T_LBRACE, T_RBRACE, T_LPAREN, T_RPAREN, T_LBRACK, T_RBRACK, T_DOT, T_ELLIPSIS, T_SEMI, T_COMMA, T_COLON, @@ -3970,6 +3970,7 @@ export function init(m: i32): void { lexFinal = 0; lastValues = 0; hyInit(); + lexReset(); setInput(inBase, 0, 0); seek(0); clearMore(); diff --git a/notes/speed-log.tsv b/notes/speed-log.tsv index 6943a7c..c7ed24b 100644 --- a/notes/speed-log.tsv +++ b/notes/speed-log.tsv @@ -1,3 +1,4 @@ hypothesis change before_ms after_ms kept drop-bug-compat-apply fix the two ts-blank-space out-of-order edit cases (property ?/!/colon-type before the name, method ? before ) in erase.zena so the edit list is always ascending; delete inPlace() and erase.zena's dual-buffer branch in strip.zena; rewrite apply() to a single forward in-place pass that skips copying kept spans entirely (offset arithmetic instead of a per-unit self-copy loop) 12.4 10.3 yes (also shrank strip.wasm: 48465 -> 47978 raw, 18090 -> 17972 gz; verified inPlace() never false across the corpus and 82304 real .ts files under ~/Developer before deleting the fallback) size-pass-6dfe829 the 11 kept size changes (notes/size-architecture.md), bench/metrics.mjs ts-stripper with build/ swapped base vs final in one session: erase lib.dom 108 -> 110 MB/s, corpus 84 -> 86 MB/s, RSS after load +1.8 -> +1.8MiB; transform lib.dom 106 -> 109, corpus 83 -> 85, RSS +1.9 -> +2.0MiB. metrics first call (one sample) read 2.70 -> 2.84ms erase and 3.08 -> 5.06ms transform; 28 fresh-process repeats on the same 2.9KB file put first call at 1.86-2.10 -> 1.70-1.97ms erase and 1.96-2.32 -> 1.85-2.20ms transform, so the 5.06 was an outlier. Interleaved min-of-21 over a 23MB vendor/oss sample: erase 381 -> 385ms, transform 396 -> 386ms (noise is about 1%) 108 110 yes +lex-replay-cache scan() was ~57% of engine CPU and ran 1.54x per distinct token (median OSS file) because every lookahead ends in reseek(mark). scan() now first checks a 256-entry direct-mapped table at LEXC (3072, 4KiB; DATA_END moves 3072 -> 7168) keyed by tb & 255, holding [tb, ts, te, tt|tk<<8|tnl<<16] of scan's own result; the old body is lexToken(). Stored only when lexToken completes (more stays 0); rescans never touch it since scan is a pure function of the bytes at a stream offset and a token completed before fin is >=3 bytes short of inEnd. init() clears keys to -1. Sizes 64/1024 measured within noise of 256. ab.mjs vs base, 3 runs each: erase oss 3000 +28.0/+34.1/+22.1%, lib.dom +14.2/+4.2/+11.8%; downlevel oss +25.9/+25.7/+25.3%, lib.dom +10.7/+10.7/+10.5%; 0 output diffs. Size: erase 27517 -> 27707 raw, 12091 -> 12208 gz; downlevel 39169 -> 39360 raw, 17194 -> 17317 gz erase oss 53.5/47.8/65.8 MB/s 68.5/64.1/80.4 MB/s yes diff --git a/scripts/gen/downlevel-text.mjs b/scripts/gen/downlevel-text.mjs index d7316e8..3b11230 100644 --- a/scripts/gen/downlevel-text.mjs +++ b/scripts/gen/downlevel-text.mjs @@ -3,7 +3,7 @@ // into build/downlevel.wasm. `@` is the name entry passed to the emitter. (Zena strings would add a // `wasm:js-string` import.) import fs from "node:fs"; -import { TEXT_BASE, DATA_END, dataModule } from "./lite-layout.mjs"; +import { TEXT_BASE, TEXT_END, dataModule } from "./lite-layout.mjs"; const TEMPLATES = { litVar: "var", @@ -75,4 +75,4 @@ const b = src.indexOf(end); if (a < 0 || b < 0) throw new Error("downlevel.zena: GENERATED markers not found"); fs.writeFileSync(file, src.slice(0, a + begin.length) + out + src.slice(b)); const wat = new URL("../../lite/downlevel-text.wat", import.meta.url); -fs.writeFileSync(wat, dataModule("downlevel-text.mjs", TEXT_BASE, Buffer.from(text, "latin1"), DATA_END)); +fs.writeFileSync(wat, dataModule("downlevel-text.mjs", TEXT_BASE, Buffer.from(text, "latin1"), TEXT_END)); diff --git a/scripts/gen/lite-keywords.mjs b/scripts/gen/lite-keywords.mjs index 54fe1a5..db7bf28 100644 --- a/scripts/gen/lite-keywords.mjs +++ b/scripts/gen/lite-keywords.mjs @@ -2,7 +2,7 @@ // lookup's table, a data segment scripts/build-lite.sh merges in). Run: node scripts/gen/lite-keywords.mjs import fs from "node:fs"; import path from "node:path"; -import { KW_BASE, TEXT_BASE, DATA_END, dataModule } from "./lite-layout.mjs"; +import { KW_BASE, TEXT_BASE, DATA_END, LEXC_BASE, LEXC_SIZE, dataModule } from "./lite-layout.mjs"; // Reserved words first: everything below K_LET cannot be an identifier. const reserved = `break case catch class const continue debugger default delete do else enum export extends false @@ -213,6 +213,9 @@ const data = Buffer.concat([slot, offs, text, Buffer.alloc(16), pidx, pent]); out += `// The static data below the frame stack: strip.zena starts the stack at DATA_END. export let DATA_END: i32 = ${DATA_END}; +// The lexer's token replay cache: LEXC_MASK + 1 entries of 16 bytes at LEXC. +export let LEXC: i32 = ${LEXC_BASE}; +export let LEXC_MASK: i32 = ${LEXC_SIZE - 1}; let KW_SLOT: i32 = ${KW_SLOT}; let KW_OFF: i32 = ${KW_OFF}; let KW_TEXT: i32 = ${KW_TEXT}; diff --git a/scripts/gen/lite-layout.mjs b/scripts/gen/lite-layout.mjs index fb21e56..418fc59 100644 --- a/scripts/gen/lite-layout.mjs +++ b/scripts/gen/lite-layout.mjs @@ -5,7 +5,11 @@ // of the ~1.3x of i64 constants in code. export const KW_BASE = 1024; // keyword and punctuator tables: lite-keywords.mjs export const TEXT_BASE = 2304; // downlevel's text templates (build/downlevel.wasm only): downlevel-text.mjs -export const DATA_END = 3072; +export const TEXT_END = 3072; +// The lexer's token replay cache (lite/lex.zena): LEXC_SIZE 16-byte entries, direct-mapped by stream offset. +export const LEXC_BASE = TEXT_END; +export const LEXC_SIZE = 256; +export const DATA_END = LEXC_BASE + LEXC_SIZE * 16; // Writes a WAT module holding `bytes` as one active segment at `base` of the imported memory. export function dataModule(comment, base, bytes, limit) { -- knotmirror 0.0.1